From 8b0398c0db987bc35d33d9bc1e4406091d6d2bcd Mon Sep 17 00:00:00 2001 From: Hugo Hakim Damer Date: Tue, 13 Aug 2024 17:26:27 +0200 Subject: [PATCH] Add support for IPv6 networks (on Linux clients) (#1459) * Feat add basic support for IPv6 networks Newly generated networks automatically generate an IPv6 prefix of size 64 within the ULA address range, devices obtain a randomly generated address within this prefix. Currently, this is Linux only and does not yet support all features (routes currently cause an error). * Fix firewall configuration for IPv6 networks * Fix routing configuration for IPv6 networks * Feat provide info on IPv6 support for specific client to mgmt server * Feat allow configuration of IPv6 support through API, improve stability * Feat add IPv6 support to new firewall implementation * Fix peer list item response not containing IPv6 address * Fix nftables breaking on IPv6 address change * Fix build issues for non-linux systems * Fix intermittent disconnections when IPv6 is enabled * Fix test issues and make some minor revisions * Fix some more testing issues * Fix more CI issues due to IPv6 * Fix more testing issues * Add inheritance of IPv6 enablement status from groups * Fix IPv6 events not having associated messages * Address first review comments regarding IPv6 support * Fix IPv6 table being created even when IPv6 is disabled Also improved stability of IPv6 route and firewall handling on client side * Fix IPv6 routes not being removed * Fix DNS IPv6 issues, limit IPv6 nameservers to IPv6 peers * Improve code for IPv6 DNS server selection, add AAAA custom records * Ensure IPv6 routes can only exist for IPv6 routing peers * Fix IPv6 network generation randomness * Fix a bunch of compilation issues and test failures * Replace method calls that are unavailable in Go 1.21 * Fix nil dereference in cleanUpDefaultForwardRules6 * Fix nil pointer dereference when persisting IPv6 network in sqlite * Clean up of client-side code changes for IPv6 * Fix nil dereference in rule mangling and compilation issues * Add a bunch of client-side test cases for IPv6 * Fix IPv6 tests running on unsupported environments * Fix import cycle in tests * Add missing method SupportsIPv6() for windows * Require IPv6 default route for IPv6 tests * Fix panics in routemanager tests on non-linux * Fix some more route manager tests concerning IPv6 * Add some final client-side tests * Add IPv6 tests for management code, small fixes * Fix linting issues * Fix small test suite issues * Fix linter issues and builds on macOS and Windows again * fix builds for iOS because of IPv6 breakage --- client/firewall/create.go | 6 + client/firewall/create_linux.go | 8 + client/firewall/iface.go | 1 + client/firewall/iptables/manager_linux.go | 8 + client/firewall/iptables/router_linux_test.go | 6 + client/firewall/manager/firewall.go | 7 + client/firewall/nftables/acl_linux.go | 566 +++++++++++++----- client/firewall/nftables/manager_linux.go | 72 ++- .../firewall/nftables/manager_linux_test.go | 395 +++++++++++- client/firewall/nftables/route_linux.go | 287 +++++++-- client/firewall/nftables/router_linux_test.go | 84 ++- client/firewall/test/cases_linux.go | 33 +- client/firewall/uspfilter/uspfilter.go | 9 + client/firewall/uspfilter/uspfilter_test.go | 4 + client/internal/acl/manager.go | 130 +++- client/internal/acl/manager_test.go | 12 + client/internal/acl/mocks/iface_mapper.go | 16 +- client/internal/connect.go | 1 + client/internal/dns/server.go | 6 +- client/internal/dns/server_test.go | 25 +- client/internal/engine.go | 32 +- client/internal/engine_test.go | 8 +- client/internal/peer/conn.go | 2 +- client/internal/peer/status.go | 1 + client/internal/routemanager/client.go | 14 +- client/internal/routemanager/manager.go | 13 + client/internal/routemanager/manager_test.go | 432 ++++++++++++- client/internal/routemanager/mock.go | 4 + .../routemanager/server_nonandroid.go | 50 +- client/internal/routemanager/systemops.go | 10 + client/internal/routemanager/systemops_ios.go | 2 +- .../internal/routemanager/systemops_linux.go | 26 +- .../routemanager/systemops_nonlinux.go | 2 +- .../internal/routemanager/systemops_test.go | 144 ++++- client/system/info.go | 1 + client/system/info_android.go | 1 + client/system/info_darwin.go | 1 + client/system/info_freebsd.go | 2 +- client/system/info_linux.go | 8 + client/system/info_windows.go | 1 + iface/iface.go | 22 + iface/iface_android.go | 11 +- iface/iface_darwin.go | 11 +- iface/iface_ios.go | 11 +- iface/iface_linux.go | 27 +- iface/iface_test.go | 324 +++++++++- iface/iface_windows.go | 11 +- iface/tun.go | 2 + iface/tun_android.go | 12 + iface/tun_darwin.go | 12 + iface/tun_ios.go | 12 + iface/tun_kernel_linux.go | 26 +- iface/tun_netstack.go | 11 + iface/tun_usp_linux.go | 11 + iface/tun_windows.go | 11 + iface/wg_configurer_kernel.go | 13 +- management/client/client_test.go | 1 + management/client/grpc.go | 1 + management/proto/management.pb.go | 533 +++++++++-------- management/proto/management.proto | 4 + management/server/account.go | 37 +- management/server/account_test.go | 274 ++++++--- management/server/activity/codes.go | 12 + management/server/dns.go | 27 +- management/server/dns_test.go | 54 +- management/server/group.go | 85 ++- management/server/group/group.go | 3 + management/server/group_test.go | 154 ++++- management/server/grpcserver.go | 19 +- .../server/http/accounts_handler_test.go | 2 +- management/server/http/api/openapi.yml | 28 + management/server/http/api/types.gen.go | 82 ++- management/server/http/groups_handler.go | 42 +- management/server/http/peers_handler.go | 31 + management/server/http/peers_handler_test.go | 6 + management/server/nameserver_test.go | 2 + management/server/network.go | 89 ++- management/server/network_test.go | 39 +- management/server/peer.go | 92 ++- management/server/peer/peer.go | 23 +- management/server/peer_test.go | 107 ++++ management/server/policy.go | 10 + management/server/policy_test.go | 9 + management/server/route.go | 86 +++ management/server/route_test.go | 297 +++++++-- 85 files changed, 4311 insertions(+), 795 deletions(-) diff --git a/client/firewall/create.go b/client/firewall/create.go index 86ce94cea..5844656e8 100644 --- a/client/firewall/create.go +++ b/client/firewall/create.go @@ -30,3 +30,9 @@ func NewFirewall(context context.Context, iface IFaceMapper) (firewall.Manager, } return fm, nil } + +// Returns true if the current firewall implementation supports IPv6. +// Currently false for anything non-linux. +func SupportsIPv6() bool { + return false +} diff --git a/client/firewall/create_linux.go b/client/firewall/create_linux.go index 92deb63dc..d0bc1b37b 100644 --- a/client/firewall/create_linux.go +++ b/client/firewall/create_linux.go @@ -70,6 +70,8 @@ func NewFirewall(context context.Context, iface IFaceMapper) (firewall.Manager, return nil, errUsp } + // Note for devs: When adding IPv6 support to userspace bind, the implementation of AllowNetbird() has to be + // adjusted accordingly. if err := fm.AllowNetbird(); err != nil { log.Errorf("failed to allow netbird interface traffic: %v", err) } @@ -83,6 +85,12 @@ func NewFirewall(context context.Context, iface IFaceMapper) (firewall.Manager, return fm, nil } +// Returns true if the current firewall implementation supports IPv6. +// Currently true if the firewall is nftables. +func SupportsIPv6() bool { + return check() == NFTABLES +} + // check returns the firewall type based on common lib checks. It returns UNKNOWN if no firewall is found. func check() FWType { useIPTABLES := false diff --git a/client/firewall/iface.go b/client/firewall/iface.go index 882daef75..8170da078 100644 --- a/client/firewall/iface.go +++ b/client/firewall/iface.go @@ -6,6 +6,7 @@ import "github.com/netbirdio/netbird/iface" type IFaceMapper interface { Name() string Address() iface.WGAddress + Address6() *iface.WGAddress IsUserspaceBind() bool SetFilter(iface.PacketFilter) error } diff --git a/client/firewall/iptables/manager_linux.go b/client/firewall/iptables/manager_linux.go index 2d231ec45..94d07136d 100644 --- a/client/firewall/iptables/manager_linux.go +++ b/client/firewall/iptables/manager_linux.go @@ -24,6 +24,14 @@ type Manager struct { router *routerManager } +func (m *Manager) ResetV6Firewall() error { + return nil +} + +func (m *Manager) V6Active() bool { + return false +} + // iFaceMapper defines subset methods of interface required for manager type iFaceMapper interface { Name() string diff --git a/client/firewall/iptables/router_linux_test.go b/client/firewall/iptables/router_linux_test.go index 79b970c36..9cc44ae96 100644 --- a/client/firewall/iptables/router_linux_test.go +++ b/client/firewall/iptables/router_linux_test.go @@ -73,6 +73,9 @@ func TestIptablesManager_InsertRoutingRules(t *testing.T) { for _, testCase := range test.InsertRuleTestCases { t.Run(testCase.Name, func(t *testing.T) { + if testCase.IsV6 { + t.Skip("Environment does not support IPv6, skipping IPv6 test...") + } iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) require.NoError(t, err, "failed to init iptables client") @@ -154,6 +157,9 @@ func TestIptablesManager_RemoveRoutingRules(t *testing.T) { for _, testCase := range test.RemoveRuleTestCases { t.Run(testCase.Name, func(t *testing.T) { + if testCase.IsV6 { + t.Skip("Environment does not support IPv6, skipping IPv6 test...") + } iptablesClient, _ := iptables.NewWithProtocol(iptables.ProtocolIPv4) manager, err := newRouterManager(context.TODO(), iptablesClient) diff --git a/client/firewall/manager/firewall.go b/client/firewall/manager/firewall.go index 6e4edb63e..4e83df475 100644 --- a/client/firewall/manager/firewall.go +++ b/client/firewall/manager/firewall.go @@ -76,6 +76,13 @@ type Manager interface { // RemoveRoutingRules removes a routing firewall rule RemoveRoutingRules(pair RouterPair) error + // ResetV6Firewall makes changes to the firewall to adapt to the IP address changes. + // It is expected that after calling this method ApplyFiltering will be called to re-add the firewall rules. + ResetV6Firewall() error + + // V6Active returns whether IPv6 rules should/may be created by upper layers. + V6Active() bool + // Reset firewall to the default state Reset() error diff --git a/client/firewall/nftables/acl_linux.go b/client/firewall/nftables/acl_linux.go index 1fa41b63a..2c68b58a4 100644 --- a/client/firewall/nftables/acl_linux.go +++ b/client/firewall/nftables/acl_linux.go @@ -34,7 +34,10 @@ const ( ) var ( - anyIP = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + anyIP = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + + nullAddress4 = []byte{0x0, 0x0, 0x0, 0x0} + nullAddress6 = []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0} postroutingMark = []byte{0xe4, 0x7, 0x0, 0x00} ) @@ -44,24 +47,32 @@ type AclManager struct { wgIface iFaceMapper routeingFwChainName string - workTable *nftables.Table - chainInputRules *nftables.Chain - chainOutputRules *nftables.Chain - chainFwFilter *nftables.Chain - chainPrerouting *nftables.Chain + workTable *nftables.Table + workTable6 *nftables.Table + v6Active bool + chainInputRules *nftables.Chain + chainOutputRules *nftables.Chain + chainFwFilter *nftables.Chain + chainPrerouting *nftables.Chain + chainInputRules6 *nftables.Chain + chainOutputRules6 *nftables.Chain + chainFwFilter6 *nftables.Chain + chainPrerouting6 *nftables.Chain - ipsetStore *ipsetStore - rules map[string]*Rule + ipsetStore *ipsetStore + ipsetStore6 *ipsetStore + rules map[string]*Rule } // iFaceMapper defines subset methods of interface required for manager type iFaceMapper interface { Name() string Address() iface.WGAddress + Address6() *iface.WGAddress IsUserspaceBind() bool } -func newAclManager(table *nftables.Table, wgIface iFaceMapper, routeingFwChainName string) (*AclManager, error) { +func newAclManager(table *nftables.Table, table6 *nftables.Table, wgIface iFaceMapper, routeingFwChainName string) (*AclManager, error) { // sConn is used for creating sets and adding/removing elements from them // it's differ then rConn (which does create new conn for each flush operation) // and is permanent. Using same connection for booth type of operations @@ -76,20 +87,75 @@ func newAclManager(table *nftables.Table, wgIface iFaceMapper, routeingFwChainNa sConn: sConn, wgIface: wgIface, workTable: table, + workTable6: table6, + v6Active: wgIface.Address6() != nil, routeingFwChainName: routeingFwChainName, - ipsetStore: newIpsetStore(), - rules: make(map[string]*Rule), + ipsetStore: newIpsetStore(), + ipsetStore6: newIpsetStore(), + rules: make(map[string]*Rule), } - err = m.createDefaultChains() + err = m.createDefaultChains(false) if err != nil { return nil, err } + if m.v6Active { + err = m.createDefaultChains(true) + if err != nil { + return nil, err + } + } + return m, nil } +// PrepareV6Reset prepares the ACL manager for a full V6 table reset (necessary if IPv6 address changes). +// Deletes all IPv6 rules and sets, and sets the V6 active status to false. +func (m *AclManager) PrepareV6Reset() (*nftables.Table, error) { + if m.workTable6 != nil { + for k, r := range m.rules { + if r.ip.To4() == nil { + err := m.DeleteRule(r) + if err != nil { + return nil, err + } + delete(m.rules, k) + } + } + sets, err := m.rConn.GetSets(m.workTable6) + if err != nil { + for _, set := range sets { + m.rConn.DelSet(set) + } + } + m.ipsetStore6 = newIpsetStore() + } + // Set to false just in case of concurrent accesses (will be set to actual values in ReinitAfterV6Reset()). + m.v6Active = false + + // return the current work table. + return m.workTable6, nil +} + +// ReinitAfterV6Reset reinitializes the IPv6 table after an IPv6 address change. +func (m *AclManager) ReinitAfterV6Reset(workTable6 *nftables.Table) error { + // If we have an IPv6 address after the address update, initialize firewall table, else not. + if m.wgIface.Address6() != nil { + m.workTable6 = workTable6 + err := m.createDefaultChains(true) + if err != nil { + return err + } + m.v6Active = true + } else { + m.workTable6 = nil + m.v6Active = false + } + return nil +} + // AddFiltering rule to the firewall // // If comment argument is empty firewall manager should set @@ -112,6 +178,9 @@ func (m *AclManager) AddFiltering( return nil, err } } + if !m.v6Active && ip.To4() == nil { + return nil, fmt.Errorf("attempted to configure filtering for IPv6 address even though IPv6 is not active") + } newRules := make([]firewall.Rule, 0, 2) ioRule, err := m.addIOFiltering(ip, proto, sPort, dPort, direction, action, ipset, comment) @@ -139,6 +208,11 @@ func (m *AclManager) DeleteRule(rule firewall.Rule) error { return fmt.Errorf("invalid rule type") } + ipsetStorage := m.ipsetStore + if r.ip.To4() == nil { + ipsetStorage = m.ipsetStore6 + } + if r.nftSet == nil { err := m.rConn.DelRule(r.nftRule) if err != nil { @@ -148,7 +222,7 @@ func (m *AclManager) DeleteRule(rule firewall.Rule) error { return m.rConn.Flush() } - ips, ok := m.ipsetStore.ips(r.nftSet.Name) + ips, ok := ipsetStorage.ips(r.nftSet.Name) if !ok { err := m.rConn.DelRule(r.nftRule) if err != nil { @@ -158,7 +232,11 @@ func (m *AclManager) DeleteRule(rule firewall.Rule) error { return m.rConn.Flush() } if _, ok := ips[r.ip.String()]; ok { - err := m.sConn.SetDeleteElements(r.nftSet, []nftables.SetElement{{Key: r.ip.To4()}}) + rawIP := r.ip.To4() + if rawIP == nil { + rawIP = r.ip.To16() + } + err := m.sConn.SetDeleteElements(r.nftSet, []nftables.SetElement{{Key: rawIP}}) if err != nil { log.Errorf("delete elements for set %q: %v", r.nftSet.Name, err) } @@ -166,7 +244,7 @@ func (m *AclManager) DeleteRule(rule firewall.Rule) error { log.Debugf("flush error of set delete element, %s", r.nftSet.Name) return err } - m.ipsetStore.DeleteIpFromSet(r.nftSet.Name, r.ip) + ipsetStorage.DeleteIpFromSet(r.nftSet.Name, r.ip) } // if after delete, set still contains other IPs, @@ -185,9 +263,9 @@ func (m *AclManager) DeleteRule(rule firewall.Rule) error { } delete(m.rules, r.GetRuleID()) - m.ipsetStore.DeleteReferenceFromIpSet(r.nftSet.Name) + ipsetStorage.DeleteReferenceFromIpSet(r.nftSet.Name) - if m.ipsetStore.HasReferenceToSet(r.nftSet.Name) { + if ipsetStorage.HasReferenceToSet(r.nftSet.Name) { return nil } @@ -195,11 +273,11 @@ func (m *AclManager) DeleteRule(rule firewall.Rule) error { // set itself and associated firewall rule too m.rConn.FlushSet(r.nftSet) m.rConn.DelSet(r.nftSet) - m.ipsetStore.deleteIpset(r.nftSet.Name) + ipsetStorage.deleteIpset(r.nftSet.Name) return nil } -// createDefaultAllowRules In case if the USP firewall manager can use the native firewall manager we must to create allow rules for +// createDefaultAllowRules In case if the USP firewall manager can use the native firewall manager we must create allow rules for // input and output chains func (m *AclManager) createDefaultAllowRules() error { expIn := []expr.Any{ @@ -282,18 +360,30 @@ func (m *AclManager) Flush() error { return err } - if err := m.refreshRuleHandles(m.chainInputRules); err != nil { + if err := m.refreshRuleHandles(m.workTable, m.chainInputRules); err != nil { log.Errorf("failed to refresh rule handles ipv4 input chain: %v", err) } - if err := m.refreshRuleHandles(m.chainOutputRules); err != nil { + if err := m.refreshRuleHandles(m.workTable, m.chainOutputRules); err != nil { log.Errorf("failed to refresh rule handles IPv4 output chain: %v", err) } - if err := m.refreshRuleHandles(m.chainPrerouting); err != nil { + if err := m.refreshRuleHandles(m.workTable, m.chainPrerouting); err != nil { log.Errorf("failed to refresh rule handles IPv4 prerouting chain: %v", err) } + if err := m.refreshRuleHandles(m.workTable6, m.chainInputRules6); err != nil { + log.Errorf("failed to refresh rule handles IPv6 input chain: %v", err) + } + + if err := m.refreshRuleHandles(m.workTable6, m.chainOutputRules6); err != nil { + log.Errorf("failed to refresh rule handles IPv6 output chain: %v", err) + } + + if err := m.refreshRuleHandles(m.workTable6, m.chainPrerouting6); err != nil { + log.Errorf("failed to refresh rule handles IPv6 prerouting chain: %v", err) + } + return nil } @@ -322,11 +412,9 @@ func (m *AclManager) addIOFiltering(ip net.IP, proto firewall.Protocol, sPort *f } if proto != firewall.ProtocolALL { - expressions = append(expressions, &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: uint32(9), - Len: uint32(1), + expressions = append(expressions, &expr.Meta{ + Key: expr.MetaKeyL4PROTO, + Register: 1, }) var protoData []byte @@ -347,14 +435,35 @@ func (m *AclManager) addIOFiltering(ip net.IP, proto firewall.Protocol, sPort *f }) } + workTable := m.workTable + // Raw bytes of IP to match (if IPv4). rawIP := ip.To4() - // check if rawIP contains zeroed IPv4 0.0.0.0 value + // source address position (in IPv4) + srcAddrOffset := uint32(12) + // destination address position (in IPv4) + dstAddrOffset := uint32(16) + // address length + addrLen := uint32(4) + // IP set storage for IPv4. + ipsetStorage := m.ipsetStore + + // If rawIP == nil, we have an IPv6 address, replace previously defined values with IPv6 counterparts. + if rawIP == nil { + rawIP = ip.To16() + srcAddrOffset = uint32(8) + dstAddrOffset = uint32(24) + addrLen = 16 + workTable = m.workTable6 + ipsetStorage = m.ipsetStore6 + } + + // check if rawIP contains zeroed IP address value // in that case not add IP match expression into the rule definition if !bytes.HasPrefix(anyIP, rawIP) { - // source address position - addrOffset := uint32(12) + + addrOffset := srcAddrOffset if direction == firewall.RuleDirectionOUT { - addrOffset += 4 // is ipv4 address length + addrOffset = dstAddrOffset } expressions = append(expressions, @@ -362,7 +471,7 @@ func (m *AclManager) addIOFiltering(ip net.IP, proto firewall.Protocol, sPort *f DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: addrOffset, - Len: 4, + Len: addrLen, }, ) // add individual IP for match if no ipset defined @@ -433,7 +542,7 @@ func (m *AclManager) addIOFiltering(ip net.IP, proto firewall.Protocol, sPort *f chain = m.chainOutputRules } nftRule := m.rConn.InsertRule(&nftables.Rule{ - Table: m.workTable, + Table: workTable, Chain: chain, Position: 0, Exprs: expressions, @@ -448,7 +557,7 @@ func (m *AclManager) addIOFiltering(ip net.IP, proto firewall.Protocol, sPort *f } m.rules[ruleId] = rule if ipset != nil { - m.ipsetStore.AddReferenceToIpset(ipset.Name) + ipsetStorage.AddReferenceToIpset(ipset.Name) } return rule, nil } @@ -466,6 +575,35 @@ func (m *AclManager) addPreroutingFiltering(ipset *nftables.Set, proto firewall. return nil, fmt.Errorf("unsupported protocol: %s", proto) } + // Raw bytes of IP to match (if IPv4). + rawIP := ip.To4() + // source address position (in IPv4) + srcAddrOffset := uint32(12) + // destination address position (in IPv4) + dstAddrOffset := uint32(16) + // address length + addrLen := uint32(4) + // Raw bytes of the wireguard interface's IPv4 address. + ifaceRawIP := m.wgIface.Address().IP.To4() + // table to insert rule in + workTable := m.workTable + // chain to insert rule in + preroutingChain := m.chainPrerouting + // IP set store to use + ipsetStorage := m.ipsetStore + + // If rawIP == nil, we have an IPv6 address, replace previously defined values with IPv6 counterparts. + if rawIP == nil { + rawIP = ip.To16() + srcAddrOffset = uint32(8) + dstAddrOffset = uint32(24) + addrLen = 16 + ifaceRawIP = m.wgIface.Address6().IP.To16() + workTable = m.workTable6 + preroutingChain = m.chainPrerouting6 + ipsetStorage = m.ipsetStore6 + } + ruleId := generateRuleIdForMangle(ipset, ip, proto, port) if r, ok := m.rules[ruleId]; ok { return &Rule{ @@ -478,7 +616,6 @@ func (m *AclManager) addPreroutingFiltering(ipset *nftables.Set, proto firewall. var ipExpression expr.Any // add individual IP for match if no ipset defined - rawIP := ip.To4() if ipset == nil { ipExpression = &expr.Cmp{ Op: expr.CmpOpEq, @@ -497,26 +634,24 @@ func (m *AclManager) addPreroutingFiltering(ipset *nftables.Set, proto firewall. &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, - Offset: 12, - Len: 4, + Offset: srcAddrOffset, + Len: addrLen, }, ipExpression, &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, - Offset: 16, - Len: 4, + Offset: dstAddrOffset, + Len: addrLen, }, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, - Data: m.wgIface.Address().IP.To4(), + Data: ifaceRawIP, }, - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: uint32(9), - Len: uint32(1), + &expr.Meta{ + Key: expr.MetaKeyL4PROTO, + Register: 1, }, &expr.Cmp{ Register: 1, @@ -554,8 +689,8 @@ func (m *AclManager) addPreroutingFiltering(ipset *nftables.Set, proto firewall. ) nftRule := m.rConn.InsertRule(&nftables.Rule{ - Table: m.workTable, - Chain: m.chainPrerouting, + Table: workTable, + Chain: preroutingChain, Position: 0, Exprs: expressions, UserData: []byte(ruleId), @@ -574,50 +709,65 @@ func (m *AclManager) addPreroutingFiltering(ipset *nftables.Set, proto firewall. m.rules[ruleId] = rule if ipset != nil { - m.ipsetStore.AddReferenceToIpset(ipset.Name) + ipsetStorage.AddReferenceToIpset(ipset.Name) } return rule, nil } -func (m *AclManager) createDefaultChains() (err error) { +func (m *AclManager) createDefaultChains(forV6 bool) (err error) { + workTable := m.workTable + if forV6 { + workTable = m.workTable6 + } + // chainNameInputRules - chain := m.createChain(chainNameInputRules) + chain := m.createChain(chainNameInputRules, workTable) err = m.rConn.Flush() if err != nil { log.Debugf("failed to create chain (%s): %s", chain.Name, err) return err } - m.chainInputRules = chain + chainInputRules := chain + if forV6 { + m.chainInputRules6 = chainInputRules + } else { + m.chainInputRules = chainInputRules + } // chainNameOutputRules - chain = m.createChain(chainNameOutputRules) + chain = m.createChain(chainNameOutputRules, workTable) err = m.rConn.Flush() if err != nil { log.Debugf("failed to create chain (%s): %s", chainNameOutputRules, err) return err } - m.chainOutputRules = chain + chainOutputRules := chain + if forV6 { + m.chainOutputRules6 = chainOutputRules + } else { + m.chainOutputRules = chainOutputRules + } // netbird-acl-input-filter // type filter hook input priority filter; policy accept; - chain = m.createFilterChainWithHook(chainNameInputFilter, nftables.ChainHookInput) + chain = m.createFilterChainWithHook(chainNameInputFilter, nftables.ChainHookInput, workTable) //netbird-acl-input-filter iifname "wt0" ip saddr 100.72.0.0/16 ip daddr != 100.72.0.0/16 accept m.addRouteAllowRule(chain, expr.MetaKeyIIFNAME) m.addFwdAllow(chain, expr.MetaKeyIIFNAME) - m.addJumpRule(chain, m.chainInputRules.Name, expr.MetaKeyIIFNAME) // to netbird-acl-input-rules + m.addJumpRule(chain, chainInputRules.Name, expr.MetaKeyIIFNAME) // to netbird-acl-input-rules m.addDropExpressions(chain, expr.MetaKeyIIFNAME) err = m.rConn.Flush() if err != nil { - log.Debugf("failed to create chain (%s): %s", chain.Name, err) + log.Debugf("failed to create chain (%s): %s", chainNameInputFilter, err) return err } // netbird-acl-output-filter // type filter hook output priority filter; policy accept; - chain = m.createFilterChainWithHook(chainNameOutputFilter, nftables.ChainHookOutput) + chain = m.createFilterChainWithHook(chainNameOutputFilter, nftables.ChainHookOutput, workTable) m.addRouteAllowRule(chain, expr.MetaKeyOIFNAME) m.addFwdAllow(chain, expr.MetaKeyOIFNAME) - m.addJumpRule(chain, m.chainOutputRules.Name, expr.MetaKeyOIFNAME) // to netbird-acl-output-rules + m.addJumpRule(chain, chainOutputRules.Name, expr.MetaKeyOIFNAME) // to netbird-acl-output-rules m.addDropExpressions(chain, expr.MetaKeyOIFNAME) err = m.rConn.Flush() if err != nil { @@ -626,29 +776,41 @@ func (m *AclManager) createDefaultChains() (err error) { } // netbird-acl-forward-filter - m.chainFwFilter = m.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward) - m.addJumpRulesToRtForward() // to - m.addMarkAccept() - m.addJumpRuleToInputChain() // to netbird-acl-input-rules - m.addDropExpressions(m.chainFwFilter, expr.MetaKeyIIFNAME) + chain = m.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward, workTable) + m.addJumpRulesToRtForward(workTable, chain) // to + m.addMarkAccept(workTable, chain) + m.addJumpRuleToInputChain(workTable, chain, chainInputRules) // to netbird-acl-input-rules + m.addDropExpressions(chain, expr.MetaKeyIIFNAME) err = m.rConn.Flush() if err != nil { log.Debugf("failed to create chain (%s): %s", chainNameForwardFilter, err) return err } + if forV6 { + m.chainFwFilter6 = chain + } else { + m.chainFwFilter = chain + } // netbird-acl-output-filter // type filter hook output priority filter; policy accept; - m.chainPrerouting = m.createPreroutingMangle() + chain = m.createPreroutingMangle(forV6) err = m.rConn.Flush() if err != nil { - log.Debugf("failed to create chain (%s): %s", m.chainPrerouting.Name, err) + log.Debugf("failed to create chain (%s): %s", chain.Name, err) return err } + if forV6 { + m.chainPrerouting6 = chain + } else { + m.chainPrerouting = chain + } + return nil } -func (m *AclManager) addJumpRulesToRtForward() { +func (m *AclManager) addJumpRulesToRtForward(table *nftables.Table, chain *nftables.Chain) { + expressions := []expr.Any{ &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, &expr.Cmp{ @@ -663,8 +825,8 @@ func (m *AclManager) addJumpRulesToRtForward() { } _ = m.rConn.AddRule(&nftables.Rule{ - Table: m.workTable, - Chain: m.chainFwFilter, + Table: table, + Chain: chain, Exprs: expressions, }) @@ -682,13 +844,13 @@ func (m *AclManager) addJumpRulesToRtForward() { } _ = m.rConn.AddRule(&nftables.Rule{ - Table: m.workTable, - Chain: m.chainFwFilter, + Table: table, + Chain: chain, Exprs: expressions, }) } -func (m *AclManager) addMarkAccept() { +func (m *AclManager) addMarkAccept(table *nftables.Table, chain *nftables.Chain) { // oifname "wt0" meta mark 0x000007e4 accept // iifname "wt0" meta mark 0x000007e4 accept ifaces := []expr.MetaKey{expr.MetaKeyIIFNAME, expr.MetaKeyOIFNAME} @@ -715,28 +877,30 @@ func (m *AclManager) addMarkAccept() { } _ = m.rConn.AddRule(&nftables.Rule{ - Table: m.workTable, - Chain: m.chainFwFilter, + Table: table, + Chain: chain, Exprs: expressions, }) } } -func (m *AclManager) createChain(name string) *nftables.Chain { +func (m *AclManager) createChain(name string, table *nftables.Table) *nftables.Chain { + chain := &nftables.Chain{ Name: name, - Table: m.workTable, + Table: table, } chain = m.rConn.AddChain(chain) + return chain } -func (m *AclManager) createFilterChainWithHook(name string, hookNum nftables.ChainHook) *nftables.Chain { +func (m *AclManager) createFilterChainWithHook(name string, hookNum nftables.ChainHook, table *nftables.Table) *nftables.Chain { polAccept := nftables.ChainPolicyAccept chain := &nftables.Chain{ Name: name, - Table: m.workTable, + Table: table, Hooknum: hookNum, Priority: nftables.ChainPriorityFilter, Type: nftables.ChainTypeFilter, @@ -746,11 +910,36 @@ func (m *AclManager) createFilterChainWithHook(name string, hookNum nftables.Cha return m.rConn.AddChain(chain) } -func (m *AclManager) createPreroutingMangle() *nftables.Chain { +func (m *AclManager) createPreroutingMangle(forV6 bool) *nftables.Chain { + workTable := m.workTable + // Raw bytes of the wireguard interface's IPv4 address. + rawIP := m.wgIface.Address().Network.IP.To4() + // Subnet mask of the wireguard interface's network. + mask := m.wgIface.Address().Network.Mask + // Length of an IPv4 address + addrLen := uint32(4) + // source address position + srcAddrOffset := uint32(12) + // destination address position + dstAddrOffset := uint32(16) + // An array representing a null address in IPv4 (0.0.0.0) + nullAddressArray := nullAddress4 + + // If prerouting mangle should be created for IPv6 table, replace previously defined values with IPv6 counterparts. + if forV6 { + workTable = m.workTable6 + rawIP = m.wgIface.Address6().Network.IP.To16() + mask = m.wgIface.Address6().Network.Mask + addrLen = 16 + srcAddrOffset = uint32(8) + dstAddrOffset = uint32(24) + nullAddressArray = nullAddress6 // corresponds to :: + } + polAccept := nftables.ChainPolicyAccept chain := &nftables.Chain{ Name: "netbird-acl-prerouting-filter", - Table: m.workTable, + Table: workTable, Hooknum: nftables.ChainHookPrerouting, Priority: nftables.ChainPriorityMangle, Type: nftables.ChainTypeFilter, @@ -759,7 +948,8 @@ func (m *AclManager) createPreroutingMangle() *nftables.Chain { chain = m.rConn.AddChain(chain) - ip, _ := netip.AddrFromSlice(m.wgIface.Address().Network.IP.To4()) + ip, _ := netip.AddrFromSlice(rawIP) + expressions := []expr.Any{ &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, &expr.Cmp{ @@ -770,15 +960,15 @@ func (m *AclManager) createPreroutingMangle() *nftables.Chain { &expr.Payload{ DestRegister: 2, Base: expr.PayloadBaseNetworkHeader, - Offset: 12, - Len: 4, + Offset: srcAddrOffset, + Len: addrLen, }, &expr.Bitwise{ SourceRegister: 2, DestRegister: 2, - Len: 4, - Xor: []byte{0x0, 0x0, 0x0, 0x0}, - Mask: m.wgIface.Address().Network.Mask, + Len: addrLen, + Xor: nullAddressArray, + Mask: mask, }, &expr.Cmp{ Op: expr.CmpOpNeq, @@ -788,13 +978,13 @@ func (m *AclManager) createPreroutingMangle() *nftables.Chain { &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, - Offset: 16, - Len: 4, + Offset: dstAddrOffset, + Len: addrLen, }, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, - Data: m.wgIface.Address().IP.To4(), + Data: rawIP, }, &expr.Immediate{ Register: 1, @@ -807,7 +997,7 @@ func (m *AclManager) createPreroutingMangle() *nftables.Chain { }, } _ = m.rConn.AddRule(&nftables.Rule{ - Table: m.workTable, + Table: workTable, Chain: chain, Exprs: expressions, }) @@ -825,14 +1015,14 @@ func (m *AclManager) addDropExpressions(chain *nftables.Chain, ifaceKey expr.Met &expr.Verdict{Kind: expr.VerdictDrop}, } _ = m.rConn.AddRule(&nftables.Rule{ - Table: m.workTable, + Table: chain.Table, Chain: chain, Exprs: expressions, }) return nil } -func (m *AclManager) addJumpRuleToInputChain() { +func (m *AclManager) addJumpRuleToInputChain(table *nftables.Table, chain *nftables.Chain, inputChain *nftables.Chain) { expressions := []expr.Any{ &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, &expr.Cmp{ @@ -842,19 +1032,43 @@ func (m *AclManager) addJumpRuleToInputChain() { }, &expr.Verdict{ Kind: expr.VerdictJump, - Chain: m.chainInputRules.Name, + Chain: inputChain.Name, }, } _ = m.rConn.AddRule(&nftables.Rule{ - Table: m.workTable, - Chain: m.chainFwFilter, + Table: table, + Chain: chain, Exprs: expressions, }) } func (m *AclManager) addRouteAllowRule(chain *nftables.Chain, netIfName expr.MetaKey) { - ip, _ := netip.AddrFromSlice(m.wgIface.Address().Network.IP.To4()) + // Raw bytes of the wireguard interface's IPv4 address. + rawIP := m.wgIface.Address().Network.IP.To4() + // Subnet mask of the wireguard interface's network. + mask := m.wgIface.Address().Network.Mask + // Length of an IPv4 address + addrLen := uint32(4) + // source address position + srcAddrOffset := uint32(12) + // destination address position + dstAddrOffset := uint32(16) + // An array representing a null address in IPv4 (0.0.0.0) + nullAddressArray := nullAddress4 + + // If route allow rule should be created for IPv6 table, replace previously defined values with IPv6 counterparts. + if chain.Table.Family == nftables.TableFamilyIPv6 { + rawIP = m.wgIface.Address6().Network.IP.To16() + mask = m.wgIface.Address6().Network.Mask + addrLen = 16 + srcAddrOffset = 8 + dstAddrOffset = 24 + nullAddressArray = nullAddress6 // corresponds to :: + } + + ip, _ := netip.AddrFromSlice(rawIP) + var srcOp, dstOp expr.CmpOp if netIfName == expr.MetaKeyIIFNAME { srcOp = expr.CmpOpEq @@ -873,15 +1087,15 @@ func (m *AclManager) addRouteAllowRule(chain *nftables.Chain, netIfName expr.Met &expr.Payload{ DestRegister: 2, Base: expr.PayloadBaseNetworkHeader, - Offset: 12, - Len: 4, + Offset: srcAddrOffset, + Len: addrLen, }, &expr.Bitwise{ SourceRegister: 2, DestRegister: 2, - Len: 4, - Xor: []byte{0x0, 0x0, 0x0, 0x0}, - Mask: m.wgIface.Address().Network.Mask, + Len: addrLen, + Xor: nullAddressArray, + Mask: mask, }, &expr.Cmp{ Op: srcOp, @@ -891,15 +1105,15 @@ func (m *AclManager) addRouteAllowRule(chain *nftables.Chain, netIfName expr.Met &expr.Payload{ DestRegister: 2, Base: expr.PayloadBaseNetworkHeader, - Offset: 16, - Len: 4, + Offset: dstAddrOffset, + Len: addrLen, }, &expr.Bitwise{ SourceRegister: 2, DestRegister: 2, - Len: 4, - Xor: []byte{0x0, 0x0, 0x0, 0x0}, - Mask: m.wgIface.Address().Network.Mask, + Len: addrLen, + Xor: nullAddressArray, + Mask: mask, }, &expr.Cmp{ Op: dstOp, @@ -918,7 +1132,31 @@ func (m *AclManager) addRouteAllowRule(chain *nftables.Chain, netIfName expr.Met } func (m *AclManager) addFwdAllow(chain *nftables.Chain, iifname expr.MetaKey) { - ip, _ := netip.AddrFromSlice(m.wgIface.Address().Network.IP.To4()) + // Raw bytes of the wireguard interface's IPv4 address. + rawIP := m.wgIface.Address().Network.IP.To4() + // Subnet mask of the wireguard interface's network. + mask := m.wgIface.Address().Network.Mask + // Length of an IPv4 address + addrLen := uint32(4) + // source address position + srcAddrOffset := uint32(12) + // destination address position + dstAddrOffset := uint32(16) + // An array representing a null address in IPv4 (0.0.0.0) + nullAddressArray := nullAddress4 + + // If forward allow rule should be created for IPv6 table, replace previously defined values with IPv6 counterparts. + if chain.Table.Family == nftables.TableFamilyIPv6 { + rawIP = m.wgIface.Address6().Network.IP.To16() + mask = m.wgIface.Address6().Network.Mask + addrLen = 16 + srcAddrOffset = 8 + dstAddrOffset = 24 + nullAddressArray = nullAddress6 // corresponds to :: + } + + ip, _ := netip.AddrFromSlice(rawIP) + var srcOp, dstOp expr.CmpOp if iifname == expr.MetaKeyIIFNAME { srcOp = expr.CmpOpNeq @@ -937,15 +1175,15 @@ func (m *AclManager) addFwdAllow(chain *nftables.Chain, iifname expr.MetaKey) { &expr.Payload{ DestRegister: 2, Base: expr.PayloadBaseNetworkHeader, - Offset: 12, - Len: 4, + Offset: srcAddrOffset, + Len: addrLen, }, &expr.Bitwise{ SourceRegister: 2, DestRegister: 2, - Len: 4, - Xor: []byte{0x0, 0x0, 0x0, 0x0}, - Mask: m.wgIface.Address().Network.Mask, + Len: addrLen, + Xor: nullAddressArray, + Mask: mask, }, &expr.Cmp{ Op: srcOp, @@ -955,15 +1193,15 @@ func (m *AclManager) addFwdAllow(chain *nftables.Chain, iifname expr.MetaKey) { &expr.Payload{ DestRegister: 2, Base: expr.PayloadBaseNetworkHeader, - Offset: 16, - Len: 4, + Offset: dstAddrOffset, + Len: addrLen, }, &expr.Bitwise{ SourceRegister: 2, DestRegister: 2, - Len: 4, - Xor: []byte{0x0, 0x0, 0x0, 0x0}, - Mask: m.wgIface.Address().Network.Mask, + Len: addrLen, + Xor: nullAddressArray, + Mask: mask, }, &expr.Cmp{ Op: dstOp, @@ -982,7 +1220,31 @@ func (m *AclManager) addFwdAllow(chain *nftables.Chain, iifname expr.MetaKey) { } func (m *AclManager) addJumpRule(chain *nftables.Chain, to string, ifaceKey expr.MetaKey) { - ip, _ := netip.AddrFromSlice(m.wgIface.Address().Network.IP.To4()) + // Raw bytes of the wireguard interface's IPv4 address. + rawIP := m.wgIface.Address().Network.IP.To4() + // Subnet mask of the wireguard interface's network. + mask := m.wgIface.Address().Network.Mask + // Length of an IPv4 address + addrLen := uint32(4) + // source address position + srcAddrOffset := uint32(12) + // destination address position + dstAddrOffset := uint32(16) + // An array representing a null address in IPv4 (0.0.0.0) + nullAddressArray := nullAddress4 + + // If jump rule should be created for IPv6 table, replace previously defined values with IPv6 counterparts. + if chain.Table.Family == nftables.TableFamilyIPv6 { + rawIP = m.wgIface.Address6().Network.IP.To16() + mask = m.wgIface.Address6().Network.Mask + addrLen = 16 + srcAddrOffset = 8 + dstAddrOffset = 24 + nullAddressArray = nullAddress6 // corresponds to :: + } + + ip, _ := netip.AddrFromSlice(rawIP) + expressions := []expr.Any{ &expr.Meta{Key: ifaceKey, Register: 1}, &expr.Cmp{ @@ -993,15 +1255,15 @@ func (m *AclManager) addJumpRule(chain *nftables.Chain, to string, ifaceKey expr &expr.Payload{ DestRegister: 2, Base: expr.PayloadBaseNetworkHeader, - Offset: 12, - Len: 4, + Offset: srcAddrOffset, + Len: addrLen, }, &expr.Bitwise{ SourceRegister: 2, DestRegister: 2, - Len: 4, - Xor: []byte{0x0, 0x0, 0x0, 0x0}, - Mask: m.wgIface.Address().Network.Mask, + Len: addrLen, + Xor: nullAddressArray, + Mask: mask, }, &expr.Cmp{ Op: expr.CmpOpEq, @@ -1011,15 +1273,15 @@ func (m *AclManager) addJumpRule(chain *nftables.Chain, to string, ifaceKey expr &expr.Payload{ DestRegister: 2, Base: expr.PayloadBaseNetworkHeader, - Offset: 16, - Len: 4, + Offset: dstAddrOffset, + Len: addrLen, }, &expr.Bitwise{ SourceRegister: 2, DestRegister: 2, - Len: 4, - Xor: []byte{0x0, 0x0, 0x0, 0x0}, - Mask: m.wgIface.Address().Network.Mask, + Len: addrLen, + Xor: nullAddressArray, + Mask: mask, }, &expr.Cmp{ Op: expr.CmpOpEq, @@ -1039,17 +1301,33 @@ func (m *AclManager) addJumpRule(chain *nftables.Chain, to string, ifaceKey expr } func (m *AclManager) addIpToSet(ipsetName string, ip net.IP) (*nftables.Set, error) { - ipset, err := m.rConn.GetSetByName(m.workTable, ipsetName) + + workTable := m.workTable + // Raw bytes of the IPv4 address to add rawIP := ip.To4() + // Type of set to add to + ipsetType := nftables.TypeIPAddr + // IP set store to use + ipsetStorage := m.ipsetStore + + // If rawIP == nil, we have an IPv6 address, replace previously defined values with IPv6 counterparts. + if rawIP == nil { + workTable = m.workTable6 + rawIP = ip.To16() + ipsetType = nftables.TypeIP6Addr + ipsetStorage = m.ipsetStore6 + } + + ipset, err := m.rConn.GetSetByName(workTable, ipsetName) if err != nil { - if ipset, err = m.createSet(m.workTable, ipsetName); err != nil { + if ipset, err = m.createSet(workTable, ipsetName, ipsetType); err != nil { return nil, fmt.Errorf("get set name: %v", err) } - m.ipsetStore.newIpset(ipset.Name) + ipsetStorage.newIpset(ipset.Name) } - if m.ipsetStore.IsIpInSet(ipset.Name, ip) { + if ipsetStorage.IsIpInSet(ipset.Name, ip) { return ipset, nil } @@ -1057,7 +1335,7 @@ func (m *AclManager) addIpToSet(ipsetName string, ip net.IP) (*nftables.Set, err return nil, fmt.Errorf("add set element for the first time: %v", err) } - m.ipsetStore.AddIpToSet(ipset.Name, ip) + ipsetStorage.AddIpToSet(ipset.Name, ip) if err := m.sConn.Flush(); err != nil { return nil, fmt.Errorf("flush add elements: %v", err) @@ -1067,12 +1345,12 @@ func (m *AclManager) addIpToSet(ipsetName string, ip net.IP) (*nftables.Set, err } // createSet in given table by name -func (m *AclManager) createSet(table *nftables.Table, name string) (*nftables.Set, error) { +func (m *AclManager) createSet(table *nftables.Table, name string, ipsetType nftables.SetDatatype) (*nftables.Set, error) { ipset := &nftables.Set{ Name: name, Table: table, Dynamic: true, - KeyType: nftables.TypeIPAddr, + KeyType: ipsetType, } if err := m.rConn.AddSet(ipset, nil); err != nil { @@ -1108,12 +1386,12 @@ func (m *AclManager) flushWithBackoff() (err error) { return } -func (m *AclManager) refreshRuleHandles(chain *nftables.Chain) error { - if m.workTable == nil || chain == nil { +func (m *AclManager) refreshRuleHandles(table *nftables.Table, chain *nftables.Chain) error { + if table == nil || chain == nil { return nil } - list, err := m.rConn.GetRules(m.workTable, chain) + list, err := m.rConn.GetRules(table, chain) if err != nil { return err } @@ -1140,6 +1418,10 @@ func generateRuleId( action firewall.Action, ipset *nftables.Set, ) string { + ipver := "v4" + if ip.To4() == nil { + ipver = "v6" + } rulesetID := ":" + strconv.Itoa(int(direction)) + ":" if sPort != nil { rulesetID += sPort.String() @@ -1153,7 +1435,7 @@ func generateRuleId( if ipset == nil { return "ip:" + ip.String() + rulesetID } - return "set:" + ipset.Name + rulesetID + return "set:" + ipver + ":" + ipset.Name + rulesetID } func generateRuleIdForMangle(ipset *nftables.Set, ip net.IP, proto firewall.Protocol, port *firewall.Port) string { // case of icmp port is empty @@ -1161,10 +1443,14 @@ func generateRuleIdForMangle(ipset *nftables.Set, ip net.IP, proto firewall.Prot if port != nil { p = port.String() } + ipver := "v4" + if (ipset != nil && ipset.Table.Family == nftables.TableFamilyIPv6) || ip.To4() == nil { + ipver = "v6" + } if ipset != nil { - return fmt.Sprintf("p:set:%s:%s:%v", ipset.Name, proto, p) + return fmt.Sprintf("p:set:%s:%s:%s:%v", ipver, ipset.Name, proto, p) } else { - return fmt.Sprintf("p:ip:%s:%s:%v", ip.String(), proto, p) + return fmt.Sprintf("p:ip:%s:%s:%s:%v", ipver, ip.String(), proto, p) } } diff --git a/client/firewall/nftables/manager_linux.go b/client/firewall/nftables/manager_linux.go index 8395fc270..6fc3e4ebb 100644 --- a/client/firewall/nftables/manager_linux.go +++ b/client/firewall/nftables/manager_linux.go @@ -36,17 +36,25 @@ func Create(context context.Context, wgIface iFaceMapper) (*Manager, error) { wgIface: wgIface, } - workTable, err := m.createWorkTable() + workTable, err := m.createWorkTable(nftables.TableFamilyIPv4) if err != nil { return nil, err } - m.router, err = newRouter(context, workTable) + var workTable6 *nftables.Table + if wgIface.Address6() != nil { + workTable6, err = m.createWorkTable(nftables.TableFamilyIPv6) + if err != nil { + return nil, err + } + } + + m.router, err = newRouter(context, workTable, workTable6) if err != nil { return nil, err } - m.aclManager, err = newAclManager(workTable, wgIface, m.router.RouteingFwChainName()) + m.aclManager, err = newAclManager(workTable, workTable6, wgIface, m.router.RouteingFwChainName()) if err != nil { return nil, err } @@ -54,6 +62,54 @@ func Create(context context.Context, wgIface iFaceMapper) (*Manager, error) { return m, nil } +// Resets the IPv6 Firewall Table to adapt to changes in IP addresses +func (m *Manager) ResetV6Firewall() error { + + // First, prepare reset by deleting all currently active rules. + workTable6, err := m.aclManager.PrepareV6Reset() + if err != nil { + return err + } + + // Depending on whether we now have an IPv6 address, we now either have to create/empty an IPv6 table, or delete it. + if m.wgIface.Address6() != nil { + if workTable6 != nil { + m.rConn.FlushTable(workTable6) + } else { + workTable6, err = m.createWorkTable(nftables.TableFamilyIPv6) + if err != nil { + return err + } + } + } else { + m.rConn.DelTable(workTable6) + workTable6 = nil + } + err = m.rConn.Flush() + if err != nil { + return err + } + + // Restore routing rules. + err = m.router.RestoreAfterV6Reset(workTable6) + if err != nil { + return err + } + + // Restore basic firewall chains (needs to happen after routes because chains from router must exist). + // Does not restore rules (will be done later during the update, when UpdateFiltering will be called at some point) + err = m.aclManager.ReinitAfterV6Reset(workTable6) + if err != nil { + return err + } + + return m.rConn.Flush() +} + +func (m *Manager) V6Active() bool { + return m.aclManager.v6Active +} + // AddFiltering rule to the firewall // // If comment argument is empty firewall manager should set @@ -72,7 +128,7 @@ func (m *Manager) AddFiltering( defer m.mutex.Unlock() rawIP := ip.To4() - if rawIP == nil { + if rawIP == nil && m.wgIface.Address6() == nil { return nil, fmt.Errorf("unsupported IP version: %s", ip.String()) } @@ -114,6 +170,8 @@ func (m *Manager) AllowNetbird() error { m.mutex.Lock() defer m.mutex.Unlock() + // Note for devs: When adding IPv6 support to uspfilter, the implementation of createDefaultAllowRules() + // must be adjusted to include IPv6 rules. err := m.aclManager.createDefaultAllowRules() if err != nil { return fmt.Errorf("failed to create default allow rules: %v", err) @@ -211,8 +269,8 @@ func (m *Manager) Flush() error { return m.aclManager.Flush() } -func (m *Manager) createWorkTable() (*nftables.Table, error) { - tables, err := m.rConn.ListTablesOfFamily(nftables.TableFamilyIPv4) +func (m *Manager) createWorkTable(tableFamily nftables.TableFamily) (*nftables.Table, error) { + tables, err := m.rConn.ListTablesOfFamily(tableFamily) if err != nil { return nil, fmt.Errorf("list of tables: %w", err) } @@ -223,7 +281,7 @@ func (m *Manager) createWorkTable() (*nftables.Table, error) { } } - table := m.rConn.AddTable(&nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv4}) + table := m.rConn.AddTable(&nftables.Table{Name: tableName, Family: tableFamily}) err = m.rConn.Flush() return table, err } diff --git a/client/firewall/nftables/manager_linux_test.go b/client/firewall/nftables/manager_linux_test.go index 1f226e315..fd7fced14 100644 --- a/client/firewall/nftables/manager_linux_test.go +++ b/client/firewall/nftables/manager_linux_test.go @@ -19,8 +19,9 @@ import ( // iFaceMapper defines subset methods of interface required for manager type iFaceMock struct { - NameFunc func() string - AddressFunc func() iface.WGAddress + NameFunc func() string + AddressFunc func() iface.WGAddress + Address6Func func() *iface.WGAddress } func (i *iFaceMock) Name() string { @@ -37,6 +38,13 @@ func (i *iFaceMock) Address() iface.WGAddress { panic("AddressFunc is not set") } +func (i *iFaceMock) Address6() *iface.WGAddress { + if i.Address6Func != nil { + return i.Address6Func() + } + panic("AddressFunc is not set") +} + func (i *iFaceMock) IsUserspaceBind() bool { return false } func TestNftablesManager(t *testing.T) { @@ -53,6 +61,7 @@ func TestNftablesManager(t *testing.T) { }, } }, + Address6Func: func() *iface.WGAddress { return nil }, } // just check on the local interface @@ -99,11 +108,9 @@ func TestNftablesManager(t *testing.T) { Register: 1, Data: ifname("lo"), }, - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: uint32(9), - Len: uint32(1), + &expr.Meta{ + Key: expr.MetaKeyL4PROTO, + Register: 1, }, &expr.Cmp{ Register: 1, @@ -152,6 +159,370 @@ func TestNftablesManager(t *testing.T) { require.NoError(t, err, "failed to reset") } +func TestNftablesManager6Disabled(t *testing.T) { + mock := &iFaceMock{ + NameFunc: func() string { + return "lo" + }, + AddressFunc: func() iface.WGAddress { + return iface.WGAddress{ + IP: net.ParseIP("100.96.0.1"), + Network: &net.IPNet{ + IP: net.ParseIP("100.96.0.0"), + Mask: net.IPv4Mask(255, 255, 255, 0), + }, + } + }, + Address6Func: func() *iface.WGAddress { return nil }, + } + + // just check on the local interface + manager, err := Create(context.Background(), mock) + require.NoError(t, err) + time.Sleep(time.Second * 3) + + defer func() { + err = manager.Reset() + require.NoError(t, err, "failed to reset") + time.Sleep(time.Second) + }() + + ip := net.ParseIP("2001:db8::fedc:ba09:8765:4321") + + testClient := &nftables.Conn{} + + _, err = manager.AddFiltering( + ip, + fw.ProtocolTCP, + nil, + &fw.Port{Values: []int{53}}, + fw.RuleDirectionIN, + fw.ActionDrop, + "", + "", + ) + require.Error(t, err, "IPv6 rule should not be added when IPv6 is disabled") + + err = manager.Flush() + require.NoError(t, err, "failed to flush") + + rules, err := testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules) + require.NoError(t, err, "failed to get rules") + + require.Len(t, rules, 0, "expected no rules") + + err = manager.Reset() + require.NoError(t, err, "failed to reset") +} + +func TestNftablesManager6(t *testing.T) { + + if !iface.SupportsIPv6() { + t.Skip("Environment does not support IPv6, skipping IPv6 test...") + } + mock := &iFaceMock{ + NameFunc: func() string { + return "lo" + }, + AddressFunc: func() iface.WGAddress { + return iface.WGAddress{ + IP: net.ParseIP("100.96.0.1"), + Network: &net.IPNet{ + IP: net.ParseIP("100.96.0.0"), + Mask: net.IPv4Mask(255, 255, 255, 0), + }, + } + }, + Address6Func: func() *iface.WGAddress { + return &iface.WGAddress{ + IP: net.ParseIP("2001:db8::0123:4567:890a:bcde"), + Network: &net.IPNet{ + IP: net.ParseIP("2001:db8::"), + Mask: net.CIDRMask(64, 128), + }, + } + }, + } + + // just check on the local interface + manager, err := Create(context.Background(), mock) + require.NoError(t, err) + time.Sleep(time.Second * 3) + + defer func() { + err = manager.Reset() + require.NoError(t, err, "failed to reset") + time.Sleep(time.Second) + }() + + require.True(t, manager.V6Active(), "IPv6 is not active even though it should be.") + + ip := net.ParseIP("2001:db8::fedc:ba09:8765:4321") + + testClient := &nftables.Conn{} + + rule, err := manager.AddFiltering( + ip, + fw.ProtocolTCP, + nil, + &fw.Port{Values: []int{53}}, + fw.RuleDirectionIN, + fw.ActionDrop, + "", + "", + ) + require.NoError(t, err, "failed to add rule") + + err = manager.Flush() + require.NoError(t, err, "failed to flush") + + rules, err := testClient.GetRules(manager.aclManager.workTable6, manager.aclManager.chainInputRules6) + require.NoError(t, err, "failed to get rules") + + require.Len(t, rules, 1, "expected 1 rules") + + ipToAdd, _ := netip.AddrFromSlice(ip) + add := ipToAdd.Unmap() + expectedExprs := []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname("lo"), + }, + &expr.Meta{ + Key: expr.MetaKeyL4PROTO, + Register: 1, + }, + &expr.Cmp{ + Register: 1, + Op: expr.CmpOpEq, + Data: []byte{unix.IPPROTO_TCP}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: 8, + Len: 16, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: add.AsSlice(), + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: 2, + Len: 2, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{0, 53}, + }, + &expr.Verdict{Kind: expr.VerdictDrop}, + } + require.ElementsMatch(t, rules[0].Exprs, expectedExprs, "expected the same expressions") + + for _, r := range rule { + err = manager.DeleteRule(r) + require.NoError(t, err, "failed to delete rule") + } + + err = manager.Flush() + require.NoError(t, err, "failed to flush") + + rules, err = testClient.GetRules(manager.aclManager.workTable6, manager.aclManager.chainInputRules6) + require.NoError(t, err, "failed to get rules") + require.Len(t, rules, 0, "expected 0 rules after deletion") + + err = manager.Reset() + require.NoError(t, err, "failed to reset") +} + +func TestNftablesManagerAddressReset6(t *testing.T) { + + if !iface.SupportsIPv6() { + t.Skip("Environment does not support IPv6, skipping IPv6 test...") + } + mock := &iFaceMock{ + NameFunc: func() string { + return "lo" + }, + AddressFunc: func() iface.WGAddress { + return iface.WGAddress{ + IP: net.ParseIP("100.96.0.1"), + Network: &net.IPNet{ + IP: net.ParseIP("100.96.0.0"), + Mask: net.IPv4Mask(255, 255, 255, 0), + }, + } + }, + Address6Func: func() *iface.WGAddress { + return &iface.WGAddress{ + IP: net.ParseIP("2001:db8::0123:4567:890a:bcde"), + Network: &net.IPNet{ + IP: net.ParseIP("2001:db8::"), + Mask: net.CIDRMask(64, 128), + }, + } + }, + } + + // just check on the local interface + manager, err := Create(context.Background(), mock) + require.NoError(t, err) + time.Sleep(time.Second * 3) + + defer func() { + err = manager.Reset() + require.NoError(t, err, "failed to reset") + time.Sleep(time.Second) + }() + + require.True(t, manager.V6Active(), "IPv6 is not active even though it should be.") + + ip := net.ParseIP("2001:db8::fedc:ba09:8765:4321") + + testClient := &nftables.Conn{} + + _, err = manager.AddFiltering( + ip, + fw.ProtocolTCP, + nil, + &fw.Port{Values: []int{53}}, + fw.RuleDirectionIN, + fw.ActionDrop, + "", + "", + ) + require.NoError(t, err, "failed to add rule") + + err = manager.Flush() + require.NoError(t, err, "failed to flush") + + rules, err := testClient.GetRules(manager.aclManager.workTable6, manager.aclManager.chainInputRules6) + require.NoError(t, err, "failed to get rules") + + require.Len(t, rules, 1, "expected 1 rules") + + mock.Address6Func = func() *iface.WGAddress { + return nil + } + + err = manager.ResetV6Firewall() + require.NoError(t, err, "failed to reset IPv6 firewall") + + err = manager.Flush() + require.NoError(t, err, "failed to flush") + + require.False(t, manager.V6Active(), "IPv6 is active even though it shouldn't be.") + + tables, err := testClient.ListTablesOfFamily(nftables.TableFamilyIPv6) + require.NoError(t, err, "failed to list IPv6 tables") + + for _, table := range tables { + if table.Name == tableName { + t.Errorf("When IPv6 is disabled, the netbird table should not exist.") + } + } + + mock.Address6Func = func() *iface.WGAddress { + return &iface.WGAddress{ + IP: net.ParseIP("2001:db8::0123:4567:890a:bcdf"), + Network: &net.IPNet{ + IP: net.ParseIP("2001:db8::"), + Mask: net.CIDRMask(64, 128), + }, + } + } + + err = manager.ResetV6Firewall() + require.NoError(t, err, "failed to reset IPv6 firewall") + + require.True(t, manager.V6Active(), "IPv6 is not active even though it should be.") + + rule, err := manager.AddFiltering( + ip, + fw.ProtocolTCP, + nil, + &fw.Port{Values: []int{53}}, + fw.RuleDirectionIN, + fw.ActionDrop, + "", + "", + ) + require.NoError(t, err, "failed to add rule") + + err = manager.Flush() + require.NoError(t, err, "failed to flush") + + rules, err = testClient.GetRules(manager.aclManager.workTable6, manager.aclManager.chainInputRules6) + require.NoError(t, err, "failed to get rules") + + require.Len(t, rules, 1, "expected 1 rule") + + ipToAdd, _ := netip.AddrFromSlice(ip) + add := ipToAdd.Unmap() + expectedExprs := []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname("lo"), + }, + &expr.Meta{ + Key: expr.MetaKeyL4PROTO, + Register: 1, + }, + &expr.Cmp{ + Register: 1, + Op: expr.CmpOpEq, + Data: []byte{unix.IPPROTO_TCP}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: 8, + Len: 16, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: add.AsSlice(), + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: 2, + Len: 2, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{0, 53}, + }, + &expr.Verdict{Kind: expr.VerdictDrop}, + } + require.ElementsMatch(t, rules[0].Exprs, expectedExprs, "expected the same expressions") + + for _, r := range rule { + err = manager.DeleteRule(r) + require.NoError(t, err, "failed to delete rule") + } + + err = manager.Flush() + require.NoError(t, err, "failed to flush") + + rules, err = testClient.GetRules(manager.aclManager.workTable6, manager.aclManager.chainInputRules6) + require.NoError(t, err, "failed to get rules") + require.Len(t, rules, 0, "expected 0 rules after deletion") + + err = manager.Reset() + require.NoError(t, err, "failed to reset") +} + func TestNFtablesCreatePerformance(t *testing.T) { mock := &iFaceMock{ NameFunc: func() string { @@ -166,6 +537,16 @@ func TestNFtablesCreatePerformance(t *testing.T) { }, } }, + Address6Func: func() *iface.WGAddress { + v6addr, v6net, _ := net.ParseCIDR("fd00:1234:dead:beef::1/64") + return &iface.WGAddress{ + IP: v6addr, + Network: &net.IPNet{ + IP: v6net.IP, + Mask: v6net.Mask, + }, + } + }, } for _, testMax := range []int{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000} { diff --git a/client/firewall/nftables/route_linux.go b/client/firewall/nftables/route_linux.go index 381136e50..862d3c24d 100644 --- a/client/firewall/nftables/route_linux.go +++ b/client/firewall/nftables/route_linux.go @@ -26,7 +26,8 @@ const ( // some presets for building nftable rules var ( - zeroXor = binaryutil.NativeEndian.PutUint32(0) + zeroXor = binaryutil.NativeEndian.PutUint32(0) + zeroXor6 = []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0} exprCounterAccept = []expr.Any{ &expr.Counter{}, @@ -39,48 +40,69 @@ var ( ) type router struct { - ctx context.Context - stop context.CancelFunc - conn *nftables.Conn - workTable *nftables.Table - filterTable *nftables.Table - chains map[string]*nftables.Chain + ctx context.Context + stop context.CancelFunc + conn *nftables.Conn + workTable *nftables.Table + workTable6 *nftables.Table + filterTable *nftables.Table + filterTable6 *nftables.Table + chains map[string]*nftables.Chain + chains6 map[string]*nftables.Chain // rules is useful to avoid duplicates and to get missing attributes that we don't have when adding new rules - rules map[string]*nftables.Rule - isDefaultFwdRulesEnabled bool + rules map[string]*nftables.Rule + rules6 map[string]*nftables.Rule + isDefaultFwdRulesEnabled bool + isDefaultFwdRulesEnabled6 bool } -func newRouter(parentCtx context.Context, workTable *nftables.Table) (*router, error) { +func newRouter(parentCtx context.Context, workTable *nftables.Table, workTable6 *nftables.Table) (*router, error) { ctx, cancel := context.WithCancel(parentCtx) r := &router{ - ctx: ctx, - stop: cancel, - conn: &nftables.Conn{}, - workTable: workTable, - chains: make(map[string]*nftables.Chain), - rules: make(map[string]*nftables.Rule), + ctx: ctx, + stop: cancel, + conn: &nftables.Conn{}, + workTable: workTable, + workTable6: workTable6, + chains: make(map[string]*nftables.Chain), + chains6: make(map[string]*nftables.Chain), + rules: make(map[string]*nftables.Rule), + rules6: make(map[string]*nftables.Rule), } var err error - r.filterTable, err = r.loadFilterTable() + r.filterTable, r.filterTable6, err = r.loadFilterTables() if err != nil { if errors.Is(err, errFilterTableNotFound) { - log.Warnf("table 'filter' not found for forward rules") + log.Warnf("table 'filter' not found for forward rules for one of the supported address families-") } else { return nil, err } } - err = r.cleanUpDefaultForwardRules() + err = r.cleanUpDefaultForwardRules(false) if err != nil { log.Errorf("failed to clean up rules from FORWARD chain: %s", err) } - err = r.createContainers() + err = r.cleanUpDefaultForwardRules(true) + if err != nil { + log.Errorf("failed to clean up rules from IPv6 FORWARD chain: %s", err) + } + + err = r.createContainers(false) if err != nil { log.Errorf("failed to create containers for route: %s", err) } + + if r.workTable6 != nil { + err = r.createContainers(true) + if err != nil { + log.Errorf("failed to create v6 containers for route: %s", err) + } + } + return r, err } @@ -90,43 +112,99 @@ func (r *router) RouteingFwChainName() string { // ResetForwardRules cleans existing nftables default forward rules from the system func (r *router) ResetForwardRules() { - err := r.cleanUpDefaultForwardRules() + err := r.cleanUpDefaultForwardRules(false) + if err != nil { + log.Errorf("failed to reset forward rules: %s", err) + } + err = r.cleanUpDefaultForwardRules(true) if err != nil { log.Errorf("failed to reset forward rules: %s", err) } } -func (r *router) loadFilterTable() (*nftables.Table, error) { +func (r *router) RestoreAfterV6Reset(newWorktable6 *nftables.Table) error { + r.workTable6 = newWorktable6 + if newWorktable6 != nil { + + err := r.cleanUpDefaultForwardRules(true) + if err != nil { + log.Errorf("failed to clean up rules from IPv6 FORWARD chain: %s", err) + } + + err = r.createContainers(true) + if err != nil { + return err + } + + for name, rule := range r.rules6 { + rule = &nftables.Rule{ + Table: r.workTable6, + Chain: r.chains6[rule.Chain.Name], + Exprs: rule.Exprs, + UserData: rule.UserData, + } + r.rules6[name] = r.conn.AddRule(rule) + } + } + return r.conn.Flush() +} + +func (r *router) loadFilterTables() (*nftables.Table, *nftables.Table, error) { tables, err := r.conn.ListTablesOfFamily(nftables.TableFamilyIPv4) if err != nil { - return nil, fmt.Errorf("nftables: unable to list tables: %v", err) + return nil, nil, fmt.Errorf("nftables: unable to list tables: %v", err) } + var table4 *nftables.Table = nil for _, table := range tables { if table.Name == "filter" { - return table, nil + table4 = table + break } } - return nil, errFilterTableNotFound + var table6 *nftables.Table = nil + tables, err = r.conn.ListTablesOfFamily(nftables.TableFamilyIPv6) + if err != nil { + return nil, nil, fmt.Errorf("nftables: unable to list tables: %v", err) + } + for _, table := range tables { + if table.Name == "filter" { + table6 = table + break + } + } + + err = nil + if table4 == nil || table6 == nil { + err = errFilterTableNotFound + } + + return table4, table6, err } -func (r *router) createContainers() error { +func (r *router) createContainers(forV6 bool) error { + workTable := r.workTable + chainStorage := r.chains + if forV6 { + workTable = r.workTable6 + chainStorage = r.chains6 + } - r.chains[chainNameRouteingFw] = r.conn.AddChain(&nftables.Chain{ + chainStorage[chainNameRouteingFw] = r.conn.AddChain(&nftables.Chain{ Name: chainNameRouteingFw, - Table: r.workTable, + Table: workTable, }) - r.chains[chainNameRoutingNat] = r.conn.AddChain(&nftables.Chain{ + chainStorage[chainNameRoutingNat] = r.conn.AddChain(&nftables.Chain{ Name: chainNameRoutingNat, - Table: r.workTable, + Table: workTable, Hooknum: nftables.ChainHookPostrouting, Priority: nftables.ChainPriorityNATSource - 1, Type: nftables.ChainTypeNAT, }) - err := r.refreshRulesMap() + err := r.refreshRulesMap(forV6) if err != nil { log.Errorf("failed to clean up rules from FORWARD chain: %s", err) } @@ -140,7 +218,13 @@ func (r *router) createContainers() error { // InsertRoutingRules inserts a nftable rule pair to the forwarding chain and if enabled, to the nat chain func (r *router) InsertRoutingRules(pair manager.RouterPair) error { - err := r.refreshRulesMap() + parsedIp, _, _ := net.ParseCIDR(pair.Source) + + if parsedIp.To4() == nil && r.workTable6 == nil { + return fmt.Errorf("nftables: attempted to add IPv6 routing rule even though IPv6 is not enabled for this host") + } + + err := r.refreshRulesMap(parsedIp.To4() == nil) if err != nil { return err } @@ -165,7 +249,11 @@ func (r *router) InsertRoutingRules(pair manager.RouterPair) error { } } - if r.filterTable != nil && !r.isDefaultFwdRulesEnabled { + filterTable := r.filterTable + if parsedIp.To4() == nil { + filterTable = r.filterTable6 + } + if filterTable != nil && !r.isDefaultFwdRulesEnabled { log.Debugf("add default accept forward rule") r.acceptForwardRule(pair.Source) } @@ -191,7 +279,13 @@ func (r *router) insertRoutingRule(format, chainName string, pair manager.Router ruleKey := manager.GenKey(format, pair.ID) - _, exists := r.rules[ruleKey] + parsedIp, _, _ := net.ParseCIDR(pair.Source) + rules := r.rules + if parsedIp.To4() == nil { + rules = r.rules6 + } + + _, exists := rules[ruleKey] if exists { err := r.removeRoutingRule(format, pair) if err != nil { @@ -199,18 +293,35 @@ func (r *router) insertRoutingRule(format, chainName string, pair manager.Router } } - r.rules[ruleKey] = r.conn.InsertRule(&nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainName], + table, chain := r.workTable, r.chains[chainName] + if parsedIp.To4() == nil { + table, chain = r.workTable6, r.chains6[chainName] + } + + newRule := r.conn.InsertRule(&nftables.Rule{ + Table: table, + Chain: chain, Exprs: expression, UserData: []byte(ruleKey), }) + + if parsedIp.To4() == nil { + r.rules[ruleKey] = newRule + } else { + r.rules6[ruleKey] = newRule + } return nil } func (r *router) acceptForwardRule(sourceNetwork string) { src := generateCIDRMatcherExpressions(true, sourceNetwork) dst := generateCIDRMatcherExpressions(false, "0.0.0.0/0") + table := r.filterTable + parsedIp, _, _ := net.ParseCIDR(sourceNetwork) + if parsedIp.To4() == nil { + dst = generateCIDRMatcherExpressions(false, "::/0") + table = r.filterTable6 + } var exprs []expr.Any exprs = append(src, append(dst, &expr.Verdict{ // nolint:gocritic @@ -218,10 +329,10 @@ func (r *router) acceptForwardRule(sourceNetwork string) { })...) rule := &nftables.Rule{ - Table: r.filterTable, + Table: table, Chain: &nftables.Chain{ Name: "FORWARD", - Table: r.filterTable, + Table: table, Type: nftables.ChainTypeFilter, Hooknum: nftables.ChainHookForward, Priority: nftables.ChainPriorityFilter, @@ -233,6 +344,9 @@ func (r *router) acceptForwardRule(sourceNetwork string) { r.conn.AddRule(rule) src = generateCIDRMatcherExpressions(true, "0.0.0.0/0") + if parsedIp.To4() == nil { + src = generateCIDRMatcherExpressions(true, "::/0") + } dst = generateCIDRMatcherExpressions(false, sourceNetwork) exprs = append(src, append(dst, &expr.Verdict{ //nolint:gocritic @@ -240,10 +354,10 @@ func (r *router) acceptForwardRule(sourceNetwork string) { })...) rule = &nftables.Rule{ - Table: r.filterTable, + Table: table, Chain: &nftables.Chain{ Name: "FORWARD", - Table: r.filterTable, + Table: table, Type: nftables.ChainTypeFilter, Hooknum: nftables.ChainHookForward, Priority: nftables.ChainPriorityFilter, @@ -252,12 +366,21 @@ func (r *router) acceptForwardRule(sourceNetwork string) { UserData: []byte(userDataAcceptForwardRuleDst), } r.conn.AddRule(rule) - r.isDefaultFwdRulesEnabled = true + if parsedIp.To4() == nil { + r.isDefaultFwdRulesEnabled6 = true + } else { + r.isDefaultFwdRulesEnabled = true + } } // RemoveRoutingRules removes a nftable rule pair from forwarding and nat chains func (r *router) RemoveRoutingRules(pair manager.RouterPair) error { - err := r.refreshRulesMap() + parsedIp, _, _ := net.ParseCIDR(pair.Source) + if parsedIp.To4() == nil && r.workTable6 == nil { + return fmt.Errorf("nftables: attempted to remove IPv6 routing rule even though IPv6 is not enabled for this host") + } + + err := r.refreshRulesMap(parsedIp.To4() == nil) if err != nil { return err } @@ -282,8 +405,12 @@ func (r *router) RemoveRoutingRules(pair manager.RouterPair) error { return err } - if len(r.rules) == 0 { - err := r.cleanUpDefaultForwardRules() + rulesList := r.rules + if parsedIp.To4() == nil { + rulesList = r.rules6 + } + if len(rulesList) == 0 { + err := r.cleanUpDefaultForwardRules(parsedIp.To4() == nil) if err != nil { log.Errorf("failed to clean up rules from FORWARD chain: %s", err) } @@ -301,7 +428,13 @@ func (r *router) RemoveRoutingRules(pair manager.RouterPair) error { func (r *router) removeRoutingRule(format string, pair manager.RouterPair) error { ruleKey := manager.GenKey(format, pair.ID) - rule, found := r.rules[ruleKey] + parsedIp, _, _ := net.ParseCIDR(pair.Source) + rules := r.rules + if parsedIp.To4() == nil { + rules = r.rules6 + } + + rule, found := rules[ruleKey] if found { ruleType := "forwarding" if rule.Chain.Type == nftables.ChainTypeNAT { @@ -315,49 +448,68 @@ func (r *router) removeRoutingRule(format string, pair manager.RouterPair) error log.Debugf("nftables: removing %s rule for %s", ruleType, pair.Destination) - delete(r.rules, ruleKey) + delete(rules, ruleKey) } return nil } // refreshRulesMap refreshes the rule map with the latest rules. this is useful to avoid // duplicates and to get missing attributes that we don't have when adding new rules -func (r *router) refreshRulesMap() error { - for _, chain := range r.chains { +func (r *router) refreshRulesMap(forV6 bool) error { + chainList := r.chains + if forV6 { + chainList = r.chains6 + } + for _, chain := range chainList { rules, err := r.conn.GetRules(chain.Table, chain) if err != nil { return fmt.Errorf("nftables: unable to list rules: %v", err) } for _, rule := range rules { if len(rule.UserData) > 0 { - r.rules[string(rule.UserData)] = rule + if forV6 { + r.rules6[string(rule.UserData)] = rule + } else { + r.rules[string(rule.UserData)] = rule + } } } } return nil } -func (r *router) cleanUpDefaultForwardRules() error { - if r.filterTable == nil { - r.isDefaultFwdRulesEnabled = false +func (r *router) cleanUpDefaultForwardRules(forV6 bool) error { + tableFamily := nftables.TableFamilyIPv4 + filterTable := r.filterTable + if forV6 { + tableFamily = nftables.TableFamilyIPv6 + filterTable = r.filterTable6 + } + + if filterTable == nil { + if forV6 { + r.isDefaultFwdRulesEnabled6 = false + } else { + r.isDefaultFwdRulesEnabled = false + } return nil } - chains, err := r.conn.ListChainsOfTableFamily(nftables.TableFamilyIPv4) + chains, err := r.conn.ListChainsOfTableFamily(tableFamily) if err != nil { return err } var rules []*nftables.Rule for _, chain := range chains { - if chain.Table.Name != r.filterTable.Name { + if chain.Table.Name != filterTable.Name { continue } if chain.Name != "FORWARD" { continue } - rules, err = r.conn.GetRules(r.filterTable, chain) + rules, err = r.conn.GetRules(filterTable, chain) if err != nil { return err } @@ -371,7 +523,12 @@ func (r *router) cleanUpDefaultForwardRules() error { } } } - r.isDefaultFwdRulesEnabled = false + + if forV6 { + r.isDefaultFwdRulesEnabled6 = false + } else { + r.isDefaultFwdRulesEnabled = false + } return r.conn.Flush() } @@ -387,6 +544,18 @@ func generateCIDRMatcherExpressions(source bool, cidr string) []expr.Any { } else { offSet = 16 // dst offset } + addrLen := uint32(4) + zeroXor := zeroXor + + if ip.To4() == nil { + if source { + offSet = 8 // src offset + } else { + offSet = 24 // dst offset + } + addrLen = 16 + zeroXor = zeroXor6 + } return []expr.Any{ // fetch src add @@ -394,13 +563,13 @@ func generateCIDRMatcherExpressions(source bool, cidr string) []expr.Any { DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: offSet, - Len: 4, + Len: addrLen, }, // net mask &expr.Bitwise{ DestRegister: 1, SourceRegister: 1, - Len: 4, + Len: addrLen, Mask: network.Mask, Xor: zeroXor, }, diff --git a/client/firewall/nftables/router_linux_test.go b/client/firewall/nftables/router_linux_test.go index aa1224a5a..23db1f74c 100644 --- a/client/firewall/nftables/router_linux_test.go +++ b/client/firewall/nftables/router_linux_test.go @@ -4,6 +4,7 @@ package nftables import ( "context" + "github.com/netbirdio/netbird/iface" "testing" "github.com/coreos/go-iptables/iptables" @@ -29,16 +30,19 @@ func TestNftablesManager_InsertRoutingRules(t *testing.T) { t.Skip("nftables not supported on this OS") } - table, err := createWorkTable() + table, table6, err := createWorkTables() if err != nil { t.Fatal(err) } - defer deleteWorkTable() + defer deleteWorkTables() for _, testCase := range test.InsertRuleTestCases { t.Run(testCase.Name, func(t *testing.T) { - manager, err := newRouter(context.TODO(), table) + if testCase.IsV6 && table6 == nil { + t.Skip("Environment does not support IPv6, skipping IPv6 test...") + } + manager, err := newRouter(context.TODO(), table, table6) require.NoError(t, err, "failed to create router") nftablesTestingClient := &nftables.Conn{} @@ -58,8 +62,13 @@ func TestNftablesManager_InsertRoutingRules(t *testing.T) { testingExpression := append(sourceExp, destExp...) //nolint:gocritic fwdRuleKey := firewall.GenKey(firewall.ForwardingFormat, testCase.InputPair.ID) + chains := manager.chains + if testCase.IsV6 { + chains = manager.chains6 + } + found := 0 - for _, chain := range manager.chains { + for _, chain := range chains { rules, err := nftablesTestingClient.GetRules(chain.Table, chain) require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name) for _, rule := range rules { @@ -75,7 +84,7 @@ func TestNftablesManager_InsertRoutingRules(t *testing.T) { if testCase.InputPair.Masquerade { natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair.ID) found := 0 - for _, chain := range manager.chains { + for _, chain := range chains { rules, err := nftablesTestingClient.GetRules(chain.Table, chain) require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name) for _, rule := range rules { @@ -94,7 +103,7 @@ func TestNftablesManager_InsertRoutingRules(t *testing.T) { inFwdRuleKey := firewall.GenKey(firewall.InForwardingFormat, testCase.InputPair.ID) found = 0 - for _, chain := range manager.chains { + for _, chain := range chains { rules, err := nftablesTestingClient.GetRules(chain.Table, chain) require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name) for _, rule := range rules { @@ -110,7 +119,7 @@ func TestNftablesManager_InsertRoutingRules(t *testing.T) { if testCase.InputPair.Masquerade { inNatRuleKey := firewall.GenKey(firewall.InNatFormat, testCase.InputPair.ID) found := 0 - for _, chain := range manager.chains { + for _, chain := range chains { rules, err := nftablesTestingClient.GetRules(chain.Table, chain) require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name) for _, rule := range rules { @@ -131,16 +140,19 @@ func TestNftablesManager_RemoveRoutingRules(t *testing.T) { t.Skip("nftables not supported on this OS") } - table, err := createWorkTable() + table, table6, err := createWorkTables() if err != nil { t.Fatal(err) } - defer deleteWorkTable() + defer deleteWorkTables() for _, testCase := range test.RemoveRuleTestCases { t.Run(testCase.Name, func(t *testing.T) { - manager, err := newRouter(context.TODO(), table) + if testCase.IsV6 && table6 == nil { + t.Skip("Environment does not support IPv6, skipping IPv6 test...") + } + manager, err := newRouter(context.TODO(), table, table6) require.NoError(t, err, "failed to create router") nftablesTestingClient := &nftables.Conn{} @@ -150,11 +162,18 @@ func TestNftablesManager_RemoveRoutingRules(t *testing.T) { sourceExp := generateCIDRMatcherExpressions(true, testCase.InputPair.Source) destExp := generateCIDRMatcherExpressions(false, testCase.InputPair.Destination) + chains := manager.chains + workTable := table + if testCase.IsV6 { + chains = manager.chains6 + workTable = table6 + } + forwardExp := append(sourceExp, append(destExp, exprCounterAccept...)...) //nolint:gocritic forwardRuleKey := firewall.GenKey(firewall.ForwardingFormat, testCase.InputPair.ID) insertedForwarding := nftablesTestingClient.InsertRule(&nftables.Rule{ - Table: manager.workTable, - Chain: manager.chains[chainNameRouteingFw], + Table: workTable, + Chain: chains[chainNameRouteingFw], Exprs: forwardExp, UserData: []byte(forwardRuleKey), }) @@ -163,8 +182,8 @@ func TestNftablesManager_RemoveRoutingRules(t *testing.T) { natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair.ID) insertedNat := nftablesTestingClient.InsertRule(&nftables.Rule{ - Table: manager.workTable, - Chain: manager.chains[chainNameRoutingNat], + Table: workTable, + Chain: chains[chainNameRoutingNat], Exprs: natExp, UserData: []byte(natRuleKey), }) @@ -175,8 +194,8 @@ func TestNftablesManager_RemoveRoutingRules(t *testing.T) { forwardExp = append(sourceExp, append(destExp, exprCounterAccept...)...) //nolint:gocritic inForwardRuleKey := firewall.GenKey(firewall.InForwardingFormat, testCase.InputPair.ID) insertedInForwarding := nftablesTestingClient.InsertRule(&nftables.Rule{ - Table: manager.workTable, - Chain: manager.chains[chainNameRouteingFw], + Table: workTable, + Chain: chains[chainNameRouteingFw], Exprs: forwardExp, UserData: []byte(inForwardRuleKey), }) @@ -185,8 +204,8 @@ func TestNftablesManager_RemoveRoutingRules(t *testing.T) { inNatRuleKey := firewall.GenKey(firewall.InNatFormat, testCase.InputPair.ID) insertedInNat := nftablesTestingClient.InsertRule(&nftables.Rule{ - Table: manager.workTable, - Chain: manager.chains[chainNameRoutingNat], + Table: workTable, + Chain: chains[chainNameRoutingNat], Exprs: natExp, UserData: []byte(inNatRuleKey), }) @@ -199,7 +218,7 @@ func TestNftablesManager_RemoveRoutingRules(t *testing.T) { err = manager.RemoveRoutingRules(testCase.InputPair) require.NoError(t, err, "shouldn't return error") - for _, chain := range manager.chains { + for _, chain := range chains { rules, err := nftablesTestingClient.GetRules(chain.Table, chain) require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name) for _, rule := range rules { @@ -238,30 +257,39 @@ func isIptablesClientAvailable(client *iptables.IPTables) bool { return err == nil } -func createWorkTable() (*nftables.Table, error) { +func createWorkTables() (*nftables.Table, *nftables.Table, error) { sConn, err := nftables.New(nftables.AsLasting()) if err != nil { - return nil, err + return nil, nil, err } tables, err := sConn.ListTablesOfFamily(nftables.TableFamilyIPv4) if err != nil { - return nil, err + return nil, nil, err } - for _, t := range tables { + tables6, err := sConn.ListTablesOfFamily(nftables.TableFamilyIPv6) + if err != nil { + return nil, nil, err + } + + for _, t := range append(tables, tables6...) { if t.Name == tableName { sConn.DelTable(t) } } table := sConn.AddTable(&nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv4}) + var table6 *nftables.Table + if iface.SupportsIPv6() { + table6 = sConn.AddTable(&nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv6}) + } err = sConn.Flush() - return table, err + return table, table6, err } -func deleteWorkTable() { +func deleteWorkTables() { sConn, err := nftables.New(nftables.AsLasting()) if err != nil { return @@ -272,6 +300,12 @@ func deleteWorkTable() { return } + tables6, err := sConn.ListTablesOfFamily(nftables.TableFamilyIPv6) + if err != nil { + return + } + tables = append(tables, tables6...) + for _, t := range tables { if t.Name == tableName { sConn.DelTable(t) diff --git a/client/firewall/test/cases_linux.go b/client/firewall/test/cases_linux.go index 432d113dd..acc71a92c 100644 --- a/client/firewall/test/cases_linux.go +++ b/client/firewall/test/cases_linux.go @@ -8,6 +8,7 @@ var ( InsertRuleTestCases = []struct { Name string InputPair firewall.RouterPair + IsV6 bool }{ { Name: "Insert Forwarding IPV4 Rule", @@ -27,12 +28,32 @@ var ( Masquerade: true, }, }, + { + Name: "Insert Forwarding IPV6 Rule", + InputPair: firewall.RouterPair{ + ID: "zxa", + Source: "2001:db8:0123:4567::1/128", + Destination: "2001:db8:0123:abcd::/64", + Masquerade: false, + }, + IsV6: true, + }, + { + Name: "Insert Forwarding And Nat IPV6 Rules", + InputPair: firewall.RouterPair{ + ID: "zxa", + Source: "2001:db8:0123:4567::1/128", + Destination: "2001:db8:0123:abcd::/64", + Masquerade: true, + }, + IsV6: true, + }, } RemoveRuleTestCases = []struct { Name string InputPair firewall.RouterPair - IpVersion string + IsV6 bool }{ { Name: "Remove Forwarding And Nat IPV4 Rules", @@ -43,5 +64,15 @@ var ( Masquerade: true, }, }, + { + Name: "Remove Forwarding And Nat IPV6 Rules", + InputPair: firewall.RouterPair{ + ID: "zxa", + Source: "2001:db8:0123:4567::1/128", + Destination: "2001:db8:0123:abcd::/64", + Masquerade: true, + }, + IsV6: true, + }, } ) diff --git a/client/firewall/uspfilter/uspfilter.go b/client/firewall/uspfilter/uspfilter.go index 427a73825..2103639e3 100644 --- a/client/firewall/uspfilter/uspfilter.go +++ b/client/firewall/uspfilter/uspfilter.go @@ -24,6 +24,7 @@ var ( type IFaceMapper interface { SetFilter(iface.PacketFilter) error Address() iface.WGAddress + Address6() *iface.WGAddress } // RuleSet is a set of rules grouped by a string key @@ -69,6 +70,14 @@ func CreateWithNativeFirewall(iface IFaceMapper, nativeFirewall firewall.Manager return mgr, nil } +func (m *Manager) ResetV6Firewall() error { + return nil +} + +func (m *Manager) V6Active() bool { + return false +} + func create(iface IFaceMapper) (*Manager, error) { m := &Manager{ decoders: sync.Pool{ diff --git a/client/firewall/uspfilter/uspfilter_test.go b/client/firewall/uspfilter/uspfilter_test.go index 514a90539..031513304 100644 --- a/client/firewall/uspfilter/uspfilter_test.go +++ b/client/firewall/uspfilter/uspfilter_test.go @@ -33,6 +33,10 @@ func (i *IFaceMock) Address() iface.WGAddress { return i.AddressFunc() } +func (i *IFaceMock) Address6() *iface.WGAddress { + return nil +} + func TestManagerCreate(t *testing.T) { ifaceMock := &IFaceMock{ SetFilterFunc: func(iface.PacketFilter) error { return nil }, diff --git a/client/internal/acl/manager.go b/client/internal/acl/manager.go index fd2c2c875..50bd9dd22 100644 --- a/client/internal/acl/manager.go +++ b/client/internal/acl/manager.go @@ -16,9 +16,10 @@ import ( mgmProto "github.com/netbirdio/netbird/management/proto" ) -// Manager is a ACL rules manager +// Manager is an ACL rules manager type Manager interface { ApplyFiltering(networkMap *mgmProto.NetworkMap) + ResetV6Acl() error } // DefaultManager uses firewall manager to handle @@ -26,16 +27,36 @@ type DefaultManager struct { firewall firewall.Manager ipsetCounter int rulesPairs map[string][]firewall.Rule + rulesPairs6 map[string][]firewall.Rule mutex sync.Mutex } func NewDefaultManager(fm firewall.Manager) *DefaultManager { return &DefaultManager{ - firewall: fm, - rulesPairs: make(map[string][]firewall.Rule), + firewall: fm, + rulesPairs: make(map[string][]firewall.Rule), + rulesPairs6: make(map[string][]firewall.Rule), } } +func (d *DefaultManager) ResetV6Acl() error { + for _, rules := range d.rulesPairs6 { + for _, r := range rules { + err := d.firewall.DeleteRule(r) + if err != nil { + return err + } + } + } + err := d.firewall.ResetV6Firewall() + if err != nil { + return err + } + d.rulesPairs6 = make(map[string][]firewall.Rule) + + return nil +} + // ApplyFiltering firewall rules to the local firewall manager processed by ACL policy. // // If allowByDefault is true it appends allow ALL traffic rules to input and output chains. @@ -83,6 +104,7 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap) { if enableSSH { rules = append(rules, &mgmProto.FirewallRule{ PeerIP: "0.0.0.0", + PeerIP6: "::", Direction: mgmProto.FirewallRule_IN, Action: mgmProto.FirewallRule_ACCEPT, Protocol: mgmProto.FirewallRule_TCP, @@ -97,12 +119,14 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap) { rules = append(rules, &mgmProto.FirewallRule{ PeerIP: "0.0.0.0", + PeerIP6: "::", Direction: mgmProto.FirewallRule_IN, Action: mgmProto.FirewallRule_ACCEPT, Protocol: mgmProto.FirewallRule_ALL, }, &mgmProto.FirewallRule{ PeerIP: "0.0.0.0", + PeerIP6: "::", Direction: mgmProto.FirewallRule_OUT, Action: mgmProto.FirewallRule_ACCEPT, Protocol: mgmProto.FirewallRule_ALL, @@ -111,6 +135,7 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap) { } newRulePairs := make(map[string][]firewall.Rule) + newRulePairs6 := make(map[string][]firewall.Rule) ipsetByRuleSelectors := make(map[string]string) for _, r := range rules { @@ -123,7 +148,7 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap) { ipsetName = fmt.Sprintf("nb%07d", d.ipsetCounter) ipsetByRuleSelectors[selector] = ipsetName } - pairID, rulePair, err := d.protoRuleToFirewallRule(r, ipsetName) + pairID, rulePair, rulePair6, err := d.protoRuleToFirewallRule(r, ipsetName) if err != nil { log.Errorf("failed to apply firewall rule: %+v, %v", r, err) d.rollBack(newRulePairs) @@ -132,6 +157,8 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap) { if len(rules) > 0 { d.rulesPairs[pairID] = rulePair newRulePairs[pairID] = rulePair + d.rulesPairs6[pairID] = rulePair6 + newRulePairs6[pairID] = rulePair6 } } @@ -146,59 +173,104 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap) { delete(d.rulesPairs, pairID) } } + for pairID, rules := range d.rulesPairs6 { + if _, ok := newRulePairs6[pairID]; !ok { + for _, rule := range rules { + if err := d.firewall.DeleteRule(rule); err != nil { + log.Errorf("failed to delete firewall rule: %v", err) + continue + } + } + delete(d.rulesPairs6, pairID) + } + } + d.rulesPairs = newRulePairs + d.rulesPairs6 = newRulePairs6 } func (d *DefaultManager) protoRuleToFirewallRule( r *mgmProto.FirewallRule, ipsetName string, -) (string, []firewall.Rule, error) { +) (string, []firewall.Rule, []firewall.Rule, error) { ip := net.ParseIP(r.PeerIP) if ip == nil { - return "", nil, fmt.Errorf("invalid IP address, skipping firewall rule") + return "", nil, nil, fmt.Errorf("invalid IP address, skipping firewall rule") + } + + var ip6 *net.IP = nil + if d.firewall.V6Active() && r.PeerIP6 != "" { + ip6tmp := net.ParseIP(r.PeerIP6) + if ip6tmp == nil { + return "", nil, nil, fmt.Errorf("invalid IP address, skipping firewall rule") + } + ip6 = &ip6tmp } protocol, err := convertToFirewallProtocol(r.Protocol) if err != nil { - return "", nil, fmt.Errorf("skipping firewall rule: %s", err) + return "", nil, nil, fmt.Errorf("skipping firewall rule: %s", err) } action, err := convertFirewallAction(r.Action) if err != nil { - return "", nil, fmt.Errorf("skipping firewall rule: %s", err) + return "", nil, nil, fmt.Errorf("skipping firewall rule: %s", err) } var port *firewall.Port if r.Port != "" { value, err := strconv.Atoi(r.Port) if err != nil { - return "", nil, fmt.Errorf("invalid port, skipping firewall rule") + return "", nil, nil, fmt.Errorf("invalid port, skipping firewall rule") } port = &firewall.Port{ Values: []int{value}, } } - ruleID := d.getRuleID(ip, protocol, int(r.Direction), port, action, "") + var rules []firewall.Rule + var rules6 []firewall.Rule + + ruleID := d.getRuleID(ip, ip6, protocol, int(r.Direction), port, action, "") if rulesPair, ok := d.rulesPairs[ruleID]; ok { - return ruleID, rulesPair, nil + rules = rulesPair + } + if rulesPair6, ok := d.rulesPairs6[ruleID]; d.firewall.V6Active() && ok && ip6 != nil { + rules6 = rulesPair6 } - var rules []firewall.Rule - switch r.Direction { - case mgmProto.FirewallRule_IN: - rules, err = d.addInRules(ip, protocol, port, action, ipsetName, "") - case mgmProto.FirewallRule_OUT: - rules, err = d.addOutRules(ip, protocol, port, action, ipsetName, "") - default: - return "", nil, fmt.Errorf("invalid direction, skipping firewall rule") + if rules == nil { + switch r.Direction { + case mgmProto.FirewallRule_IN: + rules, err = d.addInRules(ip, protocol, port, action, ipsetName, "") + case mgmProto.FirewallRule_OUT: + rules, err = d.addOutRules(ip, protocol, port, action, ipsetName, "") + default: + return "", nil, nil, fmt.Errorf("invalid direction, skipping firewall rule") + } } if err != nil { - return "", nil, err + return "", nil, nil, err } - return ruleID, rules, nil + if d.firewall.V6Active() && ip6 != nil && rules6 == nil { + switch r.Direction { + case mgmProto.FirewallRule_IN: + rules6, err = d.addInRules(*ip6, protocol, port, action, ipsetName, "") + case mgmProto.FirewallRule_OUT: + rules6, err = d.addOutRules(*ip6, protocol, port, action, ipsetName, "") + default: + return "", nil, nil, fmt.Errorf("invalid direction, skipping firewall rule") + } + + } + + if err != nil && err.Error() != "failed to add firewall rule: attempted to configure filtering for IPv6 address even though IPv6 is not active" { + return "", rules, nil, err + } + + return ruleID, rules, rules6, nil } func (d *DefaultManager) addInRules( @@ -226,8 +298,9 @@ func (d *DefaultManager) addInRules( if err != nil { return nil, fmt.Errorf("failed to add firewall rule: %v", err) } + rules = append(rules, rule...) - return append(rules, rule...), nil + return rules, nil } func (d *DefaultManager) addOutRules( @@ -255,20 +328,26 @@ func (d *DefaultManager) addOutRules( if err != nil { return nil, fmt.Errorf("failed to add firewall rule: %v", err) } + rules = append(rules, rule...) - return append(rules, rule...), nil + return rules, nil } // getRuleID() returns unique ID for the rule based on its parameters. func (d *DefaultManager) getRuleID( ip net.IP, + ip6 *net.IP, proto firewall.Protocol, direction int, port *firewall.Port, action firewall.Action, comment string, ) string { - idStr := ip.String() + string(proto) + strconv.Itoa(direction) + strconv.Itoa(int(action)) + comment + ip6Str := "" + if ip6 != nil { + ip6Str = ip6.String() + } + idStr := ip.String() + ip6Str + string(proto) + strconv.Itoa(direction) + strconv.Itoa(int(action)) + comment if port != nil { idStr += port.String() } @@ -321,6 +400,8 @@ func (d *DefaultManager) squashAcceptRules( // it means that rules for that protocol was already optimized on the // management side if r.PeerIP == "0.0.0.0" { + // I don't _think_ that IPv6 is relevant here, as any optimization that has r.PeerIP6 == "::" should also + // implicitly have r.PeerIP == "0.0.0.0". squashedRules = append(squashedRules, r) squashedProtocols[r.Protocol] = struct{}{} return @@ -364,6 +445,7 @@ func (d *DefaultManager) squashAcceptRules( // add special rule 0.0.0.0 which allows all IP's in our firewall implementations squashedRules = append(squashedRules, &mgmProto.FirewallRule{ PeerIP: "0.0.0.0", + PeerIP6: "::", Direction: direction, Action: mgmProto.FirewallRule_ACCEPT, Protocol: protocol, diff --git a/client/internal/acl/manager_test.go b/client/internal/acl/manager_test.go index 494d54bf2..e4e768176 100644 --- a/client/internal/acl/manager_test.go +++ b/client/internal/acl/manager_test.go @@ -19,6 +19,7 @@ func TestDefaultManager(t *testing.T) { FirewallRules: []*mgmProto.FirewallRule{ { PeerIP: "10.93.0.1", + PeerIP6: "2001:db8::fedc:ba09:8765:0001", Direction: mgmProto.FirewallRule_OUT, Action: mgmProto.FirewallRule_ACCEPT, Protocol: mgmProto.FirewallRule_TCP, @@ -26,6 +27,7 @@ func TestDefaultManager(t *testing.T) { }, { PeerIP: "10.93.0.2", + PeerIP6: "2001:db8::fedc:ba09:8765:0002", Direction: mgmProto.FirewallRule_OUT, Action: mgmProto.FirewallRule_DROP, Protocol: mgmProto.FirewallRule_UDP, @@ -50,6 +52,14 @@ func TestDefaultManager(t *testing.T) { IP: ip, Network: network, }).AnyTimes() + ip6, network6, err := net.ParseCIDR("2001:db8::fedc:ba09:8765:4321/64") + if err != nil { + t.Fatalf("failed to parse IP address: %v", err) + } + ifaceMock.EXPECT().Address6().Return(&iface.WGAddress{ + IP: ip6, + Network: network6, + }).AnyTimes() // we receive one rule from the management so for testing purposes ignore it fw, err := firewall.NewFirewall(context.Background(), ifaceMock) @@ -83,6 +93,7 @@ func TestDefaultManager(t *testing.T) { networkMap.FirewallRules, &mgmProto.FirewallRule{ PeerIP: "10.93.0.3", + PeerIP6: "2001:db8::fedc:ba09:8765:0003", Direction: mgmProto.FirewallRule_IN, Action: mgmProto.FirewallRule_DROP, Protocol: mgmProto.FirewallRule_ICMP, @@ -343,6 +354,7 @@ func TestDefaultManagerEnableSSHRules(t *testing.T) { IP: ip, Network: network, }).AnyTimes() + ifaceMock.EXPECT().Address6().Return(nil).AnyTimes() // we receive one rule from the management so for testing purposes ignore it fw, err := firewall.NewFirewall(context.Background(), ifaceMock) diff --git a/client/internal/acl/mocks/iface_mapper.go b/client/internal/acl/mocks/iface_mapper.go index 621b29513..037b3ad37 100644 --- a/client/internal/acl/mocks/iface_mapper.go +++ b/client/internal/acl/mocks/iface_mapper.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/netbirdio/netbird/client/internal/acl (interfaces: IFaceMapper) +// Source: ./client/firewall/iface.go // Package mocks is a generated GoMock package. package mocks @@ -48,6 +48,20 @@ func (mr *MockIFaceMapperMockRecorder) Address() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Address", reflect.TypeOf((*MockIFaceMapper)(nil).Address)) } +// Address6 mocks base method. +func (m *MockIFaceMapper) Address6() *iface.WGAddress { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Address6") + ret0, _ := ret[0].(*iface.WGAddress) + return ret0 +} + +// Address6 indicates an expected call of Address6. +func (mr *MockIFaceMapperMockRecorder) Address6() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Address6", reflect.TypeOf((*MockIFaceMapper)(nil).Address6)) +} + // IsUserspaceBind mocks base method. func (m *MockIFaceMapper) IsUserspaceBind() bool { m.ctrl.T.Helper() diff --git a/client/internal/connect.go b/client/internal/connect.go index d34d0aab0..05df9bdea 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -310,6 +310,7 @@ func createEngineConfig(key wgtypes.Key, config *Config, peerConfig *mgmProto.Pe engineConf := &EngineConfig{ WgIfaceName: config.WgIface, WgAddr: peerConfig.Address, + WgAddr6: peerConfig.Address6, IFaceBlackList: config.IFaceBlackList, DisableIPv6Discovery: config.DisableIPv6Discovery, WgPrivateKey: key, diff --git a/client/internal/dns/server.go b/client/internal/dns/server.go index 267c1ed80..5910985ed 100644 --- a/client/internal/dns/server.go +++ b/client/internal/dns/server.go @@ -485,7 +485,11 @@ func (s *DefaultServer) updateLocalResolver(update map[string]nbdns.SimpleRecord } func getNSHostPort(ns nbdns.NameServer) string { - return fmt.Sprintf("%s:%d", ns.IP.String(), ns.Port) + if ns.IP.Is4() { + return fmt.Sprintf("%s:%d", ns.IP.String(), ns.Port) + } else { + return fmt.Sprintf("[%s]:%d", ns.IP.String(), ns.Port) + } } // upstreamCallbacks returns two functions, the first one is used to deactivate diff --git a/client/internal/dns/server_test.go b/client/internal/dns/server_test.go index 22966d89c..559d60fc0 100644 --- a/client/internal/dns/server_test.go +++ b/client/internal/dns/server_test.go @@ -38,6 +38,13 @@ func (w *mocWGIface) Address() iface.WGAddress { Network: network, } } +func (w *mocWGIface) Address6() *iface.WGAddress { + ip, network, _ := net.ParseCIDR("fd00:1234:dead:beef::/64") + return &iface.WGAddress{ + IP: ip, + Network: network, + } +} func (w *mocWGIface) GetFilter() iface.PacketFilter { return w.filter @@ -261,7 +268,7 @@ func TestUpdateDNSServer(t *testing.T) { if err != nil { t.Fatal(err) } - wgIface, err := iface.NewWGIFace(fmt.Sprintf("utun230%d", n), fmt.Sprintf("100.66.100.%d/32", n+1), 33100, privKey.String(), iface.DefaultMTU, newNet, nil) + wgIface, err := iface.NewWGIFace(fmt.Sprintf("utun230%d", n), fmt.Sprintf("100.66.100.%d/32", n+1), fmt.Sprintf("fd00:1234:dead:beef::%d/128", n+1), 33100, privKey.String(), iface.DefaultMTU, newNet, nil) if err != nil { t.Fatal(err) } @@ -339,7 +346,7 @@ func TestDNSFakeResolverHandleUpdates(t *testing.T) { } privKey, _ := wgtypes.GeneratePrivateKey() - wgIface, err := iface.NewWGIFace("utun2301", "100.66.100.1/32", 33100, privKey.String(), iface.DefaultMTU, newNet, nil) + wgIface, err := iface.NewWGIFace("utun2301", "100.66.100.1/32", "", 33100, privKey.String(), iface.DefaultMTU, newNet, nil) if err != nil { t.Errorf("build interface wireguard: %v", err) return @@ -595,7 +602,7 @@ func TestDNSServerUpstreamDeactivateCallback(t *testing.T) { } func TestDNSPermanent_updateHostDNS_emptyUpstream(t *testing.T) { - wgIFace, err := createWgInterfaceWithBind(t) + wgIFace, err := createWgInterfaceWithBind(t, false) if err != nil { t.Fatal("failed to initialize wg interface") } @@ -621,7 +628,7 @@ func TestDNSPermanent_updateHostDNS_emptyUpstream(t *testing.T) { } func TestDNSPermanent_updateUpstream(t *testing.T) { - wgIFace, err := createWgInterfaceWithBind(t) + wgIFace, err := createWgInterfaceWithBind(t, false) if err != nil { t.Fatal("failed to initialize wg interface") } @@ -713,7 +720,7 @@ func TestDNSPermanent_updateUpstream(t *testing.T) { } func TestDNSPermanent_matchOnly(t *testing.T) { - wgIFace, err := createWgInterfaceWithBind(t) + wgIFace, err := createWgInterfaceWithBind(t, false) if err != nil { t.Fatal("failed to initialize wg interface") } @@ -784,7 +791,7 @@ func TestDNSPermanent_matchOnly(t *testing.T) { } } -func createWgInterfaceWithBind(t *testing.T) (*iface.WGIface, error) { +func createWgInterfaceWithBind(t *testing.T, enableV6 bool) (*iface.WGIface, error) { t.Helper() ov := os.Getenv("NB_WG_KERNEL_DISABLED") defer t.Setenv("NB_WG_KERNEL_DISABLED", ov) @@ -797,7 +804,11 @@ func createWgInterfaceWithBind(t *testing.T) (*iface.WGIface, error) { } privKey, _ := wgtypes.GeneratePrivateKey() - wgIface, err := iface.NewWGIFace("utun2301", "100.66.100.2/24", 33100, privKey.String(), iface.DefaultMTU, newNet, nil) + v6Addr := "" + if enableV6 { + v6Addr = "fd00:1234:dead:beef::1/128" + } + wgIface, err := iface.NewWGIFace("utun2301", "100.66.100.2/24", v6Addr, 33100, privKey.String(), iface.DefaultMTU, newNet, nil) if err != nil { t.Fatalf("build interface wireguard: %v", err) return nil, err diff --git a/client/internal/engine.go b/client/internal/engine.go index 83e6928f4..1721b3c2e 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -58,7 +58,8 @@ type EngineConfig struct { WgIfaceName string // WgAddr is a Wireguard local address (Netbird Network IP) - WgAddr string + WgAddr string + WgAddr6 string // WgPrivateKey is a Wireguard private key of our peer (it MUST never leave the machine) WgPrivateKey wgtypes.Key @@ -603,6 +604,32 @@ func (e *Engine) updateConfig(conf *mgmProto.PeerConfig) error { log.Infof("updated peer address from %s to %s", oldAddr, conf.Address) } + if e.wgInterface.Address6() == nil && conf.Address6 != "" || + e.wgInterface.Address6() != nil && e.wgInterface.Address6().String() != conf.Address6 { + oldAddr := "none" + if e.wgInterface.Address6() != nil { + oldAddr = e.wgInterface.Address6().String() + } + newAddr := "none" + if conf.Address6 != "" { + newAddr = conf.Address6 + } + log.Debugf("updating peer IPv6 address from %s to %s", oldAddr, newAddr) + err := e.wgInterface.UpdateAddr6(conf.Address6) + if err != nil { + return err + } + e.config.WgAddr6 = conf.Address6 + + err = e.acl.ResetV6Acl() + if err != nil { + return err + } + + e.routeManager.ResetV6Routes() + log.Infof("updated peer IPv6 address from %s to %s", oldAddr, conf.Address6) + } + if conf.GetSshConfig() != nil { err := e.updateSSH(conf.GetSshConfig()) if err != nil { @@ -612,6 +639,7 @@ func (e *Engine) updateConfig(conf *mgmProto.PeerConfig) error { e.statusRecorder.UpdateLocalPeerState(peer.LocalPeerState{ IP: e.config.WgAddr, + IP6: e.config.WgAddr6, PubKey: e.config.WgPrivateKey.PublicKey().String(), KernelInterface: iface.WireGuardModuleIsLoaded(), FQDN: conf.GetFqdn(), @@ -1238,7 +1266,7 @@ func (e *Engine) newWgIface() (*iface.WGIface, error) { default: } - return iface.NewWGIFace(e.config.WgIfaceName, e.config.WgAddr, e.config.WgPort, e.config.WgPrivateKey.String(), iface.DefaultMTU, transportNet, mArgs) + return iface.NewWGIFace(e.config.WgIfaceName, e.config.WgAddr, e.config.WgAddr6, e.config.WgPort, e.config.WgPrivateKey.String(), iface.DefaultMTU, transportNet, mArgs) } func (e *Engine) wgInterfaceCreate() (err error) { diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go index f5a98cb7f..f79354b51 100644 --- a/client/internal/engine_test.go +++ b/client/internal/engine_test.go @@ -216,7 +216,7 @@ func TestEngine_UpdateNetworkMap(t *testing.T) { if err != nil { t.Fatal(err) } - engine.wgInterface, err = iface.NewWGIFace("utun102", "100.64.0.1/24", engine.config.WgPort, key.String(), iface.DefaultMTU, newNet, nil) + engine.wgInterface, err = iface.NewWGIFace("utun102", "100.64.0.1/24", "", engine.config.WgPort, key.String(), iface.DefaultMTU, newNet, nil) if err != nil { t.Fatal(err) } @@ -565,6 +565,7 @@ func TestEngine_UpdateNetworkMapWithRoutes(t *testing.T) { engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{ WgIfaceName: wgIfaceName, WgAddr: wgAddr, + WgAddr6: "", WgPrivateKey: key, WgPort: 33100, }, MobileDependency{}, peer.NewRecorder("https://mgm")) @@ -573,7 +574,7 @@ func TestEngine_UpdateNetworkMapWithRoutes(t *testing.T) { if err != nil { t.Fatal(err) } - engine.wgInterface, err = iface.NewWGIFace(wgIfaceName, wgAddr, engine.config.WgPort, key.String(), iface.DefaultMTU, newNet, nil) + engine.wgInterface, err = iface.NewWGIFace(wgIfaceName, wgAddr, engine.config.WgAddr6, engine.config.WgPort, key.String(), iface.DefaultMTU, newNet, nil) assert.NoError(t, err, "shouldn't return error") input := struct { inputSerial uint64 @@ -735,6 +736,7 @@ func TestEngine_UpdateNetworkMapWithDNSUpdate(t *testing.T) { engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{ WgIfaceName: wgIfaceName, WgAddr: wgAddr, + WgAddr6: "", WgPrivateKey: key, WgPort: 33100, }, MobileDependency{}, peer.NewRecorder("https://mgm")) @@ -744,7 +746,7 @@ func TestEngine_UpdateNetworkMapWithDNSUpdate(t *testing.T) { if err != nil { t.Fatal(err) } - engine.wgInterface, err = iface.NewWGIFace(wgIfaceName, wgAddr, 33100, key.String(), iface.DefaultMTU, newNet, nil) + engine.wgInterface, err = iface.NewWGIFace(wgIfaceName, wgAddr, engine.config.WgAddr6, 33100, key.String(), iface.DefaultMTU, newNet, nil) assert.NoError(t, err, "shouldn't return error") mockRouteManager := &routemanager.MockManager{ diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 5f329bf7f..c7291067a 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -484,7 +484,7 @@ func (conn *Conn) configureConnection(remoteConn net.Conn, remoteWgPort int, rem log.Warnf("unable to save peer's state, got error: %v", err) } - _, ipNet, err := net.ParseCIDR(conn.config.WgConfig.AllowedIps) + _, ipNet, err := net.ParseCIDR(strings.Split(conn.config.WgConfig.AllowedIps, ",")[0]) if err != nil { return nil, err } diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index ddea7d04e..b5aba8745 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -68,6 +68,7 @@ func (s *State) GetRoutes() map[string]struct{} { // LocalPeerState contains the latest state of the local peer type LocalPeerState struct { IP string + IP6 string PubKey string KernelInterface bool FQDN string diff --git a/client/internal/routemanager/client.go b/client/internal/routemanager/client.go index e82f4b1da..27240333b 100644 --- a/client/internal/routemanager/client.go +++ b/client/internal/routemanager/client.go @@ -38,6 +38,7 @@ type clientNetwork struct { peerStateUpdate chan struct{} routePeersNotifiers map[string]chan struct{} chosenRoute *route.Route + chosenIP *net.IP network netip.Prefix updateSerial uint64 } @@ -221,6 +222,7 @@ func (c *clientNetwork) removeRouteFromWireguardPeer(peerKey string) error { func (c *clientNetwork) removeRouteFromPeerAndSystem() error { if c.chosenRoute != nil { + // TODO IPv6 (pass wgInterface) if err := removeVPNRoute(c.network, c.getAsInterface()); err != nil { return fmt.Errorf("remove route %s from system, err: %v", c.network, err) } @@ -261,10 +263,20 @@ func (c *clientNetwork) recalculateRouteAndUpdatePeerAndSystem() error { return fmt.Errorf("remove route from peer: %v", err) } } else { + // TODO recheck IPv6 + gwAddr := c.wgInterface.Address().IP + c.chosenIP = &gwAddr + if c.network.Addr().Is6() { + if c.wgInterface.Address6() == nil { + return fmt.Errorf("Could not assign IPv6 route %s for peer %s because no IPv6 address is assigned", + c.network.String(), c.wgInterface.Address().IP.String()) + } + c.chosenIP = &c.wgInterface.Address6().IP + } // otherwise add the route to the system if err := addVPNRoute(c.network, c.getAsInterface()); err != nil { return fmt.Errorf("route %s couldn't be added for peer %s, err: %v", - c.network.String(), c.wgInterface.Address().IP.String(), err) + c.network.String(), c.chosenIP.String(), err) } } diff --git a/client/internal/routemanager/manager.go b/client/internal/routemanager/manager.go index 47549f74d..69a8f043c 100644 --- a/client/internal/routemanager/manager.go +++ b/client/internal/routemanager/manager.go @@ -34,6 +34,7 @@ type Manager interface { GetRouteSelector() *routeselector.RouteSelector SetRouteChangeListener(listener listener.NetworkChangeListener) InitialRouteRange() []string + ResetV6Routes() EnableServerRouter(firewall firewall.Manager) error Stop() } @@ -148,6 +149,18 @@ func (m *DefaultManager) UpdateRoutes(updateSerial uint64, newRoutes []*route.Ro } } +// ResetV6Routes deletes all IPv6 routes (necessary if IPv6 address changes). +// It is expected that UpdateRoute is called afterwards to recreate the routing table. +func (m *DefaultManager) ResetV6Routes() { + for id, client := range m.clientNetworks { + if client.network.Addr().Is6() { + log.Debugf("stopping client network watcher due to IPv6 address change, %s", id) + client.stop() + delete(m.clientNetworks, id) + } + } +} + // SetRouteChangeListener set RouteListener for route change notifier func (m *DefaultManager) SetRouteChangeListener(listener listener.NetworkChangeListener) { m.notifier.setListener(listener) diff --git a/client/internal/routemanager/manager_test.go b/client/internal/routemanager/manager_test.go index 7eb8dd002..bdcd1a8f1 100644 --- a/client/internal/routemanager/manager_test.go +++ b/client/internal/routemanager/manager_test.go @@ -36,6 +36,7 @@ func TestManagerUpdateRoutes(t *testing.T) { serverRoutesExpected int clientNetworkWatchersExpected int clientNetworkWatchersExpectedAllowed int + isV6 bool }{ { name: "Should create 2 client networks", @@ -65,6 +66,35 @@ func TestManagerUpdateRoutes(t *testing.T) { inputSerial: 1, clientNetworkWatchersExpected: 2, }, + { + name: "Should create 2 client networks (IPv6)", + inputInitRoutes: []*route.Route{}, + inputRoutes: []*route.Route{ + { + ID: "a", + NetID: "routeA", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "b", + NetID: "routeB", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8::7890:abcd/128"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + }, + inputSerial: 1, + clientNetworkWatchersExpected: 2, + isV6: true, + }, { name: "Should Create 2 Server Routes", inputRoutes: []*route.Route{ @@ -93,6 +123,34 @@ func TestManagerUpdateRoutes(t *testing.T) { serverRoutesExpected: 2, clientNetworkWatchersExpected: 0, }, + { + name: "Should Create 2 Server Routes (IPv6)", + inputRoutes: []*route.Route{ + { + ID: "a", + NetID: "routeA", + Peer: localPeerKey, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "b", + NetID: "routeB", + Peer: localPeerKey, + Network: netip.MustParsePrefix("2001:db8::7890:abcd/128"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + }, + inputSerial: 1, + serverRoutesExpected: 2, + clientNetworkWatchersExpected: 0, + }, { name: "Should Create 1 Route For Client And Server", inputRoutes: []*route.Route{ @@ -121,6 +179,84 @@ func TestManagerUpdateRoutes(t *testing.T) { serverRoutesExpected: 1, clientNetworkWatchersExpected: 1, }, + { + name: "Should Create 1 Route For Client And Server (IPv6)", + inputRoutes: []*route.Route{ + { + ID: "a", + NetID: "routeA", + Peer: localPeerKey, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "b", + NetID: "routeB", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8::7890:abcd/128"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + }, + inputSerial: 1, + serverRoutesExpected: 1, + clientNetworkWatchersExpected: 1, + isV6: true, + }, + { + name: "Should Create 1 Route For Client And Server for each IP version", + inputRoutes: []*route.Route{ + { + ID: "a", + NetID: "routeA", + Peer: localPeerKey, + Network: netip.MustParsePrefix("100.64.30.250/30"), + NetworkType: route.IPv4Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "b", + NetID: "routeB", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("8.8.9.9/32"), + NetworkType: route.IPv4Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "a", + NetID: "routeA", + Peer: localPeerKey, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "b", + NetID: "routeB", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8::7890:abcd/128"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + }, + inputSerial: 1, + serverRoutesExpected: 2, + clientNetworkWatchersExpected: 2, + isV6: true, + }, { name: "Should Create 1 Route For Client and Skip Server Route On Empty Server Router", inputRoutes: []*route.Route{ @@ -150,6 +286,36 @@ func TestManagerUpdateRoutes(t *testing.T) { serverRoutesExpected: 0, clientNetworkWatchersExpected: 1, }, + { + name: "Should Create 1 Route For Client and Skip Server Route On Empty Server Router (IPv6)", + inputRoutes: []*route.Route{ + { + ID: "a", + NetID: "routeA", + Peer: localPeerKey, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "b", + NetID: "routeB", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8::7890:abcd/128"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + }, + inputSerial: 1, + removeSrvRouter: true, + serverRoutesExpected: 0, + clientNetworkWatchersExpected: 1, + isV6: true, + }, { name: "Should Create 1 HA Route and 1 Standalone", inputRoutes: []*route.Route{ @@ -187,6 +353,44 @@ func TestManagerUpdateRoutes(t *testing.T) { inputSerial: 1, clientNetworkWatchersExpected: 2, }, + { + name: "Should Create 1 HA Route and 1 Standalone (IPv6)", + inputRoutes: []*route.Route{ + { + ID: "a", + NetID: "routeA", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "b", + NetID: "routeA", + Peer: remotePeerKey2, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "c", + NetID: "routeB", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8::7890:abcd/128"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + }, + inputSerial: 1, + clientNetworkWatchersExpected: 2, + isV6: true, + }, { name: "No Small Client Route Should Be Added", inputRoutes: []*route.Route{ @@ -205,6 +409,25 @@ func TestManagerUpdateRoutes(t *testing.T) { clientNetworkWatchersExpected: 0, clientNetworkWatchersExpectedAllowed: 1, }, + { + name: "No Small Client Route Should Be Added (IPv6)", + inputRoutes: []*route.Route{ + { + ID: "a", + NetID: "routeA", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("::/0"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + }, + inputSerial: 1, + clientNetworkWatchersExpected: 0, + clientNetworkWatchersExpectedAllowed: 1, + isV6: true, + }, { name: "Remove 1 Client Route", inputInitRoutes: []*route.Route{ @@ -244,6 +467,46 @@ func TestManagerUpdateRoutes(t *testing.T) { inputSerial: 1, clientNetworkWatchersExpected: 1, }, + { + name: "Remove 1 Client Route (IPv6)", + inputInitRoutes: []*route.Route{ + { + ID: "a", + NetID: "routeA", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "b", + NetID: "routeB", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8::abcd:7890/128"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + }, + inputRoutes: []*route.Route{ + { + ID: "a", + NetID: "routeA", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + }, + inputSerial: 1, + clientNetworkWatchersExpected: 1, + isV6: true, + }, { name: "Update Route to HA", inputInitRoutes: []*route.Route{ @@ -293,6 +556,56 @@ func TestManagerUpdateRoutes(t *testing.T) { inputSerial: 1, clientNetworkWatchersExpected: 1, }, + { + name: "Update Route to HA (IPv6)", + inputInitRoutes: []*route.Route{ + { + ID: "a", + NetID: "routeA", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "b", + NetID: "routeB", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8::abcd:7890/128"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + }, + inputRoutes: []*route.Route{ + { + ID: "a", + NetID: "routeA", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "b", + NetID: "routeA", + Peer: remotePeerKey2, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + }, + inputSerial: 1, + clientNetworkWatchersExpected: 1, + isV6: true, + }, { name: "Remove Client Routes", inputInitRoutes: []*route.Route{ @@ -321,6 +634,35 @@ func TestManagerUpdateRoutes(t *testing.T) { inputSerial: 1, clientNetworkWatchersExpected: 0, }, + { + name: "Remove Client Routes (IPv6)", + inputInitRoutes: []*route.Route{ + { + ID: "a", + NetID: "routeA", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "b", + NetID: "routeB", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8::abcd:7890/128"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + }, + inputRoutes: []*route.Route{}, + inputSerial: 1, + clientNetworkWatchersExpected: 0, + isV6: true, + }, { name: "Remove All Routes", inputInitRoutes: []*route.Route{ @@ -350,6 +692,36 @@ func TestManagerUpdateRoutes(t *testing.T) { serverRoutesExpected: 0, clientNetworkWatchersExpected: 0, }, + { + name: "Remove All Routes (IPv6)", + inputInitRoutes: []*route.Route{ + { + ID: "a", + NetID: "routeA", + Peer: localPeerKey, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "b", + NetID: "routeB", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8::abcd:7890/128"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + }, + inputRoutes: []*route.Route{}, + inputSerial: 1, + serverRoutesExpected: 0, + clientNetworkWatchersExpected: 0, + isV6: true, + }, { name: "HA server should not register routes from the same HA group", inputRoutes: []*route.Route{ @@ -398,16 +770,74 @@ func TestManagerUpdateRoutes(t *testing.T) { serverRoutesExpected: 2, clientNetworkWatchersExpected: 1, }, + { + name: "HA server should not register routes from the same HA group (IPv6)", + inputRoutes: []*route.Route{ + { + ID: "l1", + NetID: "routeA", + Peer: localPeerKey, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "l2", + NetID: "routeA", + Peer: localPeerKey, + Network: netip.MustParsePrefix("2001:db8::abcd:7890/128"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "r1", + NetID: "routeA", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + { + ID: "r2", + NetID: "routeC", + Peer: remotePeerKey1, + Network: netip.MustParsePrefix("2001:db8::abcd:789f/128"), + NetworkType: route.IPv6Network, + Metric: 9999, + Masquerade: false, + Enabled: true, + }, + }, + inputSerial: 1, + serverRoutesExpected: 2, + clientNetworkWatchersExpected: 1, + isV6: true, + }, } for n, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { + + v6Addr := "" + //goland:noinspection GoBoolExpressions + if !iface.SupportsIPv6() && testCase.isV6 { + t.Skip("Platform does not support IPv6, skipping IPv6 test...") + } else if testCase.isV6 { + v6Addr = "2001:db8::4242:4711/128" + } + peerPrivateKey, _ := wgtypes.GeneratePrivateKey() newNet, err := stdnet.NewNet() if err != nil { t.Fatal(err) } - wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun43%d", n), "100.65.65.2/24", 33100, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil) + wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun43%d", n), "100.65.65.2/24", v6Addr, 33100, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil) require.NoError(t, err, "should create testing WGIface interface") defer wgInterface.Close() diff --git a/client/internal/routemanager/mock.go b/client/internal/routemanager/mock.go index adbef8061..5cd9604ef 100644 --- a/client/internal/routemanager/mock.go +++ b/client/internal/routemanager/mock.go @@ -64,6 +64,10 @@ func (m *MockManager) EnableServerRouter(firewall firewall.Manager) error { panic("implement me") } +func (m *MockManager) ResetV6Routes() { + panic("implement me") +} + // Stop mock implementation of Stop from Manager interface func (m *MockManager) Stop() { if m.StopFunc != nil { diff --git a/client/internal/routemanager/server_nonandroid.go b/client/internal/routemanager/server_nonandroid.go index 95672e480..bd643357b 100644 --- a/client/internal/routemanager/server_nonandroid.go +++ b/client/internal/routemanager/server_nonandroid.go @@ -70,7 +70,7 @@ func (m *defaultServerRouter) updateRoutes(routesMap map[route.ID]*route.Route) } if len(m.routes) > 0 { - err := enableIPForwarding() + err := enableIPForwarding(m.wgInterface.Address6() != nil) if err != nil { return err } @@ -79,7 +79,7 @@ func (m *defaultServerRouter) updateRoutes(routesMap map[route.ID]*route.Route) return nil } -func (m *defaultServerRouter) removeFromServerNetwork(route *route.Route) error { +func (m *defaultServerRouter) removeFromServerNetwork(rt *route.Route) error { select { case <-m.ctx.Done(): log.Infof("Not removing from server network because context is done") @@ -87,28 +87,32 @@ func (m *defaultServerRouter) removeFromServerNetwork(route *route.Route) error default: m.mux.Lock() defer m.mux.Unlock() - - routerPair, err := routeToRouterPair(m.wgInterface.Address().Masked().String(), route) + routingAddress := m.wgInterface.Address().Masked().String() + if rt.NetworkType == route.IPv6Network { + if m.wgInterface.Address6() == nil { + return fmt.Errorf("attempted to add route for IPv6 even though device has no v6 address") + } + routingAddress = m.wgInterface.Address6().Masked().String() + } + routerPair, err := routeToRouterPair(routingAddress, rt) if err != nil { return fmt.Errorf("parse prefix: %w", err) } - err = m.firewall.RemoveRoutingRules(routerPair) if err != nil { - return fmt.Errorf("remove routing rules: %w", err) + return err } - delete(m.routes, route.ID) + delete(m.routes, rt.ID) state := m.statusRecorder.GetLocalPeerState() - delete(state.Routes, route.Network.String()) + delete(state.Routes, rt.Network.String()) m.statusRecorder.UpdateLocalPeerState(state) - return nil } } -func (m *defaultServerRouter) addToServerNetwork(route *route.Route) error { +func (m *defaultServerRouter) addToServerNetwork(rt *route.Route) error { select { case <-m.ctx.Done(): log.Infof("Not adding to server network because context is done") @@ -116,8 +120,15 @@ func (m *defaultServerRouter) addToServerNetwork(route *route.Route) error { default: m.mux.Lock() defer m.mux.Unlock() + routingAddress := m.wgInterface.Address().Masked().String() + if rt.NetworkType == route.IPv6Network { + if m.wgInterface.Address6() == nil { + return fmt.Errorf("attempted to add route for IPv6 even though device has no v6 address") + } + routingAddress = m.wgInterface.Address6().Masked().String() + } - routerPair, err := routeToRouterPair(m.wgInterface.Address().Masked().String(), route) + routerPair, err := routeToRouterPair(routingAddress, rt) if err != nil { return fmt.Errorf("parse prefix: %w", err) } @@ -127,13 +138,13 @@ func (m *defaultServerRouter) addToServerNetwork(route *route.Route) error { return fmt.Errorf("insert routing rules: %w", err) } - m.routes[route.ID] = route + m.routes[rt.ID] = rt state := m.statusRecorder.GetLocalPeerState() if state.Routes == nil { state.Routes = map[string]struct{}{} } - state.Routes[route.Network.String()] = struct{}{} + state.Routes[rt.Network.String()] = struct{}{} m.statusRecorder.UpdateLocalPeerState(state) return nil @@ -144,10 +155,17 @@ func (m *defaultServerRouter) cleanUp() { m.mux.Lock() defer m.mux.Unlock() for _, r := range m.routes { - routerPair, err := routeToRouterPair(m.wgInterface.Address().Masked().String(), r) + routingAddress := m.wgInterface.Address().Masked().String() + if r.NetworkType == route.IPv6Network { + if m.wgInterface.Address6() == nil { + log.Errorf("attempted to remove route for IPv6 even though device has no v6 address") + continue + } + routingAddress = m.wgInterface.Address6().Masked().String() + } + routerPair, err := routeToRouterPair(routingAddress, r) if err != nil { - log.Errorf("Failed to convert route to router pair: %v", err) - continue + log.Errorf("parse prefix: %v", err) } err = m.firewall.RemoveRoutingRules(routerPair) diff --git a/client/internal/routemanager/systemops.go b/client/internal/routemanager/systemops.go index bc506411c..31865854e 100644 --- a/client/internal/routemanager/systemops.go +++ b/client/internal/routemanager/systemops.go @@ -122,6 +122,16 @@ func ipToAddr(ip net.IP, intf *net.Interface) (netip.Addr, error) { } func existsInRouteTable(prefix netip.Prefix) (bool, error) { + + linkLocalPrefix, err := netip.ParsePrefix("fe80::/10") + if err != nil { + return false, err + } + if prefix.Addr().Is6() && linkLocalPrefix.Contains(prefix.Addr()) { + // The link local prefix is not explicitly part of the routing table, but should be considered as such. + return true, nil + } + routes, err := getRoutesFromTable() if err != nil { return false, fmt.Errorf("get routes from table: %w", err) diff --git a/client/internal/routemanager/systemops_ios.go b/client/internal/routemanager/systemops_ios.go index 4d23d3910..1aede183b 100644 --- a/client/internal/routemanager/systemops_ios.go +++ b/client/internal/routemanager/systemops_ios.go @@ -19,7 +19,7 @@ func cleanupRouting() error { return nil } -func enableIPForwarding() error { +func enableIPForwarding(includeV6 bool) error { log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS) return nil } diff --git a/client/internal/routemanager/systemops_linux.go b/client/internal/routemanager/systemops_linux.go index ce0c07ce6..36ca7b8f5 100644 --- a/client/internal/routemanager/systemops_linux.go +++ b/client/internal/routemanager/systemops_linux.go @@ -184,12 +184,6 @@ func addVPNRoute(prefix netip.Prefix, intf *net.Interface) error { // No need to check if routes exist as main table takes precedence over the VPN table via Rule 1 - // TODO remove this once we have ipv6 support - if prefix == defaultv4 { - if err := addUnreachableRoute(defaultv6, NetbirdVPNTableID); err != nil { - return fmt.Errorf("add blackhole: %w", err) - } - } if err := addRoute(prefix, netip.Addr{}, intf, NetbirdVPNTableID); err != nil { return fmt.Errorf("add route: %w", err) } @@ -201,12 +195,6 @@ func removeVPNRoute(prefix netip.Prefix, intf *net.Interface) error { return genericRemoveVPNRoute(prefix, intf) } - // TODO remove this once we have ipv6 support - if prefix == defaultv4 { - if err := removeUnreachableRoute(defaultv6, NetbirdVPNTableID); err != nil { - return fmt.Errorf("remove unreachable route: %w", err) - } - } if err := removeRoute(prefix, netip.Addr{}, intf, NetbirdVPNTableID); err != nil { return fmt.Errorf("remove route: %w", err) } @@ -282,6 +270,9 @@ func addRoute(prefix netip.Prefix, addr netip.Addr, intf *net.Interface, tableID // addUnreachableRoute adds an unreachable route for the specified IP family and routing table. // ipFamily should be netlink.FAMILY_V4 for IPv4 or netlink.FAMILY_V6 for IPv6. // tableID specifies the routing table to which the unreachable route will be added. +// TODO should this be kept in for future use? If so, the linter needs to be told that this unreachable function should +// +// be kept func addUnreachableRoute(prefix netip.Prefix, tableID int) error { _, ipNet, err := net.ParseCIDR(prefix.String()) if err != nil { @@ -302,6 +293,9 @@ func addUnreachableRoute(prefix netip.Prefix, tableID int) error { return nil } +// TODO should this be kept in for future use? If so, the linter needs to be told that this unreachable function should +// +// be kept func removeUnreachableRoute(prefix netip.Prefix, tableID int) error { _, ipNet, err := net.ParseCIDR(prefix.String()) if err != nil { @@ -376,8 +370,14 @@ func flushRoutes(tableID, family int) error { return result.ErrorOrNil() } -func enableIPForwarding() error { +func enableIPForwarding(includeV6 bool) error { _, err := setSysctl(ipv4ForwardingPath, 1, false) + if err != nil { + return err + } + if includeV6 { + _, err = setSysctl(ipv4ForwardingPath, 1, false) + } return err } diff --git a/client/internal/routemanager/systemops_nonlinux.go b/client/internal/routemanager/systemops_nonlinux.go index 91879790a..f3b4306fa 100644 --- a/client/internal/routemanager/systemops_nonlinux.go +++ b/client/internal/routemanager/systemops_nonlinux.go @@ -10,7 +10,7 @@ import ( log "github.com/sirupsen/logrus" ) -func enableIPForwarding() error { +func enableIPForwarding(includeV6 bool) error { log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS) return nil } diff --git a/client/internal/routemanager/systemops_test.go b/client/internal/routemanager/systemops_test.go index 8bcf06dce..81d585190 100644 --- a/client/internal/routemanager/systemops_test.go +++ b/client/internal/routemanager/systemops_test.go @@ -6,6 +6,8 @@ import ( "bytes" "context" "fmt" + "github.com/google/gopacket/routing" + "github.com/netbirdio/netbird/client/firewall" "net" "net/netip" "os" @@ -46,18 +48,39 @@ func TestAddRemoveRoutes(t *testing.T) { shouldRouteToWireguard: false, shouldBeRemoved: false, }, + { + name: "Should Add And Remove Route 2001:db8:1234:5678::/64", + prefix: netip.MustParsePrefix("2001:db8:1234:5678::/64"), + shouldRouteToWireguard: true, + shouldBeRemoved: true, + }, + { + name: "Should Not Add Or Remove Route ::1/128", + prefix: netip.MustParsePrefix("::1/128"), + shouldRouteToWireguard: false, + shouldBeRemoved: false, + }, } for n, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { t.Setenv("NB_DISABLE_ROUTE_CACHE", "true") + v6Addr := "" + hasV6DefaultRoute, err := EnvironmentHasIPv6DefaultRoute() + //goland:noinspection GoBoolExpressions + if (!iface.SupportsIPv6() || !firewall.SupportsIPv6() || !hasV6DefaultRoute || err != nil) && testCase.prefix.Addr().Is6() { + t.Skip("Platform does not support IPv6, skipping IPv6 test...") + } else if testCase.prefix.Addr().Is6() { + v6Addr = "2001:db8::4242:4711/128" + } + peerPrivateKey, _ := wgtypes.GeneratePrivateKey() newNet, err := stdnet.NewNet() if err != nil { t.Fatal(err) } - wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun53%d", n), "100.65.75.2/24", 33100, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil) + wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun53%d", n), "100.65.75.2/24", v6Addr, 33100, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil) require.NoError(t, err, "should create testing WGIface interface") defer wgInterface.Close() @@ -92,6 +115,10 @@ func TestAddRemoveRoutes(t *testing.T) { internetGateway, _, err := GetNextHop(netip.MustParseAddr("0.0.0.0")) require.NoError(t, err) + if testCase.prefix.Addr().Is6() { + internetGateway, _, err = GetNextHop(netip.MustParseAddr("::/0")) + } + require.NoError(t, err) if testCase.shouldBeRemoved { require.Equal(t, internetGateway, prefixGateway, "route should be pointing to default internet gateway") @@ -151,6 +178,18 @@ func TestAddExistAndRemoveRoute(t *testing.T) { if err != nil { t.Fatal("shouldn't return error when fetching the gateway: ", err) } + var defaultGateway6 *netip.Addr + hasV6DefaultRoute, err := EnvironmentHasIPv6DefaultRoute() + //goland:noinspection GoBoolExpressions + if iface.SupportsIPv6() && firewall.SupportsIPv6() && hasV6DefaultRoute && err == nil { + gw6, _, err := GetNextHop(netip.MustParseAddr("::")) + gw6 = gw6.WithZone("") + defaultGateway6 = &gw6 + t.Log("defaultGateway6: ", defaultGateway6) + if err != nil { + t.Fatal("shouldn't return error when fetching the IPv6 gateway: ", err) + } + } testCases := []struct { name string prefix netip.Prefix @@ -185,6 +224,43 @@ func TestAddExistAndRemoveRoute(t *testing.T) { preExistingPrefix: netip.MustParsePrefix("100.100.0.0/16"), shouldAddRoute: false, }, + { + name: "Should Add And Remove random Route (IPv6)", + prefix: netip.MustParsePrefix("2001:db8::abcd/128"), + shouldAddRoute: true, + }, + { + name: "Should Add Route if bigger network exists (IPv6)", + prefix: netip.MustParsePrefix("2001:db8:b14d:abcd:1234::/96"), + preExistingPrefix: netip.MustParsePrefix("2001:db8:b14d:abcd::/64"), + shouldAddRoute: true, + }, + { + name: "Should Add Route if smaller network exists (IPv6)", + prefix: netip.MustParsePrefix("2001:db8:b14d::/48"), + preExistingPrefix: netip.MustParsePrefix("2001:db8:b14d:abcd::/64"), + shouldAddRoute: true, + }, + { + name: "Should Not Add Route if same network exists (IPv6)", + prefix: netip.MustParsePrefix("2001:db8:b14d:abcd::/64"), + preExistingPrefix: netip.MustParsePrefix("2001:db8:b14d:abcd::/64"), + shouldAddRoute: false, + }, + } + if defaultGateway6 != nil { + testCases = append(testCases, []struct { + name string + prefix netip.Prefix + preExistingPrefix netip.Prefix + shouldAddRoute bool + }{ + { + name: "Should Not Add Route if overlaps with default gateway (IPv6)", + prefix: netip.MustParsePrefix(defaultGateway6.String() + "/127"), + shouldAddRoute: false, + }, + }...) } for n, testCase := range testCases { @@ -198,12 +274,19 @@ func TestAddExistAndRemoveRoute(t *testing.T) { t.Setenv("NB_USE_LEGACY_ROUTING", "true") t.Setenv("NB_DISABLE_ROUTE_CACHE", "true") + v6Addr := "" + if testCase.prefix.Addr().Is6() && defaultGateway6 == nil { + t.Skip("Platform does not support IPv6, skipping IPv6 test...") + } else if testCase.prefix.Addr().Is6() { + v6Addr = "2001:db8::4242:4711/128" + } + peerPrivateKey, _ := wgtypes.GeneratePrivateKey() newNet, err := stdnet.NewNet() if err != nil { t.Fatal(err) } - wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun53%d", n), "100.65.75.2/24", 33100, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil) + wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun53%d", n), "100.65.75.2/24", v6Addr, 33100, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil) require.NoError(t, err, "should create testing WGIface interface") defer wgInterface.Close() @@ -249,6 +332,11 @@ func TestAddExistAndRemoveRoute(t *testing.T) { } func TestIsSubRange(t *testing.T) { + // Note: This test may fail for IPv6 in some environments, where there actually exists another route that the + // determined prefix is a sub-range of. + hasV6DefaultRoute, err := EnvironmentHasIPv6DefaultRoute() + shouldIncludeV6Routes := iface.SupportsIPv6() && firewall.SupportsIPv6() && hasV6DefaultRoute && err == nil + addresses, err := net.InterfaceAddrs() if err != nil { t.Fatal("shouldn't return error when fetching interface addresses: ", err) @@ -258,7 +346,7 @@ func TestIsSubRange(t *testing.T) { var nonSubRangeAddressPrefixes []netip.Prefix for _, address := range addresses { p := netip.MustParsePrefix(address.String()) - if !p.Addr().IsLoopback() && p.Addr().Is4() && p.Bits() < 32 { + if !p.Addr().IsLoopback() && (p.Addr().Is4() && p.Bits() < 32) || (p.Addr().Is6() && shouldIncludeV6Routes && p.Bits() < 128) { p2 := netip.PrefixFrom(p.Masked().Addr(), p.Bits()+1) subRangeAddressPrefixes = append(subRangeAddressPrefixes, p2) nonSubRangeAddressPrefixes = append(nonSubRangeAddressPrefixes, p.Masked()) @@ -286,16 +374,37 @@ func TestIsSubRange(t *testing.T) { } } +func EnvironmentHasIPv6DefaultRoute() (bool, error) { + //goland:noinspection GoBoolExpressions + if runtime.GOOS != "linux" { + // TODO when implementing IPv6 for other operating systems, this should be replaced with code that determines + // whether a default route for IPv6 exists (routing.Router panics on non-linux). + return false, nil + } + router, err := routing.New() + if err != nil { + return false, err + } + routeIface, _, _, err := router.Route(netip.MustParsePrefix("::/0").Addr().AsSlice()) + if err != nil { + return false, err + } + return routeIface != nil, nil +} + func TestExistsInRouteTable(t *testing.T) { addresses, err := net.InterfaceAddrs() if err != nil { t.Fatal("shouldn't return error when fetching interface addresses: ", err) } + hasV6DefaultRoute, err := EnvironmentHasIPv6DefaultRoute() + shouldIncludeV6Routes := iface.SupportsIPv6() && firewall.SupportsIPv6() && hasV6DefaultRoute && err == nil + var addressPrefixes []netip.Prefix for _, address := range addresses { p := netip.MustParsePrefix(address.String()) - if p.Addr().Is6() { + if p.Addr().Is6() && !shouldIncludeV6Routes { continue } // Windows sometimes has hidden interface link local addrs that don't turn up on any interface @@ -321,7 +430,7 @@ func TestExistsInRouteTable(t *testing.T) { } } -func createWGInterface(t *testing.T, interfaceName, ipAddressCIDR string, listenPort int) *iface.WGIface { +func createWGInterface(t *testing.T, interfaceName, ipAddressCIDR string, ipAddress6CIDR string, listenPort int) *iface.WGIface { t.Helper() peerPrivateKey, err := wgtypes.GeneratePrivateKey() @@ -330,7 +439,7 @@ func createWGInterface(t *testing.T, interfaceName, ipAddressCIDR string, listen newNet, err := stdnet.NewNet() require.NoError(t, err) - wgInterface, err := iface.NewWGIFace(interfaceName, ipAddressCIDR, listenPort, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil) + wgInterface, err := iface.NewWGIFace(interfaceName, ipAddressCIDR, ipAddress6CIDR, listenPort, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil) require.NoError(t, err, "should create testing WireGuard interface") err = wgInterface.Create() @@ -348,12 +457,21 @@ func setupTestEnv(t *testing.T) { setupDummyInterfacesAndRoutes(t) - wgIface := createWGInterface(t, expectedVPNint, "100.64.0.1/24", 51820) + v6Addr := "" + hasV6DefaultRoute, err := EnvironmentHasIPv6DefaultRoute() + //goland:noinspection GoBoolExpressions + if !iface.SupportsIPv6() || !firewall.SupportsIPv6() || !hasV6DefaultRoute || err != nil { + t.Skip("Platform does not support IPv6, skipping IPv6 test...") + } else { + v6Addr = "2001:db8::4242:4711/128" + } + + wgIface := createWGInterface(t, expectedVPNint, "100.64.0.1/24", v6Addr, 51820) t.Cleanup(func() { assert.NoError(t, wgIface.Close()) }) - _, _, err := setupRouting(nil, wgIface) + _, _, err = setupRouting(nil, wgIface) require.NoError(t, err, "setupRouting should not return err") t.Cleanup(func() { assert.NoError(t, cleanupRouting()) @@ -412,9 +530,15 @@ func assertWGOutInterface(t *testing.T, prefix netip.Prefix, wgIface *iface.WGIf prefixGateway, _, err := GetNextHop(prefix.Addr()) require.NoError(t, err, "GetNextHop should not return err") + + nexthop := wgIface.Address().IP.String() + if prefix.Addr().Is6() { + nexthop = wgIface.Address6().IP.String() + } + if invert { - assert.NotEqual(t, wgIface.Address().IP.String(), prefixGateway.String(), "route should not point to wireguard interface IP") + assert.NotEqual(t, nexthop, prefixGateway.String(), "route should not point to wireguard interface IP") } else { - assert.Equal(t, wgIface.Address().IP.String(), prefixGateway.String(), "route should point to wireguard interface IP") + assert.Equal(t, nexthop, prefixGateway.String(), "route should point to wireguard interface IP") } } diff --git a/client/system/info.go b/client/system/info.go index e2e057206..a5b7216f0 100644 --- a/client/system/info.go +++ b/client/system/info.go @@ -51,6 +51,7 @@ type Info struct { SystemProductName string SystemManufacturer string Environment Environment + Ipv6Supported bool } // extractUserAgent extracts Netbird's agent (client) name and version from the outgoing context diff --git a/client/system/info_android.go b/client/system/info_android.go index 7f5dd371b..94610621f 100644 --- a/client/system/info_android.go +++ b/client/system/info_android.go @@ -39,6 +39,7 @@ func GetInfo(ctx context.Context) *Info { WiretrusteeVersion: version.NetbirdVersion(), UIVersion: extractUIVersion(ctx), KernelVersion: kernelVersion, + Ipv6Supported: false, } return gio diff --git a/client/system/info_darwin.go b/client/system/info_darwin.go index 6f4ed173b..23ef19b0b 100644 --- a/client/system/info_darwin.go +++ b/client/system/info_darwin.go @@ -61,6 +61,7 @@ func GetInfo(ctx context.Context) *Info { SystemProductName: prodName, SystemManufacturer: manufacturer, Environment: env, + Ipv6Supported: false, } systemHostname, _ := os.Hostname() diff --git a/client/system/info_freebsd.go b/client/system/info_freebsd.go index b44fdee7c..4919f2689 100644 --- a/client/system/info_freebsd.go +++ b/client/system/info_freebsd.go @@ -31,7 +31,7 @@ func GetInfo(ctx context.Context) *Info { Platform: detect_platform.Detect(ctx), } - gio := &Info{Kernel: osInfo[0], Platform: runtime.GOARCH, OS: osInfo[2], GoOS: runtime.GOOS, CPUs: runtime.NumCPU(), KernelVersion: osInfo[1], Environment: env} + gio := &Info{Kernel: osInfo[0], Platform: runtime.GOARCH, OS: osInfo[2], GoOS: runtime.GOOS, CPUs: runtime.NumCPU(), KernelVersion: osInfo[1], Environment: env, Ipv6Supported: false} systemHostname, _ := os.Hostname() gio.Hostname = extractDeviceName(ctx, systemHostname) diff --git a/client/system/info_linux.go b/client/system/info_linux.go index 652bc1115..c04b6efe9 100644 --- a/client/system/info_linux.go +++ b/client/system/info_linux.go @@ -6,6 +6,8 @@ package system import ( "bytes" "context" + "github.com/netbirdio/netbird/client/firewall" + "github.com/netbirdio/netbird/iface" "os" "os/exec" "runtime" @@ -84,6 +86,7 @@ func GetInfo(ctx context.Context) *Info { SystemProductName: prodName, SystemManufacturer: manufacturer, Environment: env, + Ipv6Supported: _checkIPv6Support(), } return gio @@ -122,3 +125,8 @@ func sysInfo() (serialNumber string, productName string, manufacturer string) { si.GetSysInfo() return si.Chassis.Serial, si.Product.Name, si.Product.Vendor } + +func _checkIPv6Support() bool { + return firewall.SupportsIPv6() && + iface.SupportsIPv6() +} diff --git a/client/system/info_windows.go b/client/system/info_windows.go index 68631fe16..be062d152 100644 --- a/client/system/info_windows.go +++ b/client/system/info_windows.go @@ -75,6 +75,7 @@ func GetInfo(ctx context.Context) *Info { SystemProductName: prodName, SystemManufacturer: manufacturer, Environment: env, + Ipv6Supported: false, } systemHostname, _ := os.Hostname() diff --git a/iface/iface.go b/iface/iface.go index 3ae40ad4c..d0786830c 100644 --- a/iface/iface.go +++ b/iface/iface.go @@ -48,6 +48,11 @@ func (w *WGIface) Address() WGAddress { return w.tun.WgAddress() } +// Address6 returns the IPv6 interface address +func (w *WGIface) Address6() *WGAddress { + return w.tun.WgAddress6() +} + // Up configures a Wireguard interface // The interface must exist before calling this method (e.g. call interface.Create() before) func (w *WGIface) Up() (*bind.UniversalUDPMuxDefault, error) { @@ -70,6 +75,23 @@ func (w *WGIface) UpdateAddr(newAddr string) error { return w.tun.UpdateAddr(addr) } +// UpdateAddr6 updates the IPv6 address of the interface +func (w *WGIface) UpdateAddr6(newAddr6 string) error { + w.mu.Lock() + defer w.mu.Unlock() + + var addr *WGAddress + if newAddr6 != "" { + parsedAddr, err := parseWGAddress(newAddr6) + if err != nil { + return err + } + addr = &parsedAddr + } + + return w.tun.UpdateAddr6(addr) +} + // UpdatePeer updates existing Wireguard Peer or creates a new one if doesn't exist // Endpoint is optional func (w *WGIface) UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error { diff --git a/iface/iface_android.go b/iface/iface_android.go index d1876e495..d819698cc 100644 --- a/iface/iface_android.go +++ b/iface/iface_android.go @@ -3,11 +3,16 @@ package iface import ( "fmt" + log "github.com/sirupsen/logrus" + "github.com/pion/transport/v3" ) // NewWGIFace Creates a new WireGuard interface instance -func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *MobileIFaceArguments) (*WGIface, error) { +func NewWGIFace(iFaceName string, address string, address6 string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *MobileIFaceArguments) (*WGIface, error) { + if address6 != "" { + log.Errorf("Attempted to configure IPv6 address %s on unsupported operating system", address6) + } wgAddress, err := parseWGAddress(address) if err != nil { return nil, err @@ -38,3 +43,7 @@ func (w *WGIface) CreateOnAndroid(routes []string, dns string, searchDomains []s func (w *WGIface) Create() error { return fmt.Errorf("this function has not implemented on this platform") } + +func SupportsIPv6() bool { + return false +} diff --git a/iface/iface_darwin.go b/iface/iface_darwin.go index 4d62c6af6..673646750 100644 --- a/iface/iface_darwin.go +++ b/iface/iface_darwin.go @@ -6,13 +6,18 @@ package iface import ( "fmt" + log "github.com/sirupsen/logrus" + "github.com/pion/transport/v3" "github.com/netbirdio/netbird/iface/netstack" ) // NewWGIFace Creates a new WireGuard interface instance -func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *MobileIFaceArguments) (*WGIface, error) { +func NewWGIFace(iFaceName string, address string, address6 string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *MobileIFaceArguments) (*WGIface, error) { + if address6 != "" { + log.Errorf("Attempted to configure IPv6 address %s on unsupported operating system", address6) + } wgAddress, err := parseWGAddress(address) if err != nil { return nil, err @@ -36,3 +41,7 @@ func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, func (w *WGIface) CreateOnAndroid([]string, string, []string) error { return fmt.Errorf("this function has not implemented on this platform") } + +func SupportsIPv6() bool { + return false +} diff --git a/iface/iface_ios.go b/iface/iface_ios.go index b22e1a6a4..f6ed2fa15 100644 --- a/iface/iface_ios.go +++ b/iface/iface_ios.go @@ -5,12 +5,17 @@ package iface import ( "fmt" + log "github.com/sirupsen/logrus" "github.com/pion/transport/v3" ) // NewWGIFace Creates a new WireGuard interface instance -func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *MobileIFaceArguments) (*WGIface, error) { +func NewWGIFace(iFaceName string, address string, address6 string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *MobileIFaceArguments) (*WGIface, error) { + if address6 != "" { + log.Errorf("Attempted to configure IPv6 address %s on unsupported operating system", address6) + } + wgAddress, err := parseWGAddress(address) if err != nil { return nil, err @@ -27,3 +32,7 @@ func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, func (w *WGIface) CreateOnAndroid([]string, string, []string) error { return fmt.Errorf("this function has not implemented on this platform") } + +func SupportsIPv6() bool { + return false +} diff --git a/iface/iface_linux.go b/iface/iface_linux.go index 62ae0f0de..26d9eaaee 100644 --- a/iface/iface_linux.go +++ b/iface/iface_linux.go @@ -5,14 +5,14 @@ package iface import ( "fmt" - - "github.com/pion/transport/v3" - "github.com/netbirdio/netbird/iface/netstack" + "github.com/pion/transport/v3" + log "github.com/sirupsen/logrus" + "golang.org/x/net/nettest" ) // NewWGIFace Creates a new WireGuard interface instance -func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *MobileIFaceArguments) (*WGIface, error) { +func NewWGIFace(iFaceName string, address string, address6 string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *MobileIFaceArguments) (*WGIface, error) { wgAddress, err := parseWGAddress(address) if err != nil { return nil, err @@ -20,6 +20,10 @@ func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, wgIFace := &WGIface{} + if netstack.IsEnabled() || !WireGuardModuleIsLoaded() && address6 != "" { + log.Errorf("Attempted to configure IPv6 address %s on unsupported device implementation (netstack or tun).", address6) + } + // move the kernel/usp/netstack preference evaluation to upper layer if netstack.IsEnabled() { wgIFace.tun = newTunNetstackDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet, netstack.ListenAddr()) @@ -28,7 +32,16 @@ func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, } if WireGuardModuleIsLoaded() { - wgIFace.tun = newTunDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet) + + var wgAddress6 *WGAddress = nil + if address6 != "" { + tmpWgAddress6, err := parseWGAddress(address6) + wgAddress6 = &tmpWgAddress6 + if err != nil { + return wgIFace, err + } + } + wgIFace.tun = newTunDevice(iFaceName, wgAddress, wgAddress6, wgPort, wgPrivKey, mtu, transportNet) wgIFace.userspaceBind = false return wgIFace, nil } @@ -45,3 +58,7 @@ func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, func (w *WGIface) CreateOnAndroid([]string, string, []string) error { return fmt.Errorf("this function has not implemented on this platform") } + +func SupportsIPv6() bool { + return nettest.SupportsIPv6() && WireGuardModuleIsLoaded() && !netstack.IsEnabled() +} diff --git a/iface/iface_test.go b/iface/iface_test.go index f227eaf83..5edc52aa8 100644 --- a/iface/iface_test.go +++ b/iface/iface_test.go @@ -41,7 +41,7 @@ func TestWGIface_UpdateAddr(t *testing.T) { t.Fatal(err) } - iface, err := NewWGIFace(ifaceName, addr, wgPort, key, DefaultMTU, newNet, nil) + iface, err := NewWGIFace(ifaceName, addr, "", wgPort, key, DefaultMTU, newNet, nil) if err != nil { t.Fatal(err) } @@ -95,6 +95,64 @@ func TestWGIface_UpdateAddr(t *testing.T) { } } +func TestWGIface_UpdateAddr6(t *testing.T) { + if !SupportsIPv6() { + t.Skip("Environment does not support IPv6, skipping IPv6 test...") + } + + ifaceName := fmt.Sprintf("utun%d", WgIntNumber+4) + + addr := "100.64.0.1/8" + addr6 := "2001:db8:1234:abcd::42/64" + wgPort := 33100 + newNet, err := stdnet.NewNet() + if err != nil { + t.Fatal(err) + } + + iface, err := NewWGIFace(ifaceName, addr, addr6, wgPort, key, DefaultMTU, newNet, nil) + if err != nil { + t.Fatal(err) + } + err = iface.Create() + if err != nil { + t.Fatal(err) + } + defer func() { + err = iface.Close() + if err != nil { + t.Error(err) + } + + }() + + _, err = iface.Up() + if err != nil { + t.Fatal(err) + } + + addrs, err := getIfaceAddrs(ifaceName) + if err != nil { + t.Error(err) + } + assert.Equal(t, addr, addrs[0].String()) + + //update WireGuard address + addr = "100.64.0.2/8" + err = iface.UpdateAddr(addr) + if err != nil { + t.Fatal(err) + } + + addrs, err = getIfaceAddrs(ifaceName) + if err != nil { + t.Error(err) + } + + assert.Equal(t, addr6, addrs[1].String()) + +} + func getIfaceAddrs(ifaceName string) ([]net.Addr, error) { ief, err := net.InterfaceByName(ifaceName) if err != nil { @@ -114,7 +172,45 @@ func Test_CreateInterface(t *testing.T) { if err != nil { t.Fatal(err) } - iface, err := NewWGIFace(ifaceName, wgIP, 33100, key, DefaultMTU, newNet, nil) + iface, err := NewWGIFace(ifaceName, wgIP, "", 33100, key, DefaultMTU, newNet, nil) + if err != nil { + t.Fatal(err) + } + err = iface.Create() + if err != nil { + t.Fatal(err) + } + defer func() { + err = iface.Close() + if err != nil { + t.Error(err) + } + }() + wg, err := wgctrl.New() + if err != nil { + t.Fatal(err) + } + defer func() { + err = wg.Close() + if err != nil { + t.Error(err) + } + }() +} + +func Test_CreateInterface6(t *testing.T) { + if !SupportsIPv6() { + t.Skip("Environment does not support IPv6, skipping IPv6 test...") + } + + ifaceName := fmt.Sprintf("utun%d", WgIntNumber+1) + wgIP := "10.99.99.1/32" + wgIP6 := "2001:db8:1234:abcd::43/64" + newNet, err := stdnet.NewNet() + if err != nil { + t.Fatal(err) + } + iface, err := NewWGIFace(ifaceName, wgIP, wgIP6, 33100, key, DefaultMTU, newNet, nil) if err != nil { t.Fatal(err) } @@ -149,7 +245,46 @@ func Test_Close(t *testing.T) { t.Fatal(err) } - iface, err := NewWGIFace(ifaceName, wgIP, wgPort, key, DefaultMTU, newNet, nil) + iface, err := NewWGIFace(ifaceName, wgIP, "", wgPort, key, DefaultMTU, newNet, nil) + if err != nil { + t.Fatal(err) + } + err = iface.Create() + if err != nil { + t.Fatal(err) + } + wg, err := wgctrl.New() + if err != nil { + t.Fatal(err) + } + defer func() { + err = wg.Close() + if err != nil { + t.Error(err) + } + }() + + err = iface.Close() + if err != nil { + t.Fatal(err) + } +} + +func Test_Close6(t *testing.T) { + if !SupportsIPv6() { + t.Skip("Environment does not support IPv6, skipping IPv6 test...") + } + + ifaceName := fmt.Sprintf("utun%d", WgIntNumber+2) + wgIP := "10.99.99.2/32" + wgIP6 := "2001:db8:1234:abcd::44/64" + wgPort := 33100 + newNet, err := stdnet.NewNet() + if err != nil { + t.Fatal(err) + } + + iface, err := NewWGIFace(ifaceName, wgIP, wgIP6, wgPort, key, DefaultMTU, newNet, nil) if err != nil { t.Fatal(err) } @@ -182,7 +317,7 @@ func Test_ConfigureInterface(t *testing.T) { if err != nil { t.Fatal(err) } - iface, err := NewWGIFace(ifaceName, wgIP, wgPort, key, DefaultMTU, newNet, nil) + iface, err := NewWGIFace(ifaceName, wgIP, "", wgPort, key, DefaultMTU, newNet, nil) if err != nil { t.Fatal(err) } @@ -230,7 +365,7 @@ func Test_UpdatePeer(t *testing.T) { t.Fatal(err) } - iface, err := NewWGIFace(ifaceName, wgIP, 33100, key, DefaultMTU, newNet, nil) + iface, err := NewWGIFace(ifaceName, wgIP, "", 33100, key, DefaultMTU, newNet, nil) if err != nil { t.Fatal(err) } @@ -283,6 +418,73 @@ func Test_UpdatePeer(t *testing.T) { } } +func Test_UpdatePeer6(t *testing.T) { + if !SupportsIPv6() { + t.Skip("Environment does not support IPv6, skipping IPv6 test...") + } + + ifaceName := fmt.Sprintf("utun%d", WgIntNumber+4) + wgIP := "10.99.99.9/30" + wgIP6 := "2001:db8:1234:abcd::45/64" + newNet, err := stdnet.NewNet() + if err != nil { + t.Fatal(err) + } + + iface, err := NewWGIFace(ifaceName, wgIP, wgIP6, 33100, key, DefaultMTU, newNet, nil) + if err != nil { + t.Fatal(err) + } + err = iface.Create() + if err != nil { + t.Fatal(err) + } + defer func() { + err = iface.Close() + if err != nil { + t.Error(err) + } + }() + + _, err = iface.Up() + if err != nil { + t.Fatal(err) + } + keepAlive := 15 * time.Second + allowedIP := "10.99.99.10/32" + allowedIP6 := "2001:db8:1234:abcd::46/128" + endpoint, err := net.ResolveUDPAddr("udp", "127.0.0.1:9900") + if err != nil { + t.Fatal(err) + } + err = iface.UpdatePeer(peerPubKey, allowedIP+","+allowedIP6, keepAlive, endpoint, nil) + if err != nil { + t.Fatal(err) + } + peer, err := getPeer(ifaceName, peerPubKey) + if err != nil { + t.Fatal(err) + } + if peer.PersistentKeepaliveInterval != keepAlive { + t.Fatal("configured peer with mismatched keepalive interval value") + } + + if peer.Endpoint.String() != endpoint.String() { + t.Fatal("configured peer with mismatched endpoint") + } + + var foundAllowedIP bool + for _, aip := range peer.AllowedIPs { + if aip.String() == allowedIP6 { + foundAllowedIP = true + break + } + } + if !foundAllowedIP { + t.Fatal("configured peer with mismatched Allowed IPs") + } +} + func Test_RemovePeer(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+4) wgIP := "10.99.99.13/30" @@ -291,7 +493,7 @@ func Test_RemovePeer(t *testing.T) { t.Fatal(err) } - iface, err := NewWGIFace(ifaceName, wgIP, 33100, key, DefaultMTU, newNet, nil) + iface, err := NewWGIFace(ifaceName, wgIP, "", 33100, key, DefaultMTU, newNet, nil) if err != nil { t.Fatal(err) } @@ -345,7 +547,7 @@ func Test_ConnectPeers(t *testing.T) { t.Fatal(err) } - iface1, err := NewWGIFace(peer1ifaceName, peer1wgIP, peer1wgPort, peer1Key.String(), DefaultMTU, newNet, nil) + iface1, err := NewWGIFace(peer1ifaceName, peer1wgIP, "", peer1wgPort, peer1Key.String(), DefaultMTU, newNet, nil) if err != nil { t.Fatal(err) } @@ -368,7 +570,113 @@ func Test_ConnectPeers(t *testing.T) { if err != nil { t.Fatal(err) } - iface2, err := NewWGIFace(peer2ifaceName, peer2wgIP, peer2wgPort, peer2Key.String(), DefaultMTU, newNet, nil) + iface2, err := NewWGIFace(peer2ifaceName, peer2wgIP, "", peer2wgPort, peer2Key.String(), DefaultMTU, newNet, nil) + if err != nil { + t.Fatal(err) + } + err = iface2.Create() + if err != nil { + t.Fatal(err) + } + + _, err = iface2.Up() + if err != nil { + t.Fatal(err) + } + + peer2endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("127.0.0.1:%d", peer2wgPort)) + if err != nil { + t.Fatal(err) + } + defer func() { + err = iface1.Close() + if err != nil { + t.Error(err) + } + err = iface2.Close() + if err != nil { + t.Error(err) + } + }() + + err = iface1.UpdatePeer(peer2Key.PublicKey().String(), peer2wgIP, keepAlive, peer2endpoint, nil) + if err != nil { + t.Fatal(err) + } + err = iface2.UpdatePeer(peer1Key.PublicKey().String(), peer1wgIP, keepAlive, peer1endpoint, nil) + if err != nil { + t.Fatal(err) + } + // todo: investigate why in some tests execution we need 30s + timeout := 30 * time.Second + timeoutChannel := time.After(timeout) + + for { + select { + case <-timeoutChannel: + t.Fatalf("waiting for peer handshake timeout after %s", timeout.String()) + default: + } + + peer, gpErr := getPeer(peer1ifaceName, peer2Key.PublicKey().String()) + if gpErr != nil { + t.Fatal(gpErr) + } + if !peer.LastHandshakeTime.IsZero() { + t.Log("peers successfully handshake") + break + } + } + +} + +func Test_ConnectPeers6(t *testing.T) { + if !SupportsIPv6() { + t.Skip("Environment does not support IPv6, skipping IPv6 test...") + } + + peer1ifaceName := fmt.Sprintf("utun%d", WgIntNumber+400) + peer1wgIP := "10.99.99.17/30" + peer1wgIP6 := "2001:db8:1234:abcd::47/64" + peer1Key, _ := wgtypes.GeneratePrivateKey() + peer1wgPort := 33100 + + peer2ifaceName := "utun500" + peer2wgIP := "10.99.99.18/30" + peer2wgIP6 := "2001:db8:1234:abcd::48/64" + peer2Key, _ := wgtypes.GeneratePrivateKey() + peer2wgPort := 33200 + + keepAlive := 1 * time.Second + newNet, err := stdnet.NewNet() + if err != nil { + t.Fatal(err) + } + + iface1, err := NewWGIFace(peer1ifaceName, peer1wgIP, peer1wgIP6, peer1wgPort, peer1Key.String(), DefaultMTU, newNet, nil) + if err != nil { + t.Fatal(err) + } + err = iface1.Create() + if err != nil { + t.Fatal(err) + } + + _, err = iface1.Up() + if err != nil { + t.Fatal(err) + } + + peer1endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("127.0.0.1:%d", peer1wgPort)) + if err != nil { + t.Fatal(err) + } + + newNet, err = stdnet.NewNet() + if err != nil { + t.Fatal(err) + } + iface2, err := NewWGIFace(peer2ifaceName, peer2wgIP, peer2wgIP6, peer2wgPort, peer2Key.String(), DefaultMTU, newNet, nil) if err != nil { t.Fatal(err) } diff --git a/iface/iface_windows.go b/iface/iface_windows.go index d3a16a52f..8b9a1867c 100644 --- a/iface/iface_windows.go +++ b/iface/iface_windows.go @@ -3,13 +3,18 @@ package iface import ( "fmt" + log "github.com/sirupsen/logrus" + "github.com/pion/transport/v3" "github.com/netbirdio/netbird/iface/netstack" ) // NewWGIFace Creates a new WireGuard interface instance -func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *MobileIFaceArguments) (*WGIface, error) { +func NewWGIFace(iFaceName string, address string, address6 string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *MobileIFaceArguments) (*WGIface, error) { + if address6 != "" { + log.Errorf("Attempted to configure IPv6 address %s on unsupported operating system", address6) + } wgAddress, err := parseWGAddress(address) if err != nil { return nil, err @@ -37,3 +42,7 @@ func (w *WGIface) CreateOnAndroid([]string, string, []string) error { func (w *WGIface) GetInterfaceGUIDString() (string, error) { return w.tun.(*tunDevice).getInterfaceGUIDString() } + +func SupportsIPv6() bool { + return false +} diff --git a/iface/tun.go b/iface/tun.go index b3c0f9d80..4f3e1b69d 100644 --- a/iface/tun.go +++ b/iface/tun.go @@ -12,6 +12,8 @@ type wgTunDevice interface { Up() (*bind.UniversalUDPMuxDefault, error) UpdateAddr(address WGAddress) error WgAddress() WGAddress + UpdateAddr6(addr6 *WGAddress) error + WgAddress6() *WGAddress DeviceName() string Close() error Wrapper() *DeviceWrapper // todo eliminate this function diff --git a/iface/tun_android.go b/iface/tun_android.go index 834b2cb42..08f531c38 100644 --- a/iface/tun_android.go +++ b/iface/tun_android.go @@ -4,6 +4,7 @@ package iface import ( + "fmt" "strings" "github.com/pion/transport/v3" @@ -98,6 +99,13 @@ func (t *wgTunDevice) UpdateAddr(addr WGAddress) error { return nil } +func (t *wgTunDevice) UpdateAddr6(addr6 *WGAddress) error { + if addr6 == nil { + return nil + } + return fmt.Errorf("IPv6 is not supported on this operating system") +} + func (t *wgTunDevice) Close() error { if t.configurer != nil { t.configurer.close() @@ -127,6 +135,10 @@ func (t *wgTunDevice) WgAddress() WGAddress { return t.address } +func (t *wgTunDevice) WgAddress6() *WGAddress { + return nil +} + func (t *wgTunDevice) Wrapper() *DeviceWrapper { return t.wrapper } diff --git a/iface/tun_darwin.go b/iface/tun_darwin.go index 8dc10bd0e..979e58f51 100644 --- a/iface/tun_darwin.go +++ b/iface/tun_darwin.go @@ -3,6 +3,7 @@ package iface import ( + "fmt" "os/exec" "github.com/pion/transport/v3" @@ -88,6 +89,13 @@ func (t *tunDevice) UpdateAddr(address WGAddress) error { return t.assignAddr() } +func (t *tunDevice) UpdateAddr6(address6 *WGAddress) error { + if address6 == nil { + return nil + } + return fmt.Errorf("IPv6 is not supported on this operating system") +} + func (t *tunDevice) Close() error { if t.configurer != nil { t.configurer.close() @@ -108,6 +116,10 @@ func (t *tunDevice) WgAddress() WGAddress { return t.address } +func (t *tunDevice) WgAddress6() *WGAddress { + return nil +} + func (t *tunDevice) DeviceName() string { return t.name } diff --git a/iface/tun_ios.go b/iface/tun_ios.go index ea980818d..1c90ccb80 100644 --- a/iface/tun_ios.go +++ b/iface/tun_ios.go @@ -4,6 +4,7 @@ package iface import ( + "fmt" "os" "github.com/pion/transport/v3" @@ -123,11 +124,22 @@ func (t *tunDevice) WgAddress() WGAddress { return t.address } +func (t *tunDevice) WgAddress6() *WGAddress { + return nil +} + func (t *tunDevice) UpdateAddr(addr WGAddress) error { // todo implement return nil } +func (t *tunDevice) UpdateAddr6(address6 *WGAddress) error { + if address6 == nil { + return nil + } + return fmt.Errorf("IPv6 is not supported on this operating system") +} + func (t *tunDevice) Wrapper() *DeviceWrapper { return t.wrapper } diff --git a/iface/tun_kernel_linux.go b/iface/tun_kernel_linux.go index 12adcdf73..3d4b62e1e 100644 --- a/iface/tun_kernel_linux.go +++ b/iface/tun_kernel_linux.go @@ -19,6 +19,7 @@ import ( type tunKernelDevice struct { name string address WGAddress + address6 *WGAddress wgPort int key string mtu int @@ -31,13 +32,14 @@ type tunKernelDevice struct { udpMux *bind.UniversalUDPMuxDefault } -func newTunDevice(name string, address WGAddress, wgPort int, key string, mtu int, transportNet transport.Net) wgTunDevice { +func newTunDevice(name string, address WGAddress, address6 *WGAddress, wgPort int, key string, mtu int, transportNet transport.Net) wgTunDevice { ctx, cancel := context.WithCancel(context.Background()) return &tunKernelDevice{ ctx: ctx, ctxCancel: cancel, name: name, address: address, + address6: address6, wgPort: wgPort, key: key, mtu: mtu, @@ -136,6 +138,11 @@ func (t *tunKernelDevice) UpdateAddr(address WGAddress) error { return t.assignAddr() } +func (t *tunKernelDevice) UpdateAddr6(address6 *WGAddress) error { + t.address6 = address6 + return t.assignAddr() +} + func (t *tunKernelDevice) Close() error { if t.link == nil { return nil @@ -168,6 +175,10 @@ func (t *tunKernelDevice) WgAddress() WGAddress { return t.address } +func (t *tunKernelDevice) WgAddress6() *WGAddress { + return t.address6 +} + func (t *tunKernelDevice) DeviceName() string { return t.name } @@ -203,6 +214,19 @@ func (t *tunKernelDevice) assignAddr() error { } else if err != nil { return err } + + // Configure the optional additional IPv6 address if available. + if t.address6 != nil { + log.Debugf("adding IPv6 address %s to interface: %s", t.address6.String(), t.name) + addr6, _ := netlink.ParseAddr(t.address6.String()) + err = netlink.AddrAdd(link, addr6) + if os.IsExist(err) { + log.Infof("interface %s already has the address: %s", t.name, t.address.String()) + } else if err != nil { + return err + } + } + // On linux, the link must be brought up err = netlink.LinkSetUp(link) return err diff --git a/iface/tun_netstack.go b/iface/tun_netstack.go index e1d01ecc9..910af1e89 100644 --- a/iface/tun_netstack.go +++ b/iface/tun_netstack.go @@ -91,6 +91,13 @@ func (t *tunNetstackDevice) UpdateAddr(WGAddress) error { return nil } +func (t *tunNetstackDevice) UpdateAddr6(address6 *WGAddress) error { + if address6 == nil { + return nil + } + return fmt.Errorf("IPv6 is not supported on this operating system") +} + func (t *tunNetstackDevice) Close() error { if t.configurer != nil { t.configurer.close() @@ -110,6 +117,10 @@ func (t *tunNetstackDevice) WgAddress() WGAddress { return t.address } +func (t *tunNetstackDevice) WgAddress6() *WGAddress { + return nil +} + func (t *tunNetstackDevice) DeviceName() string { return t.name } diff --git a/iface/tun_usp_linux.go b/iface/tun_usp_linux.go index 9f0210228..df4c55057 100644 --- a/iface/tun_usp_linux.go +++ b/iface/tun_usp_linux.go @@ -98,6 +98,13 @@ func (t *tunUSPDevice) UpdateAddr(address WGAddress) error { return t.assignAddr() } +func (t *tunUSPDevice) UpdateAddr6(address6 *WGAddress) error { + if address6 == nil { + return nil + } + return fmt.Errorf("IPv6 is not supported on this operating system") +} + func (t *tunUSPDevice) Close() error { if t.configurer != nil { t.configurer.close() @@ -117,6 +124,10 @@ func (t *tunUSPDevice) WgAddress() WGAddress { return t.address } +func (t *tunUSPDevice) WgAddress6() *WGAddress { + return nil +} + func (t *tunUSPDevice) DeviceName() string { return t.name } diff --git a/iface/tun_windows.go b/iface/tun_windows.go index 900e62fc3..2cb5188c3 100644 --- a/iface/tun_windows.go +++ b/iface/tun_windows.go @@ -106,6 +106,13 @@ func (t *tunDevice) UpdateAddr(address WGAddress) error { return t.assignAddr() } +func (t *tunDevice) UpdateAddr6(address6 *WGAddress) error { + if address6 == nil { + return nil + } + return fmt.Errorf("IPv6 is not supported on this operating system") +} + func (t *tunDevice) Close() error { if t.configurer != nil { t.configurer.close() @@ -126,6 +133,10 @@ func (t *tunDevice) WgAddress() WGAddress { return t.address } +func (t *tunDevice) WgAddress6() *WGAddress { + return nil +} + func (t *tunDevice) DeviceName() string { return t.name } diff --git a/iface/wg_configurer_kernel.go b/iface/wg_configurer_kernel.go index 67bfb716d..dcffa3a87 100644 --- a/iface/wg_configurer_kernel.go +++ b/iface/wg_configurer_kernel.go @@ -5,6 +5,7 @@ package iface import ( "fmt" "net" + "strings" "time" log "github.com/sirupsen/logrus" @@ -46,9 +47,13 @@ func (c *wgKernelConfigurer) configureInterface(privateKey string, port int) err func (c *wgKernelConfigurer) updatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error { // parse allowed ips - _, ipNet, err := net.ParseCIDR(allowedIps) - if err != nil { - return err + var allowedIpNets []net.IPNet + for _, allowedIp := range strings.Split(allowedIps, ",") { + _, ipNet, err := net.ParseCIDR(allowedIp) + allowedIpNets = append(allowedIpNets, *ipNet) + if err != nil { + return err + } } peerKeyParsed, err := wgtypes.ParseKey(peerKey) @@ -58,7 +63,7 @@ func (c *wgKernelConfigurer) updatePeer(peerKey string, allowedIps string, keepA peer := wgtypes.PeerConfig{ PublicKey: peerKeyParsed, ReplaceAllowedIPs: true, - AllowedIPs: []net.IPNet{*ipNet}, + AllowedIPs: allowedIpNets, PersistentKeepaliveInterval: &keepAlive, Endpoint: endpoint, PresharedKey: preSharedKey, diff --git a/management/client/client_test.go b/management/client/client_test.go index 32ad8fce4..fa30926f1 100644 --- a/management/client/client_test.go +++ b/management/client/client_test.go @@ -379,6 +379,7 @@ func Test_SystemMetaDataFromClient(t *testing.T) { SysProductName: info.SystemProductName, SysManufacturer: info.SystemManufacturer, Environment: &mgmtProto.Environment{Cloud: info.Environment.Cloud, Platform: info.Environment.Platform}, + Ipv6Supported: info.Ipv6Supported, } assert.Equal(t, ValidKey, actualValidKey) diff --git a/management/client/grpc.go b/management/client/grpc.go index df687a160..a42c26d52 100644 --- a/management/client/grpc.go +++ b/management/client/grpc.go @@ -483,5 +483,6 @@ func infoToMetaData(info *system.Info) *proto.PeerSystemMeta { Cloud: info.Environment.Cloud, Platform: info.Environment.Platform, }, + Ipv6Supported: info.Ipv6Supported, } } diff --git a/management/proto/management.pb.go b/management/proto/management.pb.go index 18077ea89..46b04157f 100644 --- a/management/proto/management.pb.go +++ b/management/proto/management.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v4.24.3 +// protoc v4.25.2 // source: management.proto package proto @@ -668,6 +668,7 @@ type PeerSystemMeta struct { SysProductName string `protobuf:"bytes,13,opt,name=sysProductName,proto3" json:"sysProductName,omitempty"` SysManufacturer string `protobuf:"bytes,14,opt,name=sysManufacturer,proto3" json:"sysManufacturer,omitempty"` Environment *Environment `protobuf:"bytes,15,opt,name=environment,proto3" json:"environment,omitempty"` + Ipv6Supported bool `protobuf:"varint,16,opt,name=ipv6Supported,proto3" json:"ipv6Supported,omitempty"` } func (x *PeerSystemMeta) Reset() { @@ -807,6 +808,13 @@ func (x *PeerSystemMeta) GetEnvironment() *Environment { return nil } +func (x *PeerSystemMeta) GetIpv6Supported() bool { + if x != nil { + return x.Ipv6Supported + } + return false +} + type LoginResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1172,6 +1180,8 @@ type PeerConfig struct { SshConfig *SSHConfig `protobuf:"bytes,3,opt,name=sshConfig,proto3" json:"sshConfig,omitempty"` // Peer fully qualified domain name Fqdn string `protobuf:"bytes,4,opt,name=fqdn,proto3" json:"fqdn,omitempty"` + // Peer's virtual IPv6 address within the Wiretrustee VPN (a Wireguard address config) + Address6 string `protobuf:"bytes,5,opt,name=address6,proto3" json:"address6,omitempty"` } func (x *PeerConfig) Reset() { @@ -1234,6 +1244,13 @@ func (x *PeerConfig) GetFqdn() string { return "" } +func (x *PeerConfig) GetAddress6() string { + if x != nil { + return x.Address6 + } + return "" +} + // NetworkMap represents a network state of the peer with the corresponding configuration parameters to establish peer-to-peer connections type NetworkMap struct { state protoimpl.MessageState @@ -2254,6 +2271,7 @@ type FirewallRule struct { Action FirewallRuleAction `protobuf:"varint,3,opt,name=Action,proto3,enum=management.FirewallRuleAction" json:"Action,omitempty"` Protocol FirewallRuleProtocol `protobuf:"varint,4,opt,name=Protocol,proto3,enum=management.FirewallRuleProtocol" json:"Protocol,omitempty"` Port string `protobuf:"bytes,5,opt,name=Port,proto3" json:"Port,omitempty"` + PeerIP6 string `protobuf:"bytes,6,opt,name=PeerIP6,proto3" json:"PeerIP6,omitempty"` } func (x *FirewallRule) Reset() { @@ -2323,6 +2341,13 @@ func (x *FirewallRule) GetPort() string { return "" } +func (x *FirewallRule) GetPeerIP6() string { + if x != nil { + return x.PeerIP6 + } + return "" +} + type NetworkAddress struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2430,7 +2455,7 @@ var file_management_proto_rawDesc = []byte{ 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x22, 0xa9, 0x04, 0x0a, + 0x28, 0x09, 0x52, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x22, 0xcf, 0x04, 0x0a, 0x0e, 0x50, 0x65, 0x65, 0x72, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x67, @@ -2465,260 +2490,266 @@ var file_management_proto_rawDesc = []byte{ 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0b, 0x65, 0x6e, 0x76, - 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x22, 0x94, 0x01, 0x0a, 0x0d, 0x4c, 0x6f, 0x67, - 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x11, 0x77, 0x69, - 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x57, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x52, 0x11, 0x77, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, - 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x36, 0x0a, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x52, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, - 0x79, 0x0a, 0x11, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x38, 0x0a, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, - 0x73, 0x41, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, - 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, - 0x70, 0x74, 0x79, 0x22, 0xa8, 0x01, 0x0a, 0x11, 0x57, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, - 0x74, 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2c, 0x0a, 0x05, 0x73, 0x74, 0x75, - 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x52, 0x05, 0x73, 0x74, 0x75, 0x6e, 0x73, 0x12, 0x35, 0x0a, 0x05, 0x74, 0x75, 0x72, 0x6e, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x65, 0x63, 0x74, 0x65, 0x64, 0x48, 0x6f, 0x73, - 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x05, 0x74, 0x75, 0x72, 0x6e, 0x73, 0x12, 0x2e, - 0x0a, 0x06, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x69, 0x70, 0x76, 0x36, + 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x18, 0x10, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x0d, 0x69, 0x70, 0x76, 0x36, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x22, 0x94, + 0x01, 0x0a, 0x0d, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x4b, 0x0a, 0x11, 0x77, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x57, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, + 0x73, 0x74, 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x11, 0x77, 0x69, 0x72, 0x65, + 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x36, 0x0a, + 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, + 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x79, 0x0a, 0x11, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, + 0x65, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x38, 0x0a, 0x09, + 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x78, 0x70, + 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0xa8, 0x01, 0x0a, 0x11, 0x57, 0x69, + 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, + 0x2c, 0x0a, 0x05, 0x73, 0x74, 0x75, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, 0x6f, 0x73, 0x74, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x22, 0x98, - 0x01, 0x0a, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x10, 0x0a, - 0x03, 0x75, 0x72, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x69, 0x12, - 0x3b, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, - 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, - 0x6f, 0x6c, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x22, 0x3b, 0x0a, 0x08, - 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x07, 0x0a, 0x03, 0x55, 0x44, 0x50, 0x10, - 0x00, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, 0x50, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x54, - 0x54, 0x50, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x48, 0x54, 0x54, 0x50, 0x53, 0x10, 0x03, 0x12, - 0x08, 0x0a, 0x04, 0x44, 0x54, 0x4c, 0x53, 0x10, 0x04, 0x22, 0x7d, 0x0a, 0x13, 0x50, 0x72, 0x6f, - 0x74, 0x65, 0x63, 0x74, 0x65, 0x64, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x12, 0x36, 0x0a, 0x0a, 0x68, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x05, 0x73, 0x74, 0x75, 0x6e, 0x73, 0x12, 0x35, 0x0a, + 0x05, 0x74, 0x75, 0x72, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x05, 0x74, + 0x75, 0x72, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x06, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0a, 0x68, 0x6f, - 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x73, 0x65, 0x72, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x73, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, - 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x22, 0x81, 0x01, 0x0a, 0x0a, 0x50, 0x65, 0x65, - 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, - 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x12, 0x10, 0x0a, 0x03, 0x64, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x64, 0x6e, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, 0x73, - 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x22, 0xe2, 0x03, 0x0a, - 0x0a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x53, - 0x65, 0x72, 0x69, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x53, 0x65, 0x72, - 0x69, 0x61, 0x6c, 0x12, 0x36, 0x0a, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, - 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3e, 0x0a, 0x0b, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, - 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, - 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, - 0x65, 0x65, 0x72, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x29, 0x0a, 0x06, 0x52, - 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06, - 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x52, 0x09, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x40, 0x0a, 0x0c, 0x6f, - 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, - 0x0c, 0x6f, 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x12, 0x3e, 0x0a, - 0x0d, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x08, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x52, 0x0d, - 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x32, 0x0a, - 0x14, 0x66, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x49, 0x73, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x66, 0x69, 0x72, - 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x22, 0x97, 0x01, 0x0a, 0x10, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, - 0x65, 0x79, 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x49, 0x70, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x49, - 0x70, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, 0x73, 0x73, - 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x22, 0x49, 0x0a, 0x09, 0x53, - 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x73, 0x68, 0x45, - 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x73, 0x73, - 0x68, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x50, - 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x73, 0x68, - 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x22, 0x20, 0x0a, 0x1e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, - 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, - 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xbf, 0x01, 0x0a, 0x17, 0x44, 0x65, 0x76, - 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x48, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, - 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x2e, 0x70, 0x72, 0x6f, 0x76, - 0x69, 0x64, 0x65, 0x72, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x42, - 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x52, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x22, 0x16, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x0a, - 0x0a, 0x06, 0x48, 0x4f, 0x53, 0x54, 0x45, 0x44, 0x10, 0x00, 0x22, 0x1e, 0x0a, 0x1c, 0x50, 0x4b, - 0x43, 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, - 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x5b, 0x0a, 0x15, 0x50, 0x4b, - 0x43, 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, - 0x6c, 0x6f, 0x77, 0x12, 0x42, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0xea, 0x02, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, - 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x43, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x43, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x6f, - 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x6f, 0x6d, 0x61, - 0x69, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2e, - 0x0a, 0x12, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x45, 0x6e, 0x64, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x44, 0x65, 0x76, 0x69, - 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x24, - 0x0a, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x45, 0x6e, 0x64, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x55, 0x73, - 0x65, 0x49, 0x44, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, - 0x55, 0x73, 0x65, 0x49, 0x44, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x34, 0x0a, 0x15, 0x41, 0x75, - 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, - 0x69, 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x41, 0x75, 0x74, 0x68, 0x6f, - 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x12, 0x22, 0x0a, 0x0c, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x55, 0x52, 0x4c, 0x73, - 0x18, 0x0a, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, - 0x55, 0x52, 0x4c, 0x73, 0x22, 0xb5, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x0e, - 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, - 0x0a, 0x07, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x20, 0x0a, 0x0b, 0x4e, 0x65, 0x74, 0x77, - 0x6f, 0x72, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x4e, - 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, - 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x12, 0x16, - 0x0a, 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, - 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x61, 0x73, 0x71, 0x75, 0x65, - 0x72, 0x61, 0x64, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x4d, 0x61, 0x73, 0x71, - 0x75, 0x65, 0x72, 0x61, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x65, 0x74, 0x49, 0x44, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x4e, 0x65, 0x74, 0x49, 0x44, 0x22, 0xb4, 0x01, 0x0a, - 0x09, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x0d, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, - 0x12, 0x47, 0x0a, 0x10, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, 0x10, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x12, 0x38, 0x0a, 0x0b, 0x43, 0x75, 0x73, - 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x75, 0x73, 0x74, - 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, 0x52, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, - 0x6e, 0x65, 0x73, 0x22, 0x58, 0x0a, 0x0a, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, - 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x32, 0x0a, 0x07, 0x52, 0x65, 0x63, - 0x6f, 0x72, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, - 0x63, 0x6f, 0x72, 0x64, 0x52, 0x07, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, 0x74, 0x0a, - 0x0c, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x12, 0x0a, - 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x54, - 0x54, 0x4c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x54, 0x54, 0x4c, 0x12, 0x14, 0x0a, - 0x05, 0x52, 0x44, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x52, 0x44, - 0x61, 0x74, 0x61, 0x22, 0xb3, 0x01, 0x0a, 0x0f, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x38, 0x0a, 0x0b, 0x4e, 0x61, 0x6d, 0x65, 0x53, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, + 0x74, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x6c, 0x22, 0x98, 0x01, 0x0a, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x75, 0x72, 0x69, 0x12, 0x3b, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, + 0x6f, 0x6c, 0x22, 0x3b, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x07, + 0x0a, 0x03, 0x55, 0x44, 0x50, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, 0x50, 0x10, 0x01, + 0x12, 0x08, 0x0a, 0x04, 0x48, 0x54, 0x54, 0x50, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x48, 0x54, + 0x54, 0x50, 0x53, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x54, 0x4c, 0x53, 0x10, 0x04, 0x22, + 0x7d, 0x0a, 0x13, 0x50, 0x72, 0x6f, 0x74, 0x65, 0x63, 0x74, 0x65, 0x64, 0x48, 0x6f, 0x73, 0x74, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x36, 0x0a, 0x0a, 0x68, 0x6f, 0x73, 0x74, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x52, 0x0a, 0x68, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, + 0x0a, 0x04, 0x75, 0x73, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x73, + 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x22, 0x9d, + 0x01, 0x0a, 0x0a, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, + 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x64, 0x6e, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x64, 0x6e, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x73, 0x73, 0x68, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x52, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, + 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, + 0x64, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x36, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x36, 0x22, 0xe2, + 0x03, 0x0a, 0x0a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x12, 0x16, 0x0a, + 0x06, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x53, + 0x65, 0x72, 0x69, 0x61, 0x6c, 0x12, 0x36, 0x0a, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x52, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3e, 0x0a, + 0x0b, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, + 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x52, 0x0b, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x12, 0x2e, 0x0a, + 0x12, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x49, 0x73, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x72, 0x65, 0x6d, 0x6f, 0x74, + 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x29, 0x0a, + 0x06, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, + 0x52, 0x06, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x44, 0x4e, 0x53, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x52, 0x09, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x40, 0x0a, + 0x0c, 0x6f, 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x18, 0x07, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x52, 0x0c, 0x6f, 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x12, + 0x3e, 0x0a, 0x0d, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, + 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, + 0x52, 0x0d, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x12, + 0x32, 0x0a, 0x14, 0x66, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, + 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x66, + 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x49, 0x73, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x22, 0x97, 0x01, 0x0a, 0x10, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, + 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x67, 0x50, 0x75, + 0x62, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x67, 0x50, 0x75, + 0x62, 0x4b, 0x65, 0x79, 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x49, + 0x70, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, + 0x64, 0x49, 0x70, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, + 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, + 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x22, 0x49, 0x0a, + 0x09, 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x73, + 0x68, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, + 0x73, 0x73, 0x68, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x73, + 0x68, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, + 0x73, 0x68, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x22, 0x20, 0x0a, 0x1e, 0x44, 0x65, 0x76, 0x69, + 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, + 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xbf, 0x01, 0x0a, 0x17, 0x44, + 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x48, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, + 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, + 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x2e, 0x70, 0x72, + 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x12, 0x42, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x22, 0x16, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x12, 0x0a, 0x0a, 0x06, 0x48, 0x4f, 0x53, 0x54, 0x45, 0x44, 0x10, 0x00, 0x22, 0x1e, 0x0a, 0x1c, + 0x50, 0x4b, 0x43, 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x5b, 0x0a, 0x15, + 0x50, 0x4b, 0x43, 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x42, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, + 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, + 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, + 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0xea, 0x02, 0x0a, 0x0e, 0x50, 0x72, + 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, + 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, + 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, + 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x6f, + 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, + 0x12, 0x2e, 0x0a, 0x12, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x45, 0x6e, + 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x44, 0x65, + 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x45, 0x6e, + 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x0a, + 0x55, 0x73, 0x65, 0x49, 0x44, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0a, 0x55, 0x73, 0x65, 0x49, 0x44, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x34, 0x0a, 0x15, + 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, + 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x41, 0x75, 0x74, + 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x55, 0x52, + 0x4c, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, + 0x63, 0x74, 0x55, 0x52, 0x4c, 0x73, 0x22, 0xb5, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, + 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, + 0x12, 0x18, 0x0a, 0x07, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x20, 0x0a, 0x0b, 0x4e, 0x65, + 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x50, 0x65, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, + 0x12, 0x16, 0x0a, 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x61, 0x73, 0x71, + 0x75, 0x65, 0x72, 0x61, 0x64, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x4d, 0x61, + 0x73, 0x71, 0x75, 0x65, 0x72, 0x61, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x65, 0x74, 0x49, + 0x44, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x4e, 0x65, 0x74, 0x49, 0x44, 0x22, 0xb4, + 0x01, 0x0a, 0x09, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x0d, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x45, 0x6e, 0x61, 0x62, + 0x6c, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x52, 0x0b, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x73, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x07, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x44, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x44, 0x6f, - 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x44, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x14, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x44, 0x6f, 0x6d, 0x61, 0x69, - 0x6e, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, 0x48, 0x0a, 0x0a, 0x4e, 0x61, 0x6d, - 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x4e, 0x53, 0x54, 0x79, 0x70, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x4e, 0x53, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x50, - 0x6f, 0x72, 0x74, 0x22, 0xf0, 0x02, 0x0a, 0x0c, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, - 0x52, 0x75, 0x6c, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x65, 0x65, 0x72, 0x49, 0x50, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x65, 0x65, 0x72, 0x49, 0x50, 0x12, 0x40, 0x0a, 0x09, - 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x22, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, - 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x2e, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x37, - 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, - 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x2e, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3d, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, - 0x75, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x52, 0x08, 0x50, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x22, 0x1c, 0x0a, 0x09, 0x64, 0x69, - 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x06, 0x0a, 0x02, 0x49, 0x4e, 0x10, 0x00, 0x12, - 0x07, 0x0a, 0x03, 0x4f, 0x55, 0x54, 0x10, 0x01, 0x22, 0x1e, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x43, 0x43, 0x45, 0x50, 0x54, 0x10, 0x00, 0x12, 0x08, - 0x0a, 0x04, 0x44, 0x52, 0x4f, 0x50, 0x10, 0x01, 0x22, 0x3c, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, - 0x00, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x4c, 0x4c, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, - 0x50, 0x10, 0x02, 0x12, 0x07, 0x0a, 0x03, 0x55, 0x44, 0x50, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, - 0x49, 0x43, 0x4d, 0x50, 0x10, 0x04, 0x22, 0x38, 0x0a, 0x0e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, - 0x6b, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x65, 0x74, 0x49, - 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6e, 0x65, 0x74, 0x49, 0x50, 0x12, 0x10, - 0x0a, 0x03, 0x6d, 0x61, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6d, 0x61, 0x63, - 0x32, 0xd1, 0x03, 0x0a, 0x11, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, + 0x72, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, 0x10, 0x4e, 0x61, 0x6d, 0x65, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x12, 0x38, 0x0a, 0x0b, 0x43, + 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x75, + 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, 0x52, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, + 0x5a, 0x6f, 0x6e, 0x65, 0x73, 0x22, 0x58, 0x0a, 0x0a, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5a, + 0x6f, 0x6e, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x32, 0x0a, 0x07, 0x52, + 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, + 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x07, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, + 0x74, 0x0a, 0x0c, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, + 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x43, 0x6c, 0x61, 0x73, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x12, 0x10, 0x0a, + 0x03, 0x54, 0x54, 0x4c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x54, 0x54, 0x4c, 0x12, + 0x14, 0x0a, 0x05, 0x52, 0x44, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x52, 0x44, 0x61, 0x74, 0x61, 0x22, 0xb3, 0x01, 0x0a, 0x0f, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x38, 0x0a, 0x0b, 0x4e, 0x61, 0x6d, + 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x4e, 0x61, 0x6d, 0x65, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x0b, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x18, 0x0a, + 0x07, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, + 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x53, 0x65, 0x61, 0x72, 0x63, + 0x68, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x44, 0x6f, 0x6d, + 0x61, 0x69, 0x6e, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, 0x48, 0x0a, 0x0a, 0x4e, + 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x4e, 0x53, 0x54, + 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x4e, 0x53, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x04, 0x50, 0x6f, 0x72, 0x74, 0x22, 0x8a, 0x03, 0x0a, 0x0c, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, + 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x65, 0x65, 0x72, 0x49, 0x50, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x65, 0x65, 0x72, 0x49, 0x50, 0x12, 0x40, + 0x0a, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x22, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, + 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x2e, 0x64, 0x69, 0x72, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x37, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, + 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x2e, 0x61, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3d, 0x0a, 0x08, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, + 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x52, 0x08, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x18, 0x0a, 0x07, + 0x50, 0x65, 0x65, 0x72, 0x49, 0x50, 0x36, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x50, + 0x65, 0x65, 0x72, 0x49, 0x50, 0x36, 0x22, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x06, 0x0a, 0x02, 0x49, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x4f, + 0x55, 0x54, 0x10, 0x01, 0x22, 0x1e, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, + 0x0a, 0x06, 0x41, 0x43, 0x43, 0x45, 0x50, 0x54, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x52, + 0x4f, 0x50, 0x10, 0x01, 0x22, 0x3c, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, + 0x03, 0x41, 0x4c, 0x4c, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, 0x50, 0x10, 0x02, 0x12, + 0x07, 0x0a, 0x03, 0x55, 0x44, 0x50, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x43, 0x4d, 0x50, + 0x10, 0x04, 0x22, 0x38, 0x0a, 0x0e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x41, 0x64, 0x64, + 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x65, 0x74, 0x49, 0x50, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x6e, 0x65, 0x74, 0x49, 0x50, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x61, + 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6d, 0x61, 0x63, 0x32, 0xd1, 0x03, 0x0a, + 0x11, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x12, 0x45, 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1c, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, + 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x46, 0x0a, 0x04, 0x53, 0x79, 0x6e, + 0x63, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, + 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, - 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, - 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x46, 0x0a, - 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x4b, 0x65, 0x79, 0x12, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x09, 0x69, 0x73, 0x48, - 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x12, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x5a, - 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, - 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, - 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, - 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x58, 0x0a, 0x18, 0x47, 0x65, - 0x74, 0x50, 0x4b, 0x43, 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, + 0x01, 0x12, 0x42, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, + 0x79, 0x12, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x09, 0x69, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x79, 0x12, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x1a, 0x47, 0x65, + 0x74, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x58, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x50, 0x4b, 0x43, + 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, + 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, + 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, + 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, + 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/management/proto/management.proto b/management/proto/management.proto index 2cc0efa22..eeeb7b9ae 100644 --- a/management/proto/management.proto +++ b/management/proto/management.proto @@ -117,6 +117,7 @@ message PeerSystemMeta { string sysProductName = 13; string sysManufacturer = 14; Environment environment = 15; + bool ipv6Supported = 16; } message LoginResponse { @@ -182,6 +183,8 @@ message PeerConfig { SSHConfig sshConfig = 3; // Peer fully qualified domain name string fqdn = 4; + // Peer's virtual IPv6 address within the Wiretrustee VPN (a Wireguard address config) + string address6 = 5; } // NetworkMap represents a network state of the peer with the corresponding configuration parameters to establish peer-to-peer connections @@ -349,6 +352,7 @@ message FirewallRule { action Action = 3; protocol Protocol = 4; string Port = 5; + string PeerIP6 = 6; enum direction { IN = 0; diff --git a/management/server/account.go b/management/server/account.go index 984139a12..d4f59ef7a 100644 --- a/management/server/account.go +++ b/management/server/account.go @@ -280,7 +280,8 @@ func (a *Account) getRoutesToSync(peerID string, aclPeers []*nbpeer.Peer) []*rou groupListMap := a.getPeerGroups(peerID) for _, peer := range aclPeers { activeRoutes, _ := a.getRoutingPeerRoutes(peer.ID) - groupFilteredRoutes := a.filterRoutesByGroups(activeRoutes, groupListMap) + addressFamilyFilteredRoutes := a.filterRoutesByIPv6Enabled(activeRoutes, a.GetPeer(peerID).IP6 != nil && peer.IP6 != nil) + groupFilteredRoutes := a.filterRoutesByGroups(addressFamilyFilteredRoutes, groupListMap) filteredRoutes := a.filterRoutesFromPeersOfSameHAGroup(groupFilteredRoutes, peerRoutesMembership) routes = append(routes, filteredRoutes...) } @@ -315,6 +316,20 @@ func (a *Account) filterRoutesByGroups(routes []*route.Route, groupListMap looku return filteredRoutes } +// Filters out IPv6 routes if the peer does not support them. +func (a *Account) filterRoutesByIPv6Enabled(routes []*route.Route, v6Supported bool) []*route.Route { + if v6Supported { + return routes + } + var filteredRoutes []*route.Route + for _, rt := range routes { + if rt.Network.Addr().Is4() { + filteredRoutes = append(filteredRoutes, rt) + } + } + return filteredRoutes +} + // getRoutingPeerRoutes returns the enabled and disabled lists of routes that the given routing peer serves // Please mind, that the returned route.Route objects will contain Peer.Key instead of Peer.ID. // If the given is not a routing peer, then the lists are empty. @@ -348,6 +363,10 @@ func (a *Account) getRoutingPeerRoutes(peerID string) (enabledRoutes []*route.Ro } for _, r := range a.Routes { + // Skip IPv6 routes if IPv6 is currently not enabled. + if peer.IP6 == nil && r.NetworkType == route.IPv6Network { + continue + } for _, groupID := range r.PeerGroups { group := a.GetGroup(groupID) if group == nil { @@ -429,7 +448,7 @@ func (a *Account) GetPeerNetworkMap(peerID, dnsDomain string, validatedPeersMap if dnsManagementStatus { var zones []nbdns.CustomZone - peersCustomZone := getPeersCustomZone(a, dnsDomain) + peersCustomZone := getPeersCustomZone(a, dnsDomain, peer.IP6 != nil) if peersCustomZone.Domain != "" { zones = append(zones, peersCustomZone) } @@ -665,6 +684,17 @@ func (a *Account) getTakenIPs() []net.IP { return takenIps } +func (a *Account) getTakenIP6s() []net.IP { + var takenIps []net.IP + for _, existingPeer := range a.Peers { + if existingPeer.IP6 != nil { + takenIps = append(takenIps, *existingPeer.IP6) + } + } + + return takenIps +} + func (a *Account) getPeerDNSLabels() lookupMap { existingLabels := make(lookupMap) for _, peer := range a.Peers { @@ -1006,7 +1036,6 @@ func (am *DefaultAccountManager) UpdateAccountSettings(accountID, userID string, } updatedAccount := account.UpdateSettings(newSettings) - err = am.Store.SaveAccount(account) if err != nil { return nil, err @@ -2015,7 +2044,7 @@ func addAllGroup(account *Account) error { func newAccountWithId(accountID, userID, domain string) *Account { log.Debugf("creating new account") - network := NewNetwork() + network := NewNetwork(true) peers := make(map[string]*nbpeer.Peer) users := make(map[string]*User) routes := make(map[route.ID]*route.Route) diff --git a/management/server/account_test.go b/management/server/account_test.go index 38c9fabbc..590a95bdd 100644 --- a/management/server/account_test.go +++ b/management/server/account_test.go @@ -69,14 +69,15 @@ func verifyCanAddPeerToAccount(t *testing.T, manager AccountManager, account *Ac Key: "BhRPtynAAYRDy08+q4HTMsos8fs4plTP4NOSh7C1ry8=", Name: "test-host@netbird.io", Meta: nbpeer.PeerSystemMeta{ - Hostname: "test-host@netbird.io", - GoOS: "linux", - Kernel: "Linux", - Core: "21.04", - Platform: "x86_64", - OS: "Ubuntu", - WtVersion: "development", - UIVersion: "development", + Hostname: "test-host@netbird.io", + GoOS: "linux", + Kernel: "Linux", + Core: "21.04", + Platform: "x86_64", + OS: "Ubuntu", + WtVersion: "development", + UIVersion: "development", + Ipv6Supported: false, }, } @@ -964,78 +965,136 @@ func TestAccountManager_DeleteAccount(t *testing.T) { } func TestAccountManager_AddPeer(t *testing.T) { - manager, err := createManager(t) - if err != nil { - t.Fatal(err) - return + + testCases := []struct { + name string + peerIPv6Supported bool + allGroupIPv6Enabled bool + }{ + { + name: "Peer and Group IPv6 enabled", + peerIPv6Supported: true, + allGroupIPv6Enabled: true, + }, + { + name: "Peer IPv6 enabled, Group IPv6 disabled", + peerIPv6Supported: true, + allGroupIPv6Enabled: false, + }, + { + name: "Peer IPv6 disabled, Group IPv6 enabled", + peerIPv6Supported: false, + allGroupIPv6Enabled: true, + }, + { + name: "Peer and Group IPv6 disabled", + peerIPv6Supported: false, + allGroupIPv6Enabled: false, + }, } - userID := "testingUser" - account, err := createAccount(manager, "test_account", userID, "netbird.cloud") - if err != nil { - t.Fatal(err) + for _, c := range testCases { + t.Run(c.name, func(t *testing.T) { + manager, err := createManager(t) + if err != nil { + t.Fatal(err) + return + } + + userID := "testingUser" + account, err := createAccount(manager, "test_account", userID, "netbird.cloud") + if err != nil { + t.Fatal(err) + } + + serial := account.Network.CurrentSerial() // should be 0 + + setupKey, err := manager.CreateSetupKey(account.Id, "test-key", SetupKeyReusable, time.Hour, nil, 999, userID, false) + if err != nil { + t.Fatal("error creating setup key") + return + } + + if account.Network.Serial != 0 { + t.Errorf("expecting account network to have an initial Serial=0") + return + } + + account, err = manager.Store.GetAccount(account.Id) + if err != nil { + t.Fatal(err) + return + } + + if c.allGroupIPv6Enabled { + unlock := manager.Store.AcquireAccountWriteLock(account.Id) + groupAll, err := account.GetGroupAll() + if err != nil { + t.Fatal(err) + } + groupAll.IPv6Enabled = true + err = manager.Store.SaveAccount(account) + if err != nil { + t.Fatal(err) + } + unlock() + } + + key, err := wgtypes.GeneratePrivateKey() + if err != nil { + t.Fatal(err) + return + } + expectedPeerKey := key.PublicKey().String() + expectedSetupKey := setupKey.Key + + peer, _, err := manager.AddPeer(setupKey.Key, "", &nbpeer.Peer{ + Key: expectedPeerKey, + Meta: nbpeer.PeerSystemMeta{Hostname: expectedPeerKey, Ipv6Supported: c.peerIPv6Supported}, + }) + if err != nil { + t.Errorf("expecting peer to be added, got failure %v", err) + return + } + + account, err = manager.Store.GetAccount(account.Id) + if err != nil { + t.Fatal(err) + return + } + + if peer.Key != expectedPeerKey { + t.Errorf("expecting just added peer to have key = %s, got %s", expectedPeerKey, peer.Key) + } + + if !account.Network.Net.Contains(peer.IP) { + t.Errorf("expecting just added peer's IP %s to be in a network range %s", peer.IP.String(), account.Network.Net.String()) + } + + if peer.SetupKey != expectedSetupKey { + t.Errorf("expecting just added peer to have SetupKey = %s, got %s", expectedSetupKey, peer.SetupKey) + } + + if account.Network.CurrentSerial() != 1 { + t.Errorf("expecting Network Serial=%d to be incremented by 1 and be equal to %d when adding new peer to account", serial, account.Network.CurrentSerial()) + } + ev := getEvent(t, account.Id, manager, activity.PeerAddedWithSetupKey) + + assert.NotNil(t, ev) + assert.Equal(t, account.Id, ev.AccountID) + assert.Equal(t, peer.Name, ev.Meta["name"]) + assert.Equal(t, peer.FQDN(account.Domain), ev.Meta["fqdn"]) + assert.Equal(t, setupKey.Id, ev.InitiatorID) + assert.Equal(t, peer.ID, ev.TargetID) + assert.Equal(t, peer.IP.String(), fmt.Sprint(ev.Meta["ip"])) + assert.Equal(t, peer.V6Setting, nbpeer.V6Auto) + if c.peerIPv6Supported && c.allGroupIPv6Enabled { + assert.NotNil(t, peer.IP6) + } else { + assert.Nil(t, peer.IP6) + } + }) } - - serial := account.Network.CurrentSerial() // should be 0 - - setupKey, err := manager.CreateSetupKey(account.Id, "test-key", SetupKeyReusable, time.Hour, nil, 999, userID, false) - if err != nil { - t.Fatal("error creating setup key") - return - } - - if account.Network.Serial != 0 { - t.Errorf("expecting account network to have an initial Serial=0") - return - } - - key, err := wgtypes.GeneratePrivateKey() - if err != nil { - t.Fatal(err) - return - } - expectedPeerKey := key.PublicKey().String() - expectedSetupKey := setupKey.Key - - peer, _, err := manager.AddPeer(setupKey.Key, "", &nbpeer.Peer{ - Key: expectedPeerKey, - Meta: nbpeer.PeerSystemMeta{Hostname: expectedPeerKey}, - }) - if err != nil { - t.Errorf("expecting peer to be added, got failure %v", err) - return - } - - account, err = manager.Store.GetAccount(account.Id) - if err != nil { - t.Fatal(err) - return - } - - if peer.Key != expectedPeerKey { - t.Errorf("expecting just added peer to have key = %s, got %s", expectedPeerKey, peer.Key) - } - - if !account.Network.Net.Contains(peer.IP) { - t.Errorf("expecting just added peer's IP %s to be in a network range %s", peer.IP.String(), account.Network.Net.String()) - } - - if peer.SetupKey != expectedSetupKey { - t.Errorf("expecting just added peer to have SetupKey = %s, got %s", expectedSetupKey, peer.SetupKey) - } - - if account.Network.CurrentSerial() != 1 { - t.Errorf("expecting Network Serial=%d to be incremented by 1 and be equal to %d when adding new peer to account", serial, account.Network.CurrentSerial()) - } - ev := getEvent(t, account.Id, manager, activity.PeerAddedWithSetupKey) - - assert.NotNil(t, ev) - assert.Equal(t, account.Id, ev.AccountID) - assert.Equal(t, peer.Name, ev.Meta["name"]) - assert.Equal(t, peer.FQDN(account.Domain), ev.Meta["fqdn"]) - assert.Equal(t, setupKey.Id, ev.InitiatorID) - assert.Equal(t, peer.ID, ev.TargetID) - assert.Equal(t, peer.IP.String(), fmt.Sprint(ev.Meta["ip"])) } func TestAccountManager_AddPeerWithUserID(t *testing.T) { @@ -1455,9 +1514,22 @@ func TestAccount_GetRoutesToSync(t *testing.T) { if err != nil { t.Fatal(err) } + _, prefix3, err := route.ParseNetwork("2001:db8:1234:5678::/64") + if err != nil { + t.Fatal(err) + } + _, prefix4, err := route.ParseNetwork("2001:db8:1234:6789::/64") + if err != nil { + t.Fatal(err) + } + peer2IP6 := net.ParseIP("2001:db8:abcd:1234::12") account := &Account{ Peers: map[string]*nbpeer.Peer{ - "peer-1": {Key: "peer-1", Meta: nbpeer.PeerSystemMeta{GoOS: "linux"}}, "peer-2": {Key: "peer-2", Meta: nbpeer.PeerSystemMeta{GoOS: "linux"}}, "peer-3": {Key: "peer-1", Meta: nbpeer.PeerSystemMeta{GoOS: "linux"}}, + "peer-1": { + ID: "peer-1", Key: "peer-1", Meta: nbpeer.PeerSystemMeta{GoOS: "linux"}, + }, + "peer-2": {ID: "peer-2", Key: "peer-2", Meta: nbpeer.PeerSystemMeta{GoOS: "linux", Ipv6Supported: true}, IP6: &peer2IP6}, + "peer-3": {ID: "peer-3", Key: "peer-1", Meta: nbpeer.PeerSystemMeta{GoOS: "linux"}}, }, Groups: map[string]*group.Group{"group1": {ID: "group1", Peers: []string{"peer-1", "peer-2"}}}, Routes: map[route.ID]*route.Route{ @@ -1467,7 +1539,7 @@ func TestAccount_GetRoutesToSync(t *testing.T) { NetID: "network-1", Description: "network-1", Peer: "peer-1", - NetworkType: 0, + NetworkType: route.IPv4Network, Masquerade: false, Metric: 999, Enabled: true, @@ -1479,7 +1551,7 @@ func TestAccount_GetRoutesToSync(t *testing.T) { NetID: "network-2", Description: "network-2", Peer: "peer-2", - NetworkType: 0, + NetworkType: route.IPv4Network, Masquerade: false, Metric: 999, Enabled: true, @@ -1491,7 +1563,31 @@ func TestAccount_GetRoutesToSync(t *testing.T) { NetID: "network-1", Description: "network-1", Peer: "peer-2", - NetworkType: 0, + NetworkType: route.IPv4Network, + Masquerade: false, + Metric: 999, + Enabled: true, + Groups: []string{"group1"}, + }, + "route-4": { + ID: "route-4", + Network: prefix3, + NetID: "network-3", + Description: "network-3", + Peer: "peer-1", + NetworkType: route.IPv6Network, + Masquerade: false, + Metric: 999, + Enabled: true, + Groups: []string{"group1"}, + }, + "route-5": { + ID: "route-5", + Network: prefix4, + NetID: "network-4", + Description: "network-4", + Peer: "peer-2", + NetworkType: route.IPv6Network, Masquerade: false, Metric: 999, Enabled: true, @@ -1500,17 +1596,29 @@ func TestAccount_GetRoutesToSync(t *testing.T) { }, } - routes := account.getRoutesToSync("peer-2", []*nbpeer.Peer{{Key: "peer-1"}, {Key: "peer-3"}}) + routes := account.getRoutesToSync("peer-1", []*nbpeer.Peer{{ID: "peer-2", Key: "peer-2"}, {ID: "peer-3", Key: "peer-3"}}) assert.Len(t, routes, 2) routeIDs := make(map[route.ID]struct{}, 2) for _, r := range routes { routeIDs[r.ID] = struct{}{} } + assert.Contains(t, routeIDs, route.ID("route-1")) + assert.Contains(t, routeIDs, route.ID("route-2")) + + routes = account.getRoutesToSync("peer-2", []*nbpeer.Peer{{ID: "peer-1", Key: "peer-1"}, {ID: "peer-3", Key: "peer-3"}}) + + assert.Len(t, routes, 3) + + routeIDs = make(map[route.ID]struct{}, 2) + for _, r := range routes { + routeIDs[r.ID] = struct{}{} + } assert.Contains(t, routeIDs, route.ID("route-2")) assert.Contains(t, routeIDs, route.ID("route-3")) + assert.Contains(t, routeIDs, route.ID("route-5")) - emptyRoutes := account.getRoutesToSync("peer-3", []*nbpeer.Peer{{Key: "peer-1"}, {Key: "peer-2"}}) + emptyRoutes := account.getRoutesToSync("peer-3", []*nbpeer.Peer{{ID: "peer-1", Key: "peer-1"}, {ID: "peer-2", Key: "peer-2"}}) assert.Len(t, emptyRoutes, 0) } diff --git a/management/server/activity/codes.go b/management/server/activity/codes.go index 4ee57f181..e4100f379 100644 --- a/management/server/activity/codes.go +++ b/management/server/activity/codes.go @@ -139,6 +139,14 @@ const ( PostureCheckUpdated Activity = 61 // PostureCheckDeleted indicates that the user deleted a posture check PostureCheckDeleted Activity = 62 + // PeerIPv6Enabled indicates that a user enabled IPv6 for a peer + PeerIPv6Enabled Activity = 63 + // PeerIPv6Disabled indicates that a user disabled IPv6 for a peer + PeerIPv6Disabled Activity = 64 + // PeerIPv6InheritEnabled indicates that IPv6 was enabled for a peer due to a change in group memberships. + PeerIPv6InheritEnabled Activity = 65 + // PeerIPv6InheritDisabled indicates that IPv6 was disabled for a peer due to a change in group memberships. + PeerIPv6InheritDisabled Activity = 66 ) var activityMap = map[Activity]Code{ @@ -205,6 +213,10 @@ var activityMap = map[Activity]Code{ PostureCheckCreated: {"Posture check created", "posture.check.created"}, PostureCheckUpdated: {"Posture check updated", "posture.check.updated"}, PostureCheckDeleted: {"Posture check deleted", "posture.check.deleted"}, + PeerIPv6Enabled: {"Peer IPv6 enabled by user", "peer.ipv6.manual_enable"}, + PeerIPv6Disabled: {"Peer IPv6 disabled by user", "peer.ipv6.manual_disable"}, + PeerIPv6InheritDisabled: {"Peer IPv6 disabled due to change in group settings or membership", "peer.ipv6.inherit_disable"}, + PeerIPv6InheritEnabled: {"Peer IPv6 enabled due to change in group settings or membership", "peer.ipv6.inherit_enable"}, } // StringCode returns a string code of the activity diff --git a/management/server/dns.go b/management/server/dns.go index 5e2febf55..2bc5f8dcd 100644 --- a/management/server/dns.go +++ b/management/server/dns.go @@ -149,7 +149,7 @@ func toProtocolDNSConfig(update nbdns.Config) *proto.DNSConfig { return protoUpdate } -func getPeersCustomZone(account *Account, dnsDomain string) nbdns.CustomZone { +func getPeersCustomZone(account *Account, dnsDomain string, enableIPv6 bool) nbdns.CustomZone { if dnsDomain == "" { log.Errorf("no dns domain is set, returning empty zone") return nbdns.CustomZone{} @@ -172,6 +172,16 @@ func getPeersCustomZone(account *Account, dnsDomain string) nbdns.CustomZone { TTL: defaultTTL, RData: peer.IP.String(), }) + + if peer.IP6 != nil && enableIPv6 { + customZone.Records = append(customZone.Records, nbdns.SimpleRecord{ + Name: dns.Fqdn(peer.DNSLabel + "." + dnsDomain), + Type: int(dns.TypeAAAA), + Class: nbdns.DefaultClass, + TTL: defaultTTL, + RData: peer.IP6.String(), + }) + } } return customZone @@ -179,6 +189,7 @@ func getPeersCustomZone(account *Account, dnsDomain string) nbdns.CustomZone { func getPeerNSGroups(account *Account, peerID string) []*nbdns.NameServerGroup { groupList := account.getPeerGroups(peerID) + peer := account.GetPeer(peerID) var peerNSGroups []*nbdns.NameServerGroup @@ -189,8 +200,18 @@ func getPeerNSGroups(account *Account, peerID string) []*nbdns.NameServerGroup { for _, gID := range nsGroup.Groups { _, found := groupList[gID] if found { - if !peerIsNameserver(account.GetPeer(peerID), nsGroup) { - peerNSGroups = append(peerNSGroups, nsGroup.Copy()) + if !peerIsNameserver(peer, nsGroup) { + filteredNsGroup := nsGroup.Copy() + var newNameserverList []nbdns.NameServer + for _, nameserver := range filteredNsGroup.NameServers { + if nameserver.IP.Is4() || peer.IP6 != nil { + newNameserverList = append(newNameserverList, nameserver) + } + } + if len(newNameserverList) > 0 { + filteredNsGroup.NameServers = newNameserverList + peerNSGroups = append(peerNSGroups, filteredNsGroup) + } break } } diff --git a/management/server/dns_test.go b/management/server/dns_test.go index b5074e50c..509ac1920 100644 --- a/management/server/dns_test.go +++ b/management/server/dns_test.go @@ -22,6 +22,7 @@ const ( dnsAdminUserID = "testingAdminUser" dnsRegularUserID = "testingRegularUser" dnsNSGroup1 = "ns1" + dnsNSGroup2 = "ns2" ) func TestGetDNSSettings(t *testing.T) { @@ -184,7 +185,7 @@ func TestGetNetworkMap_DNSConfigSync(t *testing.T) { require.NoError(t, err) require.Len(t, peer2AccountDNSConfig.DNSConfig.CustomZones, 1, "DNS config should have one custom zone for peers not in the disabled group") require.True(t, peer2AccountDNSConfig.DNSConfig.ServiceEnable, "DNS config should have DNS service enabled for peers not in the disabled group") - require.Len(t, peer2AccountDNSConfig.DNSConfig.NameServerGroups, 1, "updated DNS config should have 1 nameserver groups since peer 2 is part of the group All") + require.Len(t, peer2AccountDNSConfig.DNSConfig.NameServerGroups, 2, "updated DNS config should have 2 nameserver groups since peer 2 is part of the group All and supports IPv6") } func createDNSManager(t *testing.T) (*DefaultAccountManager, error) { @@ -215,14 +216,15 @@ func initTestDNSAccount(t *testing.T, am *DefaultAccountManager) (*Account, erro Key: dnsPeer1Key, Name: "test-host1@netbird.io", Meta: nbpeer.PeerSystemMeta{ - Hostname: "test-host1@netbird.io", - GoOS: "linux", - Kernel: "Linux", - Core: "21.04", - Platform: "x86_64", - OS: "Ubuntu", - WtVersion: "development", - UIVersion: "development", + Hostname: "test-host1@netbird.io", + GoOS: "linux", + Kernel: "Linux", + Core: "21.04", + Platform: "x86_64", + OS: "Ubuntu", + WtVersion: "development", + UIVersion: "development", + Ipv6Supported: false, }, DNSLabel: dnsPeer1Key, } @@ -230,16 +232,18 @@ func initTestDNSAccount(t *testing.T, am *DefaultAccountManager) (*Account, erro Key: dnsPeer2Key, Name: "test-host2@netbird.io", Meta: nbpeer.PeerSystemMeta{ - Hostname: "test-host2@netbird.io", - GoOS: "linux", - Kernel: "Linux", - Core: "21.04", - Platform: "x86_64", - OS: "Ubuntu", - WtVersion: "development", - UIVersion: "development", + Hostname: "test-host2@netbird.io", + GoOS: "linux", + Kernel: "Linux", + Core: "21.04", + Platform: "x86_64", + OS: "Ubuntu", + WtVersion: "development", + UIVersion: "development", + Ipv6Supported: true, }, - DNSLabel: dnsPeer2Key, + V6Setting: nbpeer.V6Enabled, + DNSLabel: dnsPeer2Key, } domain := "example.com" @@ -312,6 +316,20 @@ func initTestDNSAccount(t *testing.T, am *DefaultAccountManager) (*Account, erro Groups: []string{allGroup.ID}, } + account.NameServerGroups[dnsNSGroup2] = &dns.NameServerGroup{ + ID: dnsNSGroup2, + Name: "ns-group-2", + NameServers: []dns.NameServer{{ + IP: netip.MustParseAddr("2001:4860:4860:0:0:0:0:8888"), // Google DNS + NSType: dns.UDPNameServerType, + Port: dns.DefaultDNSPort, + }}, + Primary: false, + Domains: []string{"example.com"}, + Enabled: true, + Groups: []string{allGroup.ID}, + } + err = am.Store.SaveAccount(account) if err != nil { return nil, err diff --git a/management/server/group.go b/management/server/group.go index 7ede2120d..13950e90e 100644 --- a/management/server/group.go +++ b/management/server/group.go @@ -149,6 +149,35 @@ func (am *DefaultAccountManager) SaveGroup(accountID, userID string, newGroup *n oldGroup, exists := account.Groups[newGroup.ID] account.Groups[newGroup.ID] = newGroup + // Determine peer difference for group. + addedPeers := make([]string, 0) + removedPeers := make([]string, 0) + if exists { + addedPeers = difference(newGroup.Peers, oldGroup.Peers) + removedPeers = difference(oldGroup.Peers, newGroup.Peers) + } else { + addedPeers = append(addedPeers, newGroup.Peers...) + } + + // Need to check whether IPv6 status has changed for all potentially affected peers. + peersToUpdate := make([]string, 0) + // If group previously had IPv6 enabled, need to check all old peers for changes in IPv6 status. + if exists && oldGroup.IPv6Enabled { + peersToUpdate = removedPeers + } + // If group IPv6 status changed, need to check all current peers, if it did not, but IPv6 is enabled, only check + // added peers, otherwise check no peers (as group can not affect IPv6 state). + if exists && oldGroup.IPv6Enabled != newGroup.IPv6Enabled { + peersToUpdate = append(peersToUpdate, newGroup.Peers...) + } else if newGroup.IPv6Enabled { + peersToUpdate = append(peersToUpdate, addedPeers...) + } + + _, err = am.updatePeerIPv6Status(account, userID, newGroup, peersToUpdate) + if err != nil { + return err + } + account.Network.IncSerial() if err = am.Store.SaveAccount(account); err != nil { return err @@ -158,13 +187,7 @@ func (am *DefaultAccountManager) SaveGroup(accountID, userID string, newGroup *n // the following snippet tracks the activity and stores the group events in the event store. // It has to happen after all the operations have been successfully performed. - addedPeers := make([]string, 0) - removedPeers := make([]string, 0) - if exists { - addedPeers = difference(newGroup.Peers, oldGroup.Peers) - removedPeers = difference(oldGroup.Peers, newGroup.Peers) - } else { - addedPeers = append(addedPeers, newGroup.Peers...) + if !exists { am.StoreEvent(userID, newGroup.ID, accountID, activity.GroupCreated, newGroup.EventMeta()) } @@ -314,6 +337,14 @@ func (am *DefaultAccountManager) DeleteGroup(accountId, userId, groupID string) delete(account.Groups, groupID) + // Update IPv6 status of all group members if necessary. + if g.IPv6Enabled { + _, err = am.updatePeerIPv6Status(account, userId, g, g.Peers) + if err != nil { + return err + } + } + account.Network.IncSerial() if err = am.Store.SaveAccount(account); err != nil { return err @@ -345,6 +376,7 @@ func (am *DefaultAccountManager) ListGroups(accountID string) ([]*nbgroup.Group, } // GroupAddPeer appends peer to the group +// TODO Question for devs: Is this method dead code? I can't seem to find any usages outside of tests... func (am *DefaultAccountManager) GroupAddPeer(accountID, groupID, peerID string) error { unlock := am.Store.AcquireAccountWriteLock(accountID) defer unlock() @@ -368,6 +400,14 @@ func (am *DefaultAccountManager) GroupAddPeer(accountID, groupID, peerID string) } if add { group.Peers = append(group.Peers, peerID) + + if group.IPv6Enabled { + // Update IPv6 status of added group member. + _, err = am.updatePeerIPv6Status(account, "", group, []string{peerID}) + if err != nil { + return err + } + } } account.Network.IncSerial() @@ -381,6 +421,7 @@ func (am *DefaultAccountManager) GroupAddPeer(accountID, groupID, peerID string) } // GroupDeletePeer removes peer from the group +// TODO Question for devs: Same as above, this seems like dead code func (am *DefaultAccountManager) GroupDeletePeer(accountID, groupID, peerID string) error { unlock := am.Store.AcquireAccountWriteLock(accountID) defer unlock() @@ -399,6 +440,15 @@ func (am *DefaultAccountManager) GroupDeletePeer(accountID, groupID, peerID stri for i, itemID := range group.Peers { if itemID == peerID { group.Peers = append(group.Peers[:i], group.Peers[i+1:]...) + + if group.IPv6Enabled { + // Update IPv6 status of deleted group member. + _, err = am.updatePeerIPv6Status(account, "", group, []string{peerID}) + if err != nil { + return err + } + } + if err := am.Store.SaveAccount(account); err != nil { return err } @@ -409,3 +459,24 @@ func (am *DefaultAccountManager) GroupDeletePeer(accountID, groupID, peerID stri return nil } + +func (am *DefaultAccountManager) updatePeerIPv6Status(account *Account, userID string, group *nbgroup.Group, peersToUpdate []string) (bool, error) { + updated := false + for _, peer := range peersToUpdate { + peerObj := account.GetPeer(peer) + update, err := am.DeterminePeerV6(account, peerObj) + if err != nil { + return false, err + } + if update { + updated = true + account.UpdatePeer(peerObj) + if peerObj.IP6 != nil { + am.StoreEvent(userID, group.ID, account.Id, activity.PeerIPv6InheritEnabled, group.EventMeta()) + } else { + am.StoreEvent(userID, group.ID, account.Id, activity.PeerIPv6InheritDisabled, group.EventMeta()) + } + } + } + return updated, nil +} diff --git a/management/server/group/group.go b/management/server/group/group.go index 79dfd995c..22f8b4788 100644 --- a/management/server/group/group.go +++ b/management/server/group/group.go @@ -25,6 +25,8 @@ type Group struct { // Peers list of the group Peers []string `gorm:"serializer:json"` + IPv6Enabled bool + IntegrationReference integration_reference.IntegrationReference `gorm:"embedded;embeddedPrefix:integration_ref_"` } @@ -38,6 +40,7 @@ func (g *Group) Copy() *Group { ID: g.ID, Name: g.Name, Issued: g.Issued, + IPv6Enabled: g.IPv6Enabled, Peers: make([]string, len(g.Peers)), IntegrationReference: g.IntegrationReference, } diff --git a/management/server/group_test.go b/management/server/group_test.go index 1c718715d..303301acd 100644 --- a/management/server/group_test.go +++ b/management/server/group_test.go @@ -2,6 +2,8 @@ package server import ( "errors" + nbpeer "github.com/netbirdio/netbird/management/server/peer" + "github.com/stretchr/testify/require" "testing" nbdns "github.com/netbirdio/netbird/dns" @@ -12,6 +14,8 @@ import ( const ( groupAdminUserID = "testingAdminUser" + groupPeer1Key = "BhRPtynAAYRDy08+q4HTMsos8fs4plTP4NOSh7C1ry8=" + groupPeer2Key = "/yF0+vCfv+mRR5k0dca0TrGdO/oiNeAI58gToZm5NyI=" ) func TestDefaultAccountManager_CreateGroup(t *testing.T) { @@ -20,7 +24,7 @@ func TestDefaultAccountManager_CreateGroup(t *testing.T) { t.Error("failed to create account manager") } - account, err := initTestGroupAccount(am) + _, account, err := initTestGroupAccount(am) if err != nil { t.Error("failed to init testing account") } @@ -55,7 +59,7 @@ func TestDefaultAccountManager_DeleteGroup(t *testing.T) { t.Error("failed to create account manager") } - account, err := initTestGroupAccount(am) + _, account, err := initTestGroupAccount(am) if err != nil { t.Error("failed to init testing account") } @@ -131,10 +135,137 @@ func TestDefaultAccountManager_DeleteGroup(t *testing.T) { } } -func initTestGroupAccount(am *DefaultAccountManager) (*Account, error) { +func TestDefaultAccountManager_GroupIPv6Consistency(t *testing.T) { + am, err := createManager(t) + if err != nil { + t.Error("failed to create account manager") + } + + peers, account, err := initTestGroupAccount(am) + peer1Id := peers[0] + peer2Id := peers[1] + if err != nil { + t.Error("failed to init testing account") + } + + group := account.GetGroup("grp-for-ipv6") + + // First, add one member to the IPv6 group before enabling IPv6. + group.Peers = append(group.Peers, peer1Id) + err = am.SaveGroup(account.Id, groupAdminUserID, group) + require.NoError(t, err, "unable to update group") + account, err = am.Store.GetAccount(account.Id) + require.NoError(t, err, "unable to update account") + group = account.GetGroup("grp-for-ipv6") + require.Nil(t, account.Peers[peer1Id].IP6, "peer1 should not have an IPv6 address if the group doesn't have it enabled.") + require.Nil(t, account.Peers[peer2Id].IP6, "peer2 should not have an IPv6 address.") + + // Now, enable IPv6. + group.IPv6Enabled = true + err = am.SaveGroup(account.Id, groupAdminUserID, group) + require.NoError(t, err, "unable to update group") + account, err = am.Store.GetAccount(account.Id) + require.NoError(t, err, "unable to update account") + group = account.GetGroup("grp-for-ipv6") + require.NotNil(t, account.Peers[peer1Id].IP6, "peer1 should have an IPv6 address as it is a member of the IPv6-enabled group.") + require.Nil(t, account.Peers[peer2Id].IP6, "peer2 should not have an IPv6 address as it is not a member of the IPv6-enabled group.") + + // Add the second peer. + group.Peers = append(group.Peers, peer2Id) + err = am.SaveGroup(account.Id, groupAdminUserID, group) + require.NoError(t, err, "unable to update group") + account, err = am.Store.GetAccount(account.Id) + require.NoError(t, err, "unable to update account") + group = account.GetGroup("grp-for-ipv6") + require.NotNil(t, account.Peers[peer1Id].IP6, "peer1 should have an IPv6 address as it is a member of the IPv6-enabled group.") + require.NotNil(t, account.Peers[peer2Id].IP6, "peer2 should have an IPv6 address as it is a member of the IPv6-enabled group.") + + // Disable IPv6 and simultaneously delete the first peer. + group.IPv6Enabled = false + group.Peers = group.Peers[1:] + err = am.SaveGroup(account.Id, groupAdminUserID, group) + require.NoError(t, err, "unable to update group") + account, err = am.Store.GetAccount(account.Id) + require.NoError(t, err, "unable to update account") + group = account.GetGroup("grp-for-ipv6") + require.Nil(t, account.Peers[peer1Id].IP6, "peer1 should not have an IPv6 address as it is not a member of any IPv6-enabled group.") + require.Nil(t, account.Peers[peer2Id].IP6, "peer2 should not have an IPv6 address as the group has IPv6 disabled.") + + // Enable IPv6 and simultaneously add the first peer again. + group.IPv6Enabled = true + group.Peers = append(group.Peers, peer1Id) + err = am.SaveGroup(account.Id, groupAdminUserID, group) + require.NoError(t, err, "unable to update group") + account, err = am.Store.GetAccount(account.Id) + require.NoError(t, err, "unable to update account") + require.NotNil(t, account.Peers[peer1Id].IP6, "peer1 should have an IPv6 address as it is a member of the IPv6-enabled group.") + require.NotNil(t, account.Peers[peer2Id].IP6, "peer2 should have an IPv6 address as it is a member of the IPv6-enabled group.") + + // Force disable IPv6. + peer1 := account.GetPeer(peer1Id) + peer2 := account.GetPeer(peer2Id) + peer1.V6Setting = nbpeer.V6Disabled + peer2.V6Setting = nbpeer.V6Disabled + _, err = am.UpdatePeer(account.Id, groupAdminUserID, peer1) + require.NoError(t, err, "unable to update peer1") + _, err = am.UpdatePeer(account.Id, groupAdminUserID, peer2) + require.NoError(t, err, "unable to update peer2") + account, err = am.Store.GetAccount(account.Id) + require.NoError(t, err, "unable to fetch updated account") + group = account.GetGroup("grp-for-ipv6") + require.Nil(t, account.GetPeer(peer1Id).IP6, "peer1 should not have an IPv6 address as it is force disabled.") + require.Nil(t, account.GetPeer(peer2Id).IP6, "peer2 should not have an IPv6 address as it is force disabled.") + + // Delete Group. + err = am.DeleteGroup(account.Id, groupAdminUserID, group.ID) + require.NoError(t, err, "unable to delete group") + account, err = am.Store.GetAccount(account.Id) + require.NoError(t, err, "unable to update account") + group = account.GetGroup("grp-for-ipv6") + require.Nil(t, group, "Group should no longer exist.") + require.Nil(t, account.Peers[peer1Id].IP6, "peer1 should not have an IPv6 address as the only IPv6-enabled group was deleted.") + require.Nil(t, account.Peers[peer2Id].IP6, "peer2 should not have an IPv6 address as the only IPv6-enabled group was deleted.") +} + +func initTestGroupAccount(am *DefaultAccountManager) ([]string, *Account, error) { accountID := "testingAcc" domain := "example.com" + peer1 := &nbpeer.Peer{ + Key: peer1Key, + Name: "peer1", + Meta: nbpeer.PeerSystemMeta{ + Hostname: "test-host1@netbird.io", + GoOS: "linux", + Kernel: "Linux", + Core: "21.04", + Platform: "x86_64", + OS: "Ubuntu", + WtVersion: "development", + UIVersion: "development", + Ipv6Supported: true, + }, + V6Setting: nbpeer.V6Auto, + DNSLabel: groupPeer1Key, + } + peer2 := &nbpeer.Peer{ + Key: peer2Key, + Name: "peer2", + Meta: nbpeer.PeerSystemMeta{ + Hostname: "test-host2@netbird.io", + GoOS: "linux", + Kernel: "Linux", + Core: "21.04", + Platform: "x86_64", + OS: "Ubuntu", + WtVersion: "development", + UIVersion: "development", + Ipv6Supported: true, + }, + V6Setting: nbpeer.V6Auto, + DNSLabel: groupPeer2Key, + } + groupForRoute := &nbgroup.Group{ ID: "grp-for-route", AccountID: "account-id", @@ -191,6 +322,14 @@ func initTestGroupAccount(am *DefaultAccountManager) (*Account, error) { Peers: make([]string, 0), } + groupForIPv6 := &nbgroup.Group{ + ID: "grp-for-ipv6", + AccountID: "account-id", + Name: "Group for IPv6", + Issued: nbgroup.GroupIssuedAPI, + Peers: make([]string, 0), + } + routeResource := &route.Route{ ID: "example route", Groups: []string{groupForRoute.ID}, @@ -235,7 +374,7 @@ func initTestGroupAccount(am *DefaultAccountManager) (*Account, error) { err := am.Store.SaveAccount(account) if err != nil { - return nil, err + return nil, nil, err } _ = am.SaveGroup(accountID, groupAdminUserID, groupForRoute) @@ -245,6 +384,11 @@ func initTestGroupAccount(am *DefaultAccountManager) (*Account, error) { _ = am.SaveGroup(accountID, groupAdminUserID, groupForSetupKeys) _ = am.SaveGroup(accountID, groupAdminUserID, groupForUsers) _ = am.SaveGroup(accountID, groupAdminUserID, groupForIntegration) + _ = am.SaveGroup(accountID, groupAdminUserID, groupForIPv6) + peer1, _, _ = am.AddPeer(setupKey.Key, user.Id, peer1) + peer2, _, _ = am.AddPeer(setupKey.Key, user.Id, peer2) - return am.Store.GetAccount(account.Id) + account, err = am.Store.GetAccount(account.Id) + + return []string{peer1.ID, peer2.ID}, account, err } diff --git a/management/server/grpcserver.go b/management/server/grpcserver.go index 32989df7d..bb11861b4 100644 --- a/management/server/grpcserver.go +++ b/management/server/grpcserver.go @@ -287,6 +287,7 @@ func extractPeerMeta(loginReq *proto.LoginRequest) nbpeer.PeerSystemMeta { Cloud: loginReq.GetMeta().GetEnvironment().GetCloud(), Platform: loginReq.GetMeta().GetEnvironment().GetPlatform(), }, + Ipv6Supported: loginReq.GetMeta().GetIpv6Supported(), } } @@ -455,21 +456,31 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *prot func toPeerConfig(peer *nbpeer.Peer, network *Network, dnsName string) *proto.PeerConfig { netmask, _ := network.Net.Mask.Size() + address6 := "" + if network.Net6 != nil && peer.IP6 != nil { + netmask6, _ := network.Net6.Mask.Size() + address6 = fmt.Sprintf("%s/%d", peer.IP6.String(), netmask6) + } fqdn := peer.FQDN(dnsName) return &proto.PeerConfig{ Address: fmt.Sprintf("%s/%d", peer.IP.String(), netmask), // take it from the network + Address6: address6, SshConfig: &proto.SSHConfig{SshEnabled: peer.SSHEnabled}, Fqdn: fqdn, } } -func toRemotePeerConfig(peers []*nbpeer.Peer, dnsName string) []*proto.RemotePeerConfig { +func toRemotePeerConfig(peers []*nbpeer.Peer, dnsName string, v6Enabled bool) []*proto.RemotePeerConfig { remotePeers := []*proto.RemotePeerConfig{} for _, rPeer := range peers { fqdn := rPeer.FQDN(dnsName) + allowedIps := []string{fmt.Sprintf(AllowedIPsFormat, rPeer.IP)} + if rPeer.IP6 != nil && v6Enabled { + allowedIps = append(allowedIps, fmt.Sprintf(AllowedIP6sFormat, *rPeer.IP6)) + } remotePeers = append(remotePeers, &proto.RemotePeerConfig{ WgPubKey: rPeer.Key, - AllowedIps: []string{fmt.Sprintf(AllowedIPsFormat, rPeer.IP)}, + AllowedIps: allowedIps, SshConfig: &proto.SSHConfig{SshPubKey: []byte(rPeer.SSHKey)}, Fqdn: fqdn, }) @@ -482,13 +493,13 @@ func toSyncResponse(config *Config, peer *nbpeer.Peer, turnCredentials *TURNCred pConfig := toPeerConfig(peer, networkMap.Network, dnsName) - remotePeers := toRemotePeerConfig(networkMap.Peers, dnsName) + remotePeers := toRemotePeerConfig(networkMap.Peers, dnsName, peer.IP6 != nil) routesUpdate := toProtocolRoutes(networkMap.Routes) dnsUpdate := toProtocolDNSConfig(networkMap.DNSConfig) - offlinePeers := toRemotePeerConfig(networkMap.OfflinePeers, dnsName) + offlinePeers := toRemotePeerConfig(networkMap.OfflinePeers, dnsName, peer.IP6 != nil) firewallRules := toProtocolFirewallRules(networkMap.FirewallRules) diff --git a/management/server/http/accounts_handler_test.go b/management/server/http/accounts_handler_test.go index 9d174d0be..b2c67c307 100644 --- a/management/server/http/accounts_handler_test.go +++ b/management/server/http/accounts_handler_test.go @@ -62,7 +62,7 @@ func TestAccounts_AccountsHandler(t *testing.T) { handler := initAccountsTestData(&server.Account{ Id: accountID, Domain: "hotmail.com", - Network: server.NewNetwork(), + Network: server.NewNetwork(true), Users: map[string]*server.User{ adminUser.Id: adminUser, }, diff --git a/management/server/http/api/openapi.yml b/management/server/http/api/openapi.yml index aeaef6f64..35fce08ae 100644 --- a/management/server/http/api/openapi.yml +++ b/management/server/http/api/openapi.yml @@ -247,10 +247,15 @@ components: description: (Cloud only) Indicates whether peer needs approval type: boolean example: true + ipv6_enabled: + type: string + enum: [enabled, disabled, auto] + example: auto required: - name - ssh_enabled - login_expiration_enabled + - ipv6_enabled PeerBase: allOf: - $ref: '#/components/schemas/PeerMinimum' @@ -264,6 +269,10 @@ components: description: Peer's public connection IP address type: string example: 35.64.0.1 + ip6: + description: Peer's IPv6 address + type: string + example: 2001:db8::0123:4567:890a:bcde connected: description: Peer to Management connection status type: boolean @@ -310,6 +319,15 @@ components: description: Peer's desktop UI version type: string example: 0.14.0 + ipv6_supported: + description: Whether this peer supports IPv6 + type: boolean + example: true + ipv6_enabled: + description: Whether IPv6 is enabled for this peer. + type: string + enum: [enabled, disabled, auto] + example: auto dns_label: description: Peer's DNS label is the parsed peer name for domain resolution. It is used to form an FQDN by appending the account's domain to the peer label. e.g. peer-dns-label.netbird.cloud type: string @@ -352,6 +370,8 @@ components: - kernel_version - last_login - last_seen + - ipv6_supported + - ipv6_enabled - login_expiration_enabled - login_expired - os @@ -627,8 +647,12 @@ components: items: type: string example: "ch8i4ug6lnn4g9hqv7m1" + ipv6_enabled: + description: Whether IPv6 should be enabled for all members with IPv6 set to "auto" + type: boolean required: - name + - ipv6_enabled Group: allOf: - $ref: '#/components/schemas/GroupMinimum' @@ -639,8 +663,12 @@ components: type: array items: $ref: '#/components/schemas/PeerMinimum' + ipv6_enabled: + description: Whether IPv6 should be enabled for all members with IPv6 set to "auto" + type: boolean required: - peers + - ipv6_enabled PolicyRuleMinimum: type: object properties: diff --git a/management/server/http/api/types.gen.go b/management/server/http/api/types.gen.go index e378213a1..87848fcd3 100644 --- a/management/server/http/api/types.gen.go +++ b/management/server/http/api/types.gen.go @@ -88,12 +88,40 @@ const ( NameserverNsTypeUdp NameserverNsType = "udp" ) +// Defines values for PeerIpv6Enabled. +const ( + PeerIpv6EnabledAuto PeerIpv6Enabled = "auto" + PeerIpv6EnabledDisabled PeerIpv6Enabled = "disabled" + PeerIpv6EnabledEnabled PeerIpv6Enabled = "enabled" +) + +// Defines values for PeerBaseIpv6Enabled. +const ( + PeerBaseIpv6EnabledAuto PeerBaseIpv6Enabled = "auto" + PeerBaseIpv6EnabledDisabled PeerBaseIpv6Enabled = "disabled" + PeerBaseIpv6EnabledEnabled PeerBaseIpv6Enabled = "enabled" +) + +// Defines values for PeerBatchIpv6Enabled. +const ( + PeerBatchIpv6EnabledAuto PeerBatchIpv6Enabled = "auto" + PeerBatchIpv6EnabledDisabled PeerBatchIpv6Enabled = "disabled" + PeerBatchIpv6EnabledEnabled PeerBatchIpv6Enabled = "enabled" +) + // Defines values for PeerNetworkRangeCheckAction. const ( PeerNetworkRangeCheckActionAllow PeerNetworkRangeCheckAction = "allow" PeerNetworkRangeCheckActionDeny PeerNetworkRangeCheckAction = "deny" ) +// Defines values for PeerRequestIpv6Enabled. +const ( + PeerRequestIpv6EnabledAuto PeerRequestIpv6Enabled = "auto" + PeerRequestIpv6EnabledDisabled PeerRequestIpv6Enabled = "disabled" + PeerRequestIpv6EnabledEnabled PeerRequestIpv6Enabled = "enabled" +) + // Defines values for PolicyRuleAction. const ( PolicyRuleActionAccept PolicyRuleAction = "accept" @@ -307,6 +335,9 @@ type Group struct { // Id Group ID Id string `json:"id"` + // Ipv6Enabled Whether IPv6 should be enabled for all members with IPv6 set to "auto" + Ipv6Enabled bool `json:"ipv6_enabled"` + // Issued How the group was issued (api, integration, jwt) Issued *GroupIssued `json:"issued,omitempty"` @@ -343,6 +374,9 @@ type GroupMinimumIssued string // GroupRequest defines model for GroupRequest. type GroupRequest struct { + // Ipv6Enabled Whether IPv6 should be enabled for all members with IPv6 set to "auto" + Ipv6Enabled bool `json:"ipv6_enabled"` + // Name Group name identifier Name string `json:"name"` @@ -502,6 +536,15 @@ type Peer struct { // Ip Peer's IP address Ip string `json:"ip"` + // Ip6 Peer's IPv6 address + Ip6 *string `json:"ip6,omitempty"` + + // Ipv6Enabled Whether IPv6 is enabled for this peer. + Ipv6Enabled PeerIpv6Enabled `json:"ipv6_enabled"` + + // Ipv6Supported Whether this peer supports IPv6 + Ipv6Supported bool `json:"ipv6_supported"` + // KernelVersion Peer's operating system kernel version KernelVersion string `json:"kernel_version"` @@ -539,6 +582,9 @@ type Peer struct { Version string `json:"version"` } +// PeerIpv6Enabled Whether IPv6 is enabled for this peer. +type PeerIpv6Enabled string + // PeerBase defines model for PeerBase. type PeerBase struct { // ApprovalRequired (Cloud only) Indicates whether peer needs approval @@ -574,6 +620,15 @@ type PeerBase struct { // Ip Peer's IP address Ip string `json:"ip"` + // Ip6 Peer's IPv6 address + Ip6 *string `json:"ip6,omitempty"` + + // Ipv6Enabled Whether IPv6 is enabled for this peer. + Ipv6Enabled PeerBaseIpv6Enabled `json:"ipv6_enabled"` + + // Ipv6Supported Whether this peer supports IPv6 + Ipv6Supported bool `json:"ipv6_supported"` + // KernelVersion Peer's operating system kernel version KernelVersion string `json:"kernel_version"` @@ -611,6 +666,9 @@ type PeerBase struct { Version string `json:"version"` } +// PeerBaseIpv6Enabled Whether IPv6 is enabled for this peer. +type PeerBaseIpv6Enabled string + // PeerBatch defines model for PeerBatch. type PeerBatch struct { // AccessiblePeersCount Number of accessible peers @@ -649,6 +707,15 @@ type PeerBatch struct { // Ip Peer's IP address Ip string `json:"ip"` + // Ip6 Peer's IPv6 address + Ip6 *string `json:"ip6,omitempty"` + + // Ipv6Enabled Whether IPv6 is enabled for this peer. + Ipv6Enabled PeerBatchIpv6Enabled `json:"ipv6_enabled"` + + // Ipv6Supported Whether this peer supports IPv6 + Ipv6Supported bool `json:"ipv6_supported"` + // KernelVersion Peer's operating system kernel version KernelVersion string `json:"kernel_version"` @@ -686,6 +753,9 @@ type PeerBatch struct { Version string `json:"version"` } +// PeerBatchIpv6Enabled Whether IPv6 is enabled for this peer. +type PeerBatchIpv6Enabled string + // PeerMinimum defines model for PeerMinimum. type PeerMinimum struct { // Id Peer ID @@ -710,12 +780,16 @@ type PeerNetworkRangeCheckAction string // PeerRequest defines model for PeerRequest. type PeerRequest struct { // ApprovalRequired (Cloud only) Indicates whether peer needs approval - ApprovalRequired *bool `json:"approval_required,omitempty"` - LoginExpirationEnabled bool `json:"login_expiration_enabled"` - Name string `json:"name"` - SshEnabled bool `json:"ssh_enabled"` + ApprovalRequired *bool `json:"approval_required,omitempty"` + Ipv6Enabled PeerRequestIpv6Enabled `json:"ipv6_enabled"` + LoginExpirationEnabled bool `json:"login_expiration_enabled"` + Name string `json:"name"` + SshEnabled bool `json:"ssh_enabled"` } +// PeerRequestIpv6Enabled defines model for PeerRequest.Ipv6Enabled. +type PeerRequestIpv6Enabled string + // PersonalAccessToken defines model for PersonalAccessToken. type PersonalAccessToken struct { // CreatedAt Date the token was created diff --git a/management/server/http/groups_handler.go b/management/server/http/groups_handler.go index 47bcf2f32..39f0c0902 100644 --- a/management/server/http/groups_handler.go +++ b/management/server/http/groups_handler.go @@ -3,6 +3,7 @@ package http import ( "encoding/json" "net/http" + "slices" "github.com/gorilla/mux" log "github.com/sirupsen/logrus" @@ -82,16 +83,6 @@ func (h *GroupsHandler) UpdateGroup(w http.ResponseWriter, r *http.Request) { return } - allGroup, err := account.GetGroupAll() - if err != nil { - util.WriteError(err, w) - return - } - if allGroup.ID == groupID { - util.WriteError(status.Errorf(status.InvalidArgument, "updating group ALL is not allowed"), w) - return - } - var req api.PutApiGroupsGroupIdJSONRequestBody err = json.NewDecoder(r.Body).Decode(&req) if err != nil { @@ -110,12 +101,42 @@ func (h *GroupsHandler) UpdateGroup(w http.ResponseWriter, r *http.Request) { } else { peers = *req.Peers } + + allGroup, err := account.GetGroupAll() + if err != nil { + util.WriteError(err, w) + return + } + + if allGroup.ID == groupID { + if len(peers) != len(allGroup.Peers) || req.Name != allGroup.Name { + util.WriteError(status.Errorf(status.InvalidArgument, "updating group ALL is not allowed"), w) + return + } + deduplicatedPeers := make(map[string]struct{}) + for _, peer := range peers { + deduplicatedPeers[peer] = struct{}{} + } + if len(deduplicatedPeers) != len(peers) { + util.WriteError(status.Errorf(status.InvalidArgument, "updating group ALL is not allowed"), w) + return + } + for peer := range deduplicatedPeers { + if slices.Contains(allGroup.Peers, peer) { + continue + } + util.WriteError(status.Errorf(status.InvalidArgument, "updating group ALL is not allowed"), w) + return + } + } + group := nbgroup.Group{ ID: groupID, Name: req.Name, Peers: peers, Issued: eg.Issued, IntegrationReference: eg.IntegrationReference, + IPv6Enabled: req.Ipv6Enabled, } if err := h.accountManager.SaveGroup(account.Id, user.Id, &group); err != nil { @@ -246,6 +267,7 @@ func toGroupResponse(account *server.Account, group *nbgroup.Group) *api.Group { Id: group.ID, Name: group.Name, Issued: (*api.GroupIssued)(&group.Issued), + Ipv6Enabled: group.IPv6Enabled, } for _, pid := range group.Peers { diff --git a/management/server/http/peers_handler.go b/management/server/http/peers_handler.go index 762576506..67ad67c49 100644 --- a/management/server/http/peers_handler.go +++ b/management/server/http/peers_handler.go @@ -85,11 +85,17 @@ func (h *PeersHandler) updatePeer(account *server.Account, user *server.User, pe return } + v6Status := nbpeer.V6Auto + if req.Ipv6Enabled != api.PeerRequestIpv6EnabledAuto { + v6Status = nbpeer.V6Status(req.Ipv6Enabled) + } + update := &nbpeer.Peer{ ID: peerID, SSHEnabled: req.SshEnabled, Name: req.Name, LoginExpirationEnabled: req.LoginExpirationEnabled, + V6Setting: v6Status, } if req.ApprovalRequired != nil { @@ -284,11 +290,22 @@ func toSinglePeerResponse(peer *nbpeer.Peer, groupsInfo []api.GroupMinimum, dnsD osVersion = peer.Meta.Core } + var ip6 *string + if peer.IP6 != nil { + ip6string := peer.IP6.String() + ip6 = &ip6string + } + v6Status := api.PeerIpv6EnabledAuto + if peer.V6Setting != nbpeer.V6Auto { + v6Status = api.PeerIpv6Enabled(peer.V6Setting) + } + return &api.Peer{ Id: peer.ID, Name: peer.Name, Ip: peer.IP.String(), ConnectionIp: peer.Location.ConnectionIP.String(), + Ip6: ip6, Connected: peer.Status.Connected, LastSeen: peer.Status.LastSeen, Os: fmt.Sprintf("%s %s", peer.Meta.OS, osVersion), @@ -300,6 +317,8 @@ func toSinglePeerResponse(peer *nbpeer.Peer, groupsInfo []api.GroupMinimum, dnsD Hostname: peer.Meta.Hostname, UserId: peer.UserID, UiVersion: peer.Meta.UIVersion, + Ipv6Supported: peer.Meta.Ipv6Supported, + Ipv6Enabled: v6Status, DnsLabel: fqdn(peer, dnsDomain), LoginExpirationEnabled: peer.LoginExpirationEnabled, LastLogin: peer.LastLogin, @@ -317,12 +336,22 @@ func toPeerListItemResponse(peer *nbpeer.Peer, groupsInfo []api.GroupMinimum, dn if osVersion == "" { osVersion = peer.Meta.Core } + var ip6 *string + if peer.IP6 != nil { + ip6string := peer.IP6.String() + ip6 = &ip6string + } + v6Status := api.PeerBatchIpv6EnabledAuto + if peer.V6Setting != nbpeer.V6Auto { + v6Status = api.PeerBatchIpv6Enabled(peer.V6Setting) + } return &api.PeerBatch{ Id: peer.ID, Name: peer.Name, Ip: peer.IP.String(), ConnectionIp: peer.Location.ConnectionIP.String(), + Ip6: ip6, Connected: peer.Status.Connected, LastSeen: peer.Status.LastSeen, Os: fmt.Sprintf("%s %s", peer.Meta.OS, osVersion), @@ -334,6 +363,8 @@ func toPeerListItemResponse(peer *nbpeer.Peer, groupsInfo []api.GroupMinimum, dn Hostname: peer.Meta.Hostname, UserId: peer.UserID, UiVersion: peer.Meta.UIVersion, + Ipv6Supported: peer.Meta.Ipv6Supported, + Ipv6Enabled: v6Status, DnsLabel: fqdn(peer, dnsDomain), LoginExpirationEnabled: peer.LoginExpirationEnabled, LastLogin: peer.LastLogin, diff --git a/management/server/http/peers_handler_test.go b/management/server/http/peers_handler_test.go index 53df5cb00..eb63dd10d 100644 --- a/management/server/http/peers_handler_test.go +++ b/management/server/http/peers_handler_test.go @@ -37,6 +37,12 @@ func initTestMetaData(peers ...*nbpeer.Peer) *PeersHandler { break } } + if p.V6Setting == nbpeer.V6Enabled && p.IP6 == nil { + ip6 := net.ParseIP("2001:db8::dead:beef") + p.IP6 = &ip6 + } else { + p.IP6 = nil + } p.SSHEnabled = update.SSHEnabled p.LoginExpirationEnabled = update.LoginExpirationEnabled p.Name = update.Name diff --git a/management/server/nameserver_test.go b/management/server/nameserver_test.go index f2921532d..6889448f5 100644 --- a/management/server/nameserver_test.go +++ b/management/server/nameserver_test.go @@ -789,6 +789,7 @@ func initTestNSAccount(t *testing.T, am *DefaultAccountManager) (*Account, error OS: "Ubuntu", WtVersion: "development", UIVersion: "development", + Ipv6Supported: false, }, } peer2 := &nbpeer.Peer{ @@ -803,6 +804,7 @@ func initTestNSAccount(t *testing.T, am *DefaultAccountManager) (*Account, error OS: "Ubuntu", WtVersion: "development", UIVersion: "development", + Ipv6Supported: false, }, } existingNSGroup := nbdns.NameServerGroup{ diff --git a/management/server/network.go b/management/server/network.go index 0e7d753a7..b75d0a9b7 100644 --- a/management/server/network.go +++ b/management/server/network.go @@ -1,13 +1,13 @@ package server import ( + crand "crypto/rand" + "encoding/binary" + "github.com/c-robinson/iplib" + "github.com/rs/xid" "math/rand" "net" "sync" - "time" - - "github.com/c-robinson/iplib" - "github.com/rs/xid" nbdns "github.com/netbirdio/netbird/dns" nbpeer "github.com/netbirdio/netbird/management/server/peer" @@ -20,11 +20,21 @@ const ( SubnetSize = 16 // NetSize is a global network size 100.64.0.0/10 NetSize = 10 + // Subnet6Size is the size of an IPv6 subnet (in Bytes, not Bits) + Subnet6Size = 8 // AllowedIPsFormat generates Wireguard AllowedIPs format (e.g. 100.64.30.1/32) AllowedIPsFormat = "%s/32" + + // AllowedIP6sFormat generates Wireguard AllowedIPs format (e.g. 2001:db8::dead:beef/128) + AllowedIP6sFormat = "%s/128" ) +// Global random number generator for IP addresses +// Accesses to the RNG must always be protected using rngLock (RNG sources are not thread-safe) +var rng = initializeRng() +var rngLock = sync.Mutex{} + type NetworkMap struct { Peers []*nbpeer.Peer Network *Network @@ -37,6 +47,7 @@ type NetworkMap struct { type Network struct { Identifier string `json:"id"` Net net.IPNet `gorm:"serializer:json"` + Net6 *net.IPNet `gorm:"serializer:json"` // Can't use gob serializer, as it cannot encode nil values. Dns string // Serial is an ID that increments by 1 when any change to the network happened (e.g. new peer has been added). // Used to synchronize state to the client apps. @@ -45,24 +56,54 @@ type Network struct { mu sync.Mutex `json:"-" gorm:"-"` } +func initializeRng() *rand.Rand { + seed := make([]byte, 8) + _, err := crand.Read(seed) + if err != nil { + return nil + } + s := rand.NewSource(int64(binary.LittleEndian.Uint64(seed))) + return rand.New(s) +} + // NewNetwork creates a new Network initializing it with a Serial=0 // It takes a random /16 subnet from 100.64.0.0/10 (64 different subnets) -func NewNetwork() *Network { +func NewNetwork(enableV6 bool) *Network { n := iplib.NewNet4(net.ParseIP("100.64.0.0"), NetSize) sub, _ := n.Subnet(SubnetSize) + rngLock.Lock() + intn := rng.Intn(len(sub)) + rngLock.Unlock() - s := rand.NewSource(time.Now().Unix()) - r := rand.New(s) - intn := r.Intn(len(sub)) + var n6 *net.IPNet = nil + if enableV6 { + n6 = GenerateNetwork6() + } return &Network{ Identifier: xid.New().String(), Net: sub[intn].IPNet, + Net6: n6, Dns: "", Serial: 0} } +func GenerateNetwork6() *net.IPNet { + addrbuf := make([]byte, 16) + addrbuf[0] = 0xfd + addrbuf[1] = 0x00 + addrbuf[2] = 0xb1 + addrbuf[3] = 0x4d + + rngLock.Lock() + _, _ = rng.Read(addrbuf[4:Subnet6Size]) + rngLock.Unlock() + + n6 := iplib.NewNet6(addrbuf, Subnet6Size*8, 0).IPNet + return &n6 +} + // IncSerial increments Serial by 1 reflecting that the network state has been changed func (n *Network) IncSerial() { n.mu.Lock() @@ -81,6 +122,7 @@ func (n *Network) Copy() *Network { return &Network{ Identifier: n.Identifier, Net: n.Net, + Net6: n.Net6, Dns: n.Dns, Serial: n.Serial, } @@ -103,13 +145,38 @@ func AllocatePeerIP(ipNet net.IPNet, takenIps []net.IP) (net.IP, error) { } // pick a random IP - s := rand.NewSource(time.Now().Unix()) - r := rand.New(s) - intn := r.Intn(len(ips)) + rngLock.Lock() + intn := rng.Intn(len(ips)) + rngLock.Unlock() return ips[intn], nil } +// AllocatePeerIP6 pics an available IPv6 from an net.IPNet. +// This method considers already taken IPs and reuses IPs if there are gaps in takenIps. +func AllocatePeerIP6(ipNet net.IPNet, takenIps []net.IP) (net.IP, error) { + + takenIPMap := make(map[string]struct{}) + takenIPMap[ipNet.IP.String()] = struct{}{} + for _, ip := range takenIps { + takenIPMap[ip.String()] = struct{}{} + } + + maskSize, _ := ipNet.Mask.Size() + + // TODO for small subnet sizes, randomly generating values until we don't get a duplicate is inefficient and could + // lead to many loop iterations, using a method similar to IPv4 would be preferable here. + + addrbuf := make(net.IP, 16) + copy(addrbuf, ipNet.IP.To16()) + for duplicate := true; duplicate; _, duplicate = takenIPMap[addrbuf.String()] { + rngLock.Lock() + _, _ = rng.Read(addrbuf[(maskSize / 8):16]) + rngLock.Unlock() + } + return addrbuf, nil +} + // generateIPs generates a list of all possible IPs of the given network excluding IPs specified in the exclusion list func generateIPs(ipNet *net.IPNet, exclusions map[string]struct{}) ([]net.IP, int) { diff --git a/management/server/network_test.go b/management/server/network_test.go index b067c4991..c4a4a34c8 100644 --- a/management/server/network_test.go +++ b/management/server/network_test.go @@ -1,6 +1,7 @@ package server import ( + "github.com/stretchr/testify/require" "net" "testing" @@ -8,11 +9,19 @@ import ( ) func TestNewNetwork(t *testing.T) { - network := NewNetwork() + network := NewNetwork(true) // generated net should be a subnet of a larger 100.64.0.0/10 net ipNet := net.IPNet{IP: net.ParseIP("100.64.0.0"), Mask: net.IPMask{255, 192, 0, 0}} - assert.Equal(t, ipNet.Contains(network.Net.IP), true) + assert.True(t, ipNet.Contains(network.Net.IP)) + + // generated IPv6 net should be a subnet of the fd00:b14d::/32 prefix. + _, ipNet6, err := net.ParseCIDR("fd00:b14d::/32") + require.NoError(t, err, "unable to parse IPv6 prefix") + assert.True(t, ipNet6.Contains(network.Net6.IP)) + // IPv6 prefix should be of size /64 + ones, _ := network.Net6.Mask.Size() + assert.Equal(t, ones, 64) } func TestAllocatePeerIP(t *testing.T) { @@ -38,6 +47,32 @@ func TestAllocatePeerIP(t *testing.T) { } } +func TestAllocatePeerIP6(t *testing.T) { + _, ipNet, err := net.ParseCIDR("2001:db8:abcd:1234::/64") + require.NoError(t, err, "unable to parse IPv6 prefix") + var ips []net.IP + // Yeah, we better not check all 2^64 possible addresses, just generating a bunch of addresses should hopefully + // reveal any possible bugs in the RNG. + for i := 0; i < 252; i++ { + ip, err := AllocatePeerIP6(*ipNet, ips) + if err != nil { + t.Fatal(err) + } + ips = append(ips, ip) + } + + assert.Len(t, ips, 252) + + uniq := make(map[string]struct{}) + for _, ip := range ips { + if _, ok := uniq[ip.String()]; !ok { + uniq[ip.String()] = struct{}{} + } else { + t.Errorf("found duplicate IP %s", ip.String()) + } + } +} + func TestGenerateIPs(t *testing.T) { ipNet := net.IPNet{IP: net.ParseIP("100.64.0.0"), Mask: net.IPMask{255, 255, 255, 0}} ips, ipsLen := generateIPs(&ipNet, map[string]struct{}{"100.64.0.0": {}}) diff --git a/management/server/peer.go b/management/server/peer.go index 13ac3801d..70a13a767 100644 --- a/management/server/peer.go +++ b/management/server/peer.go @@ -3,6 +3,7 @@ package server import ( "fmt" "net" + "slices" "strings" "time" @@ -13,6 +14,7 @@ import ( "github.com/netbirdio/netbird/management/server/activity" nbpeer "github.com/netbirdio/netbird/management/server/peer" "github.com/netbirdio/netbird/management/server/status" + nbroute "github.com/netbirdio/netbird/route" ) // PeerSync used as a data object between the gRPC API and AccountManager on Sync request. @@ -140,7 +142,66 @@ func (am *DefaultAccountManager) MarkPeerConnected(peerPubKey string, connected return nil } -// UpdatePeer updates peer. Only Peer.Name, Peer.SSHEnabled, and Peer.LoginExpirationEnabled can be updated. +// Determines the current IPv6 status of the peer (including checks for inheritance) and generates a new or removes an +// existing IPv6 address if necessary. +// Additionally, disables IPv6 routes if peer no longer has an IPv6 address. +// Note that this change does not get persisted here. +// +// Returns a boolean that indicates whether the peer and/or the account changed and needs to be updated in the data +// source. +func (am *DefaultAccountManager) DeterminePeerV6(account *Account, peer *nbpeer.Peer) (bool, error) { + v6Setting := peer.V6Setting + if peer.V6Setting == nbpeer.V6Auto { + if peer.Meta.Ipv6Supported { + for _, group := range account.Groups { + if group.IPv6Enabled && slices.Contains(group.Peers, peer.ID) { + v6Setting = nbpeer.V6Enabled + break + } + } + if v6Setting == nbpeer.V6Auto { + for _, route := range account.Routes { + if route.Peer == peer.ID && route.NetworkType == nbroute.IPv6Network { + v6Setting = nbpeer.V6Enabled + break + } + } + } + } + + if v6Setting == nbpeer.V6Auto { + v6Setting = nbpeer.V6Disabled + } + } + + if v6Setting == nbpeer.V6Enabled && peer.IP6 == nil { + if !peer.Meta.Ipv6Supported { + return false, status.Errorf(status.PreconditionFailed, "failed allocating new IPv6 for peer %s - peer does not support IPv6", peer.Name) + } + if account.Network.Net6 == nil { + account.Network.Net6 = GenerateNetwork6() + } + v6tmp, err := AllocatePeerIP6(*account.Network.Net6, account.getTakenIP6s()) + if err != nil { + return false, err + } + peer.IP6 = &v6tmp + return true, nil + } else if v6Setting == nbpeer.V6Disabled && peer.IP6 != nil { + peer.IP6 = nil + + for _, route := range account.Routes { + if route.NetworkType == nbroute.IPv6Network { + route.Enabled = false + account.Routes[route.ID] = route + } + } + return true, nil + } + return false, nil +} + +// UpdatePeer updates peer. Only Peer.Name, Peer.SSHEnabled, Peer.V6Setting and Peer.LoginExpirationEnabled can be updated. func (am *DefaultAccountManager) UpdatePeer(accountID, userID string, update *nbpeer.Peer) (*nbpeer.Peer, error) { unlock := am.Store.AcquireAccountWriteLock(accountID) defer unlock() @@ -160,6 +221,20 @@ func (am *DefaultAccountManager) UpdatePeer(accountID, userID string, update *nb return nil, err } + if peer.V6Setting != update.V6Setting { + peer.V6Setting = update.V6Setting + prevV6 := peer.IP6 + v6StatusChanged, err := am.DeterminePeerV6(account, peer) + if err != nil { + return nil, err + } + if v6StatusChanged && peer.IP6 != nil { + am.StoreEvent(userID, peer.IP6.String(), account.Id, activity.PeerIPv6Enabled, peer.EventMeta(am.GetDNSDomain())) + } else if v6StatusChanged && peer.IP6 == nil { + am.StoreEvent(userID, prevV6.String(), account.Id, activity.PeerIPv6Disabled, peer.EventMeta(am.GetDNSDomain())) + } + } + if peer.SSHEnabled != update.SSHEnabled { peer.SSHEnabled = update.SSHEnabled event := activity.PeerSSHEnabled @@ -431,6 +506,7 @@ func (am *DefaultAccountManager) AddPeer(setupKey, userID string, peer *nbpeer.P Key: peer.Key, SetupKey: upperKey, IP: nextIp, + IP6: nil, Meta: peer.Meta, Name: peer.Meta.Hostname, DNSLabel: newLabel, @@ -443,6 +519,7 @@ func (am *DefaultAccountManager) AddPeer(setupKey, userID string, peer *nbpeer.P LoginExpirationEnabled: addedByUser, Ephemeral: ephemeral, Location: peer.Location, + V6Setting: peer.V6Setting, // empty string "" corresponds to "auto" } // add peer to 'All' group @@ -475,6 +552,11 @@ func (am *DefaultAccountManager) AddPeer(setupKey, userID string, peer *nbpeer.P newPeer = am.integratedPeerValidator.PreparePeer(account.Id, newPeer, account.GetPeerGroupsList(newPeer.ID), account.Settings.Extra) + _, err = am.DeterminePeerV6(account, newPeer) + if err != nil { + return nil, nil, err + } + if addedByUser { user, err := account.FindUser(userID) if err != nil { @@ -676,6 +758,14 @@ func (am *DefaultAccountManager) LoginPeer(login PeerLogin) (*nbpeer.Peer, *Netw shouldStoreAccount = true } + updated, err = am.DeterminePeerV6(account, peer) + if err != nil { + return nil, nil, err + } + if updated { + shouldStoreAccount = true + } + peer, err = am.checkAndUpdatePeerSSHKey(peer, account, login.SSHKey) if err != nil { return nil, nil, err diff --git a/management/server/peer/peer.go b/management/server/peer/peer.go index f71f629f6..68b820d80 100644 --- a/management/server/peer/peer.go +++ b/management/server/peer/peer.go @@ -20,6 +20,8 @@ type Peer struct { SetupKey string // IP address of the Peer IP net.IP `gorm:"serializer:json"` + // IPv6 address of the Peer + IP6 *net.IP `gorm:"uniqueIndex:idx_peers_account_id_ip6"` // Meta is a Peer system meta data Meta PeerSystemMeta `gorm:"embedded;embeddedPrefix:meta_"` // Name is peer's name (machine name) @@ -44,10 +46,23 @@ type Peer struct { CreatedAt time.Time // Indicate ephemeral peer attribute Ephemeral bool - // Geo location based on connection IP + // Geolocation based on connection IP Location Location `gorm:"embedded;embeddedPrefix:location_"` + // Whether IPv6 should be enabled or not. + V6Setting V6Status } +type V6Status string + +const ( + // Inherit IPv6 settings from groups (=> if one group the peer is a member of has IPv6 enabled, it will be enabled). + V6Auto V6Status = "" + // Enable IPv6 regardless of group settings, as long as it is supported. + V6Enabled V6Status = "enabled" + // Disable IPv6 regardless of group settings. + V6Disabled V6Status = "disabled" +) + type PeerStatus struct { //nolint:revive // LastSeen is the last time peer was connected to the management service LastSeen time.Time @@ -96,6 +111,7 @@ type PeerSystemMeta struct { //nolint:revive SystemProductName string SystemManufacturer string Environment Environment `gorm:"serializer:json"` + Ipv6Supported bool } func (p PeerSystemMeta) isEqual(other PeerSystemMeta) bool { @@ -130,7 +146,8 @@ func (p PeerSystemMeta) isEqual(other PeerSystemMeta) bool { p.SystemProductName == other.SystemProductName && p.SystemManufacturer == other.SystemManufacturer && p.Environment.Cloud == other.Environment.Cloud && - p.Environment.Platform == other.Environment.Platform + p.Environment.Platform == other.Environment.Platform && + p.Ipv6Supported == other.Ipv6Supported } // AddedWithSSOLogin indicates whether this peer has been added with an SSO login by a user. @@ -150,6 +167,7 @@ func (p *Peer) Copy() *Peer { Key: p.Key, SetupKey: p.SetupKey, IP: p.IP, + IP6: p.IP6, Meta: p.Meta, Name: p.Name, DNSLabel: p.DNSLabel, @@ -162,6 +180,7 @@ func (p *Peer) Copy() *Peer { CreatedAt: p.CreatedAt, Ephemeral: p.Ephemeral, Location: p.Location, + V6Setting: p.V6Setting, } } diff --git a/management/server/peer_test.go b/management/server/peer_test.go index 6063cc2a7..ee07f2d85 100644 --- a/management/server/peer_test.go +++ b/management/server/peer_test.go @@ -1,6 +1,7 @@ package server import ( + "github.com/stretchr/testify/require" "testing" "time" @@ -136,6 +137,112 @@ func TestAccountManager_GetNetworkMap(t *testing.T) { } } +func TestDefaultAccountManager_DeterminePeerV6(t *testing.T) { + manager, err := createManager(t) + if err != nil { + t.Fatal(err) + return + } + + expectedId := "test_account" + userId := "account_creator" + account, err := createAccount(manager, expectedId, userId, "") + if err != nil { + t.Fatal(err) + } + + setupKey, err := manager.CreateSetupKey(account.Id, "test-key", SetupKeyReusable, time.Hour, nil, 999, userId, false) + if err != nil { + t.Fatal("error creating setup key") + return + } + + peerKey1, err := wgtypes.GeneratePrivateKey() + if err != nil { + t.Fatal(err) + return + } + + peer1, _, err := manager.AddPeer(setupKey.Key, "", &nbpeer.Peer{ + Key: peerKey1.PublicKey().String(), + Meta: nbpeer.PeerSystemMeta{ + Hostname: "test-peer-1", + Ipv6Supported: true, + }, + }) + if err != nil { + t.Errorf("expecting peer to be added, got failure %v", err) + return + } + + peerKey2, err := wgtypes.GeneratePrivateKey() + if err != nil { + t.Fatal(err) + return + } + + peer2, _, err := manager.AddPeer(setupKey.Key, "", &nbpeer.Peer{ + Key: peerKey2.PublicKey().String(), + Meta: nbpeer.PeerSystemMeta{ + Hostname: "test-peer-2", + Ipv6Supported: false, + }, + }) + + if err != nil { + t.Errorf("expecting peer to be added, got failure %v", err) + return + } + + account, err = manager.Store.GetAccount(account.Id) + require.NoError(t, err, "unable to fetch updated account") + + // Check if automatic setting defaults to "false". + // (Other tests for interactions between the automatic setting and group/route memberships are already covered in + // group_test.go and route_test.go) + _, err = manager.DeterminePeerV6(account, peer1) + require.NoError(t, err, "unable to determine effective peer IPv6 status") + _, err = manager.DeterminePeerV6(account, peer2) + require.NoError(t, err, "unable to determine effective peer IPv6 status") + require.Nil(t, peer1.IP6, "peer1 IPv6 address did not default to nil.") + require.Nil(t, peer1.IP6, "peer2 IPv6 address did not default to nil.") + + peer1.V6Setting = nbpeer.V6Disabled + peer2.V6Setting = nbpeer.V6Disabled + _, err = manager.DeterminePeerV6(account, peer1) + require.NoError(t, err, "unable to determine effective peer IPv6 status") + _, err = manager.DeterminePeerV6(account, peer2) + require.NoError(t, err, "unable to determine effective peer IPv6 status") + require.Nil(t, peer1.IP6, "peer1 IPv6 address is not nil even though it is force disabled.") + require.Nil(t, peer2.IP6, "peer2 IPv6 address is not nil even though it is force disabled and unsupported.") + + peer1.V6Setting = nbpeer.V6Enabled + peer2.V6Setting = nbpeer.V6Enabled + _, err = manager.DeterminePeerV6(account, peer1) + require.NoError(t, err, "unable to determine effective peer IPv6 status") + _, err = manager.DeterminePeerV6(account, peer2) + require.Error(t, err, "determining peer2 IPv6 address should fail as it is force enabled, but unsupported.") + require.NotNil(t, peer1.IP6, "peer1 IPv6 address is nil even though it is force enabled.") + require.Nil(t, peer2.IP6, "peer2 IPv6 address is not nil even though it is unsupported.") + + // Test whether disabling IPv6 will disable IPv6 routes. + allGroup, err := account.GetGroupAll() + require.NoError(t, err, "unable to retrieve all group") + route, err := manager.CreateRoute(account.Id, "2001:db8:2345:6789::/64", peer1.ID, make([]string, 0), "testroute", "testnet", false, 9999, []string{allGroup.ID}, true, userID) + require.NoError(t, err, "unable to create test IPv6 route") + require.True(t, route.Enabled, "created IPv6 test route should be enabled") + + peer1.V6Setting = nbpeer.V6Disabled + _, err = manager.UpdatePeer(account.Id, userID, peer1) + require.NoError(t, err, "unable to update peer") + + account, err = manager.Store.GetAccount(account.Id) + require.NoError(t, err, "unable to fetch updated account") + + route = account.Routes[route.ID] + require.False(t, route.Enabled, "disabling IPv6 for a peer should disable all of its IPv6 routes.") +} + func TestAccountManager_GetNetworkMapWithPolicy(t *testing.T) { // TODO: disable until we start use policy again t.Skip() diff --git a/management/server/policy.go b/management/server/policy.go index 5206df9e9..7399fe9fb 100644 --- a/management/server/policy.go +++ b/management/server/policy.go @@ -195,6 +195,8 @@ type FirewallRule struct { // PeerIP of the peer PeerIP string + PeerIP6 string + // Direction of the traffic Direction int @@ -278,8 +280,14 @@ func (a *Account) connResourcesGenerator() (func(*PolicyRule, []*nbpeer.Peer, in peersExists[peer.ID] = struct{}{} } + ip6 := "" + if peer.IP6 != nil { + ip6 = peer.IP6.String() + } + fr := FirewallRule{ PeerIP: peer.IP.String(), + PeerIP6: ip6, Direction: direction, Action: string(rule.Action), Protocol: string(rule.Protocol), @@ -287,6 +295,7 @@ func (a *Account) connResourcesGenerator() (func(*PolicyRule, []*nbpeer.Peer, in if isAll { fr.PeerIP = "0.0.0.0" + fr.PeerIP6 = "::" } ruleID := (rule.ID + fr.PeerIP + strconv.Itoa(direction) + @@ -474,6 +483,7 @@ func toProtocolFirewallRules(update []*FirewallRule) []*proto.FirewallRule { result[i] = &proto.FirewallRule{ PeerIP: update[i].PeerIP, + PeerIP6: update[i].PeerIP6, Direction: direction, Action: action, Protocol: protocol, diff --git a/management/server/policy_test.go b/management/server/policy_test.go index 1ea3bb379..03119b3c9 100644 --- a/management/server/policy_test.go +++ b/management/server/policy_test.go @@ -14,16 +14,20 @@ import ( ) func TestAccount_getPeersByPolicy(t *testing.T) { + peerAIP6 := net.ParseIP("2001:db8:abcd:1234::2") + peerBIP6 := net.ParseIP("2001:db8:abcd:1234::3") account := &Account{ Peers: map[string]*nbpeer.Peer{ "peerA": { ID: "peerA", IP: net.ParseIP("100.65.14.88"), + IP6: &peerAIP6, Status: &nbpeer.PeerStatus{}, }, "peerB": { ID: "peerB", IP: net.ParseIP("100.65.80.39"), + IP6: &peerBIP6, Status: &nbpeer.PeerStatus{}, }, "peerC": { @@ -161,6 +165,7 @@ func TestAccount_getPeersByPolicy(t *testing.T) { epectedFirewallRules := []*FirewallRule{ { PeerIP: "0.0.0.0", + PeerIP6: "::", Direction: firewallRuleDirectionIN, Action: "accept", Protocol: "all", @@ -168,6 +173,7 @@ func TestAccount_getPeersByPolicy(t *testing.T) { }, { PeerIP: "0.0.0.0", + PeerIP6: "::", Direction: firewallRuleDirectionOUT, Action: "accept", Protocol: "all", @@ -175,6 +181,7 @@ func TestAccount_getPeersByPolicy(t *testing.T) { }, { PeerIP: "100.65.14.88", + PeerIP6: "2001:db8:abcd:1234::2", Direction: firewallRuleDirectionIN, Action: "accept", Protocol: "all", @@ -182,6 +189,7 @@ func TestAccount_getPeersByPolicy(t *testing.T) { }, { PeerIP: "100.65.14.88", + PeerIP6: "2001:db8:abcd:1234::2", Direction: firewallRuleDirectionOUT, Action: "accept", Protocol: "all", @@ -678,6 +686,7 @@ func TestAccount_getPeersByPolicyPostureChecks(t *testing.T) { expectedFirewallRules := []*FirewallRule{ { PeerIP: "0.0.0.0", + PeerIP6: "::", Direction: firewallRuleDirectionOUT, Action: "accept", Protocol: "tcp", diff --git a/management/server/route.go b/management/server/route.go index 2de813d48..5ad8fb93a 100644 --- a/management/server/route.go +++ b/management/server/route.go @@ -1,6 +1,7 @@ package server import ( + nbpeer "github.com/netbirdio/netbird/management/server/peer" "net/netip" "unicode/utf8" @@ -180,6 +181,25 @@ func (am *DefaultAccountManager) CreateRoute(accountID, network, peerID string, account.Routes[newRoute.ID] = &newRoute + // IPv6 route must only be created with IPv6 enabled peers, creating an IPv6 enabled route may enable IPv6 for + // peers with V6Setting = Auto. + if peerID != "" && prefixType == route.IPv6Network && newRoute.Enabled { + peer := account.GetPeer(peerID) + if peer.V6Setting == nbpeer.V6Disabled || !peer.Meta.Ipv6Supported { + return nil, status.Errorf( + status.InvalidArgument, + "IPv6 must be enabled for peer %s to be used in route %s", + peer.Name, newPrefix.String()) + } else if peer.IP6 == nil { + _, err = am.DeterminePeerV6(account, peer) + if err != nil { + return nil, err + } + account.UpdatePeer(peer) + } + + } + account.Network.IncSerial() if err = am.Store.SaveAccount(account); err != nil { return nil, err @@ -222,6 +242,16 @@ func (am *DefaultAccountManager) SaveRoute(accountID, userID string, routeToSave return status.Errorf(status.InvalidArgument, "peer with ID and peer groups should not be provided at the same time") } + if routeToSave.Peer != "" { + peer := account.GetPeer(routeToSave.Peer) + if peer == nil { + return status.Errorf(status.InvalidArgument, "provided peer does not exist") + } + if routeToSave.NetworkType == route.IPv6Network && routeToSave.Enabled && (!peer.Meta.Ipv6Supported || peer.V6Setting == nbpeer.V6Disabled) { + return status.Errorf(status.InvalidArgument, "peer with IPv6 disabled can't be used for IPv6 route") + } + } + if len(routeToSave.PeerGroups) > 0 { err = validateGroups(routeToSave.PeerGroups, account.Groups) if err != nil { @@ -239,8 +269,49 @@ func (am *DefaultAccountManager) SaveRoute(accountID, userID string, routeToSave return err } + oldRoute := account.Routes[routeToSave.ID] + account.Routes[routeToSave.ID] = routeToSave + // Check if old peer's IPv6 status needs to be recalculated. + // Must happen if route is an IPv6 route, and either: + // - The routing peer has changed + // - The route has been disabled + // - (the route has been enabled) => caught in the next if-block + if oldRoute.Peer != "" && routeToSave.NetworkType == route.IPv6Network && ((oldRoute.Enabled && !routeToSave.Enabled) || oldRoute.Peer != routeToSave.Peer) { + oldPeer := account.GetPeer(oldRoute.Peer) + if oldPeer.V6Setting == nbpeer.V6Auto { + changed, err := am.DeterminePeerV6(account, oldPeer) + if err != nil { + return err + } + if changed { + account.UpdatePeer(oldPeer) + } + } + } + // Check if new peer's IPv6 status needs to be recalculated. + // Must happen if route is an IPv6 route, and either: + // - The routing peer has changed + // - The route has been enabled + // - (The route has been disabled) => caught in previous if-block + if oldRoute.Peer != "" && routeToSave.NetworkType == route.IPv6Network && routeToSave.Enabled && (!oldRoute.Enabled || oldRoute.Peer != routeToSave.Peer) { + newPeer := account.GetPeer(routeToSave.Peer) + if newPeer.V6Setting == nbpeer.V6Disabled || !newPeer.Meta.Ipv6Supported { + return status.Errorf( + status.InvalidArgument, + "IPv6 must be enabled for peer %s to be used in route %s", + newPeer.Name, routeToSave.Network.String()) + } else if newPeer.IP6 == nil { + _, err = am.DeterminePeerV6(account, newPeer) + if err != nil { + return err + } + account.UpdatePeer(newPeer) + } + + } + account.Network.IncSerial() if err = am.Store.SaveAccount(account); err != nil { return err @@ -269,6 +340,21 @@ func (am *DefaultAccountManager) DeleteRoute(accountID string, routeID route.ID, } delete(account.Routes, routeID) + // If the route was an IPv6 route, deleting it may update the automatic IPv6 enablement status of its routing peers, + // check if this is the case and update accordingly. + if routy.Peer != "" && routy.Enabled && routy.NetworkType == route.IPv6Network { + oldPeer := account.GetPeer(routy.Peer) + if oldPeer.V6Setting == nbpeer.V6Auto { + changed, err := am.DeterminePeerV6(account, oldPeer) + if err != nil { + return err + } + if changed { + account.UpdatePeer(oldPeer) + } + } + } + account.Network.IncSerial() if err = am.Store.SaveAccount(account); err != nil { return err diff --git a/management/server/route_test.go b/management/server/route_test.go index d28b40d48..e77109125 100644 --- a/management/server/route_test.go +++ b/management/server/route_test.go @@ -319,6 +319,88 @@ func TestCreateRoute(t *testing.T) { errFunc: require.Error, shouldCreate: false, }, + { + name: "IPv6 route on peer with disabled IPv6 should fail", + inputArgs: input{ + network: "2001:db8:7654:3210::/64", + netID: "NewId", + peerKey: peer4ID, + description: "", + masquerade: false, + metric: 9999, + enabled: true, + groups: []string{routeGroup1}, + }, + errFunc: require.Error, + shouldCreate: false, + }, + { + name: "IPv6 route on peer with unsupported IPv6 should fail", + inputArgs: input{ + network: "2001:db8:7654:3210::/64", + netID: "NewId", + peerKey: peer5ID, + description: "", + masquerade: false, + metric: 9999, + enabled: true, + groups: []string{routeGroup1}, + }, + errFunc: require.Error, + shouldCreate: false, + }, + { + name: "IPv6 route on peer with automatic IPv6 setting should succeed", + inputArgs: input{ + network: "2001:db8:7654:3210::/64", + netID: "NewId", + peerKey: peer1ID, + description: "", + masquerade: false, + metric: 9999, + enabled: true, + groups: []string{routeGroup1}, + }, + errFunc: require.NoError, + shouldCreate: true, + expectedRoute: &route.Route{ + Network: netip.MustParsePrefix("2001:db8:7654:3210::/64"), + NetworkType: route.IPv6Network, + NetID: "NewId", + Peer: peer1ID, + Description: "", + Masquerade: false, + Metric: 9999, + Enabled: true, + Groups: []string{routeGroup1}, + }, + }, + { + name: "IPv6 route on peer with force enabled IPv6 setting should succeed", + inputArgs: input{ + network: "2001:db8:7654:3211::/64", + netID: "NewId", + peerKey: peer2ID, + description: "", + masquerade: false, + metric: 9999, + enabled: true, + groups: []string{routeGroup1}, + }, + errFunc: require.NoError, + shouldCreate: true, + expectedRoute: &route.Route{ + Network: netip.MustParsePrefix("2001:db8:7654:3211::/64"), + NetworkType: route.IPv6Network, + NetID: "NewId", + Peer: peer2ID, + Description: "", + Masquerade: false, + Metric: 9999, + Enabled: true, + Groups: []string{routeGroup1}, + }, + }, } for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { @@ -377,6 +459,8 @@ func TestCreateRoute(t *testing.T) { func TestSaveRoute(t *testing.T) { validPeer := peer2ID validUsedPeer := peer5ID + ipv6DisabledPeer := peer4ID + ipv6UnsupportedPeer := peer5ID invalidPeer := "nonExisting" validPrefix := netip.MustParsePrefix("192.168.0.0/24") invalidPrefix, _ := netip.ParsePrefix("192.168.0.0/34") @@ -467,7 +551,7 @@ func TestSaveRoute(t *testing.T) { }, }, { - name: "Both peer and peers_roup Provided Should Fail", + name: "Both peer and peers_group Provided Should Fail", existingRoute: &route.Route{ ID: "testingRoute", Network: netip.MustParsePrefix("192.168.0.0/16"), @@ -517,6 +601,38 @@ func TestSaveRoute(t *testing.T) { newPeer: &invalidPeer, errFunc: require.Error, }, + { + name: "IPv6 disabled host should not be allowed as peer for IPv6 route", + existingRoute: &route.Route{ + ID: "testingRoute", + Network: netip.MustParsePrefix("2001:db8:4321:5678::/64"), + NetID: validNetID, + NetworkType: route.IPv6Network, + Description: "super", + Masquerade: false, + Metric: 9999, + Enabled: true, + Groups: []string{routeGroup1}, + }, + newPeer: &ipv6DisabledPeer, + errFunc: require.Error, + }, + { + name: "IPv6 unsupported host should not be allowed as peer for IPv6 route", + existingRoute: &route.Route{ + ID: "testingRoute", + Network: netip.MustParsePrefix("2001:db8:4321:5678::/64"), + NetID: validNetID, + NetworkType: route.IPv6Network, + Description: "super", + Masquerade: false, + Metric: 9999, + Enabled: true, + Groups: []string{routeGroup1}, + }, + newPeer: &ipv6UnsupportedPeer, + errFunc: require.Error, + }, { name: "Invalid Metric Should Fail", existingRoute: &route.Route{ @@ -772,7 +888,7 @@ func TestDeleteRoute(t *testing.T) { ID: "testingRoute", Network: netip.MustParsePrefix("192.168.0.0/16"), NetworkType: route.IPv4Network, - Peer: peer1Key, + Peer: peer1ID, Description: "super", Masquerade: false, Metric: 9999, @@ -812,6 +928,77 @@ func TestDeleteRoute(t *testing.T) { } } +func TestRouteIPv6Consistency(t *testing.T) { + testingRoute := &route.Route{ + ID: "testingRoute", + Network: netip.MustParsePrefix("2001:db8:0987:6543::/64"), + NetworkType: route.IPv6Network, + NetID: existingRouteID, + Peer: peer1ID, + Description: "super", + Masquerade: false, + Metric: 9999, + Enabled: false, + Groups: []string{routeGroup1}, + } + + am, err := createRouterManager(t) + if err != nil { + t.Error("failed to create account manager") + } + + account, err := initTestRouteAccount(t, am) + if err != nil { + t.Error("failed to init testing account") + } + + account.Routes[testingRoute.ID] = testingRoute + + err = am.Store.SaveAccount(account) + if err != nil { + t.Error("failed to save account") + } + + savedAccount, err := am.Store.GetAccount(account.Id) + if err != nil { + t.Error("failed to retrieve saved account with error: ", err) + } + + testingRoute = savedAccount.Routes[testingRoute.ID] + testingRoute.Enabled = true + + err = am.SaveRoute(account.Id, userID, testingRoute) + if err != nil { + t.Error("failed to save route") + } + + savedAccount, err = am.Store.GetAccount(account.Id) + if err != nil { + t.Error("failed to retrieve saved account with error: ", err) + } + + peer := savedAccount.GetPeer(peer1ID) + require.NotNil(t, peer.IP6, "peer with enabled IPv6 route in automatic setting must have IPv6 active.") + + err = am.DeleteRoute(account.Id, testingRoute.ID, userID) + if err != nil { + t.Error("deleting route failed with error: ", err) + } + + savedAccount, err = am.Store.GetAccount(account.Id) + if err != nil { + t.Error("failed to retrieve saved account with error: ", err) + } + + _, found := savedAccount.Routes[testingRoute.ID] + if found { + t.Error("route shouldn't be found after delete") + } + + peer = savedAccount.GetPeer(peer1ID) + require.Nil(t, peer.IP6, "disabling the only IPv6 route for a peer with automatic IPv6 setting should disable IPv6") +} + func TestGetNetworkMap_RouteSyncPeerGroups(t *testing.T) { baseRoute := &route.Route{ Network: netip.MustParsePrefix("192.168.0.0/16"), @@ -1055,14 +1242,15 @@ func initTestRouteAccount(t *testing.T, am *DefaultAccountManager) (*Account, er Name: "test-host1@netbird.io", UserID: userID, Meta: nbpeer.PeerSystemMeta{ - Hostname: "test-host1@netbird.io", - GoOS: "linux", - Kernel: "Linux", - Core: "21.04", - Platform: "x86_64", - OS: "Ubuntu", - WtVersion: "development", - UIVersion: "development", + Hostname: "test-host1@netbird.io", + GoOS: "linux", + Kernel: "Linux", + Core: "21.04", + Platform: "x86_64", + OS: "Ubuntu", + WtVersion: "development", + UIVersion: "development", + Ipv6Supported: true, }, Status: &nbpeer.PeerStatus{}, } @@ -1073,24 +1261,32 @@ func initTestRouteAccount(t *testing.T, am *DefaultAccountManager) (*Account, er if err != nil { return nil, err } + ips = account.getTakenIP6s() + peer2IP6, err := AllocatePeerIP6(account.Network.Net, ips) + if err != nil { + return nil, err + } peer2 := &nbpeer.Peer{ IP: peer2IP, + IP6: &peer2IP6, ID: peer2ID, Key: peer2Key, Name: "test-host2@netbird.io", UserID: userID, Meta: nbpeer.PeerSystemMeta{ - Hostname: "test-host2@netbird.io", - GoOS: "linux", - Kernel: "Linux", - Core: "21.04", - Platform: "x86_64", - OS: "Ubuntu", - WtVersion: "development", - UIVersion: "development", + Hostname: "test-host2@netbird.io", + GoOS: "linux", + Kernel: "Linux", + Core: "21.04", + Platform: "x86_64", + OS: "Ubuntu", + WtVersion: "development", + UIVersion: "development", + Ipv6Supported: true, }, - Status: &nbpeer.PeerStatus{}, + V6Setting: nbpeer.V6Enabled, + Status: &nbpeer.PeerStatus{}, } account.Peers[peer2.ID] = peer2 @@ -1099,24 +1295,32 @@ func initTestRouteAccount(t *testing.T, am *DefaultAccountManager) (*Account, er if err != nil { return nil, err } + ips = account.getTakenIP6s() + peer3IP6, err := AllocatePeerIP6(account.Network.Net, ips) + if err != nil { + return nil, err + } peer3 := &nbpeer.Peer{ IP: peer3IP, + IP6: &peer3IP6, ID: peer3ID, Key: peer3Key, Name: "test-host3@netbird.io", UserID: userID, Meta: nbpeer.PeerSystemMeta{ - Hostname: "test-host3@netbird.io", - GoOS: "darwin", - Kernel: "Darwin", - Core: "13.4.1", - Platform: "arm64", - OS: "darwin", - WtVersion: "development", - UIVersion: "development", + Hostname: "test-host3@netbird.io", + GoOS: "darwin", + Kernel: "Darwin", + Core: "13.4.1", + Platform: "arm64", + OS: "darwin", + WtVersion: "development", + UIVersion: "development", + Ipv6Supported: true, }, - Status: &nbpeer.PeerStatus{}, + V6Setting: nbpeer.V6Enabled, + Status: &nbpeer.PeerStatus{}, } account.Peers[peer3.ID] = peer3 @@ -1133,14 +1337,15 @@ func initTestRouteAccount(t *testing.T, am *DefaultAccountManager) (*Account, er Name: "test-host4@netbird.io", UserID: userID, Meta: nbpeer.PeerSystemMeta{ - Hostname: "test-host4@netbird.io", - GoOS: "linux", - Kernel: "Linux", - Core: "21.04", - Platform: "x86_64", - OS: "Ubuntu", - WtVersion: "development", - UIVersion: "development", + Hostname: "test-host4@netbird.io", + GoOS: "linux", + Kernel: "Linux", + Core: "21.04", + Platform: "x86_64", + OS: "Ubuntu", + WtVersion: "development", + UIVersion: "development", + Ipv6Supported: false, }, Status: &nbpeer.PeerStatus{}, } @@ -1159,16 +1364,18 @@ func initTestRouteAccount(t *testing.T, am *DefaultAccountManager) (*Account, er Name: "test-host4@netbird.io", UserID: userID, Meta: nbpeer.PeerSystemMeta{ - Hostname: "test-host4@netbird.io", - GoOS: "linux", - Kernel: "Linux", - Core: "21.04", - Platform: "x86_64", - OS: "Ubuntu", - WtVersion: "development", - UIVersion: "development", + Hostname: "test-host4@netbird.io", + GoOS: "linux", + Kernel: "Linux", + Core: "21.04", + Platform: "x86_64", + OS: "Ubuntu", + WtVersion: "development", + UIVersion: "development", + Ipv6Supported: true, }, - Status: &nbpeer.PeerStatus{}, + Status: &nbpeer.PeerStatus{}, + V6Setting: nbpeer.V6Disabled, } account.Peers[peer5.ID] = peer5