From aef8b59f227f1d9f6ff28d115ec9e3f9e2bbce21 Mon Sep 17 00:00:00 2001 From: Tim Beatham Date: Sat, 25 Nov 2023 03:15:58 +0000 Subject: [PATCH 1/2] 32-fix-routing Flooding routes into other meshes a bit like BGP. --- cmd/wgmeshd/main.go | 6 +- pkg/automerge/automerge.go | 84 +++++++++++++++++-- pkg/automerge/factory.go | 2 +- pkg/automerge/types.go | 26 +++--- pkg/ipc/ipc.go | 1 - pkg/lib/rtnetlink.go | 21 ++--- pkg/mesh/config.go | 43 +++++----- pkg/mesh/manager.go | 39 ++++----- pkg/mesh/manager_test.go | 4 - pkg/mesh/route.go | 42 +++++++--- pkg/mesh/stub_types.go | 21 +++-- pkg/mesh/types.go | 26 +++++- pkg/query/query.go | 14 +++- pkg/robin/requester.go | 31 +++---- pkg/route/route.go | 40 ++++----- .../timestamp.go => timers/timers.go} | 10 ++- 16 files changed, 255 insertions(+), 155 deletions(-) rename pkg/{timestamp/timestamp.go => timers/timers.go} (59%) diff --git a/cmd/wgmeshd/main.go b/cmd/wgmeshd/main.go index e97f378..aeab44c 100644 --- a/cmd/wgmeshd/main.go +++ b/cmd/wgmeshd/main.go @@ -13,7 +13,7 @@ import ( "github.com/tim-beatham/wgmesh/pkg/mesh" "github.com/tim-beatham/wgmesh/pkg/robin" "github.com/tim-beatham/wgmesh/pkg/sync" - "github.com/tim-beatham/wgmesh/pkg/timestamp" + timer "github.com/tim-beatham/wgmesh/pkg/timers" "golang.zx2c4.com/wireguard/wgctrl" ) @@ -57,8 +57,9 @@ func main() { syncProvider.Server = ctrlServer syncRequester := sync.NewSyncRequester(ctrlServer) syncScheduler := sync.NewSyncScheduler(ctrlServer, syncRequester) - timestampScheduler := timestamp.NewTimestampScheduler(ctrlServer) + timestampScheduler := timer.NewTimestampScheduler(ctrlServer) pruneScheduler := mesh.NewPruner(ctrlServer.MeshManager, *conf) + routeScheduler := timer.NewRouteScheduler(ctrlServer) robinIpcParams := robin.RobinIpcParams{ CtrlServer: ctrlServer, @@ -78,6 +79,7 @@ func main() { go syncScheduler.Run() go timestampScheduler.Run() go pruneScheduler.Run() + go routeScheduler.Run() closeResources := func() { logging.Log.WriteInfof("Closing resources") diff --git a/pkg/automerge/automerge.go b/pkg/automerge/automerge.go index 9e2fc3c..7baced2 100644 --- a/pkg/automerge/automerge.go +++ b/pkg/automerge/automerge.go @@ -35,7 +35,7 @@ func (c *CrdtMeshManager) AddNode(node mesh.MeshNode) { panic("node must be of type *MeshNodeCrdt") } - crdt.Routes = make(map[string]interface{}) + crdt.Routes = make(map[string]Route) crdt.Services = make(map[string]string) crdt.Timestamp = time.Now().Unix() @@ -319,7 +319,7 @@ func (m *CrdtMeshManager) RemoveService(nodeId, key string) error { } // AddRoutes: adds routes to the specific nodeId -func (m *CrdtMeshManager) AddRoutes(nodeId string, routes ...string) error { +func (m *CrdtMeshManager) AddRoutes(nodeId string, routes ...mesh.Route) error { nodeVal, err := m.doc.Path("nodes").Map().Get(nodeId) logging.Log.WriteInfof("Adding route to %s", nodeId) @@ -338,7 +338,10 @@ func (m *CrdtMeshManager) AddRoutes(nodeId string, routes ...string) error { } for _, route := range routes { - err = routeMap.Map().Set(route, struct{}{}) + err = routeMap.Map().Set(route.GetDestination().String(), Route{ + Destination: route.GetDestination().String(), + HopCount: int64(route.GetHopCount()), + }) if err != nil { return err @@ -347,6 +350,66 @@ func (m *CrdtMeshManager) AddRoutes(nodeId string, routes ...string) error { return nil } +func (m *CrdtMeshManager) getRoutes(nodeId string) ([]Route, error) { + nodeVal, err := m.doc.Path("nodes").Map().Get(nodeId) + + if err != nil { + return nil, err + } + + if nodeVal.Kind() != automerge.KindMap { + return nil, fmt.Errorf("node does not exist") + } + + routeMap, err := nodeVal.Map().Get("routes") + + if err != nil { + return nil, err + } + + if routeMap.Kind() != automerge.KindMap { + return nil, fmt.Errorf("node %s is not a map", nodeId) + } + + routes, err := automerge.As[map[string]Route](routeMap) + return lib.MapValues(routes), err +} + +func (m *CrdtMeshManager) GetRoutes(targetNode string) (map[string]mesh.Route, error) { + node, err := m.GetNode(targetNode) + + if err != nil { + return nil, err + } + + routes := make(map[string]mesh.Route) + + for _, route := range node.GetRoutes() { + routes[route.GetDestination().String()] = route + } + + for _, node := range m.GetPeers() { + nodeRoutes, err := m.getRoutes(node) + + if err != nil { + return nil, err + } + + for _, route := range nodeRoutes { + otherRoute, ok := routes[route.GetDestination().String()] + + if !ok || route.GetHopCount() < otherRoute.GetHopCount() { + routes[route.GetDestination().String()] = &Route{ + Destination: route.GetDestination().String(), + HopCount: int64(route.GetHopCount()) + 1, + } + } + } + } + + return routes, nil +} + // DeleteRoutes deletes the specified routes func (m *CrdtMeshManager) RemoveRoutes(nodeId string, routes ...string) error { nodeVal, err := m.doc.Path("nodes").Map().Get(nodeId) @@ -459,8 +522,10 @@ func (m *MeshNodeCrdt) GetTimeStamp() int64 { return m.Timestamp } -func (m *MeshNodeCrdt) GetRoutes() []string { - return lib.MapKeys(m.Routes) +func (m *MeshNodeCrdt) GetRoutes() []mesh.Route { + return lib.Map(lib.MapValues(m.Routes), func(r Route) mesh.Route { + return &r + }) } func (m *MeshNodeCrdt) GetDescription() string { @@ -516,3 +581,12 @@ func (m *MeshCrdt) GetNodes() map[string]mesh.MeshNode { return nodes } + +func (r *Route) GetDestination() *net.IPNet { + _, ipnet, _ := net.ParseCIDR(r.Destination) + return ipnet +} + +func (r *Route) GetHopCount() int { + return int(r.HopCount) +} diff --git a/pkg/automerge/factory.go b/pkg/automerge/factory.go index dd1632c..ebccf50 100644 --- a/pkg/automerge/factory.go +++ b/pkg/automerge/factory.go @@ -41,7 +41,7 @@ func (f *MeshNodeFactory) Build(params *mesh.MeshNodeFactoryParams) mesh.MeshNod WgHost: fmt.Sprintf("%s/128", params.NodeIP.String()), // Always set the routes as empty. // Routes handled by external component - Routes: map[string]interface{}{}, + Routes: make(map[string]Route), Description: "", Alias: "", Type: string(f.Config.Role), diff --git a/pkg/automerge/types.go b/pkg/automerge/types.go index 64b51c2..ae35d25 100644 --- a/pkg/automerge/types.go +++ b/pkg/automerge/types.go @@ -1,17 +1,23 @@ package crdt +// Route: Represents a CRDT of the given route +type Route struct { + Destination string `automerge:"destination"` + HopCount int64 `automerge:"hopCount"` +} + // MeshNodeCrdt: Represents a CRDT for a mesh nodes type MeshNodeCrdt struct { - HostEndpoint string `automerge:"hostEndpoint"` - WgEndpoint string `automerge:"wgEndpoint"` - PublicKey string `automerge:"publicKey"` - WgHost string `automerge:"wgHost"` - Timestamp int64 `automerge:"timestamp"` - Routes map[string]interface{} `automerge:"routes"` - Alias string `automerge:"alias"` - Description string `automerge:"description"` - Services map[string]string `automerge:"services"` - Type string `automerge:"type"` + HostEndpoint string `automerge:"hostEndpoint"` + WgEndpoint string `automerge:"wgEndpoint"` + PublicKey string `automerge:"publicKey"` + WgHost string `automerge:"wgHost"` + Timestamp int64 `automerge:"timestamp"` + Routes map[string]Route `automerge:"routes"` + Alias string `automerge:"alias"` + Description string `automerge:"description"` + Services map[string]string `automerge:"services"` + Type string `automerge:"type"` } // MeshCrdt: Represents the mesh network as a whole diff --git a/pkg/ipc/ipc.go b/pkg/ipc/ipc.go index a06515a..b095a7e 100644 --- a/pkg/ipc/ipc.go +++ b/pkg/ipc/ipc.go @@ -62,7 +62,6 @@ type MeshIpc interface { JoinMesh(args JoinMeshArgs, reply *string) error LeaveMesh(meshId string, reply *string) error GetMesh(meshId string, reply *GetMeshReply) error - EnableInterface(meshId string, reply *string) error GetDOT(meshId string, reply *string) error Query(query QueryMesh, reply *string) error PutDescription(description string, reply *string) error diff --git a/pkg/lib/rtnetlink.go b/pkg/lib/rtnetlink.go index 1ac5b63..d26905a 100644 --- a/pkg/lib/rtnetlink.go +++ b/pkg/lib/rtnetlink.go @@ -201,7 +201,7 @@ func (c *RtNetlinkConfig) DeleteRoute(ifName string, route Route) error { }) if err != nil { - return fmt.Errorf("failed to delete route %w", err) + return fmt.Errorf("failed to delete route %s", dst.IP.String()) } return nil @@ -219,22 +219,15 @@ func (r1 Route) equal(r2 Route) bool { // DeleteRoutes deletes all routes not in exclude func (c *RtNetlinkConfig) DeleteRoutes(ifName string, family uint8, exclude ...Route) error { - routes := make([]rtnetlink.RouteMessage, 0) + routes, err := c.listRoutes(ifName, family) - if len(exclude) != 0 { - lRoutes, err := c.listRoutes(ifName, family, exclude[0].Gateway) - - if err != nil { - return err - } - - routes = lRoutes + if err != nil { + return err } ifRoutes := make([]Route, 0) for _, rtRoute := range routes { - logging.Log.WriteInfof("Routes: %s", rtRoute.Attributes.Dst.String()) maskSize := 128 if family == unix.AF_INET { @@ -262,7 +255,7 @@ func (c *RtNetlinkConfig) DeleteRoutes(ifName string, family uint8, exclude ...R toDelete := Filter(ifRoutes, shouldExclude) for _, route := range toDelete { - logging.Log.WriteInfof("Deleting route %s", route.Destination.String()) + logging.Log.WriteInfof("Deleting route: %s", route.Gateway.String()) err := c.DeleteRoute(ifName, route) if err != nil { @@ -274,7 +267,7 @@ func (c *RtNetlinkConfig) DeleteRoutes(ifName string, family uint8, exclude ...R } // listRoutes lists all routes on the interface -func (c *RtNetlinkConfig) listRoutes(ifName string, family uint8, gateway net.IP) ([]rtnetlink.RouteMessage, error) { +func (c *RtNetlinkConfig) listRoutes(ifName string, family uint8) ([]rtnetlink.RouteMessage, error) { iface, err := net.InterfaceByName(ifName) if err != nil { @@ -288,7 +281,7 @@ func (c *RtNetlinkConfig) listRoutes(ifName string, family uint8, gateway net.IP } filterFunc := func(r rtnetlink.RouteMessage) bool { - return r.Attributes.Gateway.Equal(gateway) && r.Attributes.OutIface == uint32(iface.Index) + return r.Attributes.Gateway != nil && r.Attributes.OutIface == uint32(iface.Index) } routes = Filter(routes, filterFunc) diff --git a/pkg/mesh/config.go b/pkg/mesh/config.go index bd628cd..9424363 100644 --- a/pkg/mesh/config.go +++ b/pkg/mesh/config.go @@ -43,8 +43,7 @@ func (m *WgMeshConfigApplyer) convertMeshNode(node MeshNode, device *wgtypes.Dev allowedips[0] = *node.GetWgHost() for _, route := range node.GetRoutes() { - _, ipnet, _ := net.ParseCIDR(route) - allowedips = append(allowedips, *ipnet) + allowedips = append(allowedips, *route.GetDestination()) } clients, ok := peerToClients[node.GetWgHost().String()] @@ -96,39 +95,28 @@ func (m *WgMeshConfigApplyer) updateWgConf(mesh MeshProvider) error { return err } - rtnl, err := lib.NewRtNetlinkConfig() - - if err != nil { - return err - } - peerToClients := make(map[string][]net.IPNet) + routes := make([]lib.Route, 1) + for _, n := range nodes { if NodeEquals(n, self) { continue } + for _, route := range n.GetRoutes() { + + routes = append(routes, lib.Route{ + Gateway: n.GetWgHost().IP, + Destination: *route.GetDestination(), + }) + } + if n.GetType() == conf.CLIENT_ROLE && len(peers) > 0 && self.GetType() == conf.CLIENT_ROLE { peer := lib.ConsistentHash(peers, n, func(mn MeshNode) int { return lib.HashString(mn.GetWgHost().String()) }) - dev, err := mesh.GetDevice() - - if err != nil { - return err - } - - rtnl.AddRoute(dev.Name, lib.Route{ - Gateway: peer.GetWgHost().IP, - Destination: *n.GetWgHost(), - }) - - if err != nil { - return err - } - clients, ok := peerToClients[peer.GetWgHost().String()] if !ok { @@ -153,7 +141,8 @@ func (m *WgMeshConfigApplyer) updateWgConf(mesh MeshProvider) error { } cfg := wgtypes.Config{ - Peers: peerConfigs, + Peers: peerConfigs, + ReplacePeers: true, } dev, err := mesh.GetDevice() @@ -162,6 +151,12 @@ func (m *WgMeshConfigApplyer) updateWgConf(mesh MeshProvider) error { return err } + err = m.routeInstaller.InstallRoutes(dev.Name, routes...) + + if err != nil { + return err + } + return m.meshManager.GetClient().ConfigureDevice(dev.Name, cfg) } diff --git a/pkg/mesh/manager.go b/pkg/mesh/manager.go index f36d6ea..e7aa346 100644 --- a/pkg/mesh/manager.go +++ b/pkg/mesh/manager.go @@ -18,7 +18,6 @@ type MeshManager interface { AddMesh(params *AddMeshParams) error HasChanges(meshid string) bool GetMesh(meshId string) MeshProvider - EnableInterface(meshId string) error GetPublicKey(meshId string) (*wgtypes.Key, error) AddSelf(params *AddSelfParams) error LeaveMesh(meshId string) error @@ -35,6 +34,7 @@ type MeshManager interface { Close() error GetMonitor() MeshMonitor GetNode(string, string) MeshNode + GetRouteManager() RouteManager } type MeshManagerImpl struct { @@ -54,6 +54,11 @@ type MeshManagerImpl struct { Monitor MeshMonitor } +// GetRouteManager implements MeshManager. +func (m *MeshManagerImpl) GetRouteManager() RouteManager { + return m.RouteManager +} + // RemoveService implements MeshManager. func (m *MeshManagerImpl) RemoveService(service string) error { for _, mesh := range m.Meshes { @@ -200,23 +205,6 @@ func (m *MeshManagerImpl) GetMesh(meshId string) MeshProvider { return theMesh } -// EnableInterface: Enables the given WireGuard interface. -func (s *MeshManagerImpl) EnableInterface(meshId string) error { - err := s.configApplyer.ApplyConfig() - - if err != nil { - return err - } - - err = s.RouteManager.InstallRoutes() - - if err != nil { - return err - } - - return nil -} - // GetPublicKey: Gets the public key of the WireGuard mesh func (s *MeshManagerImpl) GetPublicKey(meshId string) (*wgtypes.Key, error) { if s.conf.StubWg { @@ -307,22 +295,23 @@ func (s *MeshManagerImpl) LeaveMesh(meshId string) error { return fmt.Errorf("mesh %s does not exist", meshId) } - err := s.RouteManager.RemoveRoutes(meshId) - - if err != nil { - return err - } + var err error if !s.conf.StubWg { - device, e := mesh.GetDevice() + device, err := mesh.GetDevice() - if e != nil { + if err != nil { return err } err = s.interfaceManipulator.RemoveInterface(device.Name) + + if err != nil { + return err + } } + err = s.RouteManager.RemoveRoutes(meshId) delete(s.Meshes, meshId) return err } diff --git a/pkg/mesh/manager_test.go b/pkg/mesh/manager_test.go index f254484..b1e551c 100644 --- a/pkg/mesh/manager_test.go +++ b/pkg/mesh/manager_test.go @@ -64,7 +64,6 @@ func TestAddMeshAddsAMesh(t *testing.T) { manager.AddMesh(&AddMeshParams{ MeshId: meshId, - DevName: "wg0", WgPort: 6000, MeshBytes: make([]byte, 0), }) @@ -83,7 +82,6 @@ func TestAddMeshMeshAlreadyExistsReplacesIt(t *testing.T) { for i := 0; i < 2; i++ { err := manager.AddMesh(&AddMeshParams{ MeshId: meshId, - DevName: "wg0", WgPort: 6000, MeshBytes: make([]byte, 0), }) @@ -106,7 +104,6 @@ func TestAddSelfAddsSelfToTheMesh(t *testing.T) { err := manager.AddMesh(&AddMeshParams{ MeshId: meshId, - DevName: "wg0", WgPort: 6000, MeshBytes: make([]byte, 0), }) @@ -175,7 +172,6 @@ func TestLeaveMeshDeletesMesh(t *testing.T) { err := manager.AddMesh(&AddMeshParams{ MeshId: meshId, - DevName: "wg0", WgPort: 6000, MeshBytes: make([]byte, 0), }) diff --git a/pkg/mesh/route.go b/pkg/mesh/route.go index 9b9edd8..3c54613 100644 --- a/pkg/mesh/route.go +++ b/pkg/mesh/route.go @@ -7,19 +7,16 @@ import ( "github.com/tim-beatham/wgmesh/pkg/ip" "github.com/tim-beatham/wgmesh/pkg/lib" logging "github.com/tim-beatham/wgmesh/pkg/log" - "github.com/tim-beatham/wgmesh/pkg/route" "golang.org/x/sys/unix" ) type RouteManager interface { UpdateRoutes() error - InstallRoutes() error RemoveRoutes(meshId string) error } type RouteManagerImpl struct { - meshManager MeshManager - routeInstaller route.RouteInstaller + meshManager MeshManager } func (r *RouteManagerImpl) UpdateRoutes() error { @@ -27,6 +24,24 @@ func (r *RouteManagerImpl) UpdateRoutes() error { ulaBuilder := new(ip.ULABuilder) for _, mesh1 := range meshes { + self, err := r.meshManager.GetSelf(mesh1.GetMeshId()) + + if err != nil { + return err + } + + pubKey, err := self.GetPublicKey() + + if err != nil { + return err + } + + routes, err := mesh1.GetRoutes(pubKey.String()) + + if err != nil { + return err + } + for _, mesh2 := range meshes { if mesh1 == mesh2 { continue @@ -39,13 +54,10 @@ func (r *RouteManagerImpl) UpdateRoutes() error { return err } - self, err := r.meshManager.GetSelf(mesh1.GetMeshId()) - - if err != nil { - return err - } - - err = mesh1.AddRoutes(NodeID(self), ipNet.String()) + err = mesh2.AddRoutes(NodeID(self), append(lib.MapValues(routes), &RouteStub{ + Destination: ipNet, + HopCount: 0, + })...) if err != nil { return err @@ -128,7 +140,11 @@ func (m *RouteManagerImpl) installRoute(ifName string, meshid string, node MeshN return err } - routes := lib.Map(append(node.GetRoutes(), ipNet.String()), routeMapFunc) + theRoutes := lib.Map(node.GetRoutes(), func(r Route) string { + return r.GetDestination().String() + }) + + routes := lib.Map(append(theRoutes, ipNet.String()), routeMapFunc) return m.addRoute(ifName, ipNet.String(), routes...) } @@ -180,5 +196,5 @@ func (r *RouteManagerImpl) InstallRoutes() error { } func NewRouteManager(m MeshManager) RouteManager { - return &RouteManagerImpl{meshManager: m, routeInstaller: route.NewRouteInstaller()} + return &RouteManagerImpl{meshManager: m} } diff --git a/pkg/mesh/stub_types.go b/pkg/mesh/stub_types.go index 0a007c3..53132ff 100644 --- a/pkg/mesh/stub_types.go +++ b/pkg/mesh/stub_types.go @@ -16,7 +16,7 @@ type MeshNodeStub struct { wgEndpoint string wgHost *net.IPNet timeStamp int64 - routes []string + routes []Route identifier string description string } @@ -56,7 +56,7 @@ func (m *MeshNodeStub) GetTimeStamp() int64 { return m.timeStamp } -func (m *MeshNodeStub) GetRoutes() []string { +func (m *MeshNodeStub) GetRoutes() []Route { return m.routes } @@ -81,6 +81,10 @@ type MeshProviderStub struct { snapshot *MeshSnapshotStub } +func (*MeshProviderStub) GetRoutes(targetId string) (map[string]Route, error) { + return nil, nil +} + // GetNodeIds implements MeshProvider. func (*MeshProviderStub) GetPeers() []string { return make([]string, 0) @@ -159,7 +163,7 @@ func (s *MeshProviderStub) HasChanges() bool { return false } -func (s *MeshProviderStub) AddRoutes(nodeId string, route ...string) error { +func (s *MeshProviderStub) AddRoutes(nodeId string, route ...Route) error { return nil } @@ -193,7 +197,7 @@ func (s *StubNodeFactory) Build(params *MeshNodeFactoryParams) MeshNode { wgEndpoint: fmt.Sprintf("%s:%s", params.Endpoint, s.Config.GrpcPort), wgHost: wgHost, timeStamp: time.Now().Unix(), - routes: make([]string, 0), + routes: make([]Route, 0), identifier: "abc", description: "A Mesh Node Stub", } @@ -216,6 +220,11 @@ type MeshManagerStub struct { meshes map[string]MeshProvider } +// GetRouteManager implements MeshManager. +func (*MeshManagerStub) GetRouteManager() RouteManager { + panic("unimplemented") +} + // GetNode implements MeshManager. func (*MeshManagerStub) GetNode(string, string) MeshNode { panic("unimplemented") @@ -278,10 +287,6 @@ func (m *MeshManagerStub) GetMesh(meshId string) MeshProvider { snapshot: &MeshSnapshotStub{nodes: make(map[string]MeshNode)}} } -func (m *MeshManagerStub) EnableInterface(meshId string) error { - return nil -} - func (m *MeshManagerStub) GetPublicKey(meshId string) (*wgtypes.Key, error) { key, _ := wgtypes.GenerateKey() return &key, nil diff --git a/pkg/mesh/types.go b/pkg/mesh/types.go index 39b7e67..7e4350d 100644 --- a/pkg/mesh/types.go +++ b/pkg/mesh/types.go @@ -10,6 +10,26 @@ import ( "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) +type Route interface { + // GetDestination: returns the destination of the route + GetDestination() *net.IPNet + // GetHopCount: get the total hopcount of the prefix + GetHopCount() int +} + +type RouteStub struct { + Destination *net.IPNet + HopCount int +} + +func (r *RouteStub) GetDestination() *net.IPNet { + return r.Destination +} + +func (r *RouteStub) GetHopCount() int { + return r.HopCount +} + // MeshNode represents an implementation of a node in a mesh type MeshNode interface { // GetHostEndpoint: gets the gRPC endpoint of the node @@ -23,7 +43,7 @@ type MeshNode interface { // GetTimestamp: get the UNIX time stamp of the ndoe GetTimeStamp() int64 // GetRoutes: returns the routes that the nodes provides - GetRoutes() []string + GetRoutes() []Route // GetIdentifier: returns the identifier of the node GetIdentifier() string // GetDescription: returns the description for this node @@ -82,7 +102,7 @@ type MeshProvider interface { // UpdateTimeStamp: update the timestamp of the given node UpdateTimeStamp(nodeId string) error // AddRoutes: adds routes to the given node - AddRoutes(nodeId string, route ...string) error + AddRoutes(nodeId string, route ...Route) error // DeleteRoutes: deletes the routes from the node RemoveRoutes(nodeId string, route ...string) error // GetSyncer: returns the automerge syncer for sync @@ -104,6 +124,8 @@ type MeshProvider interface { Prune(pruneAmount int) error // GetPeers: get a list of contactable peers GetPeers() []string + // GetRoutes(): Get all unique routes. Where the route with the least hop count is chosen + GetRoutes(targetNode string) (map[string]Route, error) } // HostParameters contains the IDs of a node diff --git a/pkg/query/query.go b/pkg/query/query.go index 878aa16..4884a7d 100644 --- a/pkg/query/query.go +++ b/pkg/query/query.go @@ -24,6 +24,11 @@ type QueryError struct { msg string } +type QueryRoute struct { + Destination string `json:"destination"` + HopCount int `json:"hopCount"` +} + type QueryNode struct { HostEndpoint string `json:"hostEndpoint"` PublicKey string `json:"publicKey"` @@ -31,7 +36,7 @@ type QueryNode struct { WgHost string `json:"wgHost"` Timestamp int64 `json:"timestamp"` Description string `json:"description"` - Routes []string `json:"routes"` + Routes []QueryRoute `json:"routes"` Alias string `json:"alias"` Services map[string]string `json:"services"` Type conf.NodeType `json:"type"` @@ -78,7 +83,12 @@ func MeshNodeToQueryNode(node mesh.MeshNode) *QueryNode { queryNode.WgHost = node.GetWgHost().String() queryNode.Timestamp = node.GetTimeStamp() - queryNode.Routes = node.GetRoutes() + queryNode.Routes = lib.Map(node.GetRoutes(), func(r mesh.Route) QueryRoute { + return QueryRoute{ + Destination: r.GetDestination().String(), + HopCount: r.GetHopCount(), + } + }) queryNode.Description = node.GetDescription() queryNode.Alias = node.GetAlias() queryNode.Services = node.GetServices() diff --git a/pkg/robin/requester.go b/pkg/robin/requester.go index f349b82..51aa823 100644 --- a/pkg/robin/requester.go +++ b/pkg/robin/requester.go @@ -10,6 +10,7 @@ import ( "github.com/tim-beatham/wgmesh/pkg/ctrlserver" "github.com/tim-beatham/wgmesh/pkg/ipc" + "github.com/tim-beatham/wgmesh/pkg/lib" "github.com/tim-beatham/wgmesh/pkg/mesh" "github.com/tim-beatham/wgmesh/pkg/query" "github.com/tim-beatham/wgmesh/pkg/rpc" @@ -119,19 +120,19 @@ func (n *IpcHandler) LeaveMesh(meshId string, reply *string) error { } func (n *IpcHandler) GetMesh(meshId string, reply *ipc.GetMeshReply) error { - mesh := n.Server.GetMeshManager().GetMesh(meshId) + theMesh := n.Server.GetMeshManager().GetMesh(meshId) - if mesh == nil { + if theMesh == nil { return fmt.Errorf("mesh %s does not exist", meshId) } - meshSnapshot, err := mesh.GetMesh() + meshSnapshot, err := theMesh.GetMesh() if err != nil { return err } - if mesh == nil { + if theMesh == nil { return errors.New("mesh does not exist") } @@ -151,10 +152,12 @@ func (n *IpcHandler) GetMesh(meshId string, reply *ipc.GetMeshReply) error { PublicKey: pubKey.String(), WgHost: node.GetWgHost().String(), Timestamp: node.GetTimeStamp(), - Routes: node.GetRoutes(), - Description: node.GetDescription(), - Alias: node.GetAlias(), - Services: node.GetServices(), + Routes: lib.Map(node.GetRoutes(), func(r mesh.Route) string { + return r.GetDestination().String() + }), + Description: node.GetDescription(), + Alias: node.GetAlias(), + Services: node.GetServices(), } nodes[i] = node @@ -165,18 +168,6 @@ func (n *IpcHandler) GetMesh(meshId string, reply *ipc.GetMeshReply) error { return nil } -func (n *IpcHandler) EnableInterface(meshId string, reply *string) error { - err := n.Server.GetMeshManager().EnableInterface(meshId) - - if err != nil { - *reply = err.Error() - return err - } - - *reply = "up" - return nil -} - func (n *IpcHandler) GetDOT(meshId string, reply *string) error { g := mesh.NewMeshDotConverter(n.Server.GetMeshManager()) diff --git a/pkg/route/route.go b/pkg/route/route.go index 2c2d9c7..11de7d7 100644 --- a/pkg/route/route.go +++ b/pkg/route/route.go @@ -1,22 +1,32 @@ package route import ( - "net" - "os/exec" - - logging "github.com/tim-beatham/wgmesh/pkg/log" + "github.com/tim-beatham/wgmesh/pkg/lib" + "golang.org/x/sys/unix" ) type RouteInstaller interface { - InstallRoutes(devName string, routes ...*net.IPNet) error + InstallRoutes(devName string, routes ...lib.Route) error } type RouteInstallerImpl struct{} // InstallRoutes: installs a route into the routing table -func (r *RouteInstallerImpl) InstallRoutes(devName string, routes ...*net.IPNet) error { +func (r *RouteInstallerImpl) InstallRoutes(devName string, routes ...lib.Route) error { + rtnl, err := lib.NewRtNetlinkConfig() + + if err != nil { + return err + } + + err = rtnl.DeleteRoutes(devName, unix.AF_INET6, routes...) + + if err != nil { + return err + } + for _, route := range routes { - err := r.installRoute(devName, route) + err := rtnl.AddRoute(devName, route) if err != nil { return err @@ -26,22 +36,6 @@ func (r *RouteInstallerImpl) InstallRoutes(devName string, routes ...*net.IPNet) return nil } -// installRoute: installs a route into the linux table -func (r *RouteInstallerImpl) installRoute(devName string, route *net.IPNet) error { - // TODO: Find a library that automates this - cmd := exec.Command("/usr/bin/ip", "-6", "route", "add", route.String(), "dev", devName) - - logging.Log.WriteInfof("%s %s", route.String(), devName) - - if msg, err := cmd.CombinedOutput(); err != nil { - logging.Log.WriteErrorf(err.Error()) - logging.Log.WriteErrorf(string(msg)) - return err - } - - return nil -} - func NewRouteInstaller() RouteInstaller { return &RouteInstallerImpl{} } diff --git a/pkg/timestamp/timestamp.go b/pkg/timers/timers.go similarity index 59% rename from pkg/timestamp/timestamp.go rename to pkg/timers/timers.go index 4c41289..0d9531c 100644 --- a/pkg/timestamp/timestamp.go +++ b/pkg/timers/timers.go @@ -1,4 +1,4 @@ -package timestamp +package timer import ( "github.com/tim-beatham/wgmesh/pkg/ctrlserver" @@ -12,3 +12,11 @@ func NewTimestampScheduler(ctrlServer *ctrlserver.MeshCtrlServer) lib.Timer { return *lib.NewTimer(timerFunc, ctrlServer.Conf.KeepAliveTime) } + +func NewRouteScheduler(ctrlServer *ctrlserver.MeshCtrlServer) lib.Timer { + timerFunc := func() error { + return ctrlServer.MeshManager.GetRouteManager().UpdateRoutes() + } + + return *lib.NewTimer(timerFunc, 10) +} From a2517a1e722c4f80406cf6b90847428c507e4c90 Mon Sep 17 00:00:00 2001 From: Tim Beatham Date: Mon, 27 Nov 2023 15:56:30 +0000 Subject: [PATCH 2/2] 34-fix-routing - Added mesh-to-mesh routing of hop count > 1 - If there is a tie-breaker with respect to the hop-count use consistent hashing to determine the route to take based on the public key. --- pkg/lib/hashing.go | 4 +- pkg/mesh/config.go | 112 ++++++++++++++++++++++++++++++++++++--------- pkg/mesh/types.go | 5 ++ 3 files changed, 97 insertions(+), 24 deletions(-) diff --git a/pkg/lib/hashing.go b/pkg/lib/hashing.go index 8bb40ab..824362f 100644 --- a/pkg/lib/hashing.go +++ b/pkg/lib/hashing.go @@ -18,7 +18,7 @@ func HashString(value string) int { // ConsistentHash implementation. Traverse the values until we find a key // less than ours. -func ConsistentHash[V any](values []V, client V, keyFunc func(V) int) V { +func ConsistentHash[V any, K any](values []V, client K, bucketFunc func(V) int, keyFunc func(K) int) V { if len(values) == 0 { panic("values is empty") } @@ -26,7 +26,7 @@ func ConsistentHash[V any](values []V, client V, keyFunc func(V) int) V { vs := Map(values, func(v V) consistentHashRecord[V] { return consistentHashRecord[V]{ v, - keyFunc(v), + bucketFunc(v), } }) diff --git a/pkg/mesh/config.go b/pkg/mesh/config.go index 9424363..d1c4327 100644 --- a/pkg/mesh/config.go +++ b/pkg/mesh/config.go @@ -7,6 +7,7 @@ import ( "time" "github.com/tim-beatham/wgmesh/pkg/conf" + "github.com/tim-beatham/wgmesh/pkg/ip" "github.com/tim-beatham/wgmesh/pkg/lib" "github.com/tim-beatham/wgmesh/pkg/route" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" @@ -26,7 +27,19 @@ type WgMeshConfigApplyer struct { routeInstaller route.RouteInstaller } -func (m *WgMeshConfigApplyer) convertMeshNode(node MeshNode, device *wgtypes.Device, peerToClients map[string][]net.IPNet) (*wgtypes.PeerConfig, error) { +type routeNode struct { + gateway string + route Route +} + +func (r *routeNode) equals(route2 *routeNode) bool { + return r.gateway == route2.gateway && RouteEquals(r.route, route2.route) +} + +func (m *WgMeshConfigApplyer) convertMeshNode(node MeshNode, device *wgtypes.Device, + peerToClients map[string][]net.IPNet, + routes map[string][]routeNode) (*wgtypes.PeerConfig, error) { + endpoint, err := net.ResolveUDPAddr("udp", node.GetWgEndpoint()) if err != nil { @@ -42,16 +55,36 @@ func (m *WgMeshConfigApplyer) convertMeshNode(node MeshNode, device *wgtypes.Dev allowedips := make([]net.IPNet, 1) allowedips[0] = *node.GetWgHost() - for _, route := range node.GetRoutes() { - allowedips = append(allowedips, *route.GetDestination()) - } - clients, ok := peerToClients[node.GetWgHost().String()] if ok { allowedips = append(allowedips, clients...) } + for _, route := range node.GetRoutes() { + bestRoutes := routes[route.GetDestination().String()] + + if len(bestRoutes) == 1 { + allowedips = append(allowedips, *route.GetDestination()) + } else if len(bestRoutes) > 1 { + keyFunc := func(mn MeshNode) int { + pubKey, _ := mn.GetPublicKey() + return lib.HashString(pubKey.String()) + } + + bucketFunc := func(rn routeNode) int { + return lib.HashString(rn.gateway) + } + + // Else there is more than one candidate so consistently hash + pickedRoute := lib.ConsistentHash(bestRoutes, node, bucketFunc, keyFunc) + + if pickedRoute.gateway == pubKey.String() { + allowedips = append(allowedips, *route.GetDestination()) + } + } + } + keepAlive := time.Duration(m.config.KeepAliveWg) * time.Second existing := slices.IndexFunc(device.Peers, func(p wgtypes.Peer) bool { @@ -73,6 +106,37 @@ func (m *WgMeshConfigApplyer) convertMeshNode(node MeshNode, device *wgtypes.Dev return &peerConfig, nil } +// getRoutes: finds the routes with the least hop distance. If more than one route exists +// consistently hash to evenly spread the distribution of traffic +func (m *WgMeshConfigApplyer) getRoutes(mesh MeshSnapshot) map[string][]routeNode { + routes := make(map[string][]routeNode) + + for _, node := range mesh.GetNodes() { + for _, route := range node.GetRoutes() { + destination := route.GetDestination().String() + otherRoute, ok := routes[destination] + pubKey, _ := node.GetPublicKey() + + rn := routeNode{ + gateway: pubKey.String(), + route: route, + } + + if !ok { + otherRoute = make([]routeNode, 1) + otherRoute[0] = rn + routes[destination] = otherRoute + } else if otherRoute[0].route.GetHopCount() > route.GetHopCount() { + otherRoute[0] = rn + } else if otherRoute[0].route.GetHopCount() == route.GetHopCount() { + routes[destination] = append(otherRoute, rn) + } + } + } + + return routes +} + func (m *WgMeshConfigApplyer) updateWgConf(mesh MeshProvider) error { snap, err := mesh.GetMesh() @@ -96,26 +160,19 @@ func (m *WgMeshConfigApplyer) updateWgConf(mesh MeshProvider) error { } peerToClients := make(map[string][]net.IPNet) - - routes := make([]lib.Route, 1) + routes := m.getRoutes(snap) + installedRoutes := make([]lib.Route, 0) for _, n := range nodes { if NodeEquals(n, self) { continue } - for _, route := range n.GetRoutes() { - - routes = append(routes, lib.Route{ - Gateway: n.GetWgHost().IP, - Destination: *route.GetDestination(), - }) - } - if n.GetType() == conf.CLIENT_ROLE && len(peers) > 0 && self.GetType() == conf.CLIENT_ROLE { - peer := lib.ConsistentHash(peers, n, func(mn MeshNode) int { + hashFunc := func(mn MeshNode) int { return lib.HashString(mn.GetWgHost().String()) - }) + } + peer := lib.ConsistentHash(peers, n, hashFunc, hashFunc) clients, ok := peerToClients[peer.GetWgHost().String()] @@ -129,20 +186,31 @@ func (m *WgMeshConfigApplyer) updateWgConf(mesh MeshProvider) error { } dev, _ := mesh.GetDevice() - - peer, err := m.convertMeshNode(n, dev, peerToClients) + peer, err := m.convertMeshNode(n, dev, peerToClients, routes) if err != nil { return err } + for _, route := range peer.AllowedIPs { + ula := &ip.ULABuilder{} + ipNet, _ := ula.GetIPNet(mesh.GetMeshId()) + + if !ipNet.Contains(route.IP) { + + installedRoutes = append(installedRoutes, lib.Route{ + Gateway: n.GetWgHost().IP, + Destination: route, + }) + } + } + peerConfigs[count] = *peer count++ } cfg := wgtypes.Config{ - Peers: peerConfigs, - ReplacePeers: true, + Peers: peerConfigs, } dev, err := mesh.GetDevice() @@ -151,7 +219,7 @@ func (m *WgMeshConfigApplyer) updateWgConf(mesh MeshProvider) error { return err } - err = m.routeInstaller.InstallRoutes(dev.Name, routes...) + err = m.routeInstaller.InstallRoutes(dev.Name, installedRoutes...) if err != nil { return err diff --git a/pkg/mesh/types.go b/pkg/mesh/types.go index 7e4350d..f29b03e 100644 --- a/pkg/mesh/types.go +++ b/pkg/mesh/types.go @@ -64,6 +64,11 @@ func NodeEquals(node1, node2 MeshNode) bool { return key1.String() == key2.String() } +func RouteEquals(route1, route2 Route) bool { + return route1.GetDestination().String() == route2.GetDestination().String() && + route1.GetHopCount() == route2.GetHopCount() +} + func NodeID(node MeshNode) string { key, _ := node.GetPublicKey() return key.String()