diff --git a/README.md b/README.md index 703b28e..9f725e5 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,31 @@ Redundant routing is possible to create multiple exit points to the same mesh network. In which case consistent hashing is performed to split traffic between the exit points. -## Scalability +## Message Dissemination -The prototype has been tested to a scale of 3000 peers. +A variant of the gossip protocol is used for message dissemination. Each peer +in the network is ordered lexicographically ordered by their public key. +The node with the lexicographically lowest public key is used as the leader +of the mesh. Every `heartBeatInterval` disseminates a refresh message +throughout the entirety of the group in order to prune nodes that may +have prematurely died. + +If after `3 * heartBeatInterval` a node has not received a dissemination +message then the node prunes the leader and expects one from the next +lexicographically lowest public key. + +To 'merge' updates and reconcile any conflicts a Conflict Free Replicated +Data Type (CRDT) is implemented. Consisting of an add and remove set. +Where a node is in the group if it is in the add set and there is either +no entry in the remove set or the timestamp in the remove set has a lower +vector clock value. + +## Performance + +This prototype has been tested to a scale of 3000 peers in the network. +Furthermore, the fault-tolerance has been tested to a scale 3000 nodes +to the order of 20 seconds for the entire network and 12 seconds +for the 99 percentile. ## Installation diff --git a/pkg/ctrlserver/ctrlserver.go b/pkg/ctrlserver/ctrlserver.go index 32b7d28..3e5be34 100644 --- a/pkg/ctrlserver/ctrlserver.go +++ b/pkg/ctrlserver/ctrlserver.go @@ -40,7 +40,7 @@ func NewCtrlServer(params *NewCtrlServerParams) (*MeshCtrlServer, error) { ctrlServer.timers = make([]*lib.Timer, 0) - configApplyer := mesh.NewWgMeshConfigApplyer() + configApplier := mesh.NewWgMeshConfigApplier() var syncer sync.Syncer @@ -52,7 +52,7 @@ func NewCtrlServer(params *NewCtrlServerParams) (*MeshCtrlServer, error) { IdGenerator: idGenerator, IPAllocator: ipAllocator, InterfaceManipulator: interfaceManipulator, - ConfigApplier: configApplyer, + ConfigApplier: configApplier, OnDelete: func(mesh mesh.MeshProvider) { _, err := syncer.Sync(mesh) @@ -63,7 +63,7 @@ func NewCtrlServer(params *NewCtrlServerParams) (*MeshCtrlServer, error) { } ctrlServer.MeshManager = mesh.NewMeshManager(meshManagerParams) - configApplyer.SetMeshManager(ctrlServer.MeshManager) + configApplier.SetMeshManager(ctrlServer.MeshManager) ctrlServer.Conf = params.Conf connManagerParams := conn.NewConnectionManagerParams{ diff --git a/pkg/mesh/config.go b/pkg/mesh/config.go index 5a79020..a95e1e2 100644 --- a/pkg/mesh/config.go +++ b/pkg/mesh/config.go @@ -14,16 +14,16 @@ import ( "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) -// MeshConfigApplyer abstracts applying the mesh configuration -type MeshConfigApplyer interface { +// MeshConfigApplier abstracts applying the mesh configuration +type MeshConfigApplier interface { // ApplyConfig: apply the configurtation ApplyConfig() error // SetMeshManager: sets the associated manager SetMeshManager(manager MeshManager) } -// WgMeshConfigApplyer: applies WireGuard configuration -type WgMeshConfigApplyer struct { +// WgMeshConfigApplier: applies WireGuard configuration +type WgMeshConfigApplier struct { meshManager MeshManager routeInstaller route.RouteInstaller hashFunc func(MeshNode) int @@ -42,7 +42,7 @@ type convertMeshNodeParams struct { routes map[string][]routeNode } -func (m *WgMeshConfigApplyer) convertMeshNode(params convertMeshNodeParams) (*wgtypes.PeerConfig, error) { +func (m *WgMeshConfigApplier) convertMeshNode(params convertMeshNodeParams) (*wgtypes.PeerConfig, error) { pubKey, err := params.node.GetPublicKey() if err != nil { @@ -117,7 +117,7 @@ func (m *WgMeshConfigApplyer) convertMeshNode(params convertMeshNodeParams) (*wg // getRoutes: finds the routes with the least hop distance. If more than one route exists // consistently hash to evenly spread the distribution of traffic -func (m *WgMeshConfigApplyer) getRoutes(meshProvider MeshProvider) (map[string][]routeNode, error) { +func (m *WgMeshConfigApplier) getRoutes(meshProvider MeshProvider) (map[string][]routeNode, error) { mesh, err := meshProvider.GetMesh() if err != nil { @@ -193,13 +193,13 @@ func (m *WgMeshConfigApplyer) getRoutes(meshProvider MeshProvider) (map[string][ } // getCorrespondignPeer: gets the peer corresponding to the client -func (m *WgMeshConfigApplyer) getCorrespondingPeer(peers []MeshNode, client MeshNode) MeshNode { +func (m *WgMeshConfigApplier) getCorrespondingPeer(peers []MeshNode, client MeshNode) MeshNode { peer := lib.ConsistentHash(peers, client, m.hashFunc, m.hashFunc) return peer } // getPeerCfgsToRemove: remove peer configurations that are no longer in the mesh -func (m *WgMeshConfigApplyer) getPeerCfgsToRemove(dev *wgtypes.Device, newPeers []wgtypes.PeerConfig) []wgtypes.PeerConfig { +func (m *WgMeshConfigApplier) getPeerCfgsToRemove(dev *wgtypes.Device, newPeers []wgtypes.PeerConfig) []wgtypes.PeerConfig { peers := dev.Peers peers = lib.Filter(peers, func(p1 wgtypes.Peer) bool { return !lib.Contains(newPeers, func(p2 wgtypes.PeerConfig) bool { @@ -224,7 +224,7 @@ type GetConfigParams struct { } // getClientConfig: if the node is a client get their configuration -func (m *WgMeshConfigApplyer) getClientConfig(params *GetConfigParams) (*wgtypes.Config, error) { +func (m *WgMeshConfigApplier) getClientConfig(params *GetConfigParams) (*wgtypes.Config, error) { ula := &ip.ULABuilder{} meshNet, _ := ula.GetIPNet(params.mesh.GetMeshId()) @@ -302,7 +302,7 @@ func (m *WgMeshConfigApplyer) getClientConfig(params *GetConfigParams) (*wgtypes // getRoutesToInstall: work out if the given node is advertising routes that should be installed into the // RIB -func (m *WgMeshConfigApplyer) getRoutesToInstall(wgNode *wgtypes.PeerConfig, mesh MeshProvider, node MeshNode) []lib.Route { +func (m *WgMeshConfigApplier) getRoutesToInstall(wgNode *wgtypes.PeerConfig, mesh MeshProvider, node MeshNode) []lib.Route { routes := make([]lib.Route, 0) for _, route := range wgNode.AllowedIPs { @@ -322,7 +322,7 @@ func (m *WgMeshConfigApplyer) getRoutesToInstall(wgNode *wgtypes.PeerConfig, mes } // getPeerConfig: creates the WireGuard configuration for a peer -func (m *WgMeshConfigApplyer) getPeerConfig(params *GetConfigParams) (*wgtypes.Config, error) { +func (m *WgMeshConfigApplier) getPeerConfig(params *GetConfigParams) (*wgtypes.Config, error) { peerToClients := make(map[string][]net.IPNet) installedRoutes := make([]lib.Route, 0) peerConfigs := make([]wgtypes.PeerConfig, 0) @@ -395,7 +395,7 @@ func (m *WgMeshConfigApplyer) getPeerConfig(params *GetConfigParams) (*wgtypes.C } // updateWgConf: update the WireGuard configuration -func (m *WgMeshConfigApplyer) updateWgConf(mesh MeshProvider, routes map[string][]routeNode) error { +func (m *WgMeshConfigApplier) updateWgConf(mesh MeshProvider, routes map[string][]routeNode) error { snap, err := mesh.GetMesh() if err != nil { @@ -462,7 +462,7 @@ func (m *WgMeshConfigApplyer) updateWgConf(mesh MeshProvider, routes map[string] // getAllRoutes: works out all the routes to install out of all the routes in the // set of networks the node is a part of -func (m *WgMeshConfigApplyer) getAllRoutes() (map[string][]routeNode, error) { +func (m *WgMeshConfigApplier) getAllRoutes() (map[string][]routeNode, error) { allRoutes := make(map[string][]routeNode) for _, mesh := range m.meshManager.GetMeshes() { @@ -492,7 +492,7 @@ func (m *WgMeshConfigApplyer) getAllRoutes() (map[string][]routeNode, error) { } // ApplyConfig: apply the WireGuard configuration -func (m *WgMeshConfigApplyer) ApplyConfig() error { +func (m *WgMeshConfigApplier) ApplyConfig() error { allRoutes, err := m.getAllRoutes() if err != nil { @@ -510,12 +510,12 @@ func (m *WgMeshConfigApplyer) ApplyConfig() error { return nil } -func (m *WgMeshConfigApplyer) SetMeshManager(manager MeshManager) { +func (m *WgMeshConfigApplier) SetMeshManager(manager MeshManager) { m.meshManager = manager } -func NewWgMeshConfigApplyer() MeshConfigApplyer { - return &WgMeshConfigApplyer{ +func NewWgMeshConfigApplier() MeshConfigApplier { + return &WgMeshConfigApplier{ routeInstaller: route.NewRouteInstaller(), hashFunc: func(mn MeshNode) int { pubKey, _ := mn.GetPublicKey() diff --git a/pkg/mesh/manager.go b/pkg/mesh/manager.go index b377387..3d80f9c 100644 --- a/pkg/mesh/manager.go +++ b/pkg/mesh/manager.go @@ -49,7 +49,7 @@ type MeshManagerImpl struct { conf *conf.DaemonConfiguration meshProviderFactory MeshProviderFactory nodeFactory MeshNodeFactory - configApplyer MeshConfigApplyer + configApplier MeshConfigApplier idGenerator lib.IdGenerator ipAllocator ip.IPAllocator interfaceManipulator wg.WgInterfaceManipulator @@ -411,7 +411,7 @@ func (s *MeshManagerImpl) ApplyConfig() error { if s.conf.StubWg { return nil } - return s.configApplyer.ApplyConfig() + return s.configApplier.ApplyConfig() } func (s *MeshManagerImpl) SetDescription(meshId, description string) error { @@ -513,7 +513,7 @@ type NewMeshManagerParams struct { IdGenerator lib.IdGenerator IPAllocator ip.IPAllocator InterfaceManipulator wg.WgInterfaceManipulator - ConfigApplyer MeshConfigApplyer + ConfigApplier MeshConfigApplier RouteManager RouteManager CommandRunner cmd.CmdRunner OnDelete func(MeshProvider) @@ -535,7 +535,7 @@ func NewMeshManager(params *NewMeshManagerParams) MeshManager { conf: ¶ms.Conf, } - m.configApplyer = params.ConfigApplyer + m.configApplier = params.ConfigApplier m.RouteManager = params.RouteManager if m.RouteManager == nil { diff --git a/pkg/mesh/manager_test.go b/pkg/mesh/manager_test.go index c3b23b8..81e953b 100644 --- a/pkg/mesh/manager_test.go +++ b/pkg/mesh/manager_test.go @@ -47,7 +47,7 @@ func getMeshManager() MeshManager { IdGenerator: &lib.UUIDGenerator{}, IPAllocator: &ip.ULABuilder{}, InterfaceManipulator: &wg.WgInterfaceManipulatorStub{}, - ConfigApplier: &MeshConfigApplyerStub{}, + ConfigApplier: &MeshConfigApplierStub{}, RouteManager: &RouteManagerStub{}, }) diff --git a/pkg/mesh/stub_types.go b/pkg/mesh/stub_types.go index 6c40bb1..a5fbfdc 100644 --- a/pkg/mesh/stub_types.go +++ b/pkg/mesh/stub_types.go @@ -253,17 +253,17 @@ func (s *StubNodeFactory) Build(params *MeshNodeFactoryParams) MeshNode { } } -type MeshConfigApplyerStub struct{} +type MeshConfigApplierStub struct{} -func (a *MeshConfigApplyerStub) ApplyConfig() error { +func (a *MeshConfigApplierStub) ApplyConfig() error { return nil } -func (a *MeshConfigApplyerStub) RemovePeers(meshId string) error { +func (a *MeshConfigApplierStub) RemovePeers(meshId string) error { return nil } -func (a *MeshConfigApplyerStub) SetMeshManager(manager MeshManager) { +func (a *MeshConfigApplierStub) SetMeshManager(manager MeshManager) { } type MeshManagerStub struct {