Fixed build errors

README.md

@@ -32,9 +32,31 @@ Redundant routing is possible to create multiple exit points to the same
 mesh network. In which case consistent hashing is performed to split traffic
 between the exit points.
 
-## Scalability
+## Message Dissemination
 
-The prototype has been tested to a scale of 3000 peers.
+A variant of the gossip protocol is used for message dissemination. Each peer
+in the network is ordered lexicographically ordered by their public key.
+The node with the lexicographically lowest public key is used as the leader
+of the mesh. Every `heartBeatInterval` disseminates a refresh message
+throughout the entirety of the group in order to prune nodes that may
+have prematurely died.
+
+If after `3 * heartBeatInterval` a node has not received a dissemination
+message then the node prunes the leader and expects one from the next
+lexicographically lowest public key.
+
+To 'merge' updates and reconcile any conflicts a Conflict Free Replicated
+Data Type (CRDT) is implemented. Consisting of an add and remove set.
+Where a node is in the group if it is in the add set and there is either
+no entry in the remove set or the timestamp in the remove set has a lower
+vector clock value.
+
+## Performance
+
+This prototype has been tested to a scale of 3000 peers in the network.
+Furthermore, the fault-tolerance has been tested to a scale 3000 nodes
+to the order of 20 seconds for the entire network and 12 seconds
+for the 99 percentile.
 
 ## Installation
 

pkg/ctrlserver/ctrlserver.go

@@ -40,7 +40,7 @@ func NewCtrlServer(params *NewCtrlServerParams) (*MeshCtrlServer, error) {
 
 	ctrlServer.timers = make([]*lib.Timer, 0)
 
-	configApplyer := mesh.NewWgMeshConfigApplyer()
+	configApplier := mesh.NewWgMeshConfigApplier()
 
 	var syncer sync.Syncer
 
@@ -52,7 +52,7 @@ func NewCtrlServer(params *NewCtrlServerParams) (*MeshCtrlServer, error) {
 		IdGenerator:          idGenerator,
 		IPAllocator:          ipAllocator,
 		InterfaceManipulator: interfaceManipulator,
-		ConfigApplier:        configApplyer,
+		ConfigApplier:        configApplier,
 		OnDelete: func(mesh mesh.MeshProvider) {
 			_, err := syncer.Sync(mesh)
 
@@ -63,7 +63,7 @@ func NewCtrlServer(params *NewCtrlServerParams) (*MeshCtrlServer, error) {
 	}
 
 	ctrlServer.MeshManager = mesh.NewMeshManager(meshManagerParams)
-	configApplyer.SetMeshManager(ctrlServer.MeshManager)
+	configApplier.SetMeshManager(ctrlServer.MeshManager)
 
 	ctrlServer.Conf = params.Conf
 	connManagerParams := conn.NewConnectionManagerParams{

pkg/mesh/config.go

@@ -14,16 +14,16 @@ import (
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
 
-// MeshConfigApplyer abstracts applying the mesh configuration
-type MeshConfigApplyer interface {
+// MeshConfigApplier abstracts applying the mesh configuration
+type MeshConfigApplier interface {
 	// ApplyConfig: apply the configurtation
 	ApplyConfig() error
 	// SetMeshManager: sets the associated manager
 	SetMeshManager(manager MeshManager)
 }
 
-// WgMeshConfigApplyer: applies WireGuard configuration
-type WgMeshConfigApplyer struct {
+// WgMeshConfigApplier: applies WireGuard configuration
+type WgMeshConfigApplier struct {
 	meshManager    MeshManager
 	routeInstaller route.RouteInstaller
 	hashFunc       func(MeshNode) int
@@ -42,7 +42,7 @@ type convertMeshNodeParams struct {
 	routes        map[string][]routeNode
 }
 
-func (m *WgMeshConfigApplyer) convertMeshNode(params convertMeshNodeParams) (*wgtypes.PeerConfig, error) {
+func (m *WgMeshConfigApplier) convertMeshNode(params convertMeshNodeParams) (*wgtypes.PeerConfig, error) {
 	pubKey, err := params.node.GetPublicKey()
 
 	if err != nil {
@@ -117,7 +117,7 @@ func (m *WgMeshConfigApplyer) convertMeshNode(params convertMeshNodeParams) (*wg
 
 // getRoutes: finds the routes with the least hop distance. If more than one route exists
 // consistently hash to evenly spread the distribution of traffic
-func (m *WgMeshConfigApplyer) getRoutes(meshProvider MeshProvider) (map[string][]routeNode, error) {
+func (m *WgMeshConfigApplier) getRoutes(meshProvider MeshProvider) (map[string][]routeNode, error) {
 	mesh, err := meshProvider.GetMesh()
 
 	if err != nil {
@@ -193,13 +193,13 @@ func (m *WgMeshConfigApplyer) getRoutes(meshProvider MeshProvider) (map[string][
 }
 
 // getCorrespondignPeer: gets the peer corresponding to the client
-func (m *WgMeshConfigApplyer) getCorrespondingPeer(peers []MeshNode, client MeshNode) MeshNode {
+func (m *WgMeshConfigApplier) getCorrespondingPeer(peers []MeshNode, client MeshNode) MeshNode {
 	peer := lib.ConsistentHash(peers, client, m.hashFunc, m.hashFunc)
 	return peer
 }
 
 // getPeerCfgsToRemove: remove peer configurations that are no longer in the mesh
-func (m *WgMeshConfigApplyer) getPeerCfgsToRemove(dev *wgtypes.Device, newPeers []wgtypes.PeerConfig) []wgtypes.PeerConfig {
+func (m *WgMeshConfigApplier) getPeerCfgsToRemove(dev *wgtypes.Device, newPeers []wgtypes.PeerConfig) []wgtypes.PeerConfig {
 	peers := dev.Peers
 	peers = lib.Filter(peers, func(p1 wgtypes.Peer) bool {
 		return !lib.Contains(newPeers, func(p2 wgtypes.PeerConfig) bool {
@@ -224,7 +224,7 @@ type GetConfigParams struct {
 }
 
 // getClientConfig: if the node is a client get their configuration
-func (m *WgMeshConfigApplyer) getClientConfig(params *GetConfigParams) (*wgtypes.Config, error) {
+func (m *WgMeshConfigApplier) getClientConfig(params *GetConfigParams) (*wgtypes.Config, error) {
 	ula := &ip.ULABuilder{}
 	meshNet, _ := ula.GetIPNet(params.mesh.GetMeshId())
 
@@ -302,7 +302,7 @@ func (m *WgMeshConfigApplyer) getClientConfig(params *GetConfigParams) (*wgtypes
 
 // getRoutesToInstall: work out if the given node is advertising routes that should be installed into the
 // RIB
-func (m *WgMeshConfigApplyer) getRoutesToInstall(wgNode *wgtypes.PeerConfig, mesh MeshProvider, node MeshNode) []lib.Route {
+func (m *WgMeshConfigApplier) getRoutesToInstall(wgNode *wgtypes.PeerConfig, mesh MeshProvider, node MeshNode) []lib.Route {
 	routes := make([]lib.Route, 0)
 
 	for _, route := range wgNode.AllowedIPs {
@@ -322,7 +322,7 @@ func (m *WgMeshConfigApplyer) getRoutesToInstall(wgNode *wgtypes.PeerConfig, mes
 }
 
 // getPeerConfig: creates the WireGuard configuration for a peer
-func (m *WgMeshConfigApplyer) getPeerConfig(params *GetConfigParams) (*wgtypes.Config, error) {
+func (m *WgMeshConfigApplier) getPeerConfig(params *GetConfigParams) (*wgtypes.Config, error) {
 	peerToClients := make(map[string][]net.IPNet)
 	installedRoutes := make([]lib.Route, 0)
 	peerConfigs := make([]wgtypes.PeerConfig, 0)
@@ -395,7 +395,7 @@ func (m *WgMeshConfigApplyer) getPeerConfig(params *GetConfigParams) (*wgtypes.C
 }
 
 // updateWgConf: update the WireGuard configuration
-func (m *WgMeshConfigApplyer) updateWgConf(mesh MeshProvider, routes map[string][]routeNode) error {
+func (m *WgMeshConfigApplier) updateWgConf(mesh MeshProvider, routes map[string][]routeNode) error {
 	snap, err := mesh.GetMesh()
 
 	if err != nil {
@@ -462,7 +462,7 @@ func (m *WgMeshConfigApplyer) updateWgConf(mesh MeshProvider, routes map[string]
 
 // getAllRoutes: works out all the routes to install out of all the routes in the
 // set of networks the node is a part of
-func (m *WgMeshConfigApplyer) getAllRoutes() (map[string][]routeNode, error) {
+func (m *WgMeshConfigApplier) getAllRoutes() (map[string][]routeNode, error) {
 	allRoutes := make(map[string][]routeNode)
 
 	for _, mesh := range m.meshManager.GetMeshes() {
@@ -492,7 +492,7 @@ func (m *WgMeshConfigApplyer) getAllRoutes() (map[string][]routeNode, error) {
 }
 
 // ApplyConfig: apply the WireGuard configuration
-func (m *WgMeshConfigApplyer) ApplyConfig() error {
+func (m *WgMeshConfigApplier) ApplyConfig() error {
 	allRoutes, err := m.getAllRoutes()
 
 	if err != nil {
@@ -510,12 +510,12 @@ func (m *WgMeshConfigApplyer) ApplyConfig() error {
 	return nil
 }
 
-func (m *WgMeshConfigApplyer) SetMeshManager(manager MeshManager) {
+func (m *WgMeshConfigApplier) SetMeshManager(manager MeshManager) {
 	m.meshManager = manager
 }
 
-func NewWgMeshConfigApplyer() MeshConfigApplyer {
-	return &WgMeshConfigApplyer{
+func NewWgMeshConfigApplier() MeshConfigApplier {
+	return &WgMeshConfigApplier{
 		routeInstaller: route.NewRouteInstaller(),
 		hashFunc: func(mn MeshNode) int {
 			pubKey, _ := mn.GetPublicKey()

pkg/mesh/manager.go

@@ -49,7 +49,7 @@ type MeshManagerImpl struct {
 	conf                 *conf.DaemonConfiguration
 	meshProviderFactory  MeshProviderFactory
 	nodeFactory          MeshNodeFactory
-	configApplyer        MeshConfigApplyer
+	configApplier        MeshConfigApplier
 	idGenerator          lib.IdGenerator
 	ipAllocator          ip.IPAllocator
 	interfaceManipulator wg.WgInterfaceManipulator
@@ -411,7 +411,7 @@ func (s *MeshManagerImpl) ApplyConfig() error {
 	if s.conf.StubWg {
 		return nil
 	}
-	return s.configApplyer.ApplyConfig()
+	return s.configApplier.ApplyConfig()
 }
 
 func (s *MeshManagerImpl) SetDescription(meshId, description string) error {
@@ -513,7 +513,7 @@ type NewMeshManagerParams struct {
 	IdGenerator          lib.IdGenerator
 	IPAllocator          ip.IPAllocator
 	InterfaceManipulator wg.WgInterfaceManipulator
-	ConfigApplyer        MeshConfigApplyer
+	ConfigApplier        MeshConfigApplier
 	RouteManager         RouteManager
 	CommandRunner        cmd.CmdRunner
 	OnDelete             func(MeshProvider)
@@ -535,7 +535,7 @@ func NewMeshManager(params *NewMeshManagerParams) MeshManager {
 		conf:                &params.Conf,
 	}
 
-	m.configApplyer = params.ConfigApplyer
+	m.configApplier = params.ConfigApplier
 	m.RouteManager = params.RouteManager
 
 	if m.RouteManager == nil {

pkg/mesh/manager_test.go

@@ -47,7 +47,7 @@ func getMeshManager() MeshManager {
 		IdGenerator:          &lib.UUIDGenerator{},
 		IPAllocator:          &ip.ULABuilder{},
 		InterfaceManipulator: &wg.WgInterfaceManipulatorStub{},
-		ConfigApplier:        &MeshConfigApplyerStub{},
+		ConfigApplier:        &MeshConfigApplierStub{},
 		RouteManager:         &RouteManagerStub{},
 	})
 

pkg/mesh/stub_types.go

@@ -253,17 +253,17 @@ func (s *StubNodeFactory) Build(params *MeshNodeFactoryParams) MeshNode {
 	}
 }
 
-type MeshConfigApplyerStub struct{}
+type MeshConfigApplierStub struct{}
 
-func (a *MeshConfigApplyerStub) ApplyConfig() error {
+func (a *MeshConfigApplierStub) ApplyConfig() error {
 	return nil
 }
 
-func (a *MeshConfigApplyerStub) RemovePeers(meshId string) error {
+func (a *MeshConfigApplierStub) RemovePeers(meshId string) error {
 	return nil
 }
 
-func (a *MeshConfigApplyerStub) SetMeshManager(manager MeshManager) {
+func (a *MeshConfigApplierStub) SetMeshManager(manager MeshManager) {
 }
 
 type MeshManagerStub struct {