47-default-routing

Implemented default routing and improved size of gossip. Using 64 bit
hash funciton to identify vector.

cmd/wgmeshd/main.go

@@ -45,21 +45,25 @@ func main() {
 	var robinRpc robin.WgRpc
 	var robinIpc robin.IpcHandler
 	var syncProvider sync.SyncServiceImpl
+	var syncRequester sync.SyncRequester
+	var syncer sync.Syncer
 
 	ctrlServerParams := ctrlserver.NewCtrlServerParams{
 		Conf:         conf,
 		CtrlProvider: &robinRpc,
 		SyncProvider: &syncProvider,
 		Client:       client,
+		OnDelete: func(mp mesh.MeshProvider) {
+			syncer.SyncMeshes()
+		},
 	}
 
 	ctrlServer, err := ctrlserver.NewCtrlServer(&ctrlServerParams)
 	syncProvider.Server = ctrlServer
-	syncRequester := sync.NewSyncRequester(ctrlServer)
+	syncRequester = sync.NewSyncRequester(ctrlServer)
-	syncScheduler := sync.NewSyncScheduler(ctrlServer, syncRequester)
+	syncer = sync.NewSyncer(ctrlServer.MeshManager, conf, syncRequester)
-	timestampScheduler := timer.NewTimestampScheduler(ctrlServer)
+	syncScheduler := sync.NewSyncScheduler(ctrlServer, syncRequester, syncer)
-	pruneScheduler := mesh.NewPruner(ctrlServer.MeshManager, *conf)
+	keepAlive := timer.NewTimestampScheduler(ctrlServer)
-	routeScheduler := timer.NewRouteScheduler(ctrlServer)
 
 	robinIpcParams := robin.RobinIpcParams{
 		CtrlServer: ctrlServer,
@@ -77,14 +81,12 @@ func main() {
 
 	go ipc.RunIpcHandler(&robinIpc)
 	go syncScheduler.Run()
-	go timestampScheduler.Run()
+	go keepAlive.Run()
-	go pruneScheduler.Run()
-	go routeScheduler.Run()
 
 	closeResources := func() {
 		logging.Log.WriteInfof("Closing resources")
 		syncScheduler.Stop()
-		timestampScheduler.Stop()
+		keepAlive.Stop()
 		ctrlServer.Close()
 		client.Close()
 	}

pkg/automerge/automerge.go

@@ -1,9 +1,10 @@
-package crdt
+package automerge
 
 import (
 	"errors"
 	"fmt"
 	"net"
+	"slices"
 	"strings"
 	"time"
 
@@ -289,7 +290,8 @@ func (m *CrdtMeshManager) AddService(nodeId, key, value string) error {
 		return fmt.Errorf("AddService: services property does not exist in node")
 	}
 
-	return service.Map().Set(key, value)
+	err = service.Map().Set(key, value)
+	return err
 }
 
 func (m *CrdtMeshManager) RemoveService(nodeId, key string) error {
@@ -338,6 +340,28 @@ func (m *CrdtMeshManager) AddRoutes(nodeId string, routes ...mesh.Route) error {
 	}
 
 	for _, route := range routes {
+		prevRoute, err := routeMap.Map().Get(route.GetDestination().String())
+
+		if prevRoute.Kind() == automerge.KindVoid && err != nil {
+			path, err := prevRoute.Map().Get("path")
+
+			if err != nil {
+				return err
+			}
+
+			if path.Kind() != automerge.KindList {
+				return fmt.Errorf("path is not a list")
+			}
+
+			pathStr, err := automerge.As[[]string](path)
+
+			if err != nil {
+				return err
+			}
+
+			slices.Equal(route.GetPath(), pathStr)
+		}
+
 		err = routeMap.Map().Set(route.GetDestination().String(), Route{
 			Destination: route.GetDestination().String(),
 			Path:        route.GetPath(),
@@ -385,10 +409,12 @@ func (m *CrdtMeshManager) GetRoutes(targetNode string) (map[string]mesh.Route, e
 
 	routes := make(map[string]mesh.Route)
 
+	// Add routes that the node directly has
 	for _, route := range node.GetRoutes() {
 		routes[route.GetDestination().String()] = route
 	}
 
+	// Work out the other routes in the mesh
 	for _, node := range m.GetPeers() {
 		nodeRoutes, err := m.getRoutes(node)
 
@@ -399,6 +425,12 @@ func (m *CrdtMeshManager) GetRoutes(targetNode string) (map[string]mesh.Route, e
 		for _, route := range nodeRoutes {
 			otherRoute, ok := routes[route.GetDestination().String()]
 
+			hopCount := route.GetHopCount()
+
+			if node != targetNode {
+				hopCount += 1
+			}
+
 			if !ok || route.GetHopCount()+1 < otherRoute.GetHopCount() {
 				routes[route.GetDestination().String()] = &Route{
 					Destination: route.GetDestination().String(),
@@ -411,6 +443,11 @@ func (m *CrdtMeshManager) GetRoutes(targetNode string) (map[string]mesh.Route, e
 	return routes, nil
 }
 
+func (m *CrdtMeshManager) RemoveNode(nodeId string) error {
+	err := m.doc.Path("nodes").Map().Delete(nodeId)
+	return err
+}
+
 // DeleteRoutes deletes the specified routes
 func (m *CrdtMeshManager) RemoveRoutes(nodeId string, routes ...string) error {
 	nodeVal, err := m.doc.Path("nodes").Map().Get(nodeId)
@@ -440,54 +477,54 @@ func (m *CrdtMeshManager) GetSyncer() mesh.MeshSyncer {
 	return NewAutomergeSync(m)
 }
 
-func (m *CrdtMeshManager) Prune(pruneTime int) error {
+func (m *CrdtMeshManager) Prune() error {
-	nodes, err := m.doc.Path("nodes").Get()
+	// nodes, err := m.doc.Path("nodes").Get()
 
-	if err != nil {
+	// if err != nil {
-		return err
+	// 	return err
-	}
+	// }
 
-	if nodes.Kind() != automerge.KindMap {
+	// if nodes.Kind() != automerge.KindMap {
-		return errors.New("node must be a map")
+	// 	return errors.New("node must be a map")
-	}
+	// }
 
-	values, err := nodes.Map().Values()
+	// values, err := nodes.Map().Values()
 
-	if err != nil {
+	// if err != nil {
-		return err
+	// 	return err
-	}
+	// }
 
-	deletionNodes := make([]string, 0)
+	// deletionNodes := make([]string, 0)
 
-	for nodeId, node := range values {
+	// for nodeId, node := range values {
-		if node.Kind() != automerge.KindMap {
+	// 	if node.Kind() != automerge.KindMap {
-			return errors.New("node must be a map")
+	// 		return errors.New("node must be a map")
-		}
+	// 	}
 
-		nodeMap := node.Map()
+	// 	nodeMap := node.Map()
 
-		timeStamp, err := nodeMap.Get("timestamp")
+	// 	timeStamp, err := nodeMap.Get("timestamp")
 
-		if err != nil {
+	// 	if err != nil {
-			return err
+	// 		return err
-		}
+	// 	}
 
-		if timeStamp.Kind() != automerge.KindInt64 {
+	// 	if timeStamp.Kind() != automerge.KindInt64 {
-			return errors.New("timestamp is not int64")
+	// 		return errors.New("timestamp is not int64")
-		}
+	// 	}
 
-		timeValue := timeStamp.Int64()
+	// 	timeValue := timeStamp.Int64()
-		nowValue := time.Now().Unix()
+	// 	nowValue := time.Now().Unix()
 
-		if nowValue-timeValue >= int64(pruneTime) {
+	// 	if nowValue-timeValue >= int64(pruneTime) {
-			deletionNodes = append(deletionNodes, nodeId)
+	// 		deletionNodes = append(deletionNodes, nodeId)
-		}
+	// 	}
-	}
+	// }
 
-	for _, node := range deletionNodes {
+	// for _, node := range deletionNodes {
-		logging.Log.WriteInfof("Pruning %s", node)
+	// 	logging.Log.WriteInfof("Pruning %s", node)
-		nodes.Map().Delete(node)
+	// 	nodes.Map().Delete(node)
-	}
+	// }
 
 	return nil
 }
@@ -512,7 +549,6 @@ func (m *MeshNodeCrdt) GetWgHost() *net.IPNet {
 	_, ipnet, err := net.ParseCIDR(m.WgHost)
 
 	if err != nil {
-		logging.Log.WriteErrorf("Cannot parse WgHost %s", err.Error())
 		return nil
 	}
 
@@ -540,7 +576,6 @@ func (m *MeshNodeCrdt) GetIdentifier() string {
 	ipv6 := m.WgHost[:len(m.WgHost)-4]
 
 	constituents := strings.Split(ipv6, ":")
-	logging.Log.WriteInfof(ipv6)
 	constituents = constituents[4:]
 	return strings.Join(constituents, ":")
 }

pkg/automerge/automerge_sync.go

@@ -1,4 +1,4 @@
-package crdt
+package automerge
 
 import (
 	"github.com/automerge/automerge-go"

pkg/automerge/automerge_test.go

@@ -1,4 +1,4 @@
-package crdt
+package automerge
 
 import (
 	"slices"

pkg/automerge/factory.go

@@ -1,4 +1,4 @@
-package crdt
+package automerge
 
 import (
 	"fmt"

pkg/automerge/types.go

@@ -1,4 +1,4 @@
-package crdt
+package automerge
 
 // Route: Represents a CRDT of the given route
 type Route struct {

pkg/conf/conf.go

@@ -47,6 +47,8 @@ type WgMeshConfiguration struct {
 	IPDiscovery IPDiscovery `yaml:"ipDiscovery"`
 	// AdvertiseRoutes advertises other meshes if the node is in multiple meshes
 	AdvertiseRoutes bool `yaml:"advertiseRoutes"`
+	// AdvertiseDefaultRoute advertises a default route out of the mesh.
+	AdvertiseDefaultRoute bool `yaml:"advertiseDefaults"`
 	// Endpoint is the IP in which this computer is publicly reachable.
 	// usecase is when the node has multiple IP addresses
 	Endpoint string `yaml:"publicEndpoint"`

pkg/crdt/datastore.go

@@ -0,0 +1,501 @@
+package crdt
+
+import (
+	"bytes"
+	"encoding/gob"
+	"fmt"
+	"net"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/tim-beatham/wgmesh/pkg/conf"
+	"github.com/tim-beatham/wgmesh/pkg/lib"
+	logging "github.com/tim-beatham/wgmesh/pkg/log"
+	"github.com/tim-beatham/wgmesh/pkg/mesh"
+	"golang.zx2c4.com/wireguard/wgctrl"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+)
+
+type Route struct {
+	Destination string
+	Path        []string
+}
+
+// GetDestination implements mesh.Route.
+func (r *Route) GetDestination() *net.IPNet {
+	_, ipnet, _ := net.ParseCIDR(r.Destination)
+	return ipnet
+}
+
+// GetHopCount implements mesh.Route.
+func (r *Route) GetHopCount() int {
+	return len(r.Path)
+}
+
+// GetPath implements mesh.Route.
+func (r *Route) GetPath() []string {
+	return r.Path
+}
+
+type MeshNode struct {
+	HostEndpoint string
+	WgEndpoint   string
+	PublicKey    string
+	WgHost       string
+	Timestamp    int64
+	Routes       map[string]Route
+	Alias        string
+	Description  string
+	Services     map[string]string
+	Type         string
+	Tombstone    bool
+}
+
+// Mark: marks the node is unreachable. This is not broadcast on
+// syncrhonisation
+func (m *TwoPhaseStoreMeshManager) Mark(nodeId string) {
+	m.store.Mark(nodeId)
+}
+
+// GetHostEndpoint: gets the gRPC endpoint of the node
+func (n *MeshNode) GetHostEndpoint() string {
+	return n.HostEndpoint
+}
+
+// GetPublicKey: gets the public key of the node
+func (n *MeshNode) GetPublicKey() (wgtypes.Key, error) {
+	return wgtypes.ParseKey(n.PublicKey)
+}
+
+// GetWgEndpoint(): get IP and port of the wireguard endpoint
+func (n *MeshNode) GetWgEndpoint() string {
+	return n.WgEndpoint
+}
+
+// GetWgHost: get the IP address of the WireGuard node
+func (n *MeshNode) GetWgHost() *net.IPNet {
+	_, ipnet, _ := net.ParseCIDR(n.WgHost)
+	return ipnet
+}
+
+// GetTimestamp: get the UNIX time stamp of the ndoe
+func (n *MeshNode) GetTimeStamp() int64 {
+	return n.Timestamp
+}
+
+// GetRoutes: returns the routes that the nodes provides
+func (n *MeshNode) GetRoutes() []mesh.Route {
+	routes := make([]mesh.Route, len(n.Routes))
+
+	for index, route := range lib.MapValues(n.Routes) {
+		routes[index] = &Route{
+			Destination: route.Destination,
+			Path:        route.Path,
+		}
+	}
+
+	return routes
+}
+
+// GetIdentifier: returns the identifier of the node
+func (m *MeshNode) GetIdentifier() string {
+	ipv6 := m.WgHost[:len(m.WgHost)-4]
+
+	constituents := strings.Split(ipv6, ":")
+	constituents = constituents[4:]
+	return strings.Join(constituents, ":")
+}
+
+// GetDescription: returns the description for this node
+func (n *MeshNode) GetDescription() string {
+	return n.Description
+}
+
+// GetAlias: associates the node with an alias. Potentially used
+// for DNS and so forth.
+func (n *MeshNode) GetAlias() string {
+	return n.Alias
+}
+
+// GetServices: returns a list of services offered by the node
+func (n *MeshNode) GetServices() map[string]string {
+	return n.Services
+}
+
+func (n *MeshNode) GetType() conf.NodeType {
+	return conf.NodeType(n.Type)
+}
+
+type MeshSnapshot struct {
+	Nodes map[string]MeshNode
+}
+
+// GetNodes() returns the nodes in the mesh
+func (m *MeshSnapshot) GetNodes() map[string]mesh.MeshNode {
+	newMap := make(map[string]mesh.MeshNode)
+
+	for key, value := range m.Nodes {
+		newMap[key] = &MeshNode{
+			HostEndpoint: value.HostEndpoint,
+			PublicKey:    value.PublicKey,
+			WgHost:       value.WgHost,
+			WgEndpoint:   value.WgEndpoint,
+			Timestamp:    value.Timestamp,
+			Routes:       value.Routes,
+			Alias:        value.Alias,
+			Description:  value.Description,
+			Services:     value.Services,
+			Type:         value.Type,
+		}
+	}
+
+	return newMap
+}
+
+type TwoPhaseStoreMeshManager struct {
+	MeshId    string
+	IfName    string
+	Client    *wgctrl.Client
+	LastClock uint64
+	conf      *conf.WgMeshConfiguration
+	store     *TwoPhaseMap[string, MeshNode]
+}
+
+// AddNode() adds a node to the mesh
+func (m *TwoPhaseStoreMeshManager) AddNode(node mesh.MeshNode) {
+	crdt, ok := node.(*MeshNode)
+
+	if !ok {
+		panic("node must be of type mesh node")
+	}
+
+	crdt.Routes = make(map[string]Route)
+	crdt.Services = make(map[string]string)
+	crdt.Timestamp = time.Now().Unix()
+
+	m.store.Put(crdt.PublicKey, *crdt)
+}
+
+// GetMesh() returns a snapshot of the mesh provided by the mesh provider.
+func (m *TwoPhaseStoreMeshManager) GetMesh() (mesh.MeshSnapshot, error) {
+	nodes := m.store.AsList()
+
+	snapshot := make(map[string]MeshNode)
+
+	for _, node := range nodes {
+		snapshot[node.PublicKey] = node
+	}
+
+	return &MeshSnapshot{
+		Nodes: snapshot,
+	}, nil
+}
+
+// GetMeshId() returns the ID of the mesh network
+func (m *TwoPhaseStoreMeshManager) GetMeshId() string {
+	return m.MeshId
+}
+
+// Save() saves the mesh network
+func (m *TwoPhaseStoreMeshManager) Save() []byte {
+	snapshot := m.store.Snapshot()
+
+	var buf bytes.Buffer
+	enc := gob.NewEncoder(&buf)
+
+	err := enc.Encode(*snapshot)
+
+	if err != nil {
+		logging.Log.WriteInfof(err.Error())
+	}
+
+	return buf.Bytes()
+}
+
+// Load() loads a mesh network
+func (m *TwoPhaseStoreMeshManager) Load(bs []byte) error {
+	buf := bytes.NewBuffer(bs)
+	dec := gob.NewDecoder(buf)
+
+	var snapshot TwoPhaseMapSnapshot[string, MeshNode]
+	err := dec.Decode(&snapshot)
+
+	m.store.Merge(snapshot)
+	return err
+}
+
+// GetDevice() get the device corresponding with the mesh
+func (m *TwoPhaseStoreMeshManager) GetDevice() (*wgtypes.Device, error) {
+	dev, err := m.Client.Device(m.IfName)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return dev, nil
+}
+
+// HasChanges returns true if we have changes since last time we synced
+func (m *TwoPhaseStoreMeshManager) HasChanges() bool {
+	clockValue := m.store.GetHash()
+	return clockValue != m.LastClock
+}
+
+// Record that we have changes and save the corresponding changes
+func (m *TwoPhaseStoreMeshManager) SaveChanges() {
+	clockValue := m.store.GetHash()
+	m.LastClock = clockValue
+}
+
+// UpdateTimeStamp: update the timestamp of the given node
+func (m *TwoPhaseStoreMeshManager) UpdateTimeStamp(nodeId string) error {
+	if !m.store.Contains(nodeId) {
+		return fmt.Errorf("datastore: %s does not exist in the mesh", nodeId)
+	}
+
+	// Sort nodes by their public key
+	peers := m.GetPeers()
+	slices.Sort(peers)
+
+	if len(peers) == 0 {
+		return nil
+	}
+
+	peerToUpdate := peers[0]
+
+	if uint64(time.Now().Unix())-m.store.Clock.GetTimestamp(peerToUpdate) > 3*uint64(m.conf.KeepAliveTime) {
+		m.store.Mark(peerToUpdate)
+
+		if len(peers) < 2 {
+			return nil
+		}
+
+		peerToUpdate = peers[1]
+	}
+
+	if peerToUpdate != nodeId {
+		return nil
+	}
+
+	// Refresh causing node to update it's time stamp
+	node := m.store.Get(nodeId)
+	node.Timestamp = time.Now().Unix()
+	m.store.Put(nodeId, node)
+	return nil
+}
+
+// AddRoutes: adds routes to the given node
+func (m *TwoPhaseStoreMeshManager) AddRoutes(nodeId string, routes ...mesh.Route) error {
+	if !m.store.Contains(nodeId) {
+		return fmt.Errorf("datastore: %s does not exist in the mesh", nodeId)
+	}
+
+	if len(routes) == 0 {
+		return nil
+	}
+
+	node := m.store.Get(nodeId)
+
+	changes := false
+
+	for _, route := range routes {
+		prevRoute, ok := node.Routes[route.GetDestination().String()]
+
+		if !ok || route.GetHopCount() < prevRoute.GetHopCount() {
+			changes = true
+
+			node.Routes[route.GetDestination().String()] = Route{
+				Destination: route.GetDestination().String(),
+				Path:        route.GetPath(),
+			}
+		}
+	}
+
+	if changes {
+		m.store.Put(nodeId, node)
+	}
+
+	return nil
+}
+
+// DeleteRoutes: deletes the routes from the node
+func (m *TwoPhaseStoreMeshManager) RemoveRoutes(nodeId string, routes ...string) error {
+	if !m.store.Contains(nodeId) {
+		return fmt.Errorf("datastore: %s does not exist in the mesh", nodeId)
+	}
+
+	if len(routes) == 0 {
+		return nil
+	}
+
+	node := m.store.Get(nodeId)
+
+	for _, route := range routes {
+		delete(node.Routes, route)
+	}
+
+	return nil
+}
+
+// GetSyncer: returns the automerge syncer for sync
+func (m *TwoPhaseStoreMeshManager) GetSyncer() mesh.MeshSyncer {
+	return NewTwoPhaseSyncer(m)
+}
+
+// GetNode get a particular not within the mesh
+func (m *TwoPhaseStoreMeshManager) GetNode(nodeId string) (mesh.MeshNode, error) {
+	if !m.store.Contains(nodeId) {
+		return nil, fmt.Errorf("datastore: %s does not exist in the mesh", nodeId)
+	}
+
+	node := m.store.Get(nodeId)
+	return &node, nil
+}
+
+// NodeExists: returns true if a particular node exists false otherwise
+func (m *TwoPhaseStoreMeshManager) NodeExists(nodeId string) bool {
+	return m.store.Contains(nodeId)
+}
+
+// SetDescription: sets the description of this automerge data type
+func (m *TwoPhaseStoreMeshManager) SetDescription(nodeId string, description string) error {
+	if !m.store.Contains(nodeId) {
+		return fmt.Errorf("datastore: %s does not exist in the mesh", nodeId)
+	}
+
+	node := m.store.Get(nodeId)
+	node.Description = description
+
+	m.store.Put(nodeId, node)
+	return nil
+}
+
+// SetAlias: set the alias of the nodeId
+func (m *TwoPhaseStoreMeshManager) SetAlias(nodeId string, alias string) error {
+	if !m.store.Contains(nodeId) {
+		return fmt.Errorf("datastore: %s does not exist in the mesh", nodeId)
+	}
+
+	node := m.store.Get(nodeId)
+	node.Description = alias
+
+	m.store.Put(nodeId, node)
+	return nil
+}
+
+// AddService: adds the service to the given node
+func (m *TwoPhaseStoreMeshManager) AddService(nodeId string, key string, value string) error {
+	if !m.store.Contains(nodeId) {
+		return fmt.Errorf("datastore: %s does not exist in the mesh", nodeId)
+	}
+
+	node := m.store.Get(nodeId)
+	node.Services[key] = value
+	m.store.Put(nodeId, node)
+	return nil
+}
+
+// RemoveService: removes the service form the node. throws an error if the service does not exist
+func (m *TwoPhaseStoreMeshManager) RemoveService(nodeId string, key string) error {
+	if !m.store.Contains(nodeId) {
+		return fmt.Errorf("datastore: %s does not exist in the mesh", nodeId)
+	}
+
+	node := m.store.Get(nodeId)
+	delete(node.Services, key)
+	m.store.Put(nodeId, node)
+	return nil
+}
+
+// Prune: prunes all nodes that have not updated their timestamp in
+func (m *TwoPhaseStoreMeshManager) Prune() error {
+	m.store.Prune()
+	return nil
+}
+
+// GetPeers: get a list of contactable peers
+func (m *TwoPhaseStoreMeshManager) GetPeers() []string {
+	nodes := m.store.AsList()
+	nodes = lib.Filter(nodes, func(mn MeshNode) bool {
+		if mn.Type != string(conf.PEER_ROLE) {
+			return false
+		}
+
+		// If the node is marked as unreachable don't consider it a peer.
+		// this help to optimize convergence time for unreachable nodes.
+		// However advertising it to other nodes could result in flapping.
+		if m.store.IsMarked(mn.PublicKey) {
+			return false
+		}
+
+		return true
+	})
+
+	return lib.Map(nodes, func(mn MeshNode) string {
+		return mn.PublicKey
+	})
+}
+
+func (m *TwoPhaseStoreMeshManager) getRoutes(targetNode string) (map[string]Route, error) {
+	if !m.store.Contains(targetNode) {
+		return nil, fmt.Errorf("getRoute: cannot get route %s does not exist", targetNode)
+	}
+
+	node := m.store.Get(targetNode)
+	return node.Routes, nil
+}
+
+// GetRoutes(): Get all unique routes. Where the route with the least hop count is chosen
+func (m *TwoPhaseStoreMeshManager) GetRoutes(targetNode string) (map[string]mesh.Route, error) {
+	node, err := m.GetNode(targetNode)
+
+	if err != nil {
+		return nil, err
+	}
+
+	routes := make(map[string]mesh.Route)
+
+	// Add routes that the node directly has
+	for _, route := range node.GetRoutes() {
+		routes[route.GetDestination().String()] = route
+	}
+
+	// Work out the other routes in the mesh
+	for _, node := range m.GetPeers() {
+		nodeRoutes, err := m.getRoutes(node)
+
+		if err != nil {
+			return nil, err
+		}
+
+		for _, route := range nodeRoutes {
+			otherRoute, ok := routes[route.GetDestination().String()]
+
+			hopCount := route.GetHopCount()
+
+			if node != targetNode {
+				hopCount += 1
+			}
+
+			if !ok || route.GetHopCount()+1 < otherRoute.GetHopCount() {
+				routes[route.GetDestination().String()] = &Route{
+					Destination: route.GetDestination().String(),
+					Path:        append(route.GetPath(), m.GetMeshId()),
+				}
+			}
+		}
+	}
+
+	return routes, nil
+}
+
+// RemoveNode(): remove the node from the mesh
+func (m *TwoPhaseStoreMeshManager) RemoveNode(nodeId string) error {
+	if !m.store.Contains(nodeId) {
+		return fmt.Errorf("datastore: %s does not exist in the mesh", nodeId)
+	}
+
+	m.store.Remove(nodeId)
+	return nil
+}

pkg/crdt/factory.go

@@ -0,0 +1,78 @@
+package crdt
+
+import (
+	"fmt"
+	"hash/fnv"
+
+	"github.com/tim-beatham/wgmesh/pkg/conf"
+	"github.com/tim-beatham/wgmesh/pkg/lib"
+	"github.com/tim-beatham/wgmesh/pkg/mesh"
+)
+
+type TwoPhaseMapFactory struct{}
+
+func (f *TwoPhaseMapFactory) CreateMesh(params *mesh.MeshProviderFactoryParams) (mesh.MeshProvider, error) {
+	return &TwoPhaseStoreMeshManager{
+		MeshId: params.MeshId,
+		IfName: params.DevName,
+		Client: params.Client,
+		conf:   params.Conf,
+		store: NewTwoPhaseMap[string, MeshNode](params.NodeID, func(s string) uint64 {
+			h := fnv.New64a()
+			h.Write([]byte(s))
+			return h.Sum64()
+		}, uint64(3*params.Conf.KeepAliveTime)),
+	}, nil
+}
+
+type MeshNodeFactory struct {
+	Config conf.WgMeshConfiguration
+}
+
+func (f *MeshNodeFactory) Build(params *mesh.MeshNodeFactoryParams) mesh.MeshNode {
+	hostName := f.getAddress(params)
+
+	grpcEndpoint := fmt.Sprintf("%s:%s", hostName, f.Config.GrpcPort)
+
+	if f.Config.Role == conf.CLIENT_ROLE {
+		grpcEndpoint = "-"
+	}
+
+	return &MeshNode{
+		HostEndpoint: grpcEndpoint,
+		PublicKey:    params.PublicKey.String(),
+		WgEndpoint:   fmt.Sprintf("%s:%d", hostName, params.WgPort),
+		WgHost:       fmt.Sprintf("%s/128", params.NodeIP.String()),
+		Routes:       make(map[string]Route),
+		Description:  "",
+		Alias:        "",
+		Type:         string(f.Config.Role),
+	}
+}
+
+// getAddress returns the routable address of the machine.
+func (f *MeshNodeFactory) getAddress(params *mesh.MeshNodeFactoryParams) string {
+	var hostName string = ""
+
+	if params.Endpoint != "" {
+		hostName = params.Endpoint
+	} else if len(f.Config.Endpoint) != 0 {
+		hostName = f.Config.Endpoint
+	} else {
+		ipFunc := lib.GetPublicIP
+
+		if f.Config.IPDiscovery == conf.DNS_IP_DISCOVERY {
+			ipFunc = lib.GetOutboundIP
+		}
+
+		ip, err := ipFunc()
+
+		if err != nil {
+			return ""
+		}
+
+		hostName = ip.String()
+	}
+
+	return hostName
+}

pkg/crdt/g_map.go

@@ -0,0 +1,176 @@
+// crdt is a golang implementation of a crdt
+package crdt
+
+import (
+	"cmp"
+	"sync"
+)
+
+type Bucket[D any] struct {
+	Vector     uint64
+	Contents   D
+	Gravestone bool
+}
+
+// GMap is a set that can only grow in size
+type GMap[K cmp.Ordered, D any] struct {
+	lock     sync.RWMutex
+	contents map[uint64]Bucket[D]
+	clock    *VectorClock[K]
+}
+
+func (g *GMap[K, D]) Put(key K, value D) {
+	g.lock.Lock()
+
+	clock := g.clock.IncrementClock()
+
+	g.contents[g.clock.hashFunc(key)] = Bucket[D]{
+		Vector:   clock,
+		Contents: value,
+	}
+
+	g.lock.Unlock()
+}
+
+func (g *GMap[K, D]) Contains(key K) bool {
+	return g.contains(g.clock.hashFunc(key))
+}
+
+func (g *GMap[K, D]) contains(key uint64) bool {
+	g.lock.RLock()
+
+	_, ok := g.contents[key]
+
+	g.lock.RUnlock()
+
+	return ok
+}
+
+func (g *GMap[K, D]) put(key uint64, b Bucket[D]) {
+	g.lock.Lock()
+
+	if g.contents[key].Vector < b.Vector {
+		g.contents[key] = b
+	}
+
+	g.lock.Unlock()
+}
+
+func (g *GMap[K, D]) get(key uint64) Bucket[D] {
+	g.lock.RLock()
+	bucket := g.contents[key]
+	g.lock.RUnlock()
+
+	return bucket
+}
+
+func (g *GMap[K, D]) Get(key K) D {
+	return g.get(g.clock.hashFunc(key)).Contents
+}
+
+func (g *GMap[K, D]) Mark(key K) {
+	g.lock.Lock()
+	bucket := g.contents[g.clock.hashFunc(key)]
+	bucket.Gravestone = true
+	g.contents[g.clock.hashFunc(key)] = bucket
+	g.lock.Unlock()
+}
+
+// IsMarked: returns true if the node is marked
+func (g *GMap[K, D]) IsMarked(key K) bool {
+	marked := false
+
+	g.lock.RLock()
+
+	bucket, ok := g.contents[g.clock.hashFunc(key)]
+
+	if ok {
+		marked = bucket.Gravestone
+	}
+
+	g.lock.RUnlock()
+
+	return marked
+}
+
+func (g *GMap[K, D]) Keys() []uint64 {
+	g.lock.RLock()
+
+	contents := make([]uint64, len(g.contents))
+	index := 0
+
+	for key := range g.contents {
+		contents[index] = key
+		index++
+	}
+
+	g.lock.RUnlock()
+	return contents
+}
+
+func (g *GMap[K, D]) Save() map[uint64]Bucket[D] {
+	buckets := make(map[uint64]Bucket[D])
+	g.lock.RLock()
+
+	for key, value := range g.contents {
+		buckets[key] = value
+	}
+
+	g.lock.RUnlock()
+	return buckets
+}
+
+func (g *GMap[K, D]) SaveWithKeys(keys []uint64) map[uint64]Bucket[D] {
+	buckets := make(map[uint64]Bucket[D])
+	g.lock.RLock()
+
+	for _, key := range keys {
+		buckets[key] = g.contents[key]
+	}
+
+	g.lock.RUnlock()
+	return buckets
+}
+
+func (g *GMap[K, D]) GetClock() map[uint64]uint64 {
+	clock := make(map[uint64]uint64)
+	g.lock.RLock()
+
+	for key, bucket := range g.contents {
+		clock[key] = bucket.Vector
+	}
+
+	g.lock.RUnlock()
+	return clock
+}
+
+func (g *GMap[K, D]) GetHash() uint64 {
+	hash := uint64(0)
+
+	g.lock.RLock()
+
+	for _, value := range g.contents {
+		hash += value.Vector
+	}
+
+	g.lock.RUnlock()
+	return hash
+}
+
+func (g *GMap[K, D]) Prune() {
+	stale := g.clock.getStale()
+	g.lock.Lock()
+
+	for _, outlier := range stale {
+		delete(g.contents, outlier)
+	}
+
+	g.lock.Unlock()
+}
+
+func NewGMap[K cmp.Ordered, D any](clock *VectorClock[K]) *GMap[K, D] {
+	return &GMap[K, D]{
+		contents: make(map[uint64]Bucket[D]),
+		clock:    clock,
+	}
+}

pkg/crdt/two_phase_map.go

@@ -0,0 +1,211 @@
+package crdt
+
+import (
+	"cmp"
+
+	"github.com/tim-beatham/wgmesh/pkg/lib"
+)
+
+type TwoPhaseMap[K cmp.Ordered, D any] struct {
+	addMap    *GMap[K, D]
+	removeMap *GMap[K, bool]
+	Clock     *VectorClock[K]
+	processId K
+}
+
+type TwoPhaseMapSnapshot[K cmp.Ordered, D any] struct {
+	Add    map[uint64]Bucket[D]
+	Remove map[uint64]Bucket[bool]
+}
+
+// Contains checks whether the value exists in the map
+func (m *TwoPhaseMap[K, D]) Contains(key K) bool {
+	return m.contains(m.Clock.hashFunc(key))
+}
+
+// Contains checks whether the value exists in the map
+func (m *TwoPhaseMap[K, D]) contains(key uint64) bool {
+	if !m.addMap.contains(key) {
+		return false
+	}
+
+	addValue := m.addMap.get(key)
+
+	if !m.removeMap.contains(key) {
+		return true
+	}
+
+	removeValue := m.removeMap.get(key)
+
+	return addValue.Vector >= removeValue.Vector
+}
+
+func (m *TwoPhaseMap[K, D]) Get(key K) D {
+	var result D
+
+	if !m.Contains(key) {
+		return result
+	}
+
+	return m.addMap.Get(key)
+}
+
+func (m *TwoPhaseMap[K, D]) get(key uint64) D {
+	var result D
+
+	if !m.contains(key) {
+		return result
+	}
+
+	return m.addMap.get(key).Contents
+}
+
+// Put places the key K in the map
+func (m *TwoPhaseMap[K, D]) Put(key K, data D) {
+	msgSequence := m.Clock.IncrementClock()
+	m.Clock.Put(key, msgSequence)
+	m.addMap.Put(key, data)
+}
+
+func (m *TwoPhaseMap[K, D]) Mark(key K) {
+	m.addMap.Mark(key)
+}
+
+// Remove removes the value from the map
+func (m *TwoPhaseMap[K, D]) Remove(key K) {
+	m.removeMap.Put(key, true)
+}
+
+func (m *TwoPhaseMap[K, D]) keys() []uint64 {
+	keys := make([]uint64, 0)
+
+	addKeys := m.addMap.Keys()
+
+	for _, key := range addKeys {
+		if !m.contains(key) {
+			continue
+		}
+
+		keys = append(keys, key)
+	}
+
+	return keys
+}
+
+func (m *TwoPhaseMap[K, D]) AsList() []D {
+	theList := make([]D, 0)
+
+	keys := m.keys()
+
+	for _, key := range keys {
+		theList = append(theList, m.get(key))
+	}
+
+	return theList
+}
+
+func (m *TwoPhaseMap[K, D]) Snapshot() *TwoPhaseMapSnapshot[K, D] {
+	return &TwoPhaseMapSnapshot[K, D]{
+		Add:    m.addMap.Save(),
+		Remove: m.removeMap.Save(),
+	}
+}
+
+func (m *TwoPhaseMap[K, D]) SnapShotFromState(state *TwoPhaseMapState[K]) *TwoPhaseMapSnapshot[K, D] {
+	addKeys := lib.MapKeys(state.AddContents)
+	removeKeys := lib.MapKeys(state.RemoveContents)
+
+	return &TwoPhaseMapSnapshot[K, D]{
+		Add:    m.addMap.SaveWithKeys(addKeys),
+		Remove: m.removeMap.SaveWithKeys(removeKeys),
+	}
+}
+
+type TwoPhaseMapState[K cmp.Ordered] struct {
+	Vectors        map[uint64]uint64
+	AddContents    map[uint64]uint64
+	RemoveContents map[uint64]uint64
+}
+
+func (m *TwoPhaseMap[K, D]) IsMarked(key K) bool {
+	return m.addMap.IsMarked(key)
+}
+
+// GetHash: Get the hash of the current state of the map
+// Sums the current values of the vectors. Provides good approximation
+// of increasing numbers
+func (m *TwoPhaseMap[K, D]) GetHash() uint64 {
+	return (m.addMap.GetHash() + 1) * (m.removeMap.GetHash() + 1)
+}
+
+// GetState: get the current vector clock of the add and remove
+// map
+func (m *TwoPhaseMap[K, D]) GenerateMessage() *TwoPhaseMapState[K] {
+	addContents := m.addMap.GetClock()
+	removeContents := m.removeMap.GetClock()
+
+	return &TwoPhaseMapState[K]{
+		Vectors:        m.Clock.GetClock(),
+		AddContents:    addContents,
+		RemoveContents: removeContents,
+	}
+}
+
+func (m *TwoPhaseMapState[K]) Difference(state *TwoPhaseMapState[K]) *TwoPhaseMapState[K] {
+	mapState := &TwoPhaseMapState[K]{
+		AddContents:    make(map[uint64]uint64),
+		RemoveContents: make(map[uint64]uint64),
+	}
+
+	for key, value := range state.AddContents {
+		otherValue, ok := m.AddContents[key]
+
+		if !ok || otherValue < value {
+			mapState.AddContents[key] = value
+		}
+	}
+
+	for key, value := range state.RemoveContents {
+		otherValue, ok := m.RemoveContents[key]
+
+		if !ok || otherValue < value {
+			mapState.RemoveContents[key] = value
+		}
+	}
+
+	return mapState
+}
+
+func (m *TwoPhaseMap[K, D]) Merge(snapshot TwoPhaseMapSnapshot[K, D]) {
+	for key, value := range snapshot.Add {
+		// Gravestone is local only to that node.
+		// Discover ourselves if the node is alive
+		m.addMap.put(key, value)
+		m.Clock.put(key, value.Vector)
+	}
+
+	for key, value := range snapshot.Remove {
+		m.removeMap.put(key, value)
+		m.Clock.put(key, value.Vector)
+	}
+}
+
+func (m *TwoPhaseMap[K, D]) Prune() {
+	m.addMap.Prune()
+	m.removeMap.Prune()
+	m.Clock.Prune()
+}
+
+// NewTwoPhaseMap: create a new two phase map. Consists of two maps
+// a grow map and a remove map. If both timestamps equal then favour keeping
+// it in the map
+func NewTwoPhaseMap[K cmp.Ordered, D any](processId K, hashKey func(K) uint64, staleTime uint64) *TwoPhaseMap[K, D] {
+	m := TwoPhaseMap[K, D]{
+		processId: processId,
+		Clock:     NewVectorClock(processId, hashKey, staleTime),
+	}
+
+	m.addMap = NewGMap[K, D](m.Clock)
+	m.removeMap = NewGMap[K, bool](m.Clock)
+	return &m
+}

pkg/crdt/two_phase_map_syncer.go

@@ -0,0 +1,187 @@
+package crdt
+
+import (
+	"bytes"
+	"encoding/gob"
+
+	logging "github.com/tim-beatham/wgmesh/pkg/log"
+)
+
+type SyncState int
+
+const (
+	HASH SyncState = iota
+	PREPARE
+	PRESENT
+	EXCHANGE
+	MERGE
+	FINISHED
+)
+
+// TwoPhaseSyncer is a type to sync a TwoPhase data store
+type TwoPhaseSyncer struct {
+	manager            *TwoPhaseStoreMeshManager
+	generateMessageFSM SyncFSM
+	state              SyncState
+	mapState           *TwoPhaseMapState[string]
+	peerMsg            []byte
+}
+
+type TwoPhaseHash struct {
+	Hash uint64
+}
+
+type SyncFSM map[SyncState]func(*TwoPhaseSyncer) ([]byte, bool)
+
+func hash(syncer *TwoPhaseSyncer) ([]byte, bool) {
+	hash := TwoPhaseHash{
+		Hash: syncer.manager.store.Clock.GetHash(),
+	}
+
+	var buffer bytes.Buffer
+	enc := gob.NewEncoder(&buffer)
+
+	err := enc.Encode(hash)
+
+	if err != nil {
+		logging.Log.WriteErrorf(err.Error())
+	}
+
+	syncer.IncrementState()
+	return buffer.Bytes(), true
+}
+
+func prepare(syncer *TwoPhaseSyncer) ([]byte, bool) {
+	var recvBuffer = bytes.NewBuffer(syncer.peerMsg)
+	dec := gob.NewDecoder(recvBuffer)
+
+	var hash TwoPhaseHash
+	err := dec.Decode(&hash)
+
+	if err != nil {
+		logging.Log.WriteErrorf(err.Error())
+	}
+
+	// If vector clocks are equal then no need to merge state
+	// Helps to reduce bandwidth by detecting early
+	if hash.Hash == syncer.manager.store.Clock.GetHash() {
+		return nil, false
+	}
+
+	var buffer bytes.Buffer
+	enc := gob.NewEncoder(&buffer)
+
+	err = enc.Encode(*syncer.mapState)
+
+	if err != nil {
+		logging.Log.WriteErrorf(err.Error())
+	}
+
+	syncer.IncrementState()
+	return buffer.Bytes(), true
+}
+
+func present(syncer *TwoPhaseSyncer) ([]byte, bool) {
+	if syncer.peerMsg == nil {
+		panic("peer msg is nil")
+	}
+
+	var recvBuffer = bytes.NewBuffer(syncer.peerMsg)
+	dec := gob.NewDecoder(recvBuffer)
+
+	var mapState TwoPhaseMapState[string]
+	err := dec.Decode(&mapState)
+
+	if err != nil {
+		logging.Log.WriteErrorf(err.Error())
+	}
+
+	difference := syncer.mapState.Difference(&mapState)
+	syncer.manager.store.Clock.Merge(mapState.Vectors)
+
+	var sendBuffer bytes.Buffer
+	enc := gob.NewEncoder(&sendBuffer)
+	enc.Encode(*difference)
+
+	syncer.IncrementState()
+	return sendBuffer.Bytes(), true
+}
+
+func exchange(syncer *TwoPhaseSyncer) ([]byte, bool) {
+	if syncer.peerMsg == nil {
+		panic("peer msg is nil")
+	}
+
+	var recvBuffer = bytes.NewBuffer(syncer.peerMsg)
+	dec := gob.NewDecoder(recvBuffer)
+
+	var mapState TwoPhaseMapState[string]
+	dec.Decode(&mapState)
+
+	snapshot := syncer.manager.store.SnapShotFromState(&mapState)
+
+	var sendBuffer bytes.Buffer
+	enc := gob.NewEncoder(&sendBuffer)
+	enc.Encode(*snapshot)
+
+	syncer.IncrementState()
+	return sendBuffer.Bytes(), true
+}
+
+func merge(syncer *TwoPhaseSyncer) ([]byte, bool) {
+	if syncer.peerMsg == nil {
+		panic("peer msg is nil")
+	}
+
+	var recvBuffer = bytes.NewBuffer(syncer.peerMsg)
+	dec := gob.NewDecoder(recvBuffer)
+
+	var snapshot TwoPhaseMapSnapshot[string, MeshNode]
+	dec.Decode(&snapshot)
+
+	syncer.manager.store.Merge(snapshot)
+	return nil, false
+}
+
+func (t *TwoPhaseSyncer) IncrementState() {
+	t.state = min(t.state+1, FINISHED)
+}
+
+func (t *TwoPhaseSyncer) GenerateMessage() ([]byte, bool) {
+	fsmFunc, ok := t.generateMessageFSM[t.state]
+
+	if !ok {
+		panic("state not handled")
+	}
+
+	return fsmFunc(t)
+}
+
+func (t *TwoPhaseSyncer) RecvMessage(msg []byte) error {
+	t.peerMsg = msg
+	return nil
+}
+
+func (t *TwoPhaseSyncer) Complete() {
+	logging.Log.WriteInfof("SYNC COMPLETED")
+	if t.state >= MERGE {
+		t.manager.store.Clock.IncrementClock()
+	}
+}
+
+func NewTwoPhaseSyncer(manager *TwoPhaseStoreMeshManager) *TwoPhaseSyncer {
+	var generateMessageFsm SyncFSM = SyncFSM{
+		HASH:     hash,
+		PREPARE:  prepare,
+		PRESENT:  present,
+		EXCHANGE: exchange,
+		MERGE:    merge,
+	}
+
+	return &TwoPhaseSyncer{
+		manager:            manager,
+		state:              HASH,
+		mapState:           manager.store.GenerateMessage(),
+		generateMessageFSM: generateMessageFsm,
+	}
+}

pkg/crdt/vector_clock.go

@@ -0,0 +1,149 @@
+package crdt
+
+import (
+	"cmp"
+	"sync"
+	"time"
+
+	"github.com/tim-beatham/wgmesh/pkg/lib"
+)
+
+type VectorBucket struct {
+	// clock current value of the node's clock
+	clock uint64
+	// lastUpdate we've seen
+	lastUpdate uint64
+}
+
+// Vector clock defines an abstract data type
+// for a vector clock implementation
+type VectorClock[K cmp.Ordered] struct {
+	vectors   map[uint64]*VectorBucket
+	lock      sync.RWMutex
+	processID K
+	staleTime uint64
+	hashFunc  func(K) uint64
+}
+
+// IncrementClock: increments the node's value in the vector clock
+func (m *VectorClock[K]) IncrementClock() uint64 {
+	maxClock := uint64(0)
+	m.lock.Lock()
+
+	for _, value := range m.vectors {
+		maxClock = max(maxClock, value.clock)
+	}
+
+	newBucket := VectorBucket{
+		clock:      maxClock + 1,
+		lastUpdate: uint64(time.Now().Unix()),
+	}
+
+	m.vectors[m.hashFunc(m.processID)] = &newBucket
+
+	m.lock.Unlock()
+	return maxClock
+}
+
+// GetHash: gets the hash of the vector clock used to determine if there
+// are any changes
+func (m *VectorClock[K]) GetHash() uint64 {
+	m.lock.RLock()
+
+	hash := uint64(0)
+
+	for key, bucket := range m.vectors {
+		hash += key * (bucket.clock + 1)
+	}
+
+	m.lock.RUnlock()
+	return hash
+}
+
+func (m *VectorClock[K]) Merge(vectors map[uint64]uint64) {
+	for key, value := range vectors {
+		m.put(key, value)
+	}
+}
+
+// getStale: get all entries that are stale within the mesh
+func (m *VectorClock[K]) getStale() []uint64 {
+	m.lock.RLock()
+	maxTimeStamp := lib.Reduce(0, lib.MapValues(m.vectors), func(i uint64, vb *VectorBucket) uint64 {
+		return max(i, vb.lastUpdate)
+	})
+
+	toRemove := make([]uint64, 0)
+
+	for key, bucket := range m.vectors {
+		if maxTimeStamp-bucket.lastUpdate > m.staleTime {
+			toRemove = append(toRemove, key)
+		}
+	}
+
+	m.lock.RUnlock()
+	return toRemove
+}
+
+func (m *VectorClock[K]) Prune() {
+	stale := m.getStale()
+
+	m.lock.Lock()
+
+	for _, key := range stale {
+		delete(m.vectors, key)
+	}
+
+	m.lock.Unlock()
+}
+
+func (m *VectorClock[K]) GetTimestamp(processId K) uint64 {
+	return m.vectors[m.hashFunc(m.processID)].lastUpdate
+}
+
+func (m *VectorClock[K]) Put(key K, value uint64) {
+	m.put(m.hashFunc(key), value)
+}
+
+func (m *VectorClock[K]) put(key uint64, value uint64) {
+	clockValue := uint64(0)
+
+	m.lock.Lock()
+	bucket, ok := m.vectors[key]
+
+	if ok {
+		clockValue = bucket.clock
+	}
+
+	if value > clockValue {
+		newBucket := VectorBucket{
+			clock:      value,
+			lastUpdate: uint64(time.Now().Unix()),
+		}
+		m.vectors[key] = &newBucket
+	}
+
+	m.lock.Unlock()
+}
+
+func (m *VectorClock[K]) GetClock() map[uint64]uint64 {
+	clock := make(map[uint64]uint64)
+
+	m.lock.RLock()
+
+	for key, value := range m.vectors {
+		clock[key] = value.clock
+	}
+
+	m.lock.RUnlock()
+	return clock
+}
+
+func NewVectorClock[K cmp.Ordered](processID K, hashFunc func(K) uint64, staleTime uint64) *VectorClock[K] {
+	return &VectorClock[K]{
+		vectors:   make(map[uint64]*VectorBucket),
+		processID: processID,
+		staleTime: staleTime,
+		hashFunc:  hashFunc,
+	}
+}

pkg/ctrlserver/ctrlserver.go

@@ -1,9 +1,9 @@
 package ctrlserver
 
 import (
-	crdt "github.com/tim-beatham/wgmesh/pkg/automerge"
 	"github.com/tim-beatham/wgmesh/pkg/conf"
 	"github.com/tim-beatham/wgmesh/pkg/conn"
+	"github.com/tim-beatham/wgmesh/pkg/crdt"
 	"github.com/tim-beatham/wgmesh/pkg/ip"
 	"github.com/tim-beatham/wgmesh/pkg/lib"
 	logging "github.com/tim-beatham/wgmesh/pkg/log"
@@ -21,14 +21,15 @@ type NewCtrlServerParams struct {
 	CtrlProvider rpc.MeshCtrlServerServer
 	SyncProvider rpc.SyncServiceServer
 	Querier      query.Querier
+	OnDelete     func(mesh.MeshProvider)
 }
 
 // Create a new instance of the MeshCtrlServer or error if the
 // operation failed
 func NewCtrlServer(params *NewCtrlServerParams) (*MeshCtrlServer, error) {
 	ctrlServer := new(MeshCtrlServer)
-	meshFactory := crdt.CrdtProviderFactory{}
+	meshFactory := &crdt.TwoPhaseMapFactory{}
-	nodeFactory := crdt.MeshNodeFactory{
+	nodeFactory := &crdt.MeshNodeFactory{
 		Config: *params.Conf,
 	}
 	idGenerator := &lib.IDNameGenerator{}
@@ -40,12 +41,13 @@ func NewCtrlServer(params *NewCtrlServerParams) (*MeshCtrlServer, error) {
 	meshManagerParams := &mesh.NewMeshManagerParams{
 		Conf:                 *params.Conf,
 		Client:               params.Client,
-		MeshProvider:         &meshFactory,
+		MeshProvider:         meshFactory,
-		NodeFactory:          &nodeFactory,
+		NodeFactory:          nodeFactory,
 		IdGenerator:          idGenerator,
 		IPAllocator:          ipAllocator,
 		InterfaceManipulator: interfaceManipulator,
 		ConfigApplyer:        configApplyer,
+		OnDelete:             params.OnDelete,
 	}
 
 	ctrlServer.MeshManager = mesh.NewMeshManager(meshManagerParams)

pkg/lib/conv.go

@@ -1,11 +1,13 @@
 package lib
 
+import "cmp"
+
 // MapToSlice converts a map to a slice in go
-func MapValues[K comparable, V any](m map[K]V) []V {
+func MapValues[K cmp.Ordered, V any](m map[K]V) []V {
 	return MapValuesWithExclude(m, map[K]struct{}{})
 }
 
-func MapValuesWithExclude[K comparable, V any](m map[K]V, exclude map[K]struct{}) []V {
+func MapValuesWithExclude[K cmp.Ordered, V any](m map[K]V, exclude map[K]struct{}) []V {
 	values := make([]V, len(m)-len(exclude))
 
 	i := 0
@@ -26,7 +28,7 @@ func MapValuesWithExclude[K comparable, V any](m map[K]V, exclude map[K]struct{}
 	return values
 }
 
-func MapKeys[K comparable, V any](m map[K]V) []K {
+func MapKeys[K cmp.Ordered, V any](m map[K]V) []K {
 	values := make([]K, len(m))
 
 	i := 0
@@ -76,3 +78,13 @@ func Contains[V any](list []V, proposition func(V) bool) bool {
 
 	return false
 }
+
+func Reduce[A any, V any](start A, values []V, reduce func(A, V) A) A {
+	accum := start
+
+	for _, elem := range values {
+		accum = reduce(accum, elem)
+	}
+
+	return accum
+}

pkg/lib/rtnetlink.go

@@ -248,6 +248,14 @@ func (c *RtNetlinkConfig) DeleteRoutes(ifName string, family uint8, exclude ...R
 			if route.equal(r) {
 				return false
 			}
+
+			if family == unix.AF_INET && route.Destination.IP.To4() == nil {
+				return false
+			}
+
+			if family == unix.AF_INET6 && route.Destination.IP.To16() == nil {
+				return false
+			}
 		}
 		return true
 	}
@@ -255,7 +263,7 @@ func (c *RtNetlinkConfig) DeleteRoutes(ifName string, family uint8, exclude ...R
 	toDelete := Filter(ifRoutes, shouldExclude)
 
 	for _, route := range toDelete {
-		logging.Log.WriteInfof("Deleting route: %s", route.Gateway.String())
+		logging.Log.WriteInfof("Deleting route: %s", route.Destination.String())
 		err := c.DeleteRoute(ifName, route)
 
 		if err != nil {

pkg/lib/stats.go

@@ -0,0 +1,40 @@
+// lib contains helper functions for the implementation
+package lib
+
+import (
+	"cmp"
+	"math"
+
+	"gonum.org/v1/gonum/stat"
+	"gonum.org/v1/gonum/stat/distuv"
+)
+
+// Modelling the distribution using a normal distribution get the count
+// of the outliers
+func GetOutliers[K cmp.Ordered](counts map[K]uint64, alpha float64) []K {
+	n := float64(len(counts))
+
+	keys := MapKeys(counts)
+	values := make([]float64, len(keys))
+
+	for index, key := range keys {
+		values[index] = float64(counts[key])
+	}
+
+	mean := stat.Mean(values, nil)
+	stdDev := stat.StdDev(values, nil)
+
+	moe := distuv.Normal{Mu: 0, Sigma: 1}.Quantile(1-alpha/2) * (stdDev / math.Sqrt(n))
+
+	lowerBound := mean - moe
+
+	var outliers []K
+
+	for i, count := range values {
+		if count < lowerBound {
+			outliers = append(outliers, keys[i])
+		}
+	}
+
+	return outliers
+}

pkg/mesh/config.go

@@ -4,11 +4,13 @@ import (
 	"fmt"
 	"net"
 	"slices"
+	"strings"
 	"time"
 
 	"github.com/tim-beatham/wgmesh/pkg/conf"
 	"github.com/tim-beatham/wgmesh/pkg/ip"
 	"github.com/tim-beatham/wgmesh/pkg/lib"
+	logging "github.com/tim-beatham/wgmesh/pkg/log"
 	"github.com/tim-beatham/wgmesh/pkg/route"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
@@ -32,10 +34,6 @@ type routeNode struct {
 	route   Route
 }
 
-func (r *routeNode) equals(route2 *routeNode) bool {
-	return r.gateway == route2.gateway && RouteEquals(r.route, route2.route)
-}
-
 func (m *WgMeshConfigApplyer) convertMeshNode(node MeshNode, device *wgtypes.Device,
 	peerToClients map[string][]net.IPNet,
 	routes map[string][]routeNode) (*wgtypes.PeerConfig, error) {
@@ -55,7 +53,7 @@ func (m *WgMeshConfigApplyer) convertMeshNode(node MeshNode, device *wgtypes.Dev
 	allowedips := make([]net.IPNet, 1)
 	allowedips[0] = *node.GetWgHost()
 
-	clients, ok := peerToClients[node.GetWgHost().String()]
+	clients, ok := peerToClients[pubKey.String()]
 
 	if ok {
 		allowedips = append(allowedips, clients...)
@@ -63,9 +61,10 @@ func (m *WgMeshConfigApplyer) convertMeshNode(node MeshNode, device *wgtypes.Dev
 
 	for _, route := range node.GetRoutes() {
 		bestRoutes := routes[route.GetDestination().String()]
+		var pickedRoute routeNode
 
 		if len(bestRoutes) == 1 {
-			allowedips = append(allowedips, *route.GetDestination())
+			pickedRoute = bestRoutes[0]
 		} else if len(bestRoutes) > 1 {
 			keyFunc := func(mn MeshNode) int {
 				pubKey, _ := mn.GetPublicKey()
@@ -77,11 +76,11 @@ func (m *WgMeshConfigApplyer) convertMeshNode(node MeshNode, device *wgtypes.Dev
 			}
 
 			// Else there is more than one candidate so consistently hash
-			pickedRoute := lib.ConsistentHash(bestRoutes, node, bucketFunc, keyFunc)
+			pickedRoute = lib.ConsistentHash(bestRoutes, node, bucketFunc, keyFunc)
+		}
 
-			if pickedRoute.gateway == pubKey.String() {
+		if pickedRoute.gateway == pubKey.String() {
-				allowedips = append(allowedips, *route.GetDestination())
+			allowedips = append(allowedips, *pickedRoute.route.GetDestination())
-			}
 		}
 	}
 
@@ -101,6 +100,7 @@ func (m *WgMeshConfigApplyer) convertMeshNode(node MeshNode, device *wgtypes.Dev
 		Endpoint:                    endpoint,
 		AllowedIPs:                  allowedips,
 		PersistentKeepaliveInterval: &keepAlive,
+		ReplaceAllowedIPs:           true,
 	}
 
 	return &peerConfig, nil
@@ -116,18 +116,19 @@ func (m *WgMeshConfigApplyer) getRoutes(meshProvider MeshProvider) map[string][]
 	meshPrefixes := lib.Map(lib.MapValues(m.meshManager.GetMeshes()), func(mesh MeshProvider) *net.IPNet {
 		ula := &ip.ULABuilder{}
 		ipNet, _ := ula.GetIPNet(mesh.GetMeshId())
-
 		return ipNet
 	})
 
 	for _, node := range mesh.GetNodes() {
 		pubKey, _ := node.GetPublicKey()
-		meshRoutes, _ := meshProvider.GetRoutes(pubKey.String())
 
-		for _, route := range meshRoutes {
+		for _, route := range node.GetRoutes() {
 			if lib.Contains(meshPrefixes, func(prefix *net.IPNet) bool {
-				if prefix == nil || route == nil || route.GetDestination() == nil {
+				v6Default, _, _ := net.ParseCIDR("::/0")
-					return false
+				v4Default, _, _ := net.ParseCIDR("0.0.0.0/0")
+
+				if (prefix.IP.Equal(v6Default) || prefix.IP.Equal(v4Default)) && m.config.AdvertiseDefaultRoute {
+					return true
 				}
 
 				return prefix.Contains(route.GetDestination().IP)
@@ -150,6 +151,8 @@ func (m *WgMeshConfigApplyer) getRoutes(meshProvider MeshProvider) map[string][]
 			} else if route.GetHopCount() < otherRoute[0].route.GetHopCount() {
 				otherRoute[0] = rn
 			} else if otherRoute[0].route.GetHopCount() == route.GetHopCount() {
+				logging.Log.WriteInfof("Other Route Hop: %d", otherRoute[0].route.GetHopCount())
+				logging.Log.WriteInfof("Route gateway %s, route hop %d", rn.gateway, route.GetHopCount())
 				routes[destination] = append(otherRoute, rn)
 			}
 		}
@@ -158,6 +161,142 @@ func (m *WgMeshConfigApplyer) getRoutes(meshProvider MeshProvider) map[string][]
 	return routes
 }
 
+// getCorrespondignPeer: gets the peer corresponding to the client
+func (m *WgMeshConfigApplyer) getCorrespondingPeer(peers []MeshNode, client MeshNode) MeshNode {
+	hashFunc := func(mn MeshNode) int {
+		pubKey, _ := mn.GetPublicKey()
+		return lib.HashString(pubKey.String())
+	}
+
+	peer := lib.ConsistentHash(peers, client, hashFunc, hashFunc)
+	return peer
+}
+
+func (m *WgMeshConfigApplyer) getClientConfig(mesh MeshProvider, peers []MeshNode, clients []MeshNode, dev *wgtypes.Device) (*wgtypes.Config, error) {
+	self, err := m.meshManager.GetSelf(mesh.GetMeshId())
+	ula := &ip.ULABuilder{}
+	meshNet, _ := ula.GetIPNet(mesh.GetMeshId())
+
+	routes := lib.Map(lib.MapKeys(m.getRoutes(mesh)), func(destination string) net.IPNet {
+		_, ipNet, _ := net.ParseCIDR(destination)
+		return *ipNet
+	})
+	routes = append(routes, *meshNet)
+
+	if err != nil {
+		return nil, err
+	}
+
+	peer := m.getCorrespondingPeer(peers, self)
+
+	pubKey, _ := peer.GetPublicKey()
+
+	keepAlive := time.Duration(m.config.KeepAliveWg) * time.Second
+	endpoint, err := net.ResolveUDPAddr("udp", peer.GetWgEndpoint())
+
+	if err != nil {
+		return nil, err
+	}
+
+	peerCfgs := make([]wgtypes.PeerConfig, 1)
+
+	peerCfgs[0] = wgtypes.PeerConfig{
+		PublicKey:                   pubKey,
+		Endpoint:                    endpoint,
+		PersistentKeepaliveInterval: &keepAlive,
+		AllowedIPs:                  routes,
+	}
+
+	installedRoutes := make([]lib.Route, 0)
+
+	for _, route := range peerCfgs[0].AllowedIPs {
+		installedRoutes = append(installedRoutes, lib.Route{
+			Gateway:     peer.GetWgHost().IP,
+			Destination: route,
+		})
+	}
+
+	cfg := wgtypes.Config{
+		Peers: peerCfgs,
+	}
+
+	m.routeInstaller.InstallRoutes(dev.Name, installedRoutes...)
+	return &cfg, err
+}
+
+func (m *WgMeshConfigApplyer) getPeerConfig(mesh MeshProvider, peers []MeshNode, clients []MeshNode, dev *wgtypes.Device) (*wgtypes.Config, error) {
+	peerToClients := make(map[string][]net.IPNet)
+	routes := m.getRoutes(mesh)
+	installedRoutes := make([]lib.Route, 0)
+	peerConfigs := make([]wgtypes.PeerConfig, 0)
+	self, err := m.meshManager.GetSelf(mesh.GetMeshId())
+
+	if err != nil {
+		return nil, err
+	}
+
+	for _, n := range clients {
+		if len(peers) > 0 {
+			peer := m.getCorrespondingPeer(peers, n)
+			pubKey, _ := peer.GetPublicKey()
+			clients, ok := peerToClients[pubKey.String()]
+
+			if !ok {
+				clients = make([]net.IPNet, 0)
+				peerToClients[pubKey.String()] = clients
+			}
+
+			peerToClients[pubKey.String()] = append(clients, *n.GetWgHost())
+
+			if NodeEquals(self, peer) {
+				cfg, err := m.convertMeshNode(n, dev, peerToClients, routes)
+
+				if err != nil {
+					return nil, err
+				}
+
+				peerConfigs = append(peerConfigs, *cfg)
+			}
+		}
+	}
+
+	for _, n := range peers {
+		if NodeEquals(n, self) {
+			continue
+		}
+
+		peer, err := m.convertMeshNode(n, dev, peerToClients, routes)
+
+		if err != nil {
+			return nil, err
+		}
+
+		for _, route := range peer.AllowedIPs {
+			ula := &ip.ULABuilder{}
+			ipNet, _ := ula.GetIPNet(mesh.GetMeshId())
+
+			_, defaultRoute, _ := net.ParseCIDR("::/0")
+
+			if !ipNet.Contains(route.IP) && !ipNet.IP.Equal(defaultRoute.IP) {
+				installedRoutes = append(installedRoutes, lib.Route{
+					Gateway:     n.GetWgHost().IP,
+					Destination: route,
+				})
+			}
+		}
+
+		peerConfigs = append(peerConfigs, *peer)
+	}
+
+	cfg := wgtypes.Config{
+		Peers:        peerConfigs,
+		ReplacePeers: true,
+	}
+
+	err = m.routeInstaller.InstallRoutes(dev.Name, installedRoutes...)
+	return &cfg, err
+}
+
 func (m *WgMeshConfigApplyer) updateWgConf(mesh MeshProvider) error {
 	snap, err := mesh.GetMesh()
 
@@ -166,13 +305,19 @@ func (m *WgMeshConfigApplyer) updateWgConf(mesh MeshProvider) error {
 	}
 
 	nodes := lib.MapValues(snap.GetNodes())
-	peerConfigs := make([]wgtypes.PeerConfig, len(nodes))
+	dev, _ := mesh.GetDevice()
+
+	slices.SortFunc(nodes, func(a, b MeshNode) int {
+		return strings.Compare(string(a.GetType()), string(b.GetType()))
+	})
 
 	peers := lib.Filter(nodes, func(mn MeshNode) bool {
 		return mn.GetType() == conf.PEER_ROLE
 	})
 
-	var count int = 0
+	clients := lib.Filter(nodes, func(mn MeshNode) bool {
+		return mn.GetType() == conf.CLIENT_ROLE
+	})
 
 	self, err := m.meshManager.GetSelf(mesh.GetMeshId())
 
@@ -180,73 +325,26 @@ func (m *WgMeshConfigApplyer) updateWgConf(mesh MeshProvider) error {
 		return err
 	}
 
-	peerToClients := make(map[string][]net.IPNet)
+	var cfg *wgtypes.Config = nil
-	routes := m.getRoutes(mesh)
-	installedRoutes := make([]lib.Route, 0)
 
-	for _, n := range nodes {
+	switch self.GetType() {
-		if NodeEquals(n, self) {
+	case conf.PEER_ROLE:
-			continue
+		cfg, err = m.getPeerConfig(mesh, peers, clients, dev)
-		}
+	case conf.CLIENT_ROLE:
-
+		cfg, err = m.getClientConfig(mesh, peers, clients, dev)
-		if n.GetType() == conf.CLIENT_ROLE && len(peers) > 0 && self.GetType() == conf.CLIENT_ROLE {
-			hashFunc := func(mn MeshNode) int {
-				return lib.HashString(mn.GetWgHost().String())
-			}
-			peer := lib.ConsistentHash(peers, n, hashFunc, hashFunc)
-
-			clients, ok := peerToClients[peer.GetWgHost().String()]
-
-			if !ok {
-				clients = make([]net.IPNet, 0)
-				peerToClients[peer.GetWgHost().String()] = clients
-			}
-
-			peerToClients[peer.GetWgHost().String()] = append(clients, *n.GetWgHost())
-			continue
-		}
-
-		dev, _ := mesh.GetDevice()
-		peer, err := m.convertMeshNode(n, dev, peerToClients, routes)
-
-		if err != nil {
-			return err
-		}
-
-		for _, route := range peer.AllowedIPs {
-			ula := &ip.ULABuilder{}
-			ipNet, _ := ula.GetIPNet(mesh.GetMeshId())
-
-			if !ipNet.Contains(route.IP) {
-
-				installedRoutes = append(installedRoutes, lib.Route{
-					Gateway:     n.GetWgHost().IP,
-					Destination: route,
-				})
-			}
-		}
-
-		peerConfigs[count] = *peer
-		count++
 	}
 
-	cfg := wgtypes.Config{
-		Peers: peerConfigs,
-	}
-
-	dev, err := mesh.GetDevice()
-
 	if err != nil {
 		return err
 	}
 
-	err = m.routeInstaller.InstallRoutes(dev.Name, installedRoutes...)
+	err = m.meshManager.GetClient().ConfigureDevice(dev.Name, *cfg)
 
 	if err != nil {
 		return err
 	}
 
-	return m.meshManager.GetClient().ConfigureDevice(dev.Name, cfg)
+	return nil
 }
 
 func (m *WgMeshConfigApplyer) ApplyConfig() error {
@@ -275,8 +373,8 @@ func (m *WgMeshConfigApplyer) RemovePeers(meshId string) error {
 	}
 
 	m.meshManager.GetClient().ConfigureDevice(dev.Name, wgtypes.Config{
-		ReplacePeers: true,
 		Peers:        make([]wgtypes.PeerConfig, 0),
+		ReplacePeers: true,
 	})
 
 	return nil

pkg/mesh/manager.go

@@ -3,11 +3,11 @@ package mesh
 import (
 	"errors"
 	"fmt"
+	"sync"
 
 	"github.com/tim-beatham/wgmesh/pkg/conf"
 	"github.com/tim-beatham/wgmesh/pkg/ip"
 	"github.com/tim-beatham/wgmesh/pkg/lib"
-	logging "github.com/tim-beatham/wgmesh/pkg/log"
 	"github.com/tim-beatham/wgmesh/pkg/wg"
 	"golang.zx2c4.com/wireguard/wgctrl"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
@@ -18,7 +18,7 @@ type MeshManager interface {
 	AddMesh(params *AddMeshParams) error
 	HasChanges(meshid string) bool
 	GetMesh(meshId string) MeshProvider
-	GetPublicKey(meshId string) (*wgtypes.Key, error)
+	GetPublicKey() *wgtypes.Key
 	AddSelf(params *AddSelfParams) error
 	LeaveMesh(meshId string) error
 	GetSelf(meshId string) (MeshNode, error)
@@ -38,6 +38,7 @@ type MeshManager interface {
 }
 
 type MeshManagerImpl struct {
+	lock         sync.RWMutex
 	Meshes       map[string]MeshProvider
 	RouteManager RouteManager
 	Client       *wgctrl.Client
@@ -52,6 +53,7 @@ type MeshManagerImpl struct {
 	ipAllocator          ip.IPAllocator
 	interfaceManipulator wg.WgInterfaceManipulator
 	Monitor              MeshMonitor
+	OnDelete             func(MeshProvider)
 }
 
 // GetRouteManager implements MeshManager.
@@ -109,7 +111,7 @@ func (m *MeshManagerImpl) GetMonitor() MeshMonitor {
 // Prune implements MeshManager.
 func (m *MeshManagerImpl) Prune() error {
 	for _, mesh := range m.Meshes {
-		err := mesh.Prune(m.conf.PruneTime)
+		err := mesh.Prune()
 
 		if err != nil {
 			return err
@@ -143,13 +145,16 @@ func (m *MeshManagerImpl) CreateMesh(port int) (string, error) {
 		Conf:    m.conf,
 		Client:  m.Client,
 		MeshId:  meshId,
+		NodeID:  m.HostParameters.GetPublicKey(),
 	})
 
 	if err != nil {
 		return "", fmt.Errorf("error creating mesh: %w", err)
 	}
 
+	m.lock.Lock()
 	m.Meshes[meshId] = nodeManager
+	m.lock.Unlock()
 	return meshId, nil
 }
 
@@ -178,6 +183,7 @@ func (m *MeshManagerImpl) AddMesh(params *AddMeshParams) error {
 		Conf:    m.conf,
 		Client:  m.Client,
 		MeshId:  params.MeshId,
+		NodeID:  m.HostParameters.GetPublicKey(),
 	})
 
 	if err != nil {
@@ -190,7 +196,9 @@ func (m *MeshManagerImpl) AddMesh(params *AddMeshParams) error {
 		return err
 	}
 
+	m.lock.Lock()
 	m.Meshes[params.MeshId] = meshProvider
+	m.lock.Unlock()
 	return nil
 }
 
@@ -206,25 +214,9 @@ func (m *MeshManagerImpl) GetMesh(meshId string) MeshProvider {
 }
 
 // GetPublicKey: Gets the public key of the WireGuard mesh
-func (s *MeshManagerImpl) GetPublicKey(meshId string) (*wgtypes.Key, error) {
+func (s *MeshManagerImpl) GetPublicKey() *wgtypes.Key {
-	if s.conf.StubWg {
+	key := s.HostParameters.PrivateKey.PublicKey()
-		zeroedKey := make([]byte, wgtypes.KeyLen)
+	return &key
-		return (*wgtypes.Key)(zeroedKey), nil
-	}
-
-	mesh, ok := s.Meshes[meshId]
-
-	if !ok {
-		return nil, errors.New("mesh does not exist")
-	}
-
-	dev, err := mesh.GetDevice()
-
-	if err != nil {
-		return nil, err
-	}
-
-	return &dev.PublicKey, nil
 }
 
 type AddSelfParams struct {
@@ -289,14 +281,29 @@ func (s *MeshManagerImpl) AddSelf(params *AddSelfParams) error {
 
 // LeaveMesh leaves the mesh network
 func (s *MeshManagerImpl) LeaveMesh(meshId string) error {
-	mesh, exists := s.Meshes[meshId]
+	mesh := s.GetMesh(meshId)
 
-	if !exists {
+	if mesh == nil {
 		return fmt.Errorf("mesh %s does not exist", meshId)
 	}
 
 	var err error
 
+	s.RouteManager.RemoveRoutes(meshId)
+	err = mesh.RemoveNode(s.HostParameters.GetPublicKey())
+
+	if err != nil {
+		return err
+	}
+
+	if s.OnDelete != nil {
+		s.OnDelete(mesh)
+	}
+
+	s.lock.Lock()
+	delete(s.Meshes, meshId)
+	s.lock.Unlock()
+
 	if !s.conf.StubWg {
 		device, err := mesh.GetDevice()
 
@@ -311,8 +318,6 @@ func (s *MeshManagerImpl) LeaveMesh(meshId string) error {
 		}
 	}
 
-	err = s.RouteManager.RemoveRoutes(meshId)
-	delete(s.Meshes, meshId)
 	return err
 }
 
@@ -323,7 +328,6 @@ func (s *MeshManagerImpl) GetSelf(meshId string) (MeshNode, error) {
 		return nil, fmt.Errorf("mesh %s does not exist", meshId)
 	}
 
-	logging.Log.WriteInfof(s.HostParameters.GetPublicKey())
 	node, err := meshInstance.GetNode(s.HostParameters.GetPublicKey())
 
 	if err != nil {
@@ -348,7 +352,8 @@ func (s *MeshManagerImpl) ApplyConfig() error {
 }
 
 func (s *MeshManagerImpl) SetDescription(description string) error {
-	for _, mesh := range s.Meshes {
+	meshes := s.GetMeshes()
+	for _, mesh := range meshes {
 		if mesh.NodeExists(s.HostParameters.GetPublicKey()) {
 			err := mesh.SetDescription(s.HostParameters.GetPublicKey(), description)
 
@@ -363,7 +368,8 @@ func (s *MeshManagerImpl) SetDescription(description string) error {
 
 // SetAlias implements MeshManager.
 func (s *MeshManagerImpl) SetAlias(alias string) error {
-	for _, mesh := range s.Meshes {
+	meshes := s.GetMeshes()
+	for _, mesh := range meshes {
 		if mesh.NodeExists(s.HostParameters.GetPublicKey()) {
 			err := mesh.SetAlias(s.HostParameters.GetPublicKey(), alias)
 
@@ -377,7 +383,8 @@ func (s *MeshManagerImpl) SetAlias(alias string) error {
 
 // UpdateTimeStamp updates the timestamp of this node in all meshes
 func (s *MeshManagerImpl) UpdateTimeStamp() error {
-	for _, mesh := range s.Meshes {
+	meshes := s.GetMeshes()
+	for _, mesh := range meshes {
 		if mesh.NodeExists(s.HostParameters.GetPublicKey()) {
 			err := mesh.UpdateTimeStamp(s.HostParameters.GetPublicKey())
 
@@ -395,7 +402,16 @@ func (s *MeshManagerImpl) GetClient() *wgctrl.Client {
 }
 
 func (s *MeshManagerImpl) GetMeshes() map[string]MeshProvider {
-	return s.Meshes
+	meshes := make(map[string]MeshProvider)
+
+	s.lock.RLock()
+
+	for id, mesh := range s.Meshes {
+		meshes[id] = mesh
+	}
+
+	s.lock.RUnlock()
+	return meshes
 }
 
 // Close the mesh manager
@@ -432,6 +448,7 @@ type NewMeshManagerParams struct {
 	InterfaceManipulator wg.WgInterfaceManipulator
 	ConfigApplyer        MeshConfigApplyer
 	RouteManager         RouteManager
+	OnDelete             func(MeshProvider)
 }
 
 // Creates a new instance of a mesh manager with the given parameters
@@ -454,7 +471,7 @@ func NewMeshManager(params *NewMeshManagerParams) MeshManager {
 	m.RouteManager = params.RouteManager
 
 	if m.RouteManager == nil {
-		m.RouteManager = NewRouteManager(m)
+		m.RouteManager = NewRouteManager(m, &params.Conf)
 	}
 
 	m.idGenerator = params.IdGenerator
@@ -466,5 +483,6 @@ func NewMeshManager(params *NewMeshManagerParams) MeshManager {
 	aliasManager := NewAliasManager()
 	m.Monitor.AddUpdateCallback(aliasManager.AddAliases)
 	m.Monitor.AddRemoveCallback(aliasManager.RemoveAliases)
+	m.OnDelete = params.OnDelete
 	return m
 }

pkg/mesh/route.go

@@ -1,6 +1,9 @@
 package mesh
 
 import (
+	"net"
+
+	"github.com/tim-beatham/wgmesh/pkg/conf"
 	"github.com/tim-beatham/wgmesh/pkg/ip"
 	"github.com/tim-beatham/wgmesh/pkg/lib"
 	logging "github.com/tim-beatham/wgmesh/pkg/log"
@@ -13,6 +16,7 @@ type RouteManager interface {
 
 type RouteManagerImpl struct {
 	meshManager MeshManager
+	conf        *conf.WgMeshConfiguration
 }
 
 func (r *RouteManagerImpl) UpdateRoutes() error {
@@ -32,12 +36,23 @@ func (r *RouteManagerImpl) UpdateRoutes() error {
 			return err
 		}
 
-		routes, err := mesh1.GetRoutes(pubKey.String())
+		routeMap, err := mesh1.GetRoutes(pubKey.String())
 
 		if err != nil {
 			return err
 		}
 
+		if r.conf.AdvertiseDefaultRoute {
+			_, ipv6Default, _ := net.ParseCIDR("::/0")
+
+			mesh1.AddRoutes(NodeID(self),
+				&RouteStub{
+					Destination: ipv6Default,
+					HopCount:    0,
+					Path:        make([]string, 0),
+				})
+		}
+
 		for _, mesh2 := range meshes {
 			if mesh1 == mesh2 {
 				continue
@@ -50,7 +65,9 @@ func (r *RouteManagerImpl) UpdateRoutes() error {
 				return err
 			}
 
-			err = mesh2.AddRoutes(NodeID(self), append(lib.MapValues(routes), &RouteStub{
+			routes := lib.MapValues(routeMap)
+
+			err = mesh2.AddRoutes(NodeID(self), append(routes, &RouteStub{
 				Destination: ipNet,
 				HopCount:    0,
 				Path:        make([]string, 0),
@@ -88,6 +105,6 @@ func (r *RouteManagerImpl) RemoveRoutes(meshId string) error {
 	return nil
 }
 
-func NewRouteManager(m MeshManager) RouteManager {
+func NewRouteManager(m MeshManager, conf *conf.WgMeshConfiguration) RouteManager {
-	return &RouteManagerImpl{meshManager: m}
+	return &RouteManagerImpl{meshManager: m, conf: conf}
 }

pkg/mesh/stub_types.go

@@ -81,6 +81,16 @@ type MeshProviderStub struct {
 	snapshot *MeshSnapshotStub
 }
 
+// Mark implements MeshProvider.
+func (*MeshProviderStub) Mark(nodeId string) {
+	panic("unimplemented")
+}
+
+// RemoveNode implements MeshProvider.
+func (*MeshProviderStub) RemoveNode(nodeId string) error {
+	panic("unimplemented")
+}
+
 func (*MeshProviderStub) GetRoutes(targetId string) (map[string]Route, error) {
 	return nil, nil
 }
@@ -112,7 +122,7 @@ func (*MeshProviderStub) RemoveService(nodeId string, key string) error {
 
 // SetAlias implements MeshProvider.
 func (*MeshProviderStub) SetAlias(nodeId string, alias string) error {
-	panic("unimplemented")
+	return nil
 }
 
 // RemoveRoutes implements MeshProvider.
@@ -121,7 +131,7 @@ func (*MeshProviderStub) RemoveRoutes(nodeId string, route ...string) error {
 }
 
 // Prune implements MeshProvider.
-func (*MeshProviderStub) Prune(pruneAmount int) error {
+func (*MeshProviderStub) Prune() error {
 	return nil
 }
 
@@ -287,9 +297,9 @@ func (m *MeshManagerStub) GetMesh(meshId string) MeshProvider {
 		snapshot: &MeshSnapshotStub{nodes: make(map[string]MeshNode)}}
 }
 
-func (m *MeshManagerStub) GetPublicKey(meshId string) (*wgtypes.Key, error) {
+func (m *MeshManagerStub) GetPublicKey() *wgtypes.Key {
 	key, _ := wgtypes.GenerateKey()
-	return &key, nil
+	return &key
 }
 
 func (m *MeshManagerStub) AddSelf(params *AddSelfParams) error {

pkg/mesh/types.go

@@ -131,13 +131,18 @@ type MeshProvider interface {
 	AddService(nodeId, key, value string) error
 	// RemoveService: removes the service form the node. throws an error if the service does not exist
 	RemoveService(nodeId, key string) error
-	// Prune: prunes all nodes that have not updated their timestamp in
+	// Prune: prunes all nodes that have not updated their
-	// pruneAmount seconds
+	// vector clock
-	Prune(pruneAmount int) error
+	Prune() error
 	// GetPeers: get a list of contactable peers
 	GetPeers() []string
 	// GetRoutes(): Get all unique routes. Where the route with the least hop count is chosen
 	GetRoutes(targetNode string) (map[string]Route, error)
+	// RemoveNode(): remove the node from the mesh
+	RemoveNode(nodeId string) error
+	// Mark: marks the node as unreachable. This is not broadcast to the entire
+	// this is not considered when syncing node state
+	Mark(nodeId string)
 }
 
 // HostParameters contains the IDs of a node
@@ -157,6 +162,7 @@ type MeshProviderFactoryParams struct {
 	Port    int
 	Conf    *conf.WgMeshConfiguration
 	Client  *wgctrl.Client
+	NodeID  string
 }
 
 // MeshProviderFactory creates an instance of a mesh provider

pkg/route/route.go

@@ -19,7 +19,11 @@ func (r *RouteInstallerImpl) InstallRoutes(devName string, routes ...lib.Route)
 		return err
 	}
 
-	err = rtnl.DeleteRoutes(devName, unix.AF_INET6, routes...)
+	ip6Routes := lib.Filter(routes, func(r lib.Route) bool {
+		return r.Destination.IP.To4() == nil
+	})
+
+	err = rtnl.DeleteRoutes(devName, unix.AF_INET6, ip6Routes...)
 
 	if err != nil {
 		return err

pkg/sync/syncer.go

@@ -1,8 +1,8 @@
 package sync
 
 import (
+	"io"
 	"math/rand"
-	"sync"
 	"time"
 
 	"github.com/tim-beatham/wgmesh/pkg/conf"
@@ -12,7 +12,7 @@ import (
 	"github.com/tim-beatham/wgmesh/pkg/mesh"
 )
 
-// Syncer: picks random nodes from the mesh
+// Syncer: picks random nodes from the meshs
 type Syncer interface {
 	Sync(meshId string) error
 	SyncMeshes() error
@@ -25,83 +25,95 @@ type SyncerImpl struct {
 	syncCount      int
 	cluster        conn.ConnCluster
 	conf           *conf.WgMeshConfiguration
+	lastSync       uint64
 }
 
 // Sync: Sync random nodes
 func (s *SyncerImpl) Sync(meshId string) error {
-	if !s.manager.HasChanges(meshId) && s.infectionCount == 0 {
-		logging.Log.WriteInfof("No changes for %s", meshId)
-		return nil
-	}
-
-	logging.Log.WriteInfof("UPDATING WG CONF")
-
-	if s.manager.HasChanges(meshId) {
-		err := s.manager.ApplyConfig()
-
-		if err != nil {
-			logging.Log.WriteInfof("Failed to update config %w", err)
-		}
-	}
-
-	nodeNames := s.manager.GetMesh(meshId).GetPeers()
 	self, err := s.manager.GetSelf(meshId)
 
 	if err != nil {
 		return err
 	}
 
-	selfPublickey, err := self.GetPublicKey()
+	s.manager.GetMesh(meshId).Prune()
 
-	if err != nil {
+	if self.GetType() == conf.PEER_ROLE && !s.manager.HasChanges(meshId) && s.infectionCount == 0 {
-		return err
+		logging.Log.WriteInfof("No changes for %s", meshId)
-	}
+		return nil
-
-	neighbours := s.cluster.GetNeighbours(nodeNames, selfPublickey.String())
-	randomSubset := lib.RandomSubsetOfLength(neighbours, s.conf.BranchRate)
-
-	for _, node := range randomSubset {
-		logging.Log.WriteInfof("Random node: %s", node)
 	}
 
 	before := time.Now()
+	s.manager.GetRouteManager().UpdateRoutes()
 
-	if len(nodeNames) > s.conf.ClusterSize && rand.Float64() < s.conf.InterClusterChance {
+	publicKey := s.manager.GetPublicKey()
-		logging.Log.WriteInfof("Sending to random cluster")
+
-		interCluster := s.cluster.GetInterCluster(nodeNames, selfPublickey.String())
+	logging.Log.WriteInfof(publicKey.String())
-		randomSubset = append(randomSubset, interCluster)
+
+	nodeNames := s.manager.GetMesh(meshId).GetPeers()
+
+	var gossipNodes []string
+
+	// Clients always pings its peer for configuration
+	if self.GetType() == conf.CLIENT_ROLE {
+		keyFunc := lib.HashString
+		bucketFunc := lib.HashString
+
+		neighbour := lib.ConsistentHash(nodeNames, publicKey.String(), keyFunc, bucketFunc)
+		gossipNodes = make([]string, 1)
+		gossipNodes[0] = neighbour
+	} else {
+		neighbours := s.cluster.GetNeighbours(nodeNames, publicKey.String())
+		gossipNodes = lib.RandomSubsetOfLength(neighbours, s.conf.BranchRate)
+
+		if len(nodeNames) > s.conf.ClusterSize && rand.Float64() < s.conf.InterClusterChance {
+			gossipNodes[len(gossipNodes)-1] = s.cluster.GetInterCluster(nodeNames, publicKey.String())
+		}
 	}
 
-	var waitGroup sync.WaitGroup
+	var succeeded bool = false
 
-	for index := range randomSubset {
+	// Do this synchronously to conserve bandwidth
-		waitGroup.Add(1)
+	for _, node := range gossipNodes {
+		correspondingPeer := s.manager.GetNode(meshId, node)
 
-		go func(i int) error {
+		if correspondingPeer == nil {
-			defer waitGroup.Done()
+			logging.Log.WriteErrorf("node %s does not exist", node)
+		}
 
-			correspondingPeer := s.manager.GetNode(meshId, randomSubset[i])
+		err := s.requester.SyncMesh(meshId, correspondingPeer)
 
-			if correspondingPeer == nil {
+		if err == nil || err == io.EOF {
-				logging.Log.WriteErrorf("node %s does not exist", randomSubset[i])
+			succeeded = true
-			}
+		} else {
-
+			// If the synchronisation operation has failed them mark a gravestone
-			err := s.requester.SyncMesh(meshId, correspondingPeer.GetHostEndpoint())
+			// preventing the peer from being re-contacted until it has updated
-			return err
+			// itself
-		}(index)
+			s.manager.GetMesh(meshId).Mark(node)
+		}
 	}
 
-	waitGroup.Wait()
-
 	s.syncCount++
 	logging.Log.WriteInfof("SYNC TIME: %v", time.Since(before))
 	logging.Log.WriteInfof("SYNC COUNT: %d", s.syncCount)
 
 	s.infectionCount = ((s.conf.InfectionCount + s.infectionCount - 1) % s.conf.InfectionCount)
 
-	// Check if any changes have occurred and trigger callbacks
+	if !succeeded {
-	// if changes have occurred.
+		// If could not gossip with anyone then repeat.
-	// return s.manager.GetMonitor().Trigger()
+		s.infectionCount++
+	}
+
+	s.manager.GetMesh(meshId).SaveChanges()
+	s.lastSync = uint64(time.Now().Unix())
+
+	logging.Log.WriteInfof("UPDATING WG CONF")
+	err = s.manager.ApplyConfig()
+
+	if err != nil {
+		logging.Log.WriteInfof("Failed to update config %w", err)
+	}
+
 	return nil
 }
 
@@ -111,7 +123,7 @@ func (s *SyncerImpl) SyncMeshes() error {
 		err := s.Sync(meshId)
 
 		if err != nil {
-			return err
+			logging.Log.WriteErrorf(err.Error())
 		}
 	}
 

pkg/sync/syncererror.go

@@ -17,31 +17,20 @@ type SyncErrorHandlerImpl struct {
 	meshManager mesh.MeshManager
 }
 
-func (s *SyncErrorHandlerImpl) incrementFailedCount(meshId string, endpoint string) bool {
+func (s *SyncErrorHandlerImpl) handleFailed(meshId string, nodeId string) bool {
 	mesh := s.meshManager.GetMesh(meshId)
-
+	mesh.Mark(nodeId)
-	if mesh == nil {
-		return false
-	}
-
-	// self, err := s.meshManager.GetSelf(meshId)
-
-	// if err != nil {
-	// return false
-	// }
-
-	// mesh.DecrementHealth(endpoint, self.GetHostEndpoint())
 	return true
 }
 
-func (s *SyncErrorHandlerImpl) Handle(meshId string, endpoint string, err error) bool {
+func (s *SyncErrorHandlerImpl) Handle(meshId string, nodeId string, err error) bool {
 	errStatus, _ := status.FromError(err)
 
 	logging.Log.WriteInfof("Handled gRPC error: %s", errStatus.Message())
 
 	switch errStatus.Code() {
 	case codes.Unavailable, codes.Unknown, codes.DeadlineExceeded, codes.Internal, codes.NotFound:
-		return s.incrementFailedCount(meshId, endpoint)
+		return s.handleFailed(meshId, nodeId)
 	}
 
 	return false

pkg/sync/syncrequester.go

@@ -15,7 +15,7 @@ import (
 // SyncRequester: coordinates the syncing of meshes
 type SyncRequester interface {
 	GetMesh(meshId string, ifName string, port int, endPoint string) error
-	SyncMesh(meshid string, endPoint string) error
+	SyncMesh(meshid string, meshNode mesh.MeshNode) error
 }
 
 type SyncRequesterImpl struct {
@@ -56,8 +56,8 @@ func (s *SyncRequesterImpl) GetMesh(meshId string, ifName string, port int, endP
 	return err
 }
 
-func (s *SyncRequesterImpl) handleErr(meshId, endpoint string, err error) error {
+func (s *SyncRequesterImpl) handleErr(meshId, pubKey string, err error) error {
-	ok := s.errorHdlr.Handle(meshId, endpoint, err)
+	ok := s.errorHdlr.Handle(meshId, pubKey, err)
 
 	if ok {
 		return nil
@@ -67,7 +67,10 @@ func (s *SyncRequesterImpl) handleErr(meshId, endpoint string, err error) error
 }
 
 // SyncMesh: Proactively send a sync request to the other mesh
-func (s *SyncRequesterImpl) SyncMesh(meshId, endpoint string) error {
+func (s *SyncRequesterImpl) SyncMesh(meshId string, meshNode mesh.MeshNode) error {
+	endpoint := meshNode.GetHostEndpoint()
+	pubKey, _ := meshNode.GetPublicKey()
+
 	peerConnection, err := s.server.ConnectionManager.GetConnection(endpoint)
 
 	if err != nil {
@@ -96,7 +99,7 @@ func (s *SyncRequesterImpl) SyncMesh(meshId, endpoint string) error {
 	err = s.syncMesh(mesh, ctx, c)
 
 	if err != nil {
-		return s.handleErr(meshId, endpoint, err)
+		return s.handleErr(meshId, pubKey.String(), err)
 	}
 
 	logging.Log.WriteInfof("Synced with node: %s meshId: %s\n", endpoint, meshId)

pkg/sync/syncscheduler.go

@@ -8,11 +8,11 @@ import (
 // Run implements SyncScheduler.
 func syncFunction(syncer Syncer) lib.TimerFunc {
 	return func() error {
-		return syncer.SyncMeshes()
+		syncer.SyncMeshes()
+		return nil
 	}
 }
 
-func NewSyncScheduler(s *ctrlserver.MeshCtrlServer, syncRequester SyncRequester) *lib.Timer {
+func NewSyncScheduler(s *ctrlserver.MeshCtrlServer, syncRequester SyncRequester, syncer Syncer) *lib.Timer {
-	syncer := NewSyncer(s.MeshManager, s.Conf, syncRequester)
 	return lib.NewTimer(syncFunction(syncer), int(s.Conf.SyncRate))
 }

pkg/timers/timers.go

@@ -3,20 +3,14 @@ package timer
 import (
 	"github.com/tim-beatham/wgmesh/pkg/ctrlserver"
 	"github.com/tim-beatham/wgmesh/pkg/lib"
+	logging "github.com/tim-beatham/wgmesh/pkg/log"
 )
 
 func NewTimestampScheduler(ctrlServer *ctrlserver.MeshCtrlServer) lib.Timer {
 	timerFunc := func() error {
+		logging.Log.WriteInfof("Updated Timestamp")
 		return ctrlServer.MeshManager.UpdateTimeStamp()
 	}
 
 	return *lib.NewTimer(timerFunc, ctrlServer.Conf.KeepAliveTime)
 }
-
-func NewRouteScheduler(ctrlServer *ctrlserver.MeshCtrlServer) lib.Timer {
-	timerFunc := func() error {
-		return ctrlServer.MeshManager.GetRouteManager().UpdateRoutes()
-	}
-
-	return *lib.NewTimer(timerFunc, 10)
-}

pkg/wg/wg.go

@@ -3,7 +3,6 @@ package wg
 import (
 	"crypto"
 	"crypto/rand"
-	"encoding/base64"
 	"fmt"
 
 	"github.com/tim-beatham/wgmesh/pkg/lib"
@@ -35,8 +34,7 @@ func (m *WgInterfaceManipulatorImpl) CreateInterface(port int, privKey *wgtypes.
 	}
 
 	md5 := crypto.MD5.New().Sum(randomBuf)
-
+	md5Str := fmt.Sprintf("wg%x", md5)[:hashLength]
-	md5Str := fmt.Sprintf("wg%s", base64.StdEncoding.EncodeToString(md5)[:hashLength])
 
 	err = rtnl.CreateLink(md5Str)