Bidirectional syncing

pkg/automerge/automerge.go

@@ -251,6 +251,10 @@ func (m *CrdtNodeManager) updateWgConf(devName string, nodes map[string]MeshNode
 	return nil
 }
 
+func (m *CrdtNodeManager) GetSyncer() *AutomergeSync {
+	return NewAutomergeSync(m)
+}
+
 func (n *MeshNodeCrdt) GetEscapedIP() string {
 	return fmt.Sprintf("\"%s\"", n.WgHost)
 }

pkg/automerge/automerge_sync.go

@@ -0,0 +1,33 @@
+package crdt
+
+import (
+	"github.com/automerge/automerge-go"
+)
+
+type AutomergeSync struct {
+	state *automerge.SyncState
+}
+
+func (a *AutomergeSync) GenerateMessage() ([]byte, bool) {
+	msg, valid := a.state.GenerateMessage()
+
+	if !valid {
+		return nil, false
+	}
+
+	return msg.Bytes(), true
+}
+
+func (a *AutomergeSync) RecvMessage(msg []byte) error {
+	_, err := a.state.ReceiveMessage(msg)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func NewAutomergeSync(manager *CrdtNodeManager) *AutomergeSync {
+	return &AutomergeSync{state: automerge.NewSyncState(manager.doc)}
+}

pkg/gossip/gossip_requester.go

@@ -1,28 +0,0 @@
-package gossip
-
-import (
-	"github.com/tim-beatham/wgmesh/pkg/ctrlserver"
-	"github.com/tim-beatham/wgmesh/pkg/ip"
-	"github.com/tim-beatham/wgmesh/pkg/ipc"
-)
-
-type GossipRequester struct {
-	Server      *ctrlserver.MeshCtrlServer
-	ipAlloactor ip.IPAllocator
-}
-
-func (r *GossipRequester) CreateMesh(name string, reply *string) error {
-	return nil
-}
-
-func (r *GossipRequester) ListMeshes(name string, reply string) error {
-	return nil
-}
-
-func (r *GossipRequester) JoinMesh(args ipc.JoinMeshArgs, reply *string) error {
-	return nil
-}
-
-func (r *GossipRequester) GetMesh(meshId string, reply *ipc.GetMeshReply) error {
-	return nil
-}

pkg/gossip/gossip_responser.go

@@ -1 +0,0 @@
-package gossip

pkg/rpc/syncservice.pb.go

@@ -175,6 +175,7 @@ type SyncMeshReply struct {
 	unknownFields protoimpl.UnknownFields
 
 	Success bool   `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"`
+	Changes []byte `protobuf:"bytes,2,opt,name=changes,proto3" json:"changes,omitempty"`
 }
 
 func (x *SyncMeshReply) Reset() {
@@ -216,6 +217,13 @@ func (x *SyncMeshReply) GetSuccess() bool {
 	return false
 }
 
+func (x *SyncMeshReply) GetChanges() []byte {
+	if x != nil {
+		return x.Changes
+	}
+	return nil
+}
+
 var File_pkg_grpc_ctrlserver_syncservice_proto protoreflect.FileDescriptor
 
 var file_pkg_grpc_ctrlserver_syncservice_proto_rawDesc = []byte{
@@ -231,21 +239,22 @@ var file_pkg_grpc_ctrlserver_syncservice_proto_rawDesc = []byte{
 	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x73, 0x68, 0x49, 0x64, 0x18,
 	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6d, 0x65, 0x73, 0x68, 0x49, 0x64, 0x12, 0x18, 0x0a,
 	0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07,
-	0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22, 0x29, 0x0a, 0x0d, 0x53, 0x79, 0x6e, 0x63, 0x4d,
+	0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22, 0x43, 0x0a, 0x0d, 0x53, 0x79, 0x6e, 0x63, 0x4d,
 	0x65, 0x73, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63,
 	0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65,
-	0x73, 0x73, 0x32, 0x9a, 0x01, 0x0a, 0x0b, 0x53, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69,
+	0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x02, 0x20,
-	0x63, 0x65, 0x12, 0x43, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x1b, 0x2e,
+	0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x32, 0x9e, 0x01, 0x0a,
-	0x73, 0x79, 0x6e, 0x63, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x43,
+	0x0b, 0x53, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x07,
-	0x6f, 0x6e, 0x66, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x73, 0x79, 0x6e,
+	0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x1b, 0x2e, 0x73, 0x79, 0x6e, 0x63, 0x73, 0x65,
-	0x63, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66,
+	0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x65, 0x71,
-	0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x46, 0x0a, 0x08, 0x53, 0x79, 0x6e, 0x63, 0x4d,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x73, 0x79, 0x6e, 0x63, 0x73, 0x65, 0x72, 0x76, 0x69,
-	0x65, 0x73, 0x68, 0x12, 0x1c, 0x2e, 0x73, 0x79, 0x6e, 0x63, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
+	0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22,
-	0x65, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x65, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x00, 0x12, 0x4a, 0x0a, 0x08, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x1c, 0x2e,
-	0x74, 0x1a, 0x1a, 0x2e, 0x73, 0x79, 0x6e, 0x63, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e,
+	0x73, 0x79, 0x6e, 0x63, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x53, 0x79, 0x6e, 0x63,
-	0x53, 0x79, 0x6e, 0x63, 0x4d, 0x65, 0x73, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42,
+	0x4d, 0x65, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x73, 0x79,
-	0x09, 0x5a, 0x07, 0x70, 0x6b, 0x67, 0x2f, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
+	0x6e, 0x63, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x65,
-	0x6f, 0x33,
+	0x73, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x09, 0x5a,
+	0x07, 0x70, 0x6b, 0x67, 0x2f, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

pkg/rpc/syncservice_grpc.pb.go

@@ -23,7 +23,7 @@ const _ = grpc.SupportPackageIsVersion7
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
 type SyncServiceClient interface {
 	GetConf(ctx context.Context, in *GetConfRequest, opts ...grpc.CallOption) (*GetConfReply, error)
-	SyncMesh(ctx context.Context, in *SyncMeshRequest, opts ...grpc.CallOption) (*SyncMeshReply, error)
+	SyncMesh(ctx context.Context, opts ...grpc.CallOption) (SyncService_SyncMeshClient, error)
 }
 
 type syncServiceClient struct {
@@ -43,13 +43,35 @@ func (c *syncServiceClient) GetConf(ctx context.Context, in *GetConfRequest, opt
 	return out, nil
 }
 
-func (c *syncServiceClient) SyncMesh(ctx context.Context, in *SyncMeshRequest, opts ...grpc.CallOption) (*SyncMeshReply, error) {
+func (c *syncServiceClient) SyncMesh(ctx context.Context, opts ...grpc.CallOption) (SyncService_SyncMeshClient, error) {
-	out := new(SyncMeshReply)
+	stream, err := c.cc.NewStream(ctx, &SyncService_ServiceDesc.Streams[0], "/syncservice.SyncService/SyncMesh", opts...)
-	err := c.cc.Invoke(ctx, "/syncservice.SyncService/SyncMesh", in, out, opts...)
 	if err != nil {
 		return nil, err
 	}
-	return out, nil
+	x := &syncServiceSyncMeshClient{stream}
+	return x, nil
+}
+
+type SyncService_SyncMeshClient interface {
+	Send(*SyncMeshRequest) error
+	Recv() (*SyncMeshReply, error)
+	grpc.ClientStream
+}
+
+type syncServiceSyncMeshClient struct {
+	grpc.ClientStream
+}
+
+func (x *syncServiceSyncMeshClient) Send(m *SyncMeshRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *syncServiceSyncMeshClient) Recv() (*SyncMeshReply, error) {
+	m := new(SyncMeshReply)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
 }
 
 // SyncServiceServer is the server API for SyncService service.
@@ -57,7 +79,7 @@ func (c *syncServiceClient) SyncMesh(ctx context.Context, in *SyncMeshRequest, o
 // for forward compatibility
 type SyncServiceServer interface {
 	GetConf(context.Context, *GetConfRequest) (*GetConfReply, error)
-	SyncMesh(context.Context, *SyncMeshRequest) (*SyncMeshReply, error)
+	SyncMesh(SyncService_SyncMeshServer) error
 	mustEmbedUnimplementedSyncServiceServer()
 }
 
@@ -68,8 +90,8 @@ type UnimplementedSyncServiceServer struct {
 func (UnimplementedSyncServiceServer) GetConf(context.Context, *GetConfRequest) (*GetConfReply, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GetConf not implemented")
 }
-func (UnimplementedSyncServiceServer) SyncMesh(context.Context, *SyncMeshRequest) (*SyncMeshReply, error) {
+func (UnimplementedSyncServiceServer) SyncMesh(SyncService_SyncMeshServer) error {
-	return nil, status.Errorf(codes.Unimplemented, "method SyncMesh not implemented")
+	return status.Errorf(codes.Unimplemented, "method SyncMesh not implemented")
 }
 func (UnimplementedSyncServiceServer) mustEmbedUnimplementedSyncServiceServer() {}
 
@@ -102,22 +124,30 @@ func _SyncService_GetConf_Handler(srv interface{}, ctx context.Context, dec func
 	return interceptor(ctx, in, info, handler)
 }
 
-func _SyncService_SyncMesh_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func _SyncService_SyncMesh_Handler(srv interface{}, stream grpc.ServerStream) error {
-	in := new(SyncMeshRequest)
+	return srv.(SyncServiceServer).SyncMesh(&syncServiceSyncMeshServer{stream})
-	if err := dec(in); err != nil {
+}
+
+type SyncService_SyncMeshServer interface {
+	Send(*SyncMeshReply) error
+	Recv() (*SyncMeshRequest, error)
+	grpc.ServerStream
+}
+
+type syncServiceSyncMeshServer struct {
+	grpc.ServerStream
+}
+
+func (x *syncServiceSyncMeshServer) Send(m *SyncMeshReply) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *syncServiceSyncMeshServer) Recv() (*SyncMeshRequest, error) {
+	m := new(SyncMeshRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
 		return nil, err
 	}
-	if interceptor == nil {
+	return m, nil
-		return srv.(SyncServiceServer).SyncMesh(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/syncservice.SyncService/SyncMesh",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(SyncServiceServer).SyncMesh(ctx, req.(*SyncMeshRequest))
-	}
-	return interceptor(ctx, in, info, handler)
 }
 
 // SyncService_ServiceDesc is the grpc.ServiceDesc for SyncService service.
@@ -131,11 +161,14 @@ var SyncService_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "GetConf",
 			Handler:    _SyncService_GetConf_Handler,
 		},
+	},
+	Streams: []grpc.StreamDesc{
 		{
-			MethodName: "SyncMesh",
+			StreamName:    "SyncMesh",
 			Handler:       _SyncService_SyncMesh_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
 		},
 	},
-	Streams:  []grpc.StreamDesc{},
 	Metadata: "pkg/grpc/ctrlserver/syncservice.proto",
 }

pkg/sync/syncer.go

@@ -2,6 +2,7 @@ package sync
 
 import (
 	"errors"
+	"sync"
 
 	crdt "github.com/tim-beatham/wgmesh/pkg/automerge"
 	"github.com/tim-beatham/wgmesh/pkg/lib"
@@ -54,14 +55,21 @@ func (s *SyncerImpl) Sync(meshId string) error {
 	meshNodes := lib.MapValuesWithExclude(snapshot.Nodes, excludedNodes)
 	randomSubset := lib.RandomSubsetOfLength(meshNodes, subSetLength)
 
-	for _, n := range randomSubset {
+	var waitGroup sync.WaitGroup
-		err := s.requester.SyncMesh(meshId, n.HostEndpoint)
 
-		if err != nil {
+	for _, n := range randomSubset {
+		waitGroup.Add(1)
+
+		syncMeshFunc := func() error {
+			defer waitGroup.Done()
+			err := s.requester.SyncMesh(meshId, n.HostEndpoint)
 			return err
 		}
+
+		go syncMeshFunc()
 	}
 
+	waitGroup.Wait()
 	return nil
 }
 

pkg/sync/syncrequester.go

@@ -3,8 +3,10 @@ package sync
 import (
 	"context"
 	"errors"
+	"io"
 	"time"
 
+	crdt "github.com/tim-beatham/wgmesh/pkg/automerge"
 	"github.com/tim-beatham/wgmesh/pkg/ctrlserver"
 	logging "github.com/tim-beatham/wgmesh/pkg/log"
 	"github.com/tim-beatham/wgmesh/pkg/rpc"
@@ -123,17 +125,12 @@ func (s *SyncRequesterImpl) SyncMesh(meshId, endpoint string) error {
 		return errors.New("mesh does not exist")
 	}
 
-	syncMeshRequest := rpc.SyncMeshRequest{
-		MeshId:  meshId,
-		Changes: mesh.SaveChanges(),
-	}
-
 	c := rpc.NewSyncServiceClient(client)
 
-	ctx, cancel := context.WithTimeout(authContext, time.Second)
+	ctx, cancel := context.WithTimeout(authContext, 10*time.Second)
 	defer cancel()
 
-	_, err = c.SyncMesh(ctx, &syncMeshRequest)
+	err = syncMesh(mesh, ctx, c)
 
 	if err != nil {
 		return s.handleErr(meshId, endpoint, err)
@@ -144,6 +141,50 @@ func (s *SyncRequesterImpl) SyncMesh(meshId, endpoint string) error {
 	return nil
 }
 
+func syncMesh(mesh *crdt.CrdtNodeManager, ctx context.Context, client rpc.SyncServiceClient) error {
+	stream, err := client.SyncMesh(ctx)
+
+	syncer := mesh.GetSyncer()
+
+	if err != nil {
+		return err
+	}
+
+	for {
+		msg, moreMessages := syncer.GenerateMessage()
+
+		err := stream.Send(&rpc.SyncMeshRequest{MeshId: mesh.MeshId, Changes: msg})
+
+		if err != nil {
+			return err
+		}
+
+		in, err := stream.Recv()
+
+		if err != nil && err != io.EOF {
+			logging.ErrorLog.Printf("Stream recv error: %s\n", err.Error())
+			return err
+		}
+
+		if err != io.EOF && len(in.Changes) != 0 {
+			err = syncer.RecvMessage(in.Changes)
+		}
+
+		if err != nil {
+			logging.ErrorLog.Printf("Syncer recv error: %s\n", err.Error())
+			return err
+		}
+
+		if !moreMessages {
+			break
+		}
+	}
+
+	logging.InfoLog.Println("SYNC finished")
+	stream.CloseSend()
+	return nil
+}
+
 func NewSyncRequester(s *ctrlserver.MeshCtrlServer) SyncRequester {
 	errorHdlr := NewSyncErrorHandler(s.MeshManager)
 	return &SyncRequesterImpl{server: s, errorHdlr: errorHdlr}

pkg/sync/syncservice.go

@@ -4,8 +4,11 @@ package sync
 import (
 	"context"
 	"errors"
+	"io"
 
+	crdt "github.com/tim-beatham/wgmesh/pkg/automerge"
 	"github.com/tim-beatham/wgmesh/pkg/ctrlserver"
+	logging "github.com/tim-beatham/wgmesh/pkg/log"
 	"github.com/tim-beatham/wgmesh/pkg/rpc"
 )
 
@@ -32,22 +35,60 @@ func (s *SyncServiceImpl) GetConf(context context.Context, request *rpc.GetConfR
 }
 
 // Sync: Pings a node and syncs the mesh configuration with the other node
-func (s *SyncServiceImpl) SyncMesh(conext context.Context, request *rpc.SyncMeshRequest) (*rpc.SyncMeshReply, error) {
+// SyncMesh: syncs the two streams changes
-	mesh := s.Server.MeshManager.GetMesh(request.MeshId)
+func (s *SyncServiceImpl) SyncMesh(stream rpc.SyncService_SyncMeshServer) error {
+	var meshId = ""
+	var syncer *crdt.AutomergeSync = nil
 
-	if mesh == nil {
+	for {
-		return nil, errors.New("mesh does not exist")
+		logging.InfoLog.Println("Received Attempt")
+		in, err := stream.Recv()
+		logging.InfoLog.Println("Received Worked")
+
+		if err == io.EOF {
+			return nil
 		}
 
-	err := s.Server.MeshManager.UpdateMesh(request.MeshId, request.Changes)
-
 		if err != nil {
-		return nil, err
+			return err
 		}
 
-	return &rpc.SyncMeshReply{Success: true}, nil
+		if len(meshId) == 0 {
-}
+			meshId = in.MeshId
 
+			mesh := s.Server.MeshManager.GetMesh(meshId)
+
+			if mesh == nil {
+				return errors.New("mesh does not exist")
+			}
+
+			syncer = mesh.GetSyncer()
+		} else if meshId != in.MeshId {
+			return errors.New("Differing MeshIDs")
+		}
+
+		if syncer == nil {
+			return errors.New("Syncer should not be nil")
+		}
+
+		msg, moreMessages := syncer.GenerateMessage()
+
+		if err = stream.Send(&rpc.SyncMeshReply{Success: true, Changes: msg}); err != nil {
+			return err
+		}
+
+		if len(in.Changes) != 0 {
+			if err = syncer.RecvMessage(in.Changes); err != nil {
+				return err
+			}
+		}
+
+		if !moreMessages || err == io.EOF {
+			logging.InfoLog.Println("SYNC Completed")
+			return nil
+		}
+	}
+}
 func NewSyncService(server *ctrlserver.MeshCtrlServer) *SyncServiceImpl {
 	return &SyncServiceImpl{Server: server}
 }