holm/smegmesh
1
0
Fork 0
forked from extern/smegmesh
Code Pull Requests Activity

Bidirectional syncing

Browse Source
This commit is contained in:
Tim Beatham 2023-10-23 18:13:08 +01:00
parent 360f9d3c54
commit ef2b57047d
9 changed files with 231 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/automerge/automerge.go
View File

@ -251,6 +251,10 @@ func (m *CrdtNodeManager) updateWgConf(devName string, nodes map[string]MeshNode
return nil
}
func (m *CrdtNodeManager) GetSyncer() *AutomergeSync {
return NewAutomergeSync(m)
}
func (n *MeshNodeCrdt) GetEscapedIP() string {
return fmt.Sprintf("\"%s\"", n.WgHost)
}

33
pkg/automerge/automerge_sync.go Normal file
View File

@ -0,0 +1,33 @@
package crdt
import (
"github.com/automerge/automerge-go"
)
type AutomergeSync struct {
state *automerge.SyncState
}
func (a *AutomergeSync) GenerateMessage() ([]byte, bool) {
msg, valid := a.state.GenerateMessage()
if !valid {
return nil, false
}
return msg.Bytes(), true
}
func (a *AutomergeSync) RecvMessage(msg []byte) error {
_, err := a.state.ReceiveMessage(msg)
if err != nil {
return err
}
return nil
}
func NewAutomergeSync(manager *CrdtNodeManager) *AutomergeSync {
return &AutomergeSync{state: automerge.NewSyncState(manager.doc)}
}

28
pkg/gossip/gossip_requester.go
View File

@ -1,28 +0,0 @@
package gossip
import (
"github.com/tim-beatham/wgmesh/pkg/ctrlserver"
"github.com/tim-beatham/wgmesh/pkg/ip"
"github.com/tim-beatham/wgmesh/pkg/ipc"
)
type GossipRequester struct {
Server *ctrlserver.MeshCtrlServer
ipAlloactor ip.IPAllocator
}
func (r *GossipRequester) CreateMesh(name string, reply *string) error {
return nil
}
func (r *GossipRequester) ListMeshes(name string, reply string) error {
return nil
}
func (r *GossipRequester) JoinMesh(args ipc.JoinMeshArgs, reply *string) error {
return nil
}
func (r *GossipRequester) GetMesh(meshId string, reply *ipc.GetMeshReply) error {
return nil
}

1
pkg/gossip/gossip_responser.go
View File

@ -1 +0,0 @@
package gossip

37
pkg/rpc/syncservice.pb.go
View File

@ -174,7 +174,8 @@ type SyncMeshReply struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"`
Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"`
Changes []byte `protobuf:"bytes,2,opt,name=changes,proto3" json:"changes,omitempty"`
}
func (x *SyncMeshReply) Reset() {
@ -216,6 +217,13 @@ func (x *SyncMeshReply) GetSuccess() bool {
return false
}
func (x *SyncMeshReply) GetChanges() []byte {
if x != nil {
return x.Changes
}
return nil
}
var File_pkg_grpc_ctrlserver_syncservice_proto protoreflect.FileDescriptor
var file_pkg_grpc_ctrlserver_syncservice_proto_rawDesc = []byte{
@ -231,21 +239,22 @@ var file_pkg_grpc_ctrlserver_syncservice_proto_rawDesc = []byte{
0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x73, 0x68, 0x49, 0x64, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6d, 0x65, 0x73, 0x68, 0x49, 0x64, 0x12, 0x18, 0x0a,
0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07,
0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22, 0x29, 0x0a, 0x0d, 0x53, 0x79, 0x6e, 0x63, 0x4d,
0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22, 0x43, 0x0a, 0x0d, 0x53, 0x79, 0x6e, 0x63, 0x4d,
0x65, 0x73, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63,
0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65,
0x73, 0x73, 0x32, 0x9a, 0x01, 0x0a, 0x0b, 0x53, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x12, 0x43, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x1b, 0x2e,
0x73, 0x79, 0x6e, 0x63, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x43,
0x6f, 0x6e, 0x66, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x73, 0x79, 0x6e,
0x63, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66,
0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x46, 0x0a, 0x08, 0x53, 0x79, 0x6e, 0x63, 0x4d,
0x65, 0x73, 0x68, 0x12, 0x1c, 0x2e, 0x73, 0x79, 0x6e, 0x63, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
0x65, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x65, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x1a, 0x2e, 0x73, 0x79, 0x6e, 0x63, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e,
0x53, 0x79, 0x6e, 0x63, 0x4d, 0x65, 0x73, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42,
0x09, 0x5a, 0x07, 0x70, 0x6b, 0x67, 0x2f, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x02, 0x20,
0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x32, 0x9e, 0x01, 0x0a,
0x0b, 0x53, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x07,
0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x1b, 0x2e, 0x73, 0x79, 0x6e, 0x63, 0x73, 0x65,
0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x73, 0x79, 0x6e, 0x63, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22,
0x00, 0x12, 0x4a, 0x0a, 0x08, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x1c, 0x2e,
0x73, 0x79, 0x6e, 0x63, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x53, 0x79, 0x6e, 0x63,
0x4d, 0x65, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x73, 0x79,
0x6e, 0x63, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x65,
0x73, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x09, 0x5a,
0x07, 0x70, 0x6b, 0x67, 0x2f, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (

83
pkg/rpc/syncservice_grpc.pb.go
View File

@ -23,7 +23,7 @@ const _ = grpc.SupportPackageIsVersion7
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type SyncServiceClient interface {
GetConf(ctx context.Context, in *GetConfRequest, opts ...grpc.CallOption) (*GetConfReply, error)
SyncMesh(ctx context.Context, in *SyncMeshRequest, opts ...grpc.CallOption) (*SyncMeshReply, error)
SyncMesh(ctx context.Context, opts ...grpc.CallOption) (SyncService_SyncMeshClient, error)
}
type syncServiceClient struct {
@ -43,13 +43,35 @@ func (c *syncServiceClient) GetConf(ctx context.Context, in *GetConfRequest, opt
return out, nil
}
func (c *syncServiceClient) SyncMesh(ctx context.Context, in *SyncMeshRequest, opts ...grpc.CallOption) (*SyncMeshReply, error) {
out := new(SyncMeshReply)
err := c.cc.Invoke(ctx, "/syncservice.SyncService/SyncMesh", in, out, opts...)
func (c *syncServiceClient) SyncMesh(ctx context.Context, opts ...grpc.CallOption) (SyncService_SyncMeshClient, error) {
stream, err := c.cc.NewStream(ctx, &SyncService_ServiceDesc.Streams[0], "/syncservice.SyncService/SyncMesh", opts...)
if err != nil {
return nil, err
}
return out, nil
x := &syncServiceSyncMeshClient{stream}
return x, nil
}
type SyncService_SyncMeshClient interface {
Send(*SyncMeshRequest) error
Recv() (*SyncMeshReply, error)
grpc.ClientStream
}
type syncServiceSyncMeshClient struct {
grpc.ClientStream
}
func (x *syncServiceSyncMeshClient) Send(m *SyncMeshRequest) error {
return x.ClientStream.SendMsg(m)
}
func (x *syncServiceSyncMeshClient) Recv() (*SyncMeshReply, error) {
m := new(SyncMeshReply)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
// SyncServiceServer is the server API for SyncService service.
@ -57,7 +79,7 @@ func (c *syncServiceClient) SyncMesh(ctx context.Context, in *SyncMeshRequest, o
// for forward compatibility
type SyncServiceServer interface {
GetConf(context.Context, *GetConfRequest) (*GetConfReply, error)
SyncMesh(context.Context, *SyncMeshRequest) (*SyncMeshReply, error)
SyncMesh(SyncService_SyncMeshServer) error
mustEmbedUnimplementedSyncServiceServer()
}
@ -68,8 +90,8 @@ type UnimplementedSyncServiceServer struct {
func (UnimplementedSyncServiceServer) GetConf(context.Context, *GetConfRequest) (*GetConfReply, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetConf not implemented")
}
func (UnimplementedSyncServiceServer) SyncMesh(context.Context, *SyncMeshRequest) (*SyncMeshReply, error) {
return nil, status.Errorf(codes.Unimplemented, "method SyncMesh not implemented")
func (UnimplementedSyncServiceServer) SyncMesh(SyncService_SyncMeshServer) error {
return status.Errorf(codes.Unimplemented, "method SyncMesh not implemented")
}
func (UnimplementedSyncServiceServer) mustEmbedUnimplementedSyncServiceServer() {}
@ -102,22 +124,30 @@ func _SyncService_GetConf_Handler(srv interface{}, ctx context.Context, dec func
return interceptor(ctx, in, info, handler)
}
func _SyncService_SyncMesh_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(SyncMeshRequest)
if err := dec(in); err != nil {
func _SyncService_SyncMesh_Handler(srv interface{}, stream grpc.ServerStream) error {
return srv.(SyncServiceServer).SyncMesh(&syncServiceSyncMeshServer{stream})
}
type SyncService_SyncMeshServer interface {
Send(*SyncMeshReply) error
Recv() (*SyncMeshRequest, error)
grpc.ServerStream
}
type syncServiceSyncMeshServer struct {
grpc.ServerStream
}
func (x *syncServiceSyncMeshServer) Send(m *SyncMeshReply) error {
return x.ServerStream.SendMsg(m)
}
func (x *syncServiceSyncMeshServer) Recv() (*SyncMeshRequest, error) {
m := new(SyncMeshRequest)
if err := x.ServerStream.RecvMsg(m); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SyncServiceServer).SyncMesh(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/syncservice.SyncService/SyncMesh",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SyncServiceServer).SyncMesh(ctx, req.(*SyncMeshRequest))
}
return interceptor(ctx, in, info, handler)
return m, nil
}
// SyncService_ServiceDesc is the grpc.ServiceDesc for SyncService service.
@ -131,11 +161,14 @@ var SyncService_ServiceDesc = grpc.ServiceDesc{
MethodName: "GetConf",
Handler: _SyncService_GetConf_Handler,
},
},
Streams: []grpc.StreamDesc{
{
MethodName: "SyncMesh",
Handler: _SyncService_SyncMesh_Handler,
StreamName: "SyncMesh",
Handler: _SyncService_SyncMesh_Handler,
ServerStreams: true,
ClientStreams: true,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "pkg/grpc/ctrlserver/syncservice.proto",
}

14
pkg/sync/syncer.go
View File

@ -2,6 +2,7 @@ package sync
import (
"errors"
"sync"
crdt "github.com/tim-beatham/wgmesh/pkg/automerge"
"github.com/tim-beatham/wgmesh/pkg/lib"
@ -54,14 +55,21 @@ func (s *SyncerImpl) Sync(meshId string) error {
meshNodes := lib.MapValuesWithExclude(snapshot.Nodes, excludedNodes)
randomSubset := lib.RandomSubsetOfLength(meshNodes, subSetLength)
for _, n := range randomSubset {
err := s.requester.SyncMesh(meshId, n.HostEndpoint)
var waitGroup sync.WaitGroup
if err != nil {
for _, n := range randomSubset {
waitGroup.Add(1)
syncMeshFunc := func() error {
defer waitGroup.Done()
err := s.requester.SyncMesh(meshId, n.HostEndpoint)
return err
}
go syncMeshFunc()
}
waitGroup.Wait()
return nil
}

55
pkg/sync/syncrequester.go
View File

@ -3,8 +3,10 @@ package sync
import (
"context"
"errors"
"io"
"time"
crdt "github.com/tim-beatham/wgmesh/pkg/automerge"
"github.com/tim-beatham/wgmesh/pkg/ctrlserver"
logging "github.com/tim-beatham/wgmesh/pkg/log"
"github.com/tim-beatham/wgmesh/pkg/rpc"
@ -123,17 +125,12 @@ func (s *SyncRequesterImpl) SyncMesh(meshId, endpoint string) error {
return errors.New("mesh does not exist")
}
syncMeshRequest := rpc.SyncMeshRequest{
MeshId: meshId,
Changes: mesh.SaveChanges(),
}
c := rpc.NewSyncServiceClient(client)
ctx, cancel := context.WithTimeout(authContext, time.Second)
ctx, cancel := context.WithTimeout(authContext, 10*time.Second)
defer cancel()
_, err = c.SyncMesh(ctx, &syncMeshRequest)
err = syncMesh(mesh, ctx, c)
if err != nil {
return s.handleErr(meshId, endpoint, err)
@ -144,6 +141,50 @@ func (s *SyncRequesterImpl) SyncMesh(meshId, endpoint string) error {
return nil
}
func syncMesh(mesh *crdt.CrdtNodeManager, ctx context.Context, client rpc.SyncServiceClient) error {
stream, err := client.SyncMesh(ctx)
syncer := mesh.GetSyncer()
if err != nil {
return err
}
for {
msg, moreMessages := syncer.GenerateMessage()
err := stream.Send(&rpc.SyncMeshRequest{MeshId: mesh.MeshId, Changes: msg})
if err != nil {
return err
}
in, err := stream.Recv()
if err != nil && err != io.EOF {
logging.ErrorLog.Printf("Stream recv error: %s\n", err.Error())
return err
}
if err != io.EOF && len(in.Changes) != 0 {
err = syncer.RecvMessage(in.Changes)
}
if err != nil {
logging.ErrorLog.Printf("Syncer recv error: %s\n", err.Error())
return err
}
if !moreMessages {
break
}
}
logging.InfoLog.Println("SYNC finished")
stream.CloseSend()
return nil
}
func NewSyncRequester(s *ctrlserver.MeshCtrlServer) SyncRequester {
errorHdlr := NewSyncErrorHandler(s.MeshManager)
return &SyncRequesterImpl{server: s, errorHdlr: errorHdlr}

67
pkg/sync/syncservice.go
View File

@ -4,8 +4,11 @@ package sync
import (
"context"
"errors"
"io"
crdt "github.com/tim-beatham/wgmesh/pkg/automerge"
"github.com/tim-beatham/wgmesh/pkg/ctrlserver"
logging "github.com/tim-beatham/wgmesh/pkg/log"
"github.com/tim-beatham/wgmesh/pkg/rpc"
)
@ -32,22 +35,60 @@ func (s *SyncServiceImpl) GetConf(context context.Context, request *rpc.GetConfR
}
// Sync: Pings a node and syncs the mesh configuration with the other node
func (s *SyncServiceImpl) SyncMesh(conext context.Context, request *rpc.SyncMeshRequest) (*rpc.SyncMeshReply, error) {
mesh := s.Server.MeshManager.GetMesh(request.MeshId)
// SyncMesh: syncs the two streams changes
func (s *SyncServiceImpl) SyncMesh(stream rpc.SyncService_SyncMeshServer) error {
var meshId = ""
var syncer *crdt.AutomergeSync = nil
if mesh == nil {
return nil, errors.New("mesh does not exist")
for {
logging.InfoLog.Println("Received Attempt")
in, err := stream.Recv()
logging.InfoLog.Println("Received Worked")
if err == io.EOF {
return nil
}
if err != nil {
return err
}
if len(meshId) == 0 {
meshId = in.MeshId
mesh := s.Server.MeshManager.GetMesh(meshId)
if mesh == nil {
return errors.New("mesh does not exist")
}
syncer = mesh.GetSyncer()
} else if meshId != in.MeshId {
return errors.New("Differing MeshIDs")
}
if syncer == nil {
return errors.New("Syncer should not be nil")
}
msg, moreMessages := syncer.GenerateMessage()
if err = stream.Send(&rpc.SyncMeshReply{Success: true, Changes: msg}); err != nil {
return err
}
if len(in.Changes) != 0 {
if err = syncer.RecvMessage(in.Changes); err != nil {
return err
}
}
if !moreMessages || err == io.EOF {
logging.InfoLog.Println("SYNC Completed")
return nil
}
}
err := s.Server.MeshManager.UpdateMesh(request.MeshId, request.Changes)
if err != nil {
return nil, err
}
return &rpc.SyncMeshReply{Success: true}, nil
}
func NewSyncService(server *ctrlserver.MeshCtrlServer) *SyncServiceImpl {
return &SyncServiceImpl{Server: server}
}