mirror of
https://github.com/zrepl/zrepl.git
synced 2025-05-31 07:09:47 +02:00
enforce encapsulation by breaking up replication package into packages
not perfect yet, public shouldn't be required to use 'common' package to use replication package
This commit is contained in:
Christian Schwarz
parent
c7d28fee8f
commit
38532abf45
9
Makefile
9
Makefile
@ -2,7 +2,12 @@
|
|||||||
.DEFAULT_GOAL := build
|
.DEFAULT_GOAL := build
|
||||||
|
|
||||||
ROOT := github.com/zrepl/zrepl
|
ROOT := github.com/zrepl/zrepl
|
||||||
SUBPKGS := cmd logger rpc util zfs
|
SUBPKGS := cmd
|
||||||
|
SUBPKGS += cmd/replication
|
||||||
|
SUBPKGS += cmd/replication/internal/common
|
||||||
|
SUBPKGS += cmd/replication/internal/mainfsm
|
||||||
|
SUBPKGS += cmd/replication/internal/fsfsm
|
||||||
|
SUBPKGS += logger util zfs
|
||||||
|
|
||||||
_TESTPKGS := $(ROOT) $(foreach p,$(SUBPKGS),$(ROOT)/$(p))
|
_TESTPKGS := $(ROOT) $(foreach p,$(SUBPKGS),$(ROOT)/$(p))
|
||||||
|
|
||||||
@ -26,10 +31,10 @@ vendordeps:
|
|||||||
dep ensure -v -vendor-only
|
dep ensure -v -vendor-only
|
||||||
|
|
||||||
generate: #not part of the build, must do that manually
|
generate: #not part of the build, must do that manually
|
||||||
|
protoc -I=cmd/replication/pdu --go_out=cmd/replication/pdu cmd/replication/pdu/pdu.proto
|
||||||
@for pkg in $(_TESTPKGS); do\
|
@for pkg in $(_TESTPKGS); do\
|
||||||
go generate "$$pkg" || exit 1; \
|
go generate "$$pkg" || exit 1; \
|
||||||
done;
|
done;
|
||||||
protoc -I=cmd/replication --go_out=cmd/replication cmd/replication/pdu.proto
|
|
||||||
# FIXME fix docker build!
|
# FIXME fix docker build!
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -8,7 +8,8 @@ import (
|
|||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/zrepl/zrepl/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
"sync"
|
"sync"
|
||||||
"github.com/zrepl/zrepl/cmd/replication.v2"
|
"github.com/zrepl/zrepl/cmd/replication"
|
||||||
|
"github.com/zrepl/zrepl/cmd/replication/common"
|
||||||
)
|
)
|
||||||
|
|
||||||
type LocalJob struct {
|
type LocalJob struct {
|
||||||
@ -146,7 +147,8 @@ outer:
|
|||||||
j.mainTask.Log().Debug("replicating from lhs to rhs")
|
j.mainTask.Log().Debug("replicating from lhs to rhs")
|
||||||
j.mainTask.Enter("replicate")
|
j.mainTask.Enter("replicate")
|
||||||
|
|
||||||
replication.Replicate(ctx, replication.NewEndpointPairPull(sender, receiver), nil) // FIXME
|
rep := replication.NewReplication()
|
||||||
|
rep.Drive(ctx, common.NewEndpointPairPull(sender, receiver))
|
||||||
|
|
||||||
j.mainTask.Finish()
|
j.mainTask.Finish()
|
||||||
|
|
||||||
|
|||||||
@ -13,7 +13,8 @@ import (
|
|||||||
"github.com/mitchellh/mapstructure"
|
"github.com/mitchellh/mapstructure"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/problame/go-streamrpc"
|
"github.com/problame/go-streamrpc"
|
||||||
"github.com/zrepl/zrepl/cmd/replication.v2"
|
"github.com/zrepl/zrepl/cmd/replication"
|
||||||
|
"github.com/zrepl/zrepl/cmd/replication/common"
|
||||||
)
|
)
|
||||||
|
|
||||||
type PullJob struct {
|
type PullJob struct {
|
||||||
@ -29,7 +30,7 @@ type PullJob struct {
|
|||||||
Debug JobDebugSettings
|
Debug JobDebugSettings
|
||||||
|
|
||||||
task *Task
|
task *Task
|
||||||
rep *replication.Replication
|
rep replication.Replication
|
||||||
}
|
}
|
||||||
|
|
||||||
func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j *PullJob, err error) {
|
func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j *PullJob, err error) {
|
||||||
@ -188,13 +189,12 @@ func (j *PullJob) doRun(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
ctx = replication.ContextWithLogger(ctx, replicationLogAdaptor{j.task.Log().WithField("subsystem", "replication")})
|
ctx = common.ContextWithLogger(ctx, replicationLogAdaptor{j.task.Log().WithField("subsystem", "replication")})
|
||||||
ctx = streamrpc.ContextWithLogger(ctx, streamrpcLogAdaptor{j.task.Log().WithField("subsystem", "rpc.protocol")})
|
ctx = streamrpc.ContextWithLogger(ctx, streamrpcLogAdaptor{j.task.Log().WithField("subsystem", "rpc.protocol")})
|
||||||
ctx = context.WithValue(ctx, contextKeyLog, j.task.Log().WithField("subsystem", "rpc.endpoint"))
|
ctx = context.WithValue(ctx, contextKeyLog, j.task.Log().WithField("subsystem", "rpc.endpoint"))
|
||||||
|
|
||||||
j.rep = replication.NewReplication()
|
j.rep = replication.NewReplication()
|
||||||
retryNow := make(chan struct{})
|
j.rep.Drive(ctx, common.NewEndpointPairPull(sender, puller))
|
||||||
j.rep.Drive(ctx, replication.NewEndpointPairPull(sender, puller), retryNow)
|
|
||||||
|
|
||||||
client.Close()
|
client.Close()
|
||||||
j.task.Finish()
|
j.task.Finish()
|
||||||
|
|||||||
@ -2,7 +2,8 @@ package cmd
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/zrepl/zrepl/cmd/replication.v2"
|
"github.com/zrepl/zrepl/cmd/replication/common"
|
||||||
|
"github.com/zrepl/zrepl/cmd/replication/pdu"
|
||||||
"github.com/problame/go-streamrpc"
|
"github.com/problame/go-streamrpc"
|
||||||
"github.com/zrepl/zrepl/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
"io"
|
"io"
|
||||||
@ -31,14 +32,14 @@ func NewSenderEndpoint(fsf zfs.DatasetFilter, fsvf zfs.FilesystemVersionFilter)
|
|||||||
return &SenderEndpoint{fsf, fsvf}
|
return &SenderEndpoint{fsf, fsvf}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *SenderEndpoint) ListFilesystems(ctx context.Context) ([]*replication.Filesystem, error) {
|
func (p *SenderEndpoint) ListFilesystems(ctx context.Context) ([]*pdu.Filesystem, error) {
|
||||||
fss, err := zfs.ZFSListMapping(p.FSFilter)
|
fss, err := zfs.ZFSListMapping(p.FSFilter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
rfss := make([]*replication.Filesystem, len(fss))
|
rfss := make([]*pdu.Filesystem, len(fss))
|
||||||
for i := range fss {
|
for i := range fss {
|
||||||
rfss[i] = &replication.Filesystem{
|
rfss[i] = &pdu.Filesystem{
|
||||||
Path: fss[i].ToString(),
|
Path: fss[i].ToString(),
|
||||||
// FIXME: not supporting ResumeToken yet
|
// FIXME: not supporting ResumeToken yet
|
||||||
}
|
}
|
||||||
@ -46,7 +47,7 @@ func (p *SenderEndpoint) ListFilesystems(ctx context.Context) ([]*replication.Fi
|
|||||||
return rfss, nil
|
return rfss, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *SenderEndpoint) ListFilesystemVersions(ctx context.Context, fs string) ([]*replication.FilesystemVersion, error) {
|
func (p *SenderEndpoint) ListFilesystemVersions(ctx context.Context, fs string) ([]*pdu.FilesystemVersion, error) {
|
||||||
dp, err := zfs.NewDatasetPath(fs)
|
dp, err := zfs.NewDatasetPath(fs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -56,20 +57,20 @@ func (p *SenderEndpoint) ListFilesystemVersions(ctx context.Context, fs string)
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if !pass {
|
if !pass {
|
||||||
return nil, replication.NewFilteredError(fs)
|
return nil, common.NewFilteredError(fs)
|
||||||
}
|
}
|
||||||
fsvs, err := zfs.ZFSListFilesystemVersions(dp, p.FilesystemVersionFilter)
|
fsvs, err := zfs.ZFSListFilesystemVersions(dp, p.FilesystemVersionFilter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
rfsvs := make([]*replication.FilesystemVersion, len(fsvs))
|
rfsvs := make([]*pdu.FilesystemVersion, len(fsvs))
|
||||||
for i := range fsvs {
|
for i := range fsvs {
|
||||||
rfsvs[i] = replication.FilesystemVersionFromZFS(fsvs[i])
|
rfsvs[i] = pdu.FilesystemVersionFromZFS(fsvs[i])
|
||||||
}
|
}
|
||||||
return rfsvs, nil
|
return rfsvs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *SenderEndpoint) Send(ctx context.Context, r *replication.SendReq) (*replication.SendRes, io.ReadCloser, error) {
|
func (p *SenderEndpoint) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
|
||||||
dp, err := zfs.NewDatasetPath(r.Filesystem)
|
dp, err := zfs.NewDatasetPath(r.Filesystem)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
@ -79,16 +80,16 @@ func (p *SenderEndpoint) Send(ctx context.Context, r *replication.SendReq) (*rep
|
|||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
if !pass {
|
if !pass {
|
||||||
return nil, nil, replication.NewFilteredError(r.Filesystem)
|
return nil, nil, common.NewFilteredError(r.Filesystem)
|
||||||
}
|
}
|
||||||
stream, err := zfs.ZFSSend(r.Filesystem, r.From, r.To)
|
stream, err := zfs.ZFSSend(r.Filesystem, r.From, r.To)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
return &replication.SendRes{}, stream, nil
|
return &pdu.SendRes{}, stream, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *SenderEndpoint) Receive(ctx context.Context, r *replication.ReceiveReq, sendStream io.ReadCloser) (error) {
|
func (p *SenderEndpoint) Receive(ctx context.Context, r *pdu.ReceiveReq, sendStream io.ReadCloser) (error) {
|
||||||
return fmt.Errorf("sender endpoint does not receive")
|
return fmt.Errorf("sender endpoint does not receive")
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -108,23 +109,23 @@ func NewReceiverEndpoint(fsmap *DatasetMapFilter, fsvf zfs.FilesystemVersionFilt
|
|||||||
return &ReceiverEndpoint{fsmapInv, fsmap, fsvf}, nil
|
return &ReceiverEndpoint{fsmapInv, fsmap, fsvf}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *ReceiverEndpoint) ListFilesystems(ctx context.Context) ([]*replication.Filesystem, error) {
|
func (e *ReceiverEndpoint) ListFilesystems(ctx context.Context) ([]*pdu.Filesystem, error) {
|
||||||
filtered, err := zfs.ZFSListMapping(e.fsmapInv.AsFilter())
|
filtered, err := zfs.ZFSListMapping(e.fsmapInv.AsFilter())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "error checking client permission")
|
return nil, errors.Wrap(err, "error checking client permission")
|
||||||
}
|
}
|
||||||
fss := make([]*replication.Filesystem, len(filtered))
|
fss := make([]*pdu.Filesystem, len(filtered))
|
||||||
for i, a := range filtered {
|
for i, a := range filtered {
|
||||||
mapped, err := e.fsmapInv.Map(a)
|
mapped, err := e.fsmapInv.Map(a)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
fss[i] = &replication.Filesystem{Path: mapped.ToString()}
|
fss[i] = &pdu.Filesystem{Path: mapped.ToString()}
|
||||||
}
|
}
|
||||||
return fss, nil
|
return fss, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *ReceiverEndpoint) ListFilesystemVersions(ctx context.Context, fs string) ([]*replication.FilesystemVersion, error) {
|
func (e *ReceiverEndpoint) ListFilesystemVersions(ctx context.Context, fs string) ([]*pdu.FilesystemVersion, error) {
|
||||||
p, err := zfs.NewDatasetPath(fs)
|
p, err := zfs.NewDatasetPath(fs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -142,19 +143,19 @@ func (e *ReceiverEndpoint) ListFilesystemVersions(ctx context.Context, fs string
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
rfsvs := make([]*replication.FilesystemVersion, len(fsvs))
|
rfsvs := make([]*pdu.FilesystemVersion, len(fsvs))
|
||||||
for i := range fsvs {
|
for i := range fsvs {
|
||||||
rfsvs[i] = replication.FilesystemVersionFromZFS(fsvs[i])
|
rfsvs[i] = pdu.FilesystemVersionFromZFS(fsvs[i])
|
||||||
}
|
}
|
||||||
|
|
||||||
return rfsvs, nil
|
return rfsvs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *ReceiverEndpoint) Send(ctx context.Context, req *replication.SendReq) (*replication.SendRes, io.ReadCloser, error) {
|
func (e *ReceiverEndpoint) Send(ctx context.Context, req *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
|
||||||
return nil, nil, errors.New("receiver endpoint does not send")
|
return nil, nil, errors.New("receiver endpoint does not send")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *ReceiverEndpoint) Receive(ctx context.Context, req *replication.ReceiveReq, sendStream io.ReadCloser) error {
|
func (e *ReceiverEndpoint) Receive(ctx context.Context, req *pdu.ReceiveReq, sendStream io.ReadCloser) error {
|
||||||
defer sendStream.Close()
|
defer sendStream.Close()
|
||||||
|
|
||||||
p, err := zfs.NewDatasetPath(req.Filesystem)
|
p, err := zfs.NewDatasetPath(req.Filesystem)
|
||||||
@ -236,8 +237,8 @@ type RemoteEndpoint struct {
|
|||||||
*streamrpc.Client
|
*streamrpc.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s RemoteEndpoint) ListFilesystems(ctx context.Context) ([]*replication.Filesystem, error) {
|
func (s RemoteEndpoint) ListFilesystems(ctx context.Context) ([]*pdu.Filesystem, error) {
|
||||||
req := replication.ListFilesystemReq{}
|
req := pdu.ListFilesystemReq{}
|
||||||
b, err := proto.Marshal(&req)
|
b, err := proto.Marshal(&req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -250,15 +251,15 @@ func (s RemoteEndpoint) ListFilesystems(ctx context.Context) ([]*replication.Fil
|
|||||||
rs.Close()
|
rs.Close()
|
||||||
return nil, errors.New("response contains unexpected stream")
|
return nil, errors.New("response contains unexpected stream")
|
||||||
}
|
}
|
||||||
var res replication.ListFilesystemRes
|
var res pdu.ListFilesystemRes
|
||||||
if err := proto.Unmarshal(rb.Bytes(), &res); err != nil {
|
if err := proto.Unmarshal(rb.Bytes(), &res); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return res.Filesystems, nil
|
return res.Filesystems, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s RemoteEndpoint) ListFilesystemVersions(ctx context.Context, fs string) ([]*replication.FilesystemVersion, error) {
|
func (s RemoteEndpoint) ListFilesystemVersions(ctx context.Context, fs string) ([]*pdu.FilesystemVersion, error) {
|
||||||
req := replication.ListFilesystemVersionsReq{
|
req := pdu.ListFilesystemVersionsReq{
|
||||||
Filesystem: fs,
|
Filesystem: fs,
|
||||||
}
|
}
|
||||||
b, err := proto.Marshal(&req)
|
b, err := proto.Marshal(&req)
|
||||||
@ -273,14 +274,14 @@ func (s RemoteEndpoint) ListFilesystemVersions(ctx context.Context, fs string) (
|
|||||||
rs.Close()
|
rs.Close()
|
||||||
return nil, errors.New("response contains unexpected stream")
|
return nil, errors.New("response contains unexpected stream")
|
||||||
}
|
}
|
||||||
var res replication.ListFilesystemVersionsRes
|
var res pdu.ListFilesystemVersionsRes
|
||||||
if err := proto.Unmarshal(rb.Bytes(), &res); err != nil {
|
if err := proto.Unmarshal(rb.Bytes(), &res); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return res.Versions, nil
|
return res.Versions, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s RemoteEndpoint) Send(ctx context.Context, r *replication.SendReq) (*replication.SendRes, io.ReadCloser, error) {
|
func (s RemoteEndpoint) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
|
||||||
b, err := proto.Marshal(r)
|
b, err := proto.Marshal(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
@ -292,7 +293,7 @@ func (s RemoteEndpoint) Send(ctx context.Context, r *replication.SendReq) (*repl
|
|||||||
if rs == nil {
|
if rs == nil {
|
||||||
return nil, nil, errors.New("response does not contain a stream")
|
return nil, nil, errors.New("response does not contain a stream")
|
||||||
}
|
}
|
||||||
var res replication.SendRes
|
var res pdu.SendRes
|
||||||
if err := proto.Unmarshal(rb.Bytes(), &res); err != nil {
|
if err := proto.Unmarshal(rb.Bytes(), &res); err != nil {
|
||||||
rs.Close()
|
rs.Close()
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
@ -301,7 +302,7 @@ func (s RemoteEndpoint) Send(ctx context.Context, r *replication.SendReq) (*repl
|
|||||||
return &res, rs, nil
|
return &res, rs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s RemoteEndpoint) Receive(ctx context.Context, r *replication.ReceiveReq, sendStream io.ReadCloser) (error) {
|
func (s RemoteEndpoint) Receive(ctx context.Context, r *pdu.ReceiveReq, sendStream io.ReadCloser) (error) {
|
||||||
defer sendStream.Close()
|
defer sendStream.Close()
|
||||||
b, err := proto.Marshal(r)
|
b, err := proto.Marshal(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -315,7 +316,7 @@ func (s RemoteEndpoint) Receive(ctx context.Context, r *replication.ReceiveReq,
|
|||||||
rs.Close()
|
rs.Close()
|
||||||
return errors.New("response contains unexpected stream")
|
return errors.New("response contains unexpected stream")
|
||||||
}
|
}
|
||||||
var res replication.ReceiveRes
|
var res pdu.ReceiveRes
|
||||||
if err := proto.Unmarshal(rb.Bytes(), &res); err != nil {
|
if err := proto.Unmarshal(rb.Bytes(), &res); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -323,14 +324,14 @@ func (s RemoteEndpoint) Receive(ctx context.Context, r *replication.ReceiveReq,
|
|||||||
}
|
}
|
||||||
|
|
||||||
type HandlerAdaptor struct {
|
type HandlerAdaptor struct {
|
||||||
ep replication.ReplicationEndpoint
|
ep common.ReplicationEndpoint
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *HandlerAdaptor) Handle(ctx context.Context, endpoint string, reqStructured *bytes.Buffer, reqStream io.ReadCloser) (resStructured *bytes.Buffer, resStream io.ReadCloser, err error) {
|
func (a *HandlerAdaptor) Handle(ctx context.Context, endpoint string, reqStructured *bytes.Buffer, reqStream io.ReadCloser) (resStructured *bytes.Buffer, resStream io.ReadCloser, err error) {
|
||||||
|
|
||||||
switch endpoint {
|
switch endpoint {
|
||||||
case RPCListFilesystems:
|
case RPCListFilesystems:
|
||||||
var req replication.ListFilesystemReq
|
var req pdu.ListFilesystemReq
|
||||||
if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil {
|
if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
@ -338,7 +339,7 @@ func (a *HandlerAdaptor) Handle(ctx context.Context, endpoint string, reqStructu
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
res := &replication.ListFilesystemRes{
|
res := &pdu.ListFilesystemRes{
|
||||||
Filesystems: fsses,
|
Filesystems: fsses,
|
||||||
}
|
}
|
||||||
b, err := proto.Marshal(res)
|
b, err := proto.Marshal(res)
|
||||||
@ -349,7 +350,7 @@ func (a *HandlerAdaptor) Handle(ctx context.Context, endpoint string, reqStructu
|
|||||||
|
|
||||||
case RPCListFilesystemVersions:
|
case RPCListFilesystemVersions:
|
||||||
|
|
||||||
var req replication.ListFilesystemVersionsReq
|
var req pdu.ListFilesystemVersionsReq
|
||||||
if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil {
|
if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
@ -357,7 +358,7 @@ func (a *HandlerAdaptor) Handle(ctx context.Context, endpoint string, reqStructu
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
res := &replication.ListFilesystemVersionsRes{
|
res := &pdu.ListFilesystemVersionsRes{
|
||||||
Versions: fsvs,
|
Versions: fsvs,
|
||||||
}
|
}
|
||||||
b, err := proto.Marshal(res)
|
b, err := proto.Marshal(res)
|
||||||
@ -368,7 +369,7 @@ func (a *HandlerAdaptor) Handle(ctx context.Context, endpoint string, reqStructu
|
|||||||
|
|
||||||
case RPCSend:
|
case RPCSend:
|
||||||
|
|
||||||
var req replication.SendReq
|
var req pdu.SendReq
|
||||||
if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil {
|
if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
@ -384,7 +385,7 @@ func (a *HandlerAdaptor) Handle(ctx context.Context, endpoint string, reqStructu
|
|||||||
|
|
||||||
case RPCReceive:
|
case RPCReceive:
|
||||||
|
|
||||||
var req replication.ReceiveReq
|
var req pdu.ReceiveReq
|
||||||
if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil {
|
if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
@ -392,7 +393,7 @@ func (a *HandlerAdaptor) Handle(ctx context.Context, endpoint string, reqStructu
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
b, err := proto.Marshal(&replication.ReceiveRes{})
|
b, err := proto.Marshal(&pdu.ReceiveRes{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,268 +0,0 @@
|
|||||||
package replication_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/zrepl/zrepl/cmd/replication"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func fsvlist(fsv ...string) (r []*replication.FilesystemVersion) {
|
|
||||||
|
|
||||||
r = make([]*replication.FilesystemVersion, len(fsv))
|
|
||||||
for i, f := range fsv {
|
|
||||||
|
|
||||||
// parse the id from fsvlist. it is used to derivce Guid,CreateTXG and Creation attrs
|
|
||||||
split := strings.Split(f, ",")
|
|
||||||
if len(split) != 2 {
|
|
||||||
panic("invalid fsv spec")
|
|
||||||
}
|
|
||||||
id, err := strconv.Atoi(split[1])
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.HasPrefix(f, "#") {
|
|
||||||
r[i] = &replication.FilesystemVersion{
|
|
||||||
Name: strings.TrimPrefix(f, "#"),
|
|
||||||
Type: replication.FilesystemVersion_Bookmark,
|
|
||||||
Guid: uint64(id),
|
|
||||||
CreateTXG: uint64(id),
|
|
||||||
Creation: time.Unix(0, 0).Add(time.Duration(id) * time.Second).Format(time.RFC3339),
|
|
||||||
}
|
|
||||||
} else if strings.HasPrefix(f, "@") {
|
|
||||||
r[i] = &replication.FilesystemVersion{
|
|
||||||
Name: strings.TrimPrefix(f, "@"),
|
|
||||||
Type: replication.FilesystemVersion_Snapshot,
|
|
||||||
Guid: uint64(id),
|
|
||||||
CreateTXG: uint64(id),
|
|
||||||
Creation: time.Unix(0, 0).Add(time.Duration(id) * time.Second).Format(time.RFC3339),
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
panic("invalid character")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
type incPathResult struct {
|
|
||||||
incPath []*replication.FilesystemVersion
|
|
||||||
conflict error
|
|
||||||
}
|
|
||||||
|
|
||||||
type IncrementalPathTest struct {
|
|
||||||
Msg string
|
|
||||||
Receiver, Sender []*replication.FilesystemVersion
|
|
||||||
ExpectIncPath []*replication.FilesystemVersion
|
|
||||||
ExpectNoCommonAncestor bool
|
|
||||||
ExpectDiverged *replication.ConflictDiverged
|
|
||||||
ExpectPanic bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tt *IncrementalPathTest) Test(t *testing.T) {
|
|
||||||
|
|
||||||
t.Logf("test: %s", tt.Msg)
|
|
||||||
|
|
||||||
if tt.ExpectPanic {
|
|
||||||
assert.Panics(t, func() {
|
|
||||||
replication.IncrementalPath(tt.Receiver, tt.Sender)
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
incPath, conflict := replication.IncrementalPath(tt.Receiver, tt.Sender)
|
|
||||||
|
|
||||||
if tt.ExpectIncPath != nil {
|
|
||||||
assert.Nil(t, conflict)
|
|
||||||
assert.True(t, len(incPath) == 0 || len(incPath) >= 2)
|
|
||||||
assert.Equal(t, tt.ExpectIncPath, incPath)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if conflict == nil {
|
|
||||||
t.Logf("conflict is (unexpectly) <nil>\nincPath: %#v", incPath)
|
|
||||||
}
|
|
||||||
if tt.ExpectNoCommonAncestor {
|
|
||||||
assert.IsType(t, &replication.ConflictNoCommonAncestor{}, conflict)
|
|
||||||
// TODO check sorting
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if tt.ExpectDiverged != nil {
|
|
||||||
if !assert.IsType(t, &replication.ConflictDiverged{}, conflict) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
c := conflict.(*replication.ConflictDiverged)
|
|
||||||
// TODO check sorting
|
|
||||||
assert.NotZero(t, c.CommonAncestor)
|
|
||||||
assert.NotEmpty(t, c.ReceiverOnly)
|
|
||||||
assert.Equal(t, tt.ExpectDiverged.ReceiverOnly, c.ReceiverOnly)
|
|
||||||
assert.Equal(t, tt.ExpectDiverged.SenderOnly, c.SenderOnly)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIncrementalPlan_IncrementalSnapshots(t *testing.T) {
|
|
||||||
l := fsvlist
|
|
||||||
|
|
||||||
tbl := []IncrementalPathTest{
|
|
||||||
{
|
|
||||||
Msg: "basic functionality",
|
|
||||||
Receiver: l("@a,1", "@b,2"),
|
|
||||||
Sender: l("@a,1", "@b,2", "@c,3", "@d,4"),
|
|
||||||
ExpectIncPath: l("@b,2", "@c,3", "@d,4"),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Msg: "no snaps on receiver yields no common ancestor",
|
|
||||||
Receiver: l(),
|
|
||||||
Sender: l("@a,1"),
|
|
||||||
ExpectNoCommonAncestor: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Msg: "no snapshots on sender yields empty incremental path",
|
|
||||||
Receiver: l(),
|
|
||||||
Sender: l(),
|
|
||||||
ExpectIncPath: l(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Msg: "nothing to do yields empty incremental path",
|
|
||||||
Receiver: l("@a,1"),
|
|
||||||
Sender: l("@a,1"),
|
|
||||||
ExpectIncPath: l(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Msg: "drifting apart",
|
|
||||||
Receiver: l("@a,1", "@b,2"),
|
|
||||||
Sender: l("@c,3", "@d,4"),
|
|
||||||
ExpectNoCommonAncestor: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Msg: "different snapshots on sender and receiver",
|
|
||||||
Receiver: l("@a,1", "@c,2"),
|
|
||||||
Sender: l("@a,1", "@b,3"),
|
|
||||||
ExpectDiverged: &replication.ConflictDiverged{
|
|
||||||
CommonAncestor: l("@a,1")[0],
|
|
||||||
SenderOnly: l("@b,3"),
|
|
||||||
ReceiverOnly: l("@c,2"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Msg: "snapshot on receiver not present on sender",
|
|
||||||
Receiver: l("@a,1", "@b,2"),
|
|
||||||
Sender: l("@a,1"),
|
|
||||||
ExpectDiverged: &replication.ConflictDiverged{
|
|
||||||
CommonAncestor: l("@a,1")[0],
|
|
||||||
SenderOnly: l(),
|
|
||||||
ReceiverOnly: l("@b,2"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Msg: "gaps before most recent common ancestor do not matter",
|
|
||||||
Receiver: l("@a,1", "@b,2", "@c,3"),
|
|
||||||
Sender: l("@a,1", "@c,3", "@d,4"),
|
|
||||||
ExpectIncPath: l("@c,3", "@d,4"),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, test := range tbl {
|
|
||||||
test.Test(t)
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIncrementalPlan_BookmarksSupport(t *testing.T) {
|
|
||||||
l := fsvlist
|
|
||||||
|
|
||||||
tbl := []IncrementalPathTest{
|
|
||||||
{
|
|
||||||
Msg: "bookmarks are used",
|
|
||||||
Receiver: l("@a,1"),
|
|
||||||
Sender: l("#a,1", "@b,2"),
|
|
||||||
ExpectIncPath: l("#a,1", "@b,2"),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Msg: "boomarks are stripped from incPath (cannot send incrementally)",
|
|
||||||
Receiver: l("@a,1"),
|
|
||||||
Sender: l("#a,1", "#b,2", "@c,3"),
|
|
||||||
ExpectIncPath: l("#a,1", "@c,3"),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Msg: "bookmarks are preferred over snapshots for start of incPath",
|
|
||||||
Receiver: l("@a,1"),
|
|
||||||
Sender: l("#a,1", "@a,1", "@b,2"),
|
|
||||||
ExpectIncPath: l("#a,1", "@b,2"),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Msg: "bookmarks are preferred over snapshots for start of incPath (regardless of order)",
|
|
||||||
Receiver: l("@a,1"),
|
|
||||||
Sender: l("@a,1", "#a,1", "@b,2"),
|
|
||||||
ExpectIncPath: l("#a,1", "@b,2"),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, test := range tbl {
|
|
||||||
test.Test(t)
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSortVersionListByCreateTXGThenBookmarkLTSnapshot(t *testing.T) {
|
|
||||||
|
|
||||||
type Test struct {
|
|
||||||
Msg string
|
|
||||||
Input, Output []*replication.FilesystemVersion
|
|
||||||
}
|
|
||||||
|
|
||||||
l := fsvlist
|
|
||||||
|
|
||||||
tbl := []Test{
|
|
||||||
{
|
|
||||||
"snapshot sorting already sorted",
|
|
||||||
l("@a,1", "@b,2"),
|
|
||||||
l("@a,1", "@b,2"),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"bookmark sorting already sorted",
|
|
||||||
l("#a,1", "#b,2"),
|
|
||||||
l("#a,1", "#b,2"),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"snapshot sorting",
|
|
||||||
l("@b,2", "@a,1"),
|
|
||||||
l("@a,1", "@b,2"),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"bookmark sorting",
|
|
||||||
l("#b,2", "#a,1"),
|
|
||||||
l("#a,1", "#b,2"),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, test := range tbl {
|
|
||||||
t.Logf("test: %s", test.Msg)
|
|
||||||
inputlen := len(test.Input)
|
|
||||||
sorted := replication.SortVersionListByCreateTXGThenBookmarkLTSnapshot(test.Input)
|
|
||||||
if len(sorted) != inputlen {
|
|
||||||
t.Errorf("lenghts of input and output do not match: %d vs %d", inputlen, len(sorted))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !assert.Equal(t, test.Output, sorted) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
last := sorted[0]
|
|
||||||
for _, s := range sorted[1:] {
|
|
||||||
if s.CreateTXG < last.CreateTXG {
|
|
||||||
t.Errorf("must be sorted ascending, got:\n\t%#v", sorted)
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if s.CreateTXG == last.CreateTXG {
|
|
||||||
if last.Type == replication.FilesystemVersion_Bookmark && s.Type != replication.FilesystemVersion_Snapshot {
|
|
||||||
t.Errorf("snapshots must come after bookmarks")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
last = s
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,666 +0,0 @@
|
|||||||
package replication
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"math/bits"
|
|
||||||
"net"
|
|
||||||
"sort"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
//go:generate stringer -type=ReplicationState
|
|
||||||
type ReplicationState uint
|
|
||||||
|
|
||||||
const (
|
|
||||||
Planning ReplicationState = 1 << iota
|
|
||||||
PlanningError
|
|
||||||
Working
|
|
||||||
WorkingWait
|
|
||||||
Completed
|
|
||||||
ContextDone
|
|
||||||
)
|
|
||||||
|
|
||||||
func (s ReplicationState) rsf() replicationStateFunc {
|
|
||||||
idx := bits.TrailingZeros(uint(s))
|
|
||||||
if idx == bits.UintSize {
|
|
||||||
panic(s) // invalid value
|
|
||||||
}
|
|
||||||
m := []replicationStateFunc{
|
|
||||||
rsfPlanning,
|
|
||||||
rsfPlanningError,
|
|
||||||
rsfWorking,
|
|
||||||
rsfWorkingWait,
|
|
||||||
nil,
|
|
||||||
nil,
|
|
||||||
}
|
|
||||||
return m[idx]
|
|
||||||
}
|
|
||||||
|
|
||||||
type replicationQueueItem struct {
|
|
||||||
retriesSinceLastError int
|
|
||||||
// duplicates fsr.state to avoid accessing and locking fsr
|
|
||||||
state FSReplicationState
|
|
||||||
// duplicates fsr.current.nextStepDate to avoid accessing & locking fsr
|
|
||||||
nextStepDate time.Time
|
|
||||||
|
|
||||||
fsr *FSReplication
|
|
||||||
}
|
|
||||||
|
|
||||||
type Replication struct {
|
|
||||||
// lock protects all fields of this struct (but not the fields behind pointers!)
|
|
||||||
lock sync.Mutex
|
|
||||||
|
|
||||||
state ReplicationState
|
|
||||||
|
|
||||||
// Working, WorkingWait, Completed, ContextDone
|
|
||||||
queue *replicationQueue
|
|
||||||
completed []*FSReplication
|
|
||||||
active *FSReplication
|
|
||||||
|
|
||||||
// PlanningError
|
|
||||||
planningError error
|
|
||||||
|
|
||||||
// ContextDone
|
|
||||||
contextError error
|
|
||||||
|
|
||||||
// PlanningError, WorkingWait
|
|
||||||
sleepUntil time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
type replicationUpdater func(func(*Replication)) (newState ReplicationState)
|
|
||||||
type replicationStateFunc func(context.Context, EndpointPair, replicationUpdater) replicationStateFunc
|
|
||||||
|
|
||||||
//go:generate stringer -type=FSReplicationState
|
|
||||||
type FSReplicationState uint
|
|
||||||
|
|
||||||
const (
|
|
||||||
FSReady FSReplicationState = 1 << iota
|
|
||||||
FSRetryWait
|
|
||||||
FSPermanentError
|
|
||||||
FSCompleted
|
|
||||||
)
|
|
||||||
|
|
||||||
func (s FSReplicationState) fsrsf() fsrsf {
|
|
||||||
idx := bits.TrailingZeros(uint(s))
|
|
||||||
if idx == bits.UintSize {
|
|
||||||
panic(s)
|
|
||||||
}
|
|
||||||
m := []fsrsf{
|
|
||||||
fsrsfReady,
|
|
||||||
fsrsfRetryWait,
|
|
||||||
nil,
|
|
||||||
nil,
|
|
||||||
}
|
|
||||||
return m[idx]
|
|
||||||
}
|
|
||||||
|
|
||||||
type FSReplication struct {
|
|
||||||
// lock protects all fields in this struct, but not the data behind pointers
|
|
||||||
lock sync.Mutex
|
|
||||||
state FSReplicationState
|
|
||||||
fs *Filesystem
|
|
||||||
err error
|
|
||||||
retryWaitUntil time.Time
|
|
||||||
completed, pending []*FSReplicationStep
|
|
||||||
current *FSReplicationStep
|
|
||||||
}
|
|
||||||
|
|
||||||
func newReplicationQueueItemPermanentError(fs *Filesystem, err error) *replicationQueueItem {
|
|
||||||
return &replicationQueueItem{
|
|
||||||
retriesSinceLastError: 0,
|
|
||||||
state: FSPermanentError,
|
|
||||||
fsr: &FSReplication{
|
|
||||||
state: FSPermanentError,
|
|
||||||
fs: fs,
|
|
||||||
err: err,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *replicationQueueItemBuilder) Complete() *replicationQueueItem {
|
|
||||||
if len(b.r.pending) > 0 {
|
|
||||||
b.r.state = FSReady
|
|
||||||
} else {
|
|
||||||
b.r.state = FSCompleted
|
|
||||||
}
|
|
||||||
r := b.r
|
|
||||||
return &replicationQueueItem{0, b.r.state, time.Time{}, r}
|
|
||||||
}
|
|
||||||
|
|
||||||
//go:generate stringer -type=FSReplicationStepState
|
|
||||||
type FSReplicationStepState uint
|
|
||||||
|
|
||||||
const (
|
|
||||||
StepReady FSReplicationStepState = 1 << iota
|
|
||||||
StepRetry
|
|
||||||
StepPermanentError
|
|
||||||
StepCompleted
|
|
||||||
)
|
|
||||||
|
|
||||||
type FSReplicationStep struct {
|
|
||||||
// only protects state, err
|
|
||||||
// from, to and fsrep are assumed to be immutable
|
|
||||||
lock sync.Mutex
|
|
||||||
|
|
||||||
state FSReplicationStepState
|
|
||||||
from, to *FilesystemVersion
|
|
||||||
fsrep *FSReplication
|
|
||||||
|
|
||||||
// both retry and permanent error
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Replication) Drive(ctx context.Context, ep EndpointPair, retryNow chan struct{}) {
|
|
||||||
|
|
||||||
var u replicationUpdater = func(f func(*Replication)) ReplicationState {
|
|
||||||
r.lock.Lock()
|
|
||||||
defer r.lock.Unlock()
|
|
||||||
if f != nil {
|
|
||||||
f(r)
|
|
||||||
}
|
|
||||||
return r.state
|
|
||||||
}
|
|
||||||
|
|
||||||
var s replicationStateFunc = rsfPlanning
|
|
||||||
var pre, post ReplicationState
|
|
||||||
for s != nil {
|
|
||||||
preTime := time.Now()
|
|
||||||
pre = u(nil)
|
|
||||||
s = s(ctx, ep, u)
|
|
||||||
delta := time.Now().Sub(preTime)
|
|
||||||
post = u(nil)
|
|
||||||
getLogger(ctx).
|
|
||||||
WithField("transition", fmt.Sprintf("%s => %s", pre, post)).
|
|
||||||
WithField("duration", delta).
|
|
||||||
Debug("main state transition")
|
|
||||||
}
|
|
||||||
|
|
||||||
getLogger(ctx).
|
|
||||||
WithField("final_state", post).
|
|
||||||
Debug("main final state")
|
|
||||||
}
|
|
||||||
|
|
||||||
func rsfPlanning(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
|
|
||||||
|
|
||||||
log := getLogger(ctx)
|
|
||||||
|
|
||||||
handlePlanningError := func(err error) replicationStateFunc {
|
|
||||||
return u(func(r *Replication) {
|
|
||||||
r.planningError = err
|
|
||||||
r.state = PlanningError
|
|
||||||
}).rsf()
|
|
||||||
}
|
|
||||||
|
|
||||||
sfss, err := ep.Sender().ListFilesystems(ctx)
|
|
||||||
if err != nil {
|
|
||||||
log.WithError(err).Error("error listing sender filesystems")
|
|
||||||
return handlePlanningError(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
rfss, err := ep.Receiver().ListFilesystems(ctx)
|
|
||||||
if err != nil {
|
|
||||||
log.WithError(err).Error("error listing receiver filesystems")
|
|
||||||
return handlePlanningError(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
q := newReplicationQueue()
|
|
||||||
mainlog := log
|
|
||||||
for _, fs := range sfss {
|
|
||||||
|
|
||||||
log := mainlog.WithField("filesystem", fs.Path)
|
|
||||||
|
|
||||||
log.Info("assessing filesystem")
|
|
||||||
|
|
||||||
sfsvs, err := ep.Sender().ListFilesystemVersions(ctx, fs.Path)
|
|
||||||
if err != nil {
|
|
||||||
log.WithError(err).Error("cannot get remote filesystem versions")
|
|
||||||
return handlePlanningError(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(sfsvs) <= 1 {
|
|
||||||
err := errors.New("sender does not have any versions")
|
|
||||||
log.Error(err.Error())
|
|
||||||
q.Add(newReplicationQueueItemPermanentError(fs, err))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
receiverFSExists := false
|
|
||||||
for _, rfs := range rfss {
|
|
||||||
if rfs.Path == fs.Path {
|
|
||||||
receiverFSExists = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var rfsvs []*FilesystemVersion
|
|
||||||
if receiverFSExists {
|
|
||||||
rfsvs, err = ep.Receiver().ListFilesystemVersions(ctx, fs.Path)
|
|
||||||
if err != nil {
|
|
||||||
if _, ok := err.(FilteredError); ok {
|
|
||||||
log.Info("receiver ignores filesystem")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
log.WithError(err).Error("receiver error")
|
|
||||||
return handlePlanningError(err)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
rfsvs = []*FilesystemVersion{}
|
|
||||||
}
|
|
||||||
|
|
||||||
path, conflict := IncrementalPath(rfsvs, sfsvs)
|
|
||||||
if conflict != nil {
|
|
||||||
var msg string
|
|
||||||
path, msg = resolveConflict(conflict) // no shadowing allowed!
|
|
||||||
if path != nil {
|
|
||||||
log.WithField("conflict", conflict).Info("conflict")
|
|
||||||
log.WithField("resolution", msg).Info("automatically resolved")
|
|
||||||
} else {
|
|
||||||
log.WithField("conflict", conflict).Error("conflict")
|
|
||||||
log.WithField("problem", msg).Error("cannot resolve conflict")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if path == nil {
|
|
||||||
q.Add(newReplicationQueueItemPermanentError(fs, conflict))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
builder := buildReplicationQueueItem(fs)
|
|
||||||
if len(path) == 1 {
|
|
||||||
builder.AddStep(nil, path[0])
|
|
||||||
} else {
|
|
||||||
for i := 0; i < len(path)-1; i++ {
|
|
||||||
builder.AddStep(path[i], path[i+1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
qitem := builder.Complete()
|
|
||||||
q.Add(qitem)
|
|
||||||
}
|
|
||||||
|
|
||||||
return u(func(r *Replication) {
|
|
||||||
r.completed = nil
|
|
||||||
r.queue = q
|
|
||||||
r.planningError = nil
|
|
||||||
r.state = Working
|
|
||||||
}).rsf()
|
|
||||||
}
|
|
||||||
|
|
||||||
func rsfPlanningError(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
|
|
||||||
sleepTime := 10 * time.Second
|
|
||||||
u(func(r *Replication) {
|
|
||||||
r.sleepUntil = time.Now().Add(sleepTime)
|
|
||||||
})
|
|
||||||
t := time.NewTimer(sleepTime) // FIXME make constant onfigurable
|
|
||||||
defer t.Stop()
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return u(func(r *Replication) {
|
|
||||||
r.state = ContextDone
|
|
||||||
r.contextError = ctx.Err()
|
|
||||||
}).rsf()
|
|
||||||
case <-t.C:
|
|
||||||
return u(func(r *Replication) {
|
|
||||||
r.state = Planning
|
|
||||||
}).rsf()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func rsfWorking(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
|
|
||||||
|
|
||||||
var active *replicationQueueItemHandle
|
|
||||||
|
|
||||||
rsfNext := u(func(r *Replication) {
|
|
||||||
done, next := r.queue.GetNext()
|
|
||||||
r.completed = append(r.completed, done...)
|
|
||||||
if next == nil {
|
|
||||||
r.state = Completed
|
|
||||||
}
|
|
||||||
active = next
|
|
||||||
}).rsf()
|
|
||||||
|
|
||||||
if active == nil {
|
|
||||||
return rsfNext
|
|
||||||
}
|
|
||||||
|
|
||||||
state, nextStepDate := active.GetFSReplication().takeStep(ctx, ep)
|
|
||||||
|
|
||||||
return u(func(r *Replication) {
|
|
||||||
active.Update(state, nextStepDate)
|
|
||||||
}).rsf()
|
|
||||||
}
|
|
||||||
|
|
||||||
func rsfWorkingWait(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
|
|
||||||
sleepTime := 10 * time.Second
|
|
||||||
u(func(r *Replication) {
|
|
||||||
r.sleepUntil = time.Now().Add(sleepTime)
|
|
||||||
})
|
|
||||||
t := time.NewTimer(sleepTime)
|
|
||||||
defer t.Stop()
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return u(func(r *Replication) {
|
|
||||||
r.state = ContextDone
|
|
||||||
r.contextError = ctx.Err()
|
|
||||||
}).rsf()
|
|
||||||
case <-t.C:
|
|
||||||
return u(func(r *Replication) {
|
|
||||||
r.state = Working
|
|
||||||
}).rsf()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type replicationQueueItemBuilder struct {
|
|
||||||
r *FSReplication
|
|
||||||
steps []*FSReplicationStep
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildReplicationQueueItem(fs *Filesystem) *replicationQueueItemBuilder {
|
|
||||||
return &replicationQueueItemBuilder{
|
|
||||||
r: &FSReplication{
|
|
||||||
fs: fs,
|
|
||||||
pending: make([]*FSReplicationStep, 0),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *replicationQueueItemBuilder) AddStep(from, to *FilesystemVersion) *replicationQueueItemBuilder {
|
|
||||||
step := &FSReplicationStep{
|
|
||||||
state: StepReady,
|
|
||||||
fsrep: b.r,
|
|
||||||
from: from,
|
|
||||||
to: to,
|
|
||||||
}
|
|
||||||
b.r.pending = append(b.r.pending, step)
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
type replicationQueue []*replicationQueueItem
|
|
||||||
|
|
||||||
func newReplicationQueue() *replicationQueue {
|
|
||||||
q := make(replicationQueue, 0)
|
|
||||||
return &q
|
|
||||||
}
|
|
||||||
|
|
||||||
func (q replicationQueue) Len() int { return len(q) }
|
|
||||||
func (q replicationQueue) Swap(i, j int) { q[i], q[j] = q[j], q[i] }
|
|
||||||
func (q replicationQueue) Less(i, j int) bool {
|
|
||||||
a, b := q[i], q[j]
|
|
||||||
statePrio := func(x *replicationQueueItem) int {
|
|
||||||
if x.state&(FSReady|FSRetryWait) == 0 {
|
|
||||||
panic(x)
|
|
||||||
}
|
|
||||||
if x.state == FSReady {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
aprio, bprio := statePrio(a), statePrio(b)
|
|
||||||
if aprio != bprio {
|
|
||||||
return aprio < bprio
|
|
||||||
}
|
|
||||||
// now we know they are the same state
|
|
||||||
if a.state == FSReady {
|
|
||||||
return a.nextStepDate.Before(b.nextStepDate)
|
|
||||||
}
|
|
||||||
if a.state == FSRetryWait {
|
|
||||||
return a.retriesSinceLastError < b.retriesSinceLastError
|
|
||||||
}
|
|
||||||
panic("should not be reached")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (q *replicationQueue) sort() (done []*FSReplication) {
|
|
||||||
// pre-scan for everything that is not ready
|
|
||||||
newq := make(replicationQueue, 0, len(*q))
|
|
||||||
done = make([]*FSReplication, 0, len(*q))
|
|
||||||
for _, qitem := range *q {
|
|
||||||
if qitem.state&(FSReady|FSRetryWait) == 0 {
|
|
||||||
done = append(done, qitem.fsr)
|
|
||||||
} else {
|
|
||||||
newq = append(newq, qitem)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.SortStable(newq) // stable to avoid flickering in reports
|
|
||||||
*q = newq
|
|
||||||
return done
|
|
||||||
}
|
|
||||||
|
|
||||||
// next remains valid until the next call to GetNext()
|
|
||||||
func (q *replicationQueue) GetNext() (done []*FSReplication, next *replicationQueueItemHandle) {
|
|
||||||
done = q.sort()
|
|
||||||
if len(*q) == 0 {
|
|
||||||
return done, nil
|
|
||||||
}
|
|
||||||
next = &replicationQueueItemHandle{(*q)[0]}
|
|
||||||
return done, next
|
|
||||||
}
|
|
||||||
|
|
||||||
func (q *replicationQueue) Add(qitem *replicationQueueItem) {
|
|
||||||
*q = append(*q, qitem)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (q *replicationQueue) Foreach(fu func(*replicationQueueItemHandle)) {
|
|
||||||
for _, qitem := range *q {
|
|
||||||
fu(&replicationQueueItemHandle{qitem})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type replicationQueueItemHandle struct {
|
|
||||||
i *replicationQueueItem
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h replicationQueueItemHandle) GetFSReplication() *FSReplication {
|
|
||||||
return h.i.fsr
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h replicationQueueItemHandle) Update(newState FSReplicationState, nextStepDate time.Time) {
|
|
||||||
h.i.state = newState
|
|
||||||
h.i.nextStepDate = nextStepDate
|
|
||||||
if h.i.state&FSReady != 0 {
|
|
||||||
h.i.retriesSinceLastError = 0
|
|
||||||
} else if h.i.state&FSRetryWait != 0 {
|
|
||||||
h.i.retriesSinceLastError++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *FSReplication) takeStep(ctx context.Context, ep EndpointPair) (post FSReplicationState, nextStepDate time.Time) {
|
|
||||||
|
|
||||||
var u fsrUpdater = func(fu func(*FSReplication)) FSReplicationState {
|
|
||||||
f.lock.Lock()
|
|
||||||
defer f.lock.Unlock()
|
|
||||||
if fu != nil {
|
|
||||||
fu(f)
|
|
||||||
}
|
|
||||||
return f.state
|
|
||||||
}
|
|
||||||
var s fsrsf = u(nil).fsrsf()
|
|
||||||
|
|
||||||
pre := u(nil)
|
|
||||||
preTime := time.Now()
|
|
||||||
s = s(ctx, ep, u)
|
|
||||||
delta := time.Now().Sub(preTime)
|
|
||||||
post = u(func(f *FSReplication) {
|
|
||||||
if f.state != FSReady {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ct, err := f.current.to.CreationAsTime()
|
|
||||||
if err != nil {
|
|
||||||
panic(err) // FIXME
|
|
||||||
}
|
|
||||||
nextStepDate = ct
|
|
||||||
})
|
|
||||||
|
|
||||||
getLogger(ctx).
|
|
||||||
WithField("fs", f.fs.Path).
|
|
||||||
WithField("transition", fmt.Sprintf("%s => %s", pre, post)).
|
|
||||||
WithField("duration", delta).
|
|
||||||
Debug("fsr step taken")
|
|
||||||
|
|
||||||
return post, nextStepDate
|
|
||||||
}
|
|
||||||
|
|
||||||
type fsrUpdater func(func(fsr *FSReplication)) FSReplicationState
|
|
||||||
type fsrsf func(ctx context.Context, ep EndpointPair, u fsrUpdater) fsrsf
|
|
||||||
|
|
||||||
func fsrsfReady(ctx context.Context, ep EndpointPair, u fsrUpdater) fsrsf {
|
|
||||||
|
|
||||||
var current *FSReplicationStep
|
|
||||||
s := u(func(f *FSReplication) {
|
|
||||||
if f.current == nil {
|
|
||||||
if len(f.pending) == 0 {
|
|
||||||
f.state = FSCompleted
|
|
||||||
return
|
|
||||||
}
|
|
||||||
f.current = f.pending[0]
|
|
||||||
f.pending = f.pending[1:]
|
|
||||||
}
|
|
||||||
current = f.current
|
|
||||||
})
|
|
||||||
if s != FSReady {
|
|
||||||
return s.fsrsf()
|
|
||||||
}
|
|
||||||
|
|
||||||
stepState := current.do(ctx, ep)
|
|
||||||
|
|
||||||
return u(func(f *FSReplication) {
|
|
||||||
switch stepState {
|
|
||||||
case StepCompleted:
|
|
||||||
f.completed = append(f.completed, f.current)
|
|
||||||
f.current = nil
|
|
||||||
if len(f.pending) > 0 {
|
|
||||||
f.state = FSReady
|
|
||||||
} else {
|
|
||||||
f.state = FSCompleted
|
|
||||||
}
|
|
||||||
case StepRetry:
|
|
||||||
f.retryWaitUntil = time.Now().Add(10 * time.Second) // FIXME make configurable
|
|
||||||
f.state = FSRetryWait
|
|
||||||
case StepPermanentError:
|
|
||||||
f.state = FSPermanentError
|
|
||||||
f.err = errors.New("a replication step failed with a permanent error")
|
|
||||||
default:
|
|
||||||
panic(f)
|
|
||||||
}
|
|
||||||
}).fsrsf()
|
|
||||||
}
|
|
||||||
|
|
||||||
func fsrsfRetryWait(ctx context.Context, ep EndpointPair, u fsrUpdater) fsrsf {
|
|
||||||
var sleepUntil time.Time
|
|
||||||
u(func(f *FSReplication) {
|
|
||||||
sleepUntil = f.retryWaitUntil
|
|
||||||
})
|
|
||||||
t := time.NewTimer(sleepUntil.Sub(time.Now()))
|
|
||||||
defer t.Stop()
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return u(func(f *FSReplication) {
|
|
||||||
f.state = FSPermanentError
|
|
||||||
f.err = ctx.Err()
|
|
||||||
}).fsrsf()
|
|
||||||
case <-t.C:
|
|
||||||
}
|
|
||||||
return u(func(f *FSReplication) {
|
|
||||||
f.state = FSReady
|
|
||||||
}).fsrsf()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *FSReplicationStep) do(ctx context.Context, ep EndpointPair) FSReplicationStepState {
|
|
||||||
|
|
||||||
fs := s.fsrep.fs
|
|
||||||
|
|
||||||
log := getLogger(ctx).
|
|
||||||
WithField("filesystem", fs.Path).
|
|
||||||
WithField("step", s.String())
|
|
||||||
|
|
||||||
updateStateError := func(err error) FSReplicationStepState {
|
|
||||||
s.lock.Lock()
|
|
||||||
defer s.lock.Unlock()
|
|
||||||
|
|
||||||
s.err = err
|
|
||||||
switch err {
|
|
||||||
case io.EOF:
|
|
||||||
fallthrough
|
|
||||||
case io.ErrUnexpectedEOF:
|
|
||||||
fallthrough
|
|
||||||
case io.ErrClosedPipe:
|
|
||||||
s.state = StepRetry
|
|
||||||
return s.state
|
|
||||||
}
|
|
||||||
if _, ok := err.(net.Error); ok {
|
|
||||||
s.state = StepRetry
|
|
||||||
return s.state
|
|
||||||
}
|
|
||||||
s.state = StepPermanentError
|
|
||||||
return s.state
|
|
||||||
}
|
|
||||||
|
|
||||||
updateStateCompleted := func() FSReplicationStepState {
|
|
||||||
s.lock.Lock()
|
|
||||||
defer s.lock.Unlock()
|
|
||||||
s.err = nil
|
|
||||||
s.state = StepCompleted
|
|
||||||
return s.state
|
|
||||||
}
|
|
||||||
|
|
||||||
// FIXME refresh fs resume token
|
|
||||||
fs.ResumeToken = ""
|
|
||||||
|
|
||||||
var sr *SendReq
|
|
||||||
if fs.ResumeToken != "" {
|
|
||||||
sr = &SendReq{
|
|
||||||
Filesystem: fs.Path,
|
|
||||||
ResumeToken: fs.ResumeToken,
|
|
||||||
}
|
|
||||||
} else if s.from == nil {
|
|
||||||
sr = &SendReq{
|
|
||||||
Filesystem: fs.Path,
|
|
||||||
From: s.to.RelName(), // FIXME fix protocol to use To, like zfs does internally
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
sr = &SendReq{
|
|
||||||
Filesystem: fs.Path,
|
|
||||||
From: s.from.RelName(),
|
|
||||||
To: s.to.RelName(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.WithField("request", sr).Debug("initiate send request")
|
|
||||||
sres, sstream, err := ep.Sender().Send(ctx, sr)
|
|
||||||
if err != nil {
|
|
||||||
log.WithError(err).Error("send request failed")
|
|
||||||
return updateStateError(err)
|
|
||||||
}
|
|
||||||
if sstream == nil {
|
|
||||||
err := errors.New("send request did not return a stream, broken endpoint implementation")
|
|
||||||
return updateStateError(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
rr := &ReceiveReq{
|
|
||||||
Filesystem: fs.Path,
|
|
||||||
ClearResumeToken: !sres.UsedResumeToken,
|
|
||||||
}
|
|
||||||
log.WithField("request", rr).Debug("initiate receive request")
|
|
||||||
err = ep.Receiver().Receive(ctx, rr, sstream)
|
|
||||||
if err != nil {
|
|
||||||
log.WithError(err).Error("receive request failed (might also be error on sender)")
|
|
||||||
sstream.Close()
|
|
||||||
// This failure could be due to
|
|
||||||
// - an unexpected exit of ZFS on the sending side
|
|
||||||
// - an unexpected exit of ZFS on the receiving side
|
|
||||||
// - a connectivity issue
|
|
||||||
return updateStateError(err)
|
|
||||||
}
|
|
||||||
log.Info("receive finished")
|
|
||||||
return updateStateCompleted()
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *FSReplicationStep) String() string {
|
|
||||||
if s.from == nil { // FIXME: ZFS semantics are that to is nil on non-incremental send
|
|
||||||
return fmt.Sprintf("%s%s (full)", s.fsrep.fs.Path, s.to.RelName())
|
|
||||||
} else {
|
|
||||||
return fmt.Sprintf("%s(%s => %s)", s.fsrep.fs.Path, s.from.RelName(), s.to.RelName())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,181 +0,0 @@
|
|||||||
package replication
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"syscall"
|
|
||||||
"encoding/json"
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"github.com/zrepl/zrepl/logger"
|
|
||||||
"io"
|
|
||||||
"os/signal"
|
|
||||||
)
|
|
||||||
|
|
||||||
type ReplicationEndpoint interface {
|
|
||||||
// Does not include placeholder filesystems
|
|
||||||
ListFilesystems(ctx context.Context) ([]*Filesystem, error)
|
|
||||||
ListFilesystemVersions(ctx context.Context, fs string) ([]*FilesystemVersion, error) // fix depS
|
|
||||||
Send(ctx context.Context, r *SendReq) (*SendRes, io.ReadCloser, error)
|
|
||||||
Receive(ctx context.Context, r *ReceiveReq, sendStream io.ReadCloser) error
|
|
||||||
}
|
|
||||||
|
|
||||||
type FilteredError struct{ fs string }
|
|
||||||
|
|
||||||
func NewFilteredError(fs string) FilteredError {
|
|
||||||
return FilteredError{fs}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f FilteredError) Error() string { return "endpoint does not allow access to filesystem " + f.fs }
|
|
||||||
|
|
||||||
type ReplicationMode int
|
|
||||||
|
|
||||||
const (
|
|
||||||
ReplicationModePull ReplicationMode = iota
|
|
||||||
ReplicationModePush
|
|
||||||
)
|
|
||||||
|
|
||||||
type EndpointPair struct {
|
|
||||||
a, b ReplicationEndpoint
|
|
||||||
m ReplicationMode
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewEndpointPairPull(sender, receiver ReplicationEndpoint) EndpointPair {
|
|
||||||
return EndpointPair{sender, receiver, ReplicationModePull}
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewEndpointPairPush(sender, receiver ReplicationEndpoint) EndpointPair {
|
|
||||||
return EndpointPair{receiver, sender, ReplicationModePush}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p EndpointPair) Sender() ReplicationEndpoint {
|
|
||||||
switch p.m {
|
|
||||||
case ReplicationModePull:
|
|
||||||
return p.a
|
|
||||||
case ReplicationModePush:
|
|
||||||
return p.b
|
|
||||||
}
|
|
||||||
panic("should not be reached")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p EndpointPair) Receiver() ReplicationEndpoint {
|
|
||||||
switch p.m {
|
|
||||||
case ReplicationModePull:
|
|
||||||
return p.b
|
|
||||||
case ReplicationModePush:
|
|
||||||
return p.a
|
|
||||||
}
|
|
||||||
panic("should not be reached")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p EndpointPair) Mode() ReplicationMode {
|
|
||||||
return p.m
|
|
||||||
}
|
|
||||||
|
|
||||||
type contextKey int
|
|
||||||
|
|
||||||
const (
|
|
||||||
contextKeyLog contextKey = iota
|
|
||||||
)
|
|
||||||
|
|
||||||
//type Logger interface {
|
|
||||||
// Infof(fmt string, args ...interface{})
|
|
||||||
// Errorf(fmt string, args ...interface{})
|
|
||||||
//}
|
|
||||||
|
|
||||||
//var _ Logger = nullLogger{}
|
|
||||||
|
|
||||||
//type nullLogger struct{}
|
|
||||||
//
|
|
||||||
//func (nullLogger) Infof(fmt string, args ...interface{}) {}
|
|
||||||
//func (nullLogger) Errorf(fmt string, args ...interface{}) {}
|
|
||||||
|
|
||||||
type Logger = logger.Logger
|
|
||||||
|
|
||||||
func ContextWithLogger(ctx context.Context, l Logger) context.Context {
|
|
||||||
return context.WithValue(ctx, contextKeyLog, l)
|
|
||||||
}
|
|
||||||
|
|
||||||
func getLogger(ctx context.Context) Logger {
|
|
||||||
l, ok := ctx.Value(contextKeyLog).(Logger)
|
|
||||||
if !ok {
|
|
||||||
l = logger.NewNullLogger()
|
|
||||||
}
|
|
||||||
return l
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveConflict(conflict error) (path []*FilesystemVersion, msg string) {
|
|
||||||
if noCommonAncestor, ok := conflict.(*ConflictNoCommonAncestor); ok {
|
|
||||||
if len(noCommonAncestor.SortedReceiverVersions) == 0 {
|
|
||||||
// FIXME hard-coded replication policy: most recent
|
|
||||||
// snapshot as source
|
|
||||||
var mostRecentSnap *FilesystemVersion
|
|
||||||
for n := len(noCommonAncestor.SortedSenderVersions) - 1; n >= 0; n-- {
|
|
||||||
if noCommonAncestor.SortedSenderVersions[n].Type == FilesystemVersion_Snapshot {
|
|
||||||
mostRecentSnap = noCommonAncestor.SortedSenderVersions[n]
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if mostRecentSnap == nil {
|
|
||||||
return nil, "no snapshots available on sender side"
|
|
||||||
}
|
|
||||||
return []*FilesystemVersion{mostRecentSnap}, fmt.Sprintf("start replication at most recent snapshot %s", mostRecentSnap.RelName())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil, "no automated way to handle conflict type"
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewReplication() *Replication {
|
|
||||||
r := Replication{
|
|
||||||
state: Planning,
|
|
||||||
}
|
|
||||||
return &r
|
|
||||||
}
|
|
||||||
// Replicate replicates filesystems from ep.Sender() to ep.Receiver().
|
|
||||||
//
|
|
||||||
// All filesystems presented by the sending side are replicated,
|
|
||||||
// unless the receiver rejects a Receive request with a *FilteredError.
|
|
||||||
//
|
|
||||||
// If an error occurs when replicating a filesystem, that error is logged to the logger in ctx.
|
|
||||||
// Replicate continues with the replication of the remaining file systems.
|
|
||||||
// Depending on the type of error, failed replications are retried in an unspecified order (currently FIFO).
|
|
||||||
func Replicate(ctx context.Context, ep EndpointPair, retryNow chan struct{}) {
|
|
||||||
r := Replication{
|
|
||||||
state: Planning,
|
|
||||||
}
|
|
||||||
|
|
||||||
c := make(chan os.Signal)
|
|
||||||
defer close(c)
|
|
||||||
signal.Notify(c, syscall.SIGHUP)
|
|
||||||
go func() {
|
|
||||||
f, err := os.OpenFile("/tmp/report", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
|
|
||||||
if err != nil {
|
|
||||||
getLogger(ctx).WithError(err).Error("cannot open report file")
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case sig := <-c:
|
|
||||||
if sig == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
report := r.Report()
|
|
||||||
enc := json.NewEncoder(f)
|
|
||||||
enc.SetIndent(" ", " ")
|
|
||||||
if err := enc.Encode(report); err != nil {
|
|
||||||
getLogger(ctx).WithError(err).Error("cannot encode report")
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
f.Write([]byte("\n"))
|
|
||||||
f.Sync()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
r.Drive(ctx, ep, retryNow)
|
|
||||||
}
|
|
||||||
|
|
||||||
@ -1,181 +0,0 @@
|
|||||||
package replication_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/zrepl/zrepl/cmd/replication"
|
|
||||||
"io"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
type IncrementalPathSequenceStep struct {
|
|
||||||
SendRequest *replication.SendReq
|
|
||||||
SendResponse *replication.SendRes
|
|
||||||
SendReader io.ReadCloser
|
|
||||||
SendError error
|
|
||||||
ReceiveRequest *replication.ReceiveReq
|
|
||||||
ReceiveError error
|
|
||||||
}
|
|
||||||
|
|
||||||
type MockIncrementalPathRecorder struct {
|
|
||||||
T *testing.T
|
|
||||||
Sequence []IncrementalPathSequenceStep
|
|
||||||
Pos int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *MockIncrementalPathRecorder) Receive(ctx context.Context, r *replication.ReceiveReq, rs io.ReadCloser) (error) {
|
|
||||||
if m.Pos >= len(m.Sequence) {
|
|
||||||
m.T.Fatal("unexpected Receive")
|
|
||||||
}
|
|
||||||
i := m.Sequence[m.Pos]
|
|
||||||
m.Pos++
|
|
||||||
if !assert.Equal(m.T, i.ReceiveRequest, r) {
|
|
||||||
m.T.FailNow()
|
|
||||||
}
|
|
||||||
return i.ReceiveError
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *MockIncrementalPathRecorder) Send(ctx context.Context, r *replication.SendReq) (*replication.SendRes, io.ReadCloser, error) {
|
|
||||||
if m.Pos >= len(m.Sequence) {
|
|
||||||
m.T.Fatal("unexpected Send")
|
|
||||||
}
|
|
||||||
i := m.Sequence[m.Pos]
|
|
||||||
m.Pos++
|
|
||||||
if !assert.Equal(m.T, i.SendRequest, r) {
|
|
||||||
m.T.FailNow()
|
|
||||||
}
|
|
||||||
return i.SendResponse, i.SendReader, i.SendError
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *MockIncrementalPathRecorder) Finished() bool {
|
|
||||||
return m.Pos == len(m.Sequence)
|
|
||||||
}
|
|
||||||
|
|
||||||
//type IncrementalPathReplicatorTest struct {
|
|
||||||
// Msg string
|
|
||||||
// Filesystem *replication.Filesystem
|
|
||||||
// Path []*replication.FilesystemVersion
|
|
||||||
// Steps []IncrementalPathSequenceStep
|
|
||||||
//}
|
|
||||||
//
|
|
||||||
//func (test *IncrementalPathReplicatorTest) Test(t *testing.T) {
|
|
||||||
//
|
|
||||||
// t.Log(test.Msg)
|
|
||||||
//
|
|
||||||
// rec := &MockIncrementalPathRecorder{
|
|
||||||
// T: t,
|
|
||||||
// Sequence: test.Steps,
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// ctx := replication.ContextWithLogger(context.Background(), testLog{t})
|
|
||||||
//
|
|
||||||
// ipr := replication.NewIncrementalPathReplicator()
|
|
||||||
// ipr.Replicate(
|
|
||||||
// ctx,
|
|
||||||
// rec,
|
|
||||||
// rec,
|
|
||||||
// DiscardCopier{},
|
|
||||||
// test.Filesystem,
|
|
||||||
// test.Path,
|
|
||||||
// )
|
|
||||||
//
|
|
||||||
// assert.True(t, rec.Finished())
|
|
||||||
//
|
|
||||||
//}
|
|
||||||
|
|
||||||
//type testLog struct {
|
|
||||||
// t *testing.T
|
|
||||||
//}
|
|
||||||
//
|
|
||||||
//var _ replication.Logger = testLog{}
|
|
||||||
//
|
|
||||||
//func (t testLog) Infof(fmt string, args ...interface{}) {
|
|
||||||
// t.t.Logf(fmt, args)
|
|
||||||
//}
|
|
||||||
//func (t testLog) Errorf(fmt string, args ...interface{}) {
|
|
||||||
// t.t.Logf(fmt, args)
|
|
||||||
//}
|
|
||||||
|
|
||||||
|
|
||||||
//func TestIncrementalPathReplicator_Replicate(t *testing.T) {
|
|
||||||
//
|
|
||||||
// tbl := []IncrementalPathReplicatorTest{
|
|
||||||
// {
|
|
||||||
// Msg: "generic happy place with resume token",
|
|
||||||
// Filesystem: &replication.Filesystem{
|
|
||||||
// Path: "foo/bar",
|
|
||||||
// ResumeToken: "blafoo",
|
|
||||||
// },
|
|
||||||
// Path: fsvlist("@a,1", "@b,2", "@c,3"),
|
|
||||||
// Steps: []IncrementalPathSequenceStep{
|
|
||||||
// {
|
|
||||||
// SendRequest: &replication.SendReq{
|
|
||||||
// Filesystem: "foo/bar",
|
|
||||||
// From: "@a,1",
|
|
||||||
// To: "@b,2",
|
|
||||||
// ResumeToken: "blafoo",
|
|
||||||
// },
|
|
||||||
// SendResponse: &replication.SendRes{
|
|
||||||
// UsedResumeToken: true,
|
|
||||||
// },
|
|
||||||
// },
|
|
||||||
// {
|
|
||||||
// ReceiveRequest: &replication.ReceiveReq{
|
|
||||||
// Filesystem: "foo/bar",
|
|
||||||
// ClearResumeToken: false,
|
|
||||||
// },
|
|
||||||
// },
|
|
||||||
// {
|
|
||||||
// SendRequest: &replication.SendReq{
|
|
||||||
// Filesystem: "foo/bar",
|
|
||||||
// From: "@b,2",
|
|
||||||
// To: "@c,3",
|
|
||||||
// },
|
|
||||||
// },
|
|
||||||
// {
|
|
||||||
// ReceiveRequest: &replication.ReceiveReq{
|
|
||||||
// Filesystem: "foo/bar",
|
|
||||||
// },
|
|
||||||
// },
|
|
||||||
// },
|
|
||||||
// },
|
|
||||||
// {
|
|
||||||
// Msg: "no action on empty sequence",
|
|
||||||
// Filesystem: &replication.Filesystem{
|
|
||||||
// Path: "foo/bar",
|
|
||||||
// },
|
|
||||||
// Path: fsvlist(),
|
|
||||||
// Steps: []IncrementalPathSequenceStep{},
|
|
||||||
// },
|
|
||||||
// {
|
|
||||||
// Msg: "full send on single entry path",
|
|
||||||
// Filesystem: &replication.Filesystem{
|
|
||||||
// Path: "foo/bar",
|
|
||||||
// },
|
|
||||||
// Path: fsvlist("@justone,1"),
|
|
||||||
// Steps: []IncrementalPathSequenceStep{
|
|
||||||
// {
|
|
||||||
// SendRequest: &replication.SendReq{
|
|
||||||
// Filesystem: "foo/bar",
|
|
||||||
// From: "@justone,1",
|
|
||||||
// To: "", // empty means full send
|
|
||||||
// },
|
|
||||||
// SendResponse: &replication.SendRes{
|
|
||||||
// UsedResumeToken: false,
|
|
||||||
// },
|
|
||||||
// },
|
|
||||||
// {
|
|
||||||
// ReceiveRequest: &replication.ReceiveReq{
|
|
||||||
// Filesystem: "foo/bar",
|
|
||||||
// ClearResumeToken: false,
|
|
||||||
// },
|
|
||||||
// },
|
|
||||||
// },
|
|
||||||
// },
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// for _, test := range tbl {
|
|
||||||
// test.Test(t)
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
//}
|
|
||||||
@ -1,94 +0,0 @@
|
|||||||
package replication
|
|
||||||
|
|
||||||
type Report struct {
|
|
||||||
Status string
|
|
||||||
Problem string
|
|
||||||
Completed []*FilesystemReplicationReport
|
|
||||||
Pending []*FilesystemReplicationReport
|
|
||||||
Active *FilesystemReplicationReport
|
|
||||||
}
|
|
||||||
|
|
||||||
type StepReport struct {
|
|
||||||
From, To string
|
|
||||||
Status string
|
|
||||||
Problem string
|
|
||||||
}
|
|
||||||
|
|
||||||
type FilesystemReplicationReport struct {
|
|
||||||
Filesystem string
|
|
||||||
Status string
|
|
||||||
Problem string
|
|
||||||
Steps []*StepReport
|
|
||||||
}
|
|
||||||
|
|
||||||
func stepReportFromStep(step *FSReplicationStep) *StepReport {
|
|
||||||
var from string // FIXME follow same convention as ZFS: to should be nil on full send
|
|
||||||
if step.from != nil {
|
|
||||||
from = step.from.RelName()
|
|
||||||
}
|
|
||||||
rep := StepReport{
|
|
||||||
From: from,
|
|
||||||
To: step.to.RelName(),
|
|
||||||
Status: step.state.String(),
|
|
||||||
}
|
|
||||||
return &rep
|
|
||||||
}
|
|
||||||
|
|
||||||
// access to fsr's members must be exclusive
|
|
||||||
func filesystemReplicationReport(fsr *FSReplication) *FilesystemReplicationReport {
|
|
||||||
fsr.lock.Lock()
|
|
||||||
defer fsr.lock.Unlock()
|
|
||||||
|
|
||||||
rep := FilesystemReplicationReport{
|
|
||||||
Filesystem: fsr.fs.Path,
|
|
||||||
Status: fsr.state.String(),
|
|
||||||
}
|
|
||||||
|
|
||||||
if fsr.state&FSPermanentError != 0 {
|
|
||||||
rep.Problem = fsr.err.Error()
|
|
||||||
return &rep
|
|
||||||
}
|
|
||||||
|
|
||||||
rep.Steps = make([]*StepReport, 0, len(fsr.completed)+len(fsr.pending) + 1)
|
|
||||||
for _, step := range fsr.completed {
|
|
||||||
rep.Steps = append(rep.Steps, stepReportFromStep(step))
|
|
||||||
}
|
|
||||||
if fsr.current != nil {
|
|
||||||
rep.Steps = append(rep.Steps, stepReportFromStep(fsr.current))
|
|
||||||
}
|
|
||||||
for _, step := range fsr.pending {
|
|
||||||
rep.Steps = append(rep.Steps, stepReportFromStep(step))
|
|
||||||
}
|
|
||||||
return &rep
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Replication) Report() *Report {
|
|
||||||
r.lock.Lock()
|
|
||||||
defer r.lock.Unlock()
|
|
||||||
|
|
||||||
rep := Report{
|
|
||||||
Status: r.state.String(),
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.state&(Planning|PlanningError|ContextDone) != 0 {
|
|
||||||
switch r.state {
|
|
||||||
case PlanningError:
|
|
||||||
rep.Problem = r.planningError.Error()
|
|
||||||
case ContextDone:
|
|
||||||
rep.Problem = r.contextError.Error()
|
|
||||||
}
|
|
||||||
return &rep
|
|
||||||
}
|
|
||||||
|
|
||||||
rep.Pending = make([]*FilesystemReplicationReport, 0, r.queue.Len())
|
|
||||||
rep.Completed = make([]*FilesystemReplicationReport, 0, len(r.completed)) // room for active (potentially)
|
|
||||||
|
|
||||||
r.queue.Foreach(func (h *replicationQueueItemHandle){
|
|
||||||
rep.Pending = append(rep.Pending, filesystemReplicationReport(h.GetFSReplication()))
|
|
||||||
})
|
|
||||||
for _, fsr := range r.completed {
|
|
||||||
rep.Completed = append(rep.Completed, filesystemReplicationReport(fsr))
|
|
||||||
}
|
|
||||||
|
|
||||||
return &rep
|
|
||||||
}
|
|
||||||
91
cmd/replication/common/common.go
Normal file
91
cmd/replication/common/common.go
Normal file
@ -0,0 +1,91 @@
|
|||||||
|
package common
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"github.com/zrepl/zrepl/cmd/replication/pdu"
|
||||||
|
"github.com/zrepl/zrepl/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
type contextKey int
|
||||||
|
|
||||||
|
const (
|
||||||
|
contextKeyLog contextKey = iota
|
||||||
|
)
|
||||||
|
|
||||||
|
type Logger = logger.Logger
|
||||||
|
|
||||||
|
func ContextWithLogger(ctx context.Context, l Logger) context.Context {
|
||||||
|
return context.WithValue(ctx, contextKeyLog, l)
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetLogger(ctx context.Context) Logger {
|
||||||
|
l, ok := ctx.Value(contextKeyLog).(Logger)
|
||||||
|
if !ok {
|
||||||
|
l = logger.NewNullLogger()
|
||||||
|
}
|
||||||
|
return l
|
||||||
|
}
|
||||||
|
|
||||||
|
type ReplicationEndpoint interface {
|
||||||
|
// Does not include placeholder filesystems
|
||||||
|
ListFilesystems(ctx context.Context) ([]*pdu.Filesystem, error)
|
||||||
|
ListFilesystemVersions(ctx context.Context, fs string) ([]*pdu.FilesystemVersion, error) // fix depS
|
||||||
|
Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error)
|
||||||
|
Receive(ctx context.Context, r *pdu.ReceiveReq, sendStream io.ReadCloser) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type FilteredError struct{ fs string }
|
||||||
|
|
||||||
|
func NewFilteredError(fs string) *FilteredError {
|
||||||
|
return &FilteredError{fs}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f FilteredError) Error() string { return "endpoint does not allow access to filesystem " + f.fs }
|
||||||
|
|
||||||
|
type ReplicationMode int
|
||||||
|
|
||||||
|
const (
|
||||||
|
ReplicationModePull ReplicationMode = iota
|
||||||
|
ReplicationModePush
|
||||||
|
)
|
||||||
|
|
||||||
|
type EndpointPair struct {
|
||||||
|
a, b ReplicationEndpoint
|
||||||
|
m ReplicationMode
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewEndpointPairPull(sender, receiver ReplicationEndpoint) EndpointPair {
|
||||||
|
return EndpointPair{sender, receiver, ReplicationModePull}
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewEndpointPairPush(sender, receiver ReplicationEndpoint) EndpointPair {
|
||||||
|
return EndpointPair{receiver, sender, ReplicationModePush}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p EndpointPair) Sender() ReplicationEndpoint {
|
||||||
|
switch p.m {
|
||||||
|
case ReplicationModePull:
|
||||||
|
return p.a
|
||||||
|
case ReplicationModePush:
|
||||||
|
return p.b
|
||||||
|
}
|
||||||
|
panic("should not be reached")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p EndpointPair) Receiver() ReplicationEndpoint {
|
||||||
|
switch p.m {
|
||||||
|
case ReplicationModePull:
|
||||||
|
return p.b
|
||||||
|
case ReplicationModePush:
|
||||||
|
return p.a
|
||||||
|
}
|
||||||
|
panic("should not be reached")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p EndpointPair) Mode() ReplicationMode {
|
||||||
|
return p.m
|
||||||
|
}
|
||||||
361
cmd/replication/internal/fsfsm/fsfsm.go
Normal file
361
cmd/replication/internal/fsfsm/fsfsm.go
Normal file
@ -0,0 +1,361 @@
|
|||||||
|
package fsfsm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"math/bits"
|
||||||
|
"net"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/zrepl/zrepl/cmd/replication/pdu"
|
||||||
|
. "github.com/zrepl/zrepl/cmd/replication/common"
|
||||||
|
)
|
||||||
|
|
||||||
|
type StepReport struct {
|
||||||
|
From, To string
|
||||||
|
Status string
|
||||||
|
Problem string
|
||||||
|
}
|
||||||
|
|
||||||
|
type FilesystemReplicationReport struct {
|
||||||
|
Filesystem string
|
||||||
|
Status string
|
||||||
|
Problem string
|
||||||
|
Completed,Pending []*StepReport
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
//go:generate stringer -type=FSReplicationState
|
||||||
|
type FSReplicationState uint
|
||||||
|
|
||||||
|
const (
|
||||||
|
FSReady FSReplicationState = 1 << iota
|
||||||
|
FSRetryWait
|
||||||
|
FSPermanentError
|
||||||
|
FSCompleted
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s FSReplicationState) fsrsf() fsrsf {
|
||||||
|
idx := bits.TrailingZeros(uint(s))
|
||||||
|
if idx == bits.UintSize {
|
||||||
|
panic(s)
|
||||||
|
}
|
||||||
|
m := []fsrsf{
|
||||||
|
fsrsfReady,
|
||||||
|
fsrsfRetryWait,
|
||||||
|
nil,
|
||||||
|
nil,
|
||||||
|
}
|
||||||
|
return m[idx]
|
||||||
|
}
|
||||||
|
|
||||||
|
type FSReplication struct {
|
||||||
|
// lock protects all fields in this struct, but not the data behind pointers
|
||||||
|
lock sync.Mutex
|
||||||
|
state FSReplicationState
|
||||||
|
fs string
|
||||||
|
err error
|
||||||
|
retryWaitUntil time.Time
|
||||||
|
completed, pending []*FSReplicationStep
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *FSReplication) State() FSReplicationState {
|
||||||
|
f.lock.Lock()
|
||||||
|
defer f.lock.Unlock()
|
||||||
|
return f.state
|
||||||
|
}
|
||||||
|
|
||||||
|
type FSReplicationBuilder struct {
|
||||||
|
r *FSReplication
|
||||||
|
}
|
||||||
|
|
||||||
|
func BuildFSReplication(fs string) *FSReplicationBuilder {
|
||||||
|
return &FSReplicationBuilder{&FSReplication{fs: fs}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *FSReplicationBuilder) AddStep(from, to FilesystemVersion) *FSReplicationBuilder {
|
||||||
|
step := &FSReplicationStep{
|
||||||
|
state: StepReady,
|
||||||
|
fsrep: b.r,
|
||||||
|
from: from,
|
||||||
|
to: to,
|
||||||
|
}
|
||||||
|
b.r.pending = append(b.r.pending, step)
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *FSReplicationBuilder) Done() (r *FSReplication) {
|
||||||
|
if len(b.r.pending) > 0 {
|
||||||
|
b.r.state = FSReady
|
||||||
|
} else {
|
||||||
|
b.r.state = FSCompleted
|
||||||
|
}
|
||||||
|
r = b.r
|
||||||
|
b.r = nil
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewFSReplicationWithPermanentError(fs string, err error) *FSReplication {
|
||||||
|
return &FSReplication{
|
||||||
|
state: FSPermanentError,
|
||||||
|
fs: fs,
|
||||||
|
err: err,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
//go:generate stringer -type=FSReplicationStepState
|
||||||
|
type FSReplicationStepState uint
|
||||||
|
|
||||||
|
const (
|
||||||
|
StepReady FSReplicationStepState = 1 << iota
|
||||||
|
StepRetry
|
||||||
|
StepPermanentError
|
||||||
|
StepCompleted
|
||||||
|
)
|
||||||
|
|
||||||
|
type FilesystemVersion interface {
|
||||||
|
SnapshotTime() time.Time
|
||||||
|
RelName() string
|
||||||
|
}
|
||||||
|
|
||||||
|
type FSReplicationStep struct {
|
||||||
|
// only protects state, err
|
||||||
|
// from, to and fsrep are assumed to be immutable
|
||||||
|
lock sync.Mutex
|
||||||
|
|
||||||
|
state FSReplicationStepState
|
||||||
|
from, to FilesystemVersion
|
||||||
|
fsrep *FSReplication
|
||||||
|
|
||||||
|
// both retry and permanent error
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *FSReplication) TakeStep(ctx context.Context, ep EndpointPair) (post FSReplicationState, nextStepDate time.Time) {
|
||||||
|
|
||||||
|
var u fsrUpdater = func(fu func(*FSReplication)) FSReplicationState {
|
||||||
|
f.lock.Lock()
|
||||||
|
defer f.lock.Unlock()
|
||||||
|
if fu != nil {
|
||||||
|
fu(f)
|
||||||
|
}
|
||||||
|
return f.state
|
||||||
|
}
|
||||||
|
var s fsrsf = u(nil).fsrsf()
|
||||||
|
|
||||||
|
pre := u(nil)
|
||||||
|
preTime := time.Now()
|
||||||
|
s = s(ctx, ep, u)
|
||||||
|
delta := time.Now().Sub(preTime)
|
||||||
|
post = u(func(f *FSReplication) {
|
||||||
|
if len(f.pending) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
nextStepDate = f.pending[0].to.SnapshotTime()
|
||||||
|
})
|
||||||
|
|
||||||
|
GetLogger(ctx).
|
||||||
|
WithField("fs", f.fs).
|
||||||
|
WithField("transition", fmt.Sprintf("%s => %s", pre, post)).
|
||||||
|
WithField("duration", delta).
|
||||||
|
Debug("fsr step taken")
|
||||||
|
|
||||||
|
return post, nextStepDate
|
||||||
|
}
|
||||||
|
|
||||||
|
type fsrUpdater func(func(fsr *FSReplication)) FSReplicationState
|
||||||
|
|
||||||
|
type fsrsf func(ctx context.Context, ep EndpointPair, u fsrUpdater) fsrsf
|
||||||
|
|
||||||
|
func fsrsfReady(ctx context.Context, ep EndpointPair, u fsrUpdater) fsrsf {
|
||||||
|
|
||||||
|
var current *FSReplicationStep
|
||||||
|
s := u(func(f *FSReplication) {
|
||||||
|
if len(f.pending) == 0 {
|
||||||
|
f.state = FSCompleted
|
||||||
|
return
|
||||||
|
}
|
||||||
|
current = f.pending[0]
|
||||||
|
})
|
||||||
|
if s != FSReady {
|
||||||
|
return s.fsrsf()
|
||||||
|
}
|
||||||
|
|
||||||
|
stepState := current.do(ctx, ep)
|
||||||
|
|
||||||
|
return u(func(f *FSReplication) {
|
||||||
|
switch stepState {
|
||||||
|
case StepCompleted:
|
||||||
|
f.completed = append(f.completed, current)
|
||||||
|
f.pending = f.pending[1:]
|
||||||
|
if len(f.pending) > 0 {
|
||||||
|
f.state = FSReady
|
||||||
|
} else {
|
||||||
|
f.state = FSCompleted
|
||||||
|
}
|
||||||
|
case StepRetry:
|
||||||
|
f.retryWaitUntil = time.Now().Add(10 * time.Second) // FIXME make configurable
|
||||||
|
f.state = FSRetryWait
|
||||||
|
case StepPermanentError:
|
||||||
|
f.state = FSPermanentError
|
||||||
|
f.err = errors.New("a replication step failed with a permanent error")
|
||||||
|
default:
|
||||||
|
panic(f)
|
||||||
|
}
|
||||||
|
}).fsrsf()
|
||||||
|
}
|
||||||
|
|
||||||
|
func fsrsfRetryWait(ctx context.Context, ep EndpointPair, u fsrUpdater) fsrsf {
|
||||||
|
var sleepUntil time.Time
|
||||||
|
u(func(f *FSReplication) {
|
||||||
|
sleepUntil = f.retryWaitUntil
|
||||||
|
})
|
||||||
|
t := time.NewTimer(sleepUntil.Sub(time.Now()))
|
||||||
|
defer t.Stop()
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return u(func(f *FSReplication) {
|
||||||
|
f.state = FSPermanentError
|
||||||
|
f.err = ctx.Err()
|
||||||
|
}).fsrsf()
|
||||||
|
case <-t.C:
|
||||||
|
}
|
||||||
|
return u(func(f *FSReplication) {
|
||||||
|
f.state = FSReady
|
||||||
|
}).fsrsf()
|
||||||
|
}
|
||||||
|
|
||||||
|
// access to fsr's members must be exclusive
|
||||||
|
func (fsr *FSReplication) Report() *FilesystemReplicationReport {
|
||||||
|
fsr.lock.Lock()
|
||||||
|
defer fsr.lock.Unlock()
|
||||||
|
|
||||||
|
rep := FilesystemReplicationReport{
|
||||||
|
Filesystem: fsr.fs,
|
||||||
|
Status: fsr.state.String(),
|
||||||
|
}
|
||||||
|
|
||||||
|
if fsr.state&FSPermanentError != 0 {
|
||||||
|
rep.Problem = fsr.err.Error()
|
||||||
|
return &rep
|
||||||
|
}
|
||||||
|
|
||||||
|
rep.Completed = make([]*StepReport, len(fsr.completed))
|
||||||
|
for i := range fsr.completed {
|
||||||
|
rep.Completed[i] = fsr.completed[i].Report()
|
||||||
|
}
|
||||||
|
rep.Pending = make([]*StepReport, len(fsr.pending))
|
||||||
|
for i := range fsr.pending {
|
||||||
|
rep.Pending[i] = fsr.pending[i].Report()
|
||||||
|
}
|
||||||
|
return &rep
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *FSReplicationStep) do(ctx context.Context, ep EndpointPair) FSReplicationStepState {
|
||||||
|
|
||||||
|
fs := s.fsrep.fs
|
||||||
|
|
||||||
|
log := GetLogger(ctx).
|
||||||
|
WithField("filesystem", fs).
|
||||||
|
WithField("step", s.String())
|
||||||
|
|
||||||
|
updateStateError := func(err error) FSReplicationStepState {
|
||||||
|
s.lock.Lock()
|
||||||
|
defer s.lock.Unlock()
|
||||||
|
|
||||||
|
s.err = err
|
||||||
|
switch err {
|
||||||
|
case io.EOF:
|
||||||
|
fallthrough
|
||||||
|
case io.ErrUnexpectedEOF:
|
||||||
|
fallthrough
|
||||||
|
case io.ErrClosedPipe:
|
||||||
|
s.state = StepRetry
|
||||||
|
return s.state
|
||||||
|
}
|
||||||
|
if _, ok := err.(net.Error); ok {
|
||||||
|
s.state = StepRetry
|
||||||
|
return s.state
|
||||||
|
}
|
||||||
|
s.state = StepPermanentError
|
||||||
|
return s.state
|
||||||
|
}
|
||||||
|
|
||||||
|
updateStateCompleted := func() FSReplicationStepState {
|
||||||
|
s.lock.Lock()
|
||||||
|
defer s.lock.Unlock()
|
||||||
|
s.err = nil
|
||||||
|
s.state = StepCompleted
|
||||||
|
return s.state
|
||||||
|
}
|
||||||
|
|
||||||
|
var sr *pdu.SendReq
|
||||||
|
if s.from == nil {
|
||||||
|
sr = &pdu.SendReq{
|
||||||
|
Filesystem: fs,
|
||||||
|
From: s.to.RelName(), // FIXME fix protocol to use To, like zfs does internally
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
sr = &pdu.SendReq{
|
||||||
|
Filesystem: fs,
|
||||||
|
From: s.from.RelName(),
|
||||||
|
To: s.to.RelName(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.WithField("request", sr).Debug("initiate send request")
|
||||||
|
sres, sstream, err := ep.Sender().Send(ctx, sr)
|
||||||
|
if err != nil {
|
||||||
|
log.WithError(err).Error("send request failed")
|
||||||
|
return updateStateError(err)
|
||||||
|
}
|
||||||
|
if sstream == nil {
|
||||||
|
err := errors.New("send request did not return a stream, broken endpoint implementation")
|
||||||
|
return updateStateError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rr := &pdu.ReceiveReq{
|
||||||
|
Filesystem: fs,
|
||||||
|
ClearResumeToken: !sres.UsedResumeToken,
|
||||||
|
}
|
||||||
|
log.WithField("request", rr).Debug("initiate receive request")
|
||||||
|
err = ep.Receiver().Receive(ctx, rr, sstream)
|
||||||
|
if err != nil {
|
||||||
|
log.WithError(err).Error("receive request failed (might also be error on sender)")
|
||||||
|
sstream.Close()
|
||||||
|
// This failure could be due to
|
||||||
|
// - an unexpected exit of ZFS on the sending side
|
||||||
|
// - an unexpected exit of ZFS on the receiving side
|
||||||
|
// - a connectivity issue
|
||||||
|
return updateStateError(err)
|
||||||
|
}
|
||||||
|
log.Info("receive finished")
|
||||||
|
return updateStateCompleted()
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *FSReplicationStep) String() string {
|
||||||
|
if s.from == nil { // FIXME: ZFS semantics are that to is nil on non-incremental send
|
||||||
|
return fmt.Sprintf("%s%s (full)", s.fsrep.fs, s.to.RelName())
|
||||||
|
} else {
|
||||||
|
return fmt.Sprintf("%s(%s => %s)", s.fsrep.fs, s.from, s.to.RelName())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (step *FSReplicationStep) Report() *StepReport {
|
||||||
|
var from string // FIXME follow same convention as ZFS: to should be nil on full send
|
||||||
|
if step.from != nil {
|
||||||
|
from = step.from.RelName()
|
||||||
|
}
|
||||||
|
rep := StepReport{
|
||||||
|
From: from,
|
||||||
|
To: step.to.RelName(),
|
||||||
|
Status: step.state.String(),
|
||||||
|
}
|
||||||
|
return &rep
|
||||||
|
}
|
||||||
|
|
||||||
@ -1,6 +1,6 @@
|
|||||||
// Code generated by "stringer -type=FSReplicationState"; DO NOT EDIT.
|
// Code generated by "stringer -type=FSReplicationState"; DO NOT EDIT.
|
||||||
|
|
||||||
package replication
|
package fsfsm
|
||||||
|
|
||||||
import "strconv"
|
import "strconv"
|
||||||
|
|
||||||
@ -1,6 +1,6 @@
|
|||||||
// Code generated by "stringer -type=FSReplicationStepState"; DO NOT EDIT.
|
// Code generated by "stringer -type=FSReplicationStepState"; DO NOT EDIT.
|
||||||
|
|
||||||
package replication
|
package fsfsm
|
||||||
|
|
||||||
import "strconv"
|
import "strconv"
|
||||||
|
|
||||||
@ -1,7 +1,9 @@
|
|||||||
package replication
|
package mainfsm
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"sort"
|
"sort"
|
||||||
|
|
||||||
|
. "github.com/zrepl/zrepl/cmd/replication/pdu"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ConflictNoCommonAncestor struct {
|
type ConflictNoCommonAncestor struct {
|
||||||
342
cmd/replication/internal/mainfsm/mainfsm.go
Normal file
342
cmd/replication/internal/mainfsm/mainfsm.go
Normal file
@ -0,0 +1,342 @@
|
|||||||
|
package mainfsm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"math/bits"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
. "github.com/zrepl/zrepl/cmd/replication/common"
|
||||||
|
"github.com/zrepl/zrepl/cmd/replication/pdu"
|
||||||
|
"github.com/zrepl/zrepl/cmd/replication/internal/fsfsm"
|
||||||
|
. "github.com/zrepl/zrepl/cmd/replication/internal/mainfsm/queue"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:generate stringer -type=ReplicationState
|
||||||
|
type ReplicationState uint
|
||||||
|
|
||||||
|
const (
|
||||||
|
Planning ReplicationState = 1 << iota
|
||||||
|
PlanningError
|
||||||
|
Working
|
||||||
|
WorkingWait
|
||||||
|
Completed
|
||||||
|
ContextDone
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s ReplicationState) rsf() replicationStateFunc {
|
||||||
|
idx := bits.TrailingZeros(uint(s))
|
||||||
|
if idx == bits.UintSize {
|
||||||
|
panic(s) // invalid value
|
||||||
|
}
|
||||||
|
m := []replicationStateFunc{
|
||||||
|
rsfPlanning,
|
||||||
|
rsfPlanningError,
|
||||||
|
rsfWorking,
|
||||||
|
rsfWorkingWait,
|
||||||
|
nil,
|
||||||
|
nil,
|
||||||
|
}
|
||||||
|
return m[idx]
|
||||||
|
}
|
||||||
|
|
||||||
|
type Replication struct {
|
||||||
|
// lock protects all fields of this struct (but not the fields behind pointers!)
|
||||||
|
lock sync.Mutex
|
||||||
|
|
||||||
|
state ReplicationState
|
||||||
|
|
||||||
|
// Working, WorkingWait, Completed, ContextDone
|
||||||
|
queue *ReplicationQueue
|
||||||
|
completed []*fsfsm.FSReplication
|
||||||
|
active *ReplicationQueueItemHandle
|
||||||
|
|
||||||
|
// PlanningError
|
||||||
|
planningError error
|
||||||
|
|
||||||
|
// ContextDone
|
||||||
|
contextError error
|
||||||
|
|
||||||
|
// PlanningError, WorkingWait
|
||||||
|
sleepUntil time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
type Report struct {
|
||||||
|
Status string
|
||||||
|
Problem string
|
||||||
|
Completed []*fsfsm.FilesystemReplicationReport
|
||||||
|
Pending []*fsfsm.FilesystemReplicationReport
|
||||||
|
Active *fsfsm.FilesystemReplicationReport
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
func NewReplication() *Replication {
|
||||||
|
r := Replication{
|
||||||
|
state: Planning,
|
||||||
|
}
|
||||||
|
return &r
|
||||||
|
}
|
||||||
|
|
||||||
|
type replicationUpdater func(func(*Replication)) (newState ReplicationState)
|
||||||
|
type replicationStateFunc func(context.Context, EndpointPair, replicationUpdater) replicationStateFunc
|
||||||
|
|
||||||
|
func (r *Replication) Drive(ctx context.Context, ep EndpointPair) {
|
||||||
|
|
||||||
|
var u replicationUpdater = func(f func(*Replication)) ReplicationState {
|
||||||
|
r.lock.Lock()
|
||||||
|
defer r.lock.Unlock()
|
||||||
|
if f != nil {
|
||||||
|
f(r)
|
||||||
|
}
|
||||||
|
return r.state
|
||||||
|
}
|
||||||
|
|
||||||
|
var s replicationStateFunc = rsfPlanning
|
||||||
|
var pre, post ReplicationState
|
||||||
|
for s != nil {
|
||||||
|
preTime := time.Now()
|
||||||
|
pre = u(nil)
|
||||||
|
s = s(ctx, ep, u)
|
||||||
|
delta := time.Now().Sub(preTime)
|
||||||
|
post = u(nil)
|
||||||
|
GetLogger(ctx).
|
||||||
|
WithField("transition", fmt.Sprintf("%s => %s", pre, post)).
|
||||||
|
WithField("duration", delta).
|
||||||
|
Debug("main state transition")
|
||||||
|
}
|
||||||
|
|
||||||
|
GetLogger(ctx).
|
||||||
|
WithField("final_state", post).
|
||||||
|
Debug("main final state")
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveConflict(conflict error) (path []*pdu.FilesystemVersion, msg string) {
|
||||||
|
if noCommonAncestor, ok := conflict.(*ConflictNoCommonAncestor); ok {
|
||||||
|
if len(noCommonAncestor.SortedReceiverVersions) == 0 {
|
||||||
|
// FIXME hard-coded replication policy: most recent
|
||||||
|
// snapshot as source
|
||||||
|
var mostRecentSnap *pdu.FilesystemVersion
|
||||||
|
for n := len(noCommonAncestor.SortedSenderVersions) - 1; n >= 0; n-- {
|
||||||
|
if noCommonAncestor.SortedSenderVersions[n].Type == pdu.FilesystemVersion_Snapshot {
|
||||||
|
mostRecentSnap = noCommonAncestor.SortedSenderVersions[n]
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if mostRecentSnap == nil {
|
||||||
|
return nil, "no snapshots available on sender side"
|
||||||
|
}
|
||||||
|
return []*pdu.FilesystemVersion{mostRecentSnap}, fmt.Sprintf("start replication at most recent snapshot %s", mostRecentSnap.RelName())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, "no automated way to handle conflict type"
|
||||||
|
}
|
||||||
|
|
||||||
|
func rsfPlanning(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
|
||||||
|
|
||||||
|
log := GetLogger(ctx)
|
||||||
|
|
||||||
|
handlePlanningError := func(err error) replicationStateFunc {
|
||||||
|
return u(func(r *Replication) {
|
||||||
|
r.planningError = err
|
||||||
|
r.state = PlanningError
|
||||||
|
}).rsf()
|
||||||
|
}
|
||||||
|
|
||||||
|
sfss, err := ep.Sender().ListFilesystems(ctx)
|
||||||
|
if err != nil {
|
||||||
|
log.WithError(err).Error("error listing sender filesystems")
|
||||||
|
return handlePlanningError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rfss, err := ep.Receiver().ListFilesystems(ctx)
|
||||||
|
if err != nil {
|
||||||
|
log.WithError(err).Error("error listing receiver filesystems")
|
||||||
|
return handlePlanningError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
q := NewReplicationQueue()
|
||||||
|
mainlog := log
|
||||||
|
for _, fs := range sfss {
|
||||||
|
|
||||||
|
log := mainlog.WithField("filesystem", fs.Path)
|
||||||
|
|
||||||
|
log.Info("assessing filesystem")
|
||||||
|
|
||||||
|
sfsvs, err := ep.Sender().ListFilesystemVersions(ctx, fs.Path)
|
||||||
|
if err != nil {
|
||||||
|
log.WithError(err).Error("cannot get remote filesystem versions")
|
||||||
|
return handlePlanningError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(sfsvs) <= 1 {
|
||||||
|
err := errors.New("sender does not have any versions")
|
||||||
|
log.Error(err.Error())
|
||||||
|
q.Add(fsfsm.NewFSReplicationWithPermanentError(fs.Path, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
receiverFSExists := false
|
||||||
|
for _, rfs := range rfss {
|
||||||
|
if rfs.Path == fs.Path {
|
||||||
|
receiverFSExists = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var rfsvs []*pdu.FilesystemVersion
|
||||||
|
if receiverFSExists {
|
||||||
|
rfsvs, err = ep.Receiver().ListFilesystemVersions(ctx, fs.Path)
|
||||||
|
if err != nil {
|
||||||
|
if _, ok := err.(*FilteredError); ok {
|
||||||
|
log.Info("receiver ignores filesystem")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.WithError(err).Error("receiver error")
|
||||||
|
return handlePlanningError(err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
rfsvs = []*pdu.FilesystemVersion{}
|
||||||
|
}
|
||||||
|
|
||||||
|
path, conflict := IncrementalPath(rfsvs, sfsvs)
|
||||||
|
if conflict != nil {
|
||||||
|
var msg string
|
||||||
|
path, msg = resolveConflict(conflict) // no shadowing allowed!
|
||||||
|
if path != nil {
|
||||||
|
log.WithField("conflict", conflict).Info("conflict")
|
||||||
|
log.WithField("resolution", msg).Info("automatically resolved")
|
||||||
|
} else {
|
||||||
|
log.WithField("conflict", conflict).Error("conflict")
|
||||||
|
log.WithField("problem", msg).Error("cannot resolve conflict")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if path == nil {
|
||||||
|
q.Add(fsfsm.NewFSReplicationWithPermanentError(fs.Path, conflict))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
fsrfsm := fsfsm.BuildFSReplication(fs.Path)
|
||||||
|
if len(path) == 1 {
|
||||||
|
fsrfsm.AddStep(nil, path[0])
|
||||||
|
} else {
|
||||||
|
for i := 0; i < len(path)-1; i++ {
|
||||||
|
fsrfsm.AddStep(path[i], path[i+1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
qitem := fsrfsm.Done()
|
||||||
|
q.Add(qitem)
|
||||||
|
}
|
||||||
|
|
||||||
|
return u(func(r *Replication) {
|
||||||
|
r.completed = nil
|
||||||
|
r.queue = q
|
||||||
|
r.planningError = nil
|
||||||
|
r.state = Working
|
||||||
|
}).rsf()
|
||||||
|
}
|
||||||
|
|
||||||
|
func rsfPlanningError(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
|
||||||
|
sleepTime := 10 * time.Second
|
||||||
|
u(func(r *Replication) {
|
||||||
|
r.sleepUntil = time.Now().Add(sleepTime)
|
||||||
|
})
|
||||||
|
t := time.NewTimer(sleepTime) // FIXME make constant onfigurable
|
||||||
|
defer t.Stop()
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return u(func(r *Replication) {
|
||||||
|
r.state = ContextDone
|
||||||
|
r.contextError = ctx.Err()
|
||||||
|
}).rsf()
|
||||||
|
case <-t.C:
|
||||||
|
return u(func(r *Replication) {
|
||||||
|
r.state = Planning
|
||||||
|
}).rsf()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func rsfWorking(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
|
||||||
|
|
||||||
|
var active *ReplicationQueueItemHandle
|
||||||
|
rsfNext := u(func(r *Replication) {
|
||||||
|
done, next := r.queue.GetNext()
|
||||||
|
r.completed = append(r.completed, done...)
|
||||||
|
if next == nil {
|
||||||
|
r.state = Completed
|
||||||
|
}
|
||||||
|
r.active = next
|
||||||
|
active = next
|
||||||
|
}).rsf()
|
||||||
|
|
||||||
|
if active == nil {
|
||||||
|
return rsfNext
|
||||||
|
}
|
||||||
|
|
||||||
|
state, nextStepDate := active.GetFSReplication().TakeStep(ctx, ep)
|
||||||
|
|
||||||
|
return u(func(r *Replication) {
|
||||||
|
active.Update(state, nextStepDate)
|
||||||
|
r.active = nil
|
||||||
|
}).rsf()
|
||||||
|
}
|
||||||
|
|
||||||
|
func rsfWorkingWait(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
|
||||||
|
sleepTime := 10 * time.Second
|
||||||
|
u(func(r *Replication) {
|
||||||
|
r.sleepUntil = time.Now().Add(sleepTime)
|
||||||
|
})
|
||||||
|
t := time.NewTimer(sleepTime)
|
||||||
|
defer t.Stop()
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return u(func(r *Replication) {
|
||||||
|
r.state = ContextDone
|
||||||
|
r.contextError = ctx.Err()
|
||||||
|
}).rsf()
|
||||||
|
case <-t.C:
|
||||||
|
return u(func(r *Replication) {
|
||||||
|
r.state = Working
|
||||||
|
}).rsf()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Replication) Report() *Report {
|
||||||
|
r.lock.Lock()
|
||||||
|
defer r.lock.Unlock()
|
||||||
|
|
||||||
|
rep := Report{
|
||||||
|
Status: r.state.String(),
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.state&(Planning|PlanningError|ContextDone) != 0 {
|
||||||
|
switch r.state {
|
||||||
|
case PlanningError:
|
||||||
|
rep.Problem = r.planningError.Error()
|
||||||
|
case ContextDone:
|
||||||
|
rep.Problem = r.contextError.Error()
|
||||||
|
}
|
||||||
|
return &rep
|
||||||
|
}
|
||||||
|
|
||||||
|
rep.Pending = make([]*fsfsm.FilesystemReplicationReport, 0, r.queue.Len())
|
||||||
|
rep.Completed = make([]*fsfsm.FilesystemReplicationReport, 0, len(r.completed)) // room for active (potentially)
|
||||||
|
|
||||||
|
var active *fsfsm.FSReplication
|
||||||
|
if r.active != nil {
|
||||||
|
active = r.active.GetFSReplication()
|
||||||
|
rep.Active = active.Report()
|
||||||
|
}
|
||||||
|
r.queue.Foreach(func (h *ReplicationQueueItemHandle){
|
||||||
|
fsr := h.GetFSReplication()
|
||||||
|
if active != fsr {
|
||||||
|
rep.Pending = append(rep.Pending, fsr.Report())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
for _, fsr := range r.completed {
|
||||||
|
rep.Completed = append(rep.Completed, fsr.Report())
|
||||||
|
}
|
||||||
|
|
||||||
|
return &rep
|
||||||
|
}
|
||||||
|
|
||||||
124
cmd/replication/internal/mainfsm/queue/queue.go
Normal file
124
cmd/replication/internal/mainfsm/queue/queue.go
Normal file
@ -0,0 +1,124 @@
|
|||||||
|
package queue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
. "github.com/zrepl/zrepl/cmd/replication/internal/fsfsm"
|
||||||
|
)
|
||||||
|
|
||||||
|
type replicationQueueItem struct {
|
||||||
|
retriesSinceLastError int
|
||||||
|
// duplicates fsr.state to avoid accessing and locking fsr
|
||||||
|
state FSReplicationState
|
||||||
|
// duplicates fsr.current.nextStepDate to avoid accessing & locking fsr
|
||||||
|
nextStepDate time.Time
|
||||||
|
|
||||||
|
fsr *FSReplication
|
||||||
|
}
|
||||||
|
|
||||||
|
type ReplicationQueue []*replicationQueueItem
|
||||||
|
|
||||||
|
func NewReplicationQueue() *ReplicationQueue {
|
||||||
|
q := make(ReplicationQueue, 0)
|
||||||
|
return &q
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q ReplicationQueue) Len() int { return len(q) }
|
||||||
|
func (q ReplicationQueue) Swap(i, j int) { q[i], q[j] = q[j], q[i] }
|
||||||
|
|
||||||
|
type lessmapEntry struct{
|
||||||
|
prio int
|
||||||
|
less func(a,b *replicationQueueItem) bool
|
||||||
|
}
|
||||||
|
|
||||||
|
var lessmap = map[FSReplicationState]lessmapEntry {
|
||||||
|
FSReady: {
|
||||||
|
prio: 0,
|
||||||
|
less: func(a, b *replicationQueueItem) bool {
|
||||||
|
return a.nextStepDate.Before(b.nextStepDate)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
FSRetryWait: {
|
||||||
|
prio: 1,
|
||||||
|
less: func(a, b *replicationQueueItem) bool {
|
||||||
|
return a.retriesSinceLastError < b.retriesSinceLastError
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q ReplicationQueue) Less(i, j int) bool {
|
||||||
|
|
||||||
|
a, b := q[i], q[j]
|
||||||
|
al, aok := lessmap[a.state]
|
||||||
|
if !aok {
|
||||||
|
panic(a)
|
||||||
|
}
|
||||||
|
bl, bok := lessmap[b.state]
|
||||||
|
if !bok {
|
||||||
|
panic(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
if al.prio != bl.prio {
|
||||||
|
return al.prio < bl.prio
|
||||||
|
}
|
||||||
|
|
||||||
|
return al.less(a, b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *ReplicationQueue) sort() (done []*FSReplication) {
|
||||||
|
// pre-scan for everything that is not ready
|
||||||
|
newq := make(ReplicationQueue, 0, len(*q))
|
||||||
|
done = make([]*FSReplication, 0, len(*q))
|
||||||
|
for _, qitem := range *q {
|
||||||
|
if _, ok := lessmap[qitem.state]; !ok {
|
||||||
|
done = append(done, qitem.fsr)
|
||||||
|
} else {
|
||||||
|
newq = append(newq, qitem)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Stable(newq) // stable to avoid flickering in reports
|
||||||
|
*q = newq
|
||||||
|
return done
|
||||||
|
}
|
||||||
|
|
||||||
|
// next remains valid until the next call to GetNext()
|
||||||
|
func (q *ReplicationQueue) GetNext() (done []*FSReplication, next *ReplicationQueueItemHandle) {
|
||||||
|
done = q.sort()
|
||||||
|
if len(*q) == 0 {
|
||||||
|
return done, nil
|
||||||
|
}
|
||||||
|
next = &ReplicationQueueItemHandle{(*q)[0]}
|
||||||
|
return done, next
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *ReplicationQueue) Add(fsr *FSReplication) {
|
||||||
|
*q = append(*q, &replicationQueueItem{
|
||||||
|
fsr: fsr,
|
||||||
|
state: fsr.State(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *ReplicationQueue) Foreach(fu func(*ReplicationQueueItemHandle)) {
|
||||||
|
for _, qitem := range *q {
|
||||||
|
fu(&ReplicationQueueItemHandle{qitem})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type ReplicationQueueItemHandle struct {
|
||||||
|
i *replicationQueueItem
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h ReplicationQueueItemHandle) GetFSReplication() *FSReplication {
|
||||||
|
return h.i.fsr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h ReplicationQueueItemHandle) Update(newState FSReplicationState, nextStepDate time.Time) {
|
||||||
|
h.i.state = newState
|
||||||
|
h.i.nextStepDate = nextStepDate
|
||||||
|
if h.i.state&FSReady != 0 {
|
||||||
|
h.i.retriesSinceLastError = 0
|
||||||
|
} else if h.i.state&FSRetryWait != 0 {
|
||||||
|
h.i.retriesSinceLastError++
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,6 +1,6 @@
|
|||||||
// Code generated by "stringer -type=ReplicationState"; DO NOT EDIT.
|
// Code generated by "stringer -type=ReplicationState"; DO NOT EDIT.
|
||||||
|
|
||||||
package replication
|
package mainfsm
|
||||||
|
|
||||||
import "strconv"
|
import "strconv"
|
||||||
|
|
||||||
@ -2,7 +2,7 @@
|
|||||||
// source: pdu.proto
|
// source: pdu.proto
|
||||||
|
|
||||||
/*
|
/*
|
||||||
Package replication is a generated protocol buffer package.
|
Package pdu is a generated protocol buffer package.
|
||||||
|
|
||||||
It is generated from these files:
|
It is generated from these files:
|
||||||
pdu.proto
|
pdu.proto
|
||||||
@ -20,7 +20,7 @@ It has these top-level messages:
|
|||||||
ReceiveReq
|
ReceiveReq
|
||||||
ReceiveRes
|
ReceiveRes
|
||||||
*/
|
*/
|
||||||
package replication
|
package pdu
|
||||||
|
|
||||||
import proto "github.com/golang/protobuf/proto"
|
import proto "github.com/golang/protobuf/proto"
|
||||||
import fmt "fmt"
|
import fmt "fmt"
|
||||||
@ -141,7 +141,7 @@ func (m *ListFilesystemVersionsRes) GetVersions() []*FilesystemVersion {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type FilesystemVersion struct {
|
type FilesystemVersion struct {
|
||||||
Type FilesystemVersion_VersionType `protobuf:"varint,1,opt,name=Type,enum=replication.FilesystemVersion_VersionType" json:"Type,omitempty"`
|
Type FilesystemVersion_VersionType `protobuf:"varint,1,opt,name=Type,enum=pdu.FilesystemVersion_VersionType" json:"Type,omitempty"`
|
||||||
Name string `protobuf:"bytes,2,opt,name=Name" json:"Name,omitempty"`
|
Name string `protobuf:"bytes,2,opt,name=Name" json:"Name,omitempty"`
|
||||||
Guid uint64 `protobuf:"varint,3,opt,name=Guid" json:"Guid,omitempty"`
|
Guid uint64 `protobuf:"varint,3,opt,name=Guid" json:"Guid,omitempty"`
|
||||||
CreateTXG uint64 `protobuf:"varint,4,opt,name=CreateTXG" json:"CreateTXG,omitempty"`
|
CreateTXG uint64 `protobuf:"varint,4,opt,name=CreateTXG" json:"CreateTXG,omitempty"`
|
||||||
@ -191,7 +191,8 @@ func (m *FilesystemVersion) GetCreation() string {
|
|||||||
type SendReq struct {
|
type SendReq struct {
|
||||||
Filesystem string `protobuf:"bytes,1,opt,name=Filesystem" json:"Filesystem,omitempty"`
|
Filesystem string `protobuf:"bytes,1,opt,name=Filesystem" json:"Filesystem,omitempty"`
|
||||||
From string `protobuf:"bytes,2,opt,name=From" json:"From,omitempty"`
|
From string `protobuf:"bytes,2,opt,name=From" json:"From,omitempty"`
|
||||||
To string `protobuf:"bytes,3,opt,name=To" json:"To,omitempty"`
|
// May be empty / null to request a full transfer of From
|
||||||
|
To string `protobuf:"bytes,3,opt,name=To" json:"To,omitempty"`
|
||||||
// If ResumeToken is not empty, the resume token that CAN be tried for 'zfs send' by the sender.
|
// If ResumeToken is not empty, the resume token that CAN be tried for 'zfs send' by the sender.
|
||||||
// The sender MUST indicate in SendRes.UsedResumeToken
|
// The sender MUST indicate in SendRes.UsedResumeToken
|
||||||
// If it does not work, the sender SHOULD clear the resume token on their side
|
// If it does not work, the sender SHOULD clear the resume token on their side
|
||||||
@ -334,51 +335,50 @@ func (*ReceiveRes) ProtoMessage() {}
|
|||||||
func (*ReceiveRes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
|
func (*ReceiveRes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
proto.RegisterType((*ListFilesystemReq)(nil), "replication.ListFilesystemReq")
|
proto.RegisterType((*ListFilesystemReq)(nil), "pdu.ListFilesystemReq")
|
||||||
proto.RegisterType((*ListFilesystemRes)(nil), "replication.ListFilesystemRes")
|
proto.RegisterType((*ListFilesystemRes)(nil), "pdu.ListFilesystemRes")
|
||||||
proto.RegisterType((*Filesystem)(nil), "replication.Filesystem")
|
proto.RegisterType((*Filesystem)(nil), "pdu.Filesystem")
|
||||||
proto.RegisterType((*ListFilesystemVersionsReq)(nil), "replication.ListFilesystemVersionsReq")
|
proto.RegisterType((*ListFilesystemVersionsReq)(nil), "pdu.ListFilesystemVersionsReq")
|
||||||
proto.RegisterType((*ListFilesystemVersionsRes)(nil), "replication.ListFilesystemVersionsRes")
|
proto.RegisterType((*ListFilesystemVersionsRes)(nil), "pdu.ListFilesystemVersionsRes")
|
||||||
proto.RegisterType((*FilesystemVersion)(nil), "replication.FilesystemVersion")
|
proto.RegisterType((*FilesystemVersion)(nil), "pdu.FilesystemVersion")
|
||||||
proto.RegisterType((*SendReq)(nil), "replication.SendReq")
|
proto.RegisterType((*SendReq)(nil), "pdu.SendReq")
|
||||||
proto.RegisterType((*Property)(nil), "replication.Property")
|
proto.RegisterType((*Property)(nil), "pdu.Property")
|
||||||
proto.RegisterType((*SendRes)(nil), "replication.SendRes")
|
proto.RegisterType((*SendRes)(nil), "pdu.SendRes")
|
||||||
proto.RegisterType((*ReceiveReq)(nil), "replication.ReceiveReq")
|
proto.RegisterType((*ReceiveReq)(nil), "pdu.ReceiveReq")
|
||||||
proto.RegisterType((*ReceiveRes)(nil), "replication.ReceiveRes")
|
proto.RegisterType((*ReceiveRes)(nil), "pdu.ReceiveRes")
|
||||||
proto.RegisterEnum("replication.FilesystemVersion_VersionType", FilesystemVersion_VersionType_name, FilesystemVersion_VersionType_value)
|
proto.RegisterEnum("pdu.FilesystemVersion_VersionType", FilesystemVersion_VersionType_name, FilesystemVersion_VersionType_value)
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() { proto.RegisterFile("pdu.proto", fileDescriptor0) }
|
func init() { proto.RegisterFile("pdu.proto", fileDescriptor0) }
|
||||||
|
|
||||||
var fileDescriptor0 = []byte{
|
var fileDescriptor0 = []byte{
|
||||||
// 454 bytes of a gzipped FileDescriptorProto
|
// 445 bytes of a gzipped FileDescriptorProto
|
||||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0x4d, 0x6f, 0xd3, 0x40,
|
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0xc1, 0x6e, 0xd3, 0x40,
|
||||||
0x10, 0x65, 0x53, 0xa7, 0x38, 0xe3, 0xaa, 0xa4, 0x4b, 0x11, 0x06, 0xa1, 0x2a, 0xda, 0x53, 0xe8,
|
0x10, 0x65, 0x13, 0xa7, 0x38, 0x93, 0xd2, 0xa6, 0x4b, 0x85, 0x0c, 0x42, 0x28, 0xda, 0x53, 0x40,
|
||||||
0x21, 0x87, 0x02, 0x07, 0x40, 0xe2, 0xd0, 0xa2, 0xf6, 0x82, 0xaa, 0x6a, 0x6b, 0x4a, 0xaf, 0xa6,
|
0x22, 0x12, 0x01, 0x71, 0xe1, 0xd6, 0xa2, 0xf4, 0x82, 0xa0, 0xda, 0x9a, 0xaa, 0x57, 0x17, 0x8f,
|
||||||
0x1e, 0xa9, 0x4b, 0x62, 0xaf, 0xbb, 0x63, 0x23, 0xe5, 0xe7, 0xf0, 0xcf, 0xf8, 0x29, 0xc8, 0x53,
|
0x54, 0x2b, 0xb1, 0x77, 0xbb, 0x63, 0x23, 0xe5, 0x73, 0xf8, 0x2b, 0x3e, 0x07, 0x79, 0x6a, 0x27,
|
||||||
0x3b, 0xd9, 0x26, 0x2a, 0xca, 0xc9, 0xf3, 0xde, 0x7c, 0xbd, 0x7d, 0xeb, 0x85, 0x41, 0x99, 0xd5,
|
0x4b, 0x0c, 0x52, 0x4e, 0x99, 0xf7, 0x66, 0x32, 0xf3, 0xe6, 0xcd, 0x1a, 0x86, 0x36, 0xad, 0x66,
|
||||||
0x93, 0xd2, 0xd9, 0xca, 0xca, 0xc8, 0x61, 0x39, 0x33, 0x37, 0x69, 0x65, 0x6c, 0xa1, 0x9e, 0xc3,
|
0xd6, 0x99, 0xd2, 0xc8, 0xbe, 0x4d, 0x2b, 0xf5, 0x14, 0x4e, 0xbe, 0x64, 0x54, 0x2e, 0xb2, 0x15,
|
||||||
0xde, 0x37, 0x43, 0xd5, 0xa9, 0x99, 0x21, 0xcd, 0xa9, 0xc2, 0x5c, 0xe3, 0x9d, 0x3a, 0x5f, 0x27,
|
0xd2, 0x9a, 0x4a, 0xcc, 0x35, 0xde, 0xab, 0x45, 0x97, 0x24, 0xf9, 0x0e, 0x46, 0x5b, 0x82, 0x22,
|
||||||
0x49, 0x7e, 0x84, 0x68, 0x49, 0x50, 0x2c, 0x46, 0x5b, 0xe3, 0xe8, 0xe8, 0xe5, 0xc4, 0x1b, 0x36,
|
0x31, 0xe9, 0x4f, 0x47, 0xf3, 0xe3, 0x59, 0xdd, 0xcf, 0x2b, 0xf4, 0x6b, 0xd4, 0x19, 0xc0, 0x16,
|
||||||
0xf1, 0x1a, 0xfc, 0x5a, 0x75, 0x0c, 0xb0, 0x84, 0x52, 0x42, 0x70, 0x91, 0x56, 0xb7, 0xb1, 0x18,
|
0x4a, 0x09, 0xc1, 0x65, 0x52, 0xde, 0x45, 0x62, 0x22, 0xa6, 0x43, 0xcd, 0xb1, 0x9c, 0xc0, 0x48,
|
||||||
0x89, 0xf1, 0x40, 0x73, 0x2c, 0x47, 0x10, 0x69, 0xa4, 0x3a, 0xc7, 0xc4, 0x4e, 0xb1, 0x88, 0x7b,
|
0x23, 0x55, 0x39, 0xc6, 0x66, 0x89, 0x45, 0xd4, 0xe3, 0x94, 0x4f, 0xa9, 0x4f, 0xf0, 0xfc, 0x6f,
|
||||||
0x9c, 0xf2, 0x29, 0xf5, 0x19, 0x5e, 0x3d, 0xd4, 0x74, 0x85, 0x8e, 0x8c, 0x2d, 0x48, 0xe3, 0x9d,
|
0x2d, 0xd7, 0xe8, 0x28, 0x33, 0x05, 0x69, 0xbc, 0x97, 0xaf, 0xfc, 0x01, 0x4d, 0x63, 0x8f, 0x51,
|
||||||
0x3c, 0xf0, 0x17, 0xb4, 0x83, 0x3d, 0x46, 0xfd, 0x78, 0xbc, 0x99, 0xe4, 0x27, 0x08, 0x3b, 0xd8,
|
0xdf, 0xfe, 0xff, 0x67, 0x92, 0x73, 0x08, 0x5b, 0xd8, 0x6c, 0xf3, 0x6c, 0x67, 0x9b, 0x26, 0xad,
|
||||||
0x9e, 0xea, 0xe0, 0x91, 0x53, 0xb5, 0x65, 0x7a, 0x51, 0xaf, 0xfe, 0x0a, 0xd8, 0x5b, 0xcb, 0xcb,
|
0x37, 0x75, 0xea, 0xb7, 0x80, 0x93, 0x4e, 0x5e, 0x7e, 0x84, 0x20, 0x5e, 0x5b, 0x64, 0x01, 0x47,
|
||||||
0x2f, 0x10, 0x24, 0xf3, 0x12, 0x59, 0xc8, 0xee, 0xd1, 0xe1, 0xff, 0xa7, 0x4d, 0xda, 0x6f, 0xd3,
|
0x73, 0xf5, 0xef, 0x2e, 0xb3, 0xe6, 0xb7, 0xae, 0xd4, 0x5c, 0x5f, 0x3b, 0xf2, 0x35, 0xc9, 0xb1,
|
||||||
0xa1, 0xb9, 0xaf, 0x71, 0xe8, 0x3c, 0xcd, 0xb1, 0xb5, 0x81, 0xe3, 0x86, 0x3b, 0xab, 0x4d, 0x16,
|
0x59, 0x9b, 0xe3, 0x9a, 0xbb, 0xa8, 0xb2, 0x34, 0xea, 0x4f, 0xc4, 0x34, 0xd0, 0x1c, 0xcb, 0x97,
|
||||||
0x6f, 0x8d, 0xc4, 0x38, 0xd0, 0x1c, 0xcb, 0x37, 0x30, 0x38, 0x71, 0x98, 0x56, 0x98, 0x5c, 0x9f,
|
0x30, 0x3c, 0x77, 0x98, 0x94, 0x18, 0xdf, 0x5c, 0x44, 0x01, 0x27, 0xb6, 0x84, 0x7c, 0x01, 0x21,
|
||||||
0xc5, 0x01, 0x27, 0x96, 0x84, 0x7c, 0x0d, 0x21, 0x03, 0x63, 0x8b, 0xb8, 0xcf, 0x93, 0x16, 0x58,
|
0x83, 0xcc, 0x14, 0xd1, 0x80, 0x3b, 0x6d, 0xb0, 0x7a, 0x0d, 0x23, 0x6f, 0xac, 0x3c, 0x84, 0xf0,
|
||||||
0xbd, 0x85, 0xc8, 0x5b, 0x2b, 0x77, 0x20, 0xbc, 0x2c, 0xd2, 0x92, 0x6e, 0x6d, 0x35, 0x7c, 0xd2,
|
0xaa, 0x48, 0x2c, 0xdd, 0x99, 0x72, 0xfc, 0xa8, 0x46, 0x67, 0xc6, 0x2c, 0xf3, 0xc4, 0x2d, 0xc7,
|
||||||
0xa0, 0x63, 0x6b, 0xa7, 0x79, 0xea, 0xa6, 0x43, 0xa1, 0xfe, 0x08, 0x78, 0x7a, 0x89, 0x45, 0xb6,
|
0x42, 0xfd, 0x12, 0xf0, 0xf8, 0x0a, 0x8b, 0x74, 0x0f, 0x5f, 0x6b, 0x91, 0x0b, 0x67, 0xf2, 0x56,
|
||||||
0x81, 0xcf, 0x8d, 0xc8, 0x53, 0x67, 0xf3, 0x4e, 0x78, 0x13, 0xcb, 0x5d, 0xe8, 0x25, 0x96, 0x65,
|
0x78, 0x1d, 0xcb, 0x23, 0xe8, 0xc5, 0x86, 0x65, 0x0f, 0x75, 0x2f, 0x36, 0xbb, 0xa7, 0x0d, 0x3a,
|
||||||
0x0f, 0x74, 0x2f, 0xb1, 0xab, 0x57, 0x1d, 0xac, 0x5d, 0x35, 0x0b, 0xb7, 0x79, 0xe9, 0x90, 0x88,
|
0xa7, 0x65, 0xe1, 0x26, 0xb7, 0x0e, 0x89, 0x58, 0x78, 0xa8, 0x37, 0x58, 0x9e, 0xc2, 0xe0, 0x33,
|
||||||
0x85, 0x87, 0x7a, 0x81, 0xe5, 0x3e, 0xf4, 0xbf, 0x62, 0x56, 0x97, 0xf1, 0x36, 0x27, 0xee, 0x81,
|
0xa6, 0x95, 0x8d, 0x0e, 0x38, 0xf1, 0x00, 0xd4, 0x07, 0x08, 0x2f, 0x9d, 0xb1, 0xe8, 0xca, 0xf5,
|
||||||
0x7a, 0x0f, 0xe1, 0x85, 0xb3, 0x25, 0xba, 0x6a, 0xbe, 0x30, 0x4f, 0x78, 0xe6, 0xed, 0x43, 0xff,
|
0xc6, 0x3c, 0xe1, 0x99, 0x77, 0x0a, 0x83, 0xeb, 0x64, 0x55, 0xb5, 0x8e, 0x3e, 0x00, 0x75, 0xdb,
|
||||||
0x2a, 0x9d, 0xd5, 0x9d, 0xa3, 0xf7, 0x40, 0xfd, 0xea, 0x0e, 0x46, 0x72, 0x0c, 0xcf, 0xbe, 0x13,
|
0x2e, 0x46, 0x72, 0x0a, 0xc7, 0xdf, 0x09, 0x53, 0x5f, 0x98, 0xe0, 0x01, 0xbb, 0xb4, 0x7c, 0x0b,
|
||||||
0x66, 0xbe, 0x30, 0xc1, 0x0b, 0x56, 0x69, 0xf9, 0x01, 0xa0, 0x5d, 0x65, 0x90, 0xe2, 0x1e, 0xff,
|
0xd0, 0x8c, 0xca, 0x90, 0xa2, 0x1e, 0xbf, 0x8f, 0x27, 0x7c, 0xd9, 0x56, 0x81, 0xf6, 0x0a, 0xd4,
|
||||||
0x2f, 0x2f, 0x1e, 0xdc, 0x70, 0xa7, 0x44, 0x7b, 0x85, 0xea, 0x1a, 0x40, 0xe3, 0x0d, 0x9a, 0xdf,
|
0x0d, 0x80, 0xc6, 0x1f, 0x98, 0xfd, 0xc4, 0x7d, 0xfc, 0x7b, 0x03, 0xe3, 0xf3, 0x15, 0x26, 0x6e,
|
||||||
0xb8, 0x89, 0x8f, 0x87, 0x30, 0x3c, 0x99, 0x61, 0xea, 0x56, 0xdf, 0x44, 0xa8, 0xd7, 0x78, 0xb5,
|
0xf7, 0xed, 0x87, 0xba, 0xc3, 0xab, 0x43, 0xaf, 0x33, 0xdd, 0x1e, 0xf0, 0xb7, 0xfb, 0xfe, 0x4f,
|
||||||
0xe3, 0x4d, 0xa6, 0x9f, 0xdb, 0xfc, 0xc6, 0xdf, 0xfd, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xa4, 0x5a,
|
0x00, 0x00, 0x00, 0xff, 0xff, 0xfd, 0xc4, 0x8d, 0xb7, 0xc8, 0x03, 0x00, 0x00,
|
||||||
0xf6, 0xa7, 0xf0, 0x03, 0x00, 0x00,
|
|
||||||
}
|
}
|
||||||
@ -1,6 +1,6 @@
|
|||||||
syntax = "proto3";
|
syntax = "proto3";
|
||||||
|
|
||||||
package replication;
|
package pdu;
|
||||||
|
|
||||||
message ListFilesystemReq {}
|
message ListFilesystemReq {}
|
||||||
|
|
||||||
@ -1,4 +1,4 @@
|
|||||||
package replication
|
package pdu
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@ -45,6 +45,15 @@ func (v *FilesystemVersion) CreationAsTime() (time.Time, error) {
|
|||||||
return time.Parse(time.RFC3339, v.Creation)
|
return time.Parse(time.RFC3339, v.Creation)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// implement fsfsm.FilesystemVersion
|
||||||
|
func (v *FilesystemVersion) SnapshotTime() time.Time {
|
||||||
|
t, err := v.CreationAsTime()
|
||||||
|
if err != nil {
|
||||||
|
panic(err) // FIXME
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
func (v *FilesystemVersion) ZFSFilesystemVersion() *zfs.FilesystemVersion {
|
func (v *FilesystemVersion) ZFSFilesystemVersion() *zfs.FilesystemVersion {
|
||||||
ct := time.Time{}
|
ct := time.Time{}
|
||||||
if v.Creation != "" {
|
if v.Creation != "" {
|
||||||
@ -61,4 +70,4 @@ func (v *FilesystemVersion) ZFSFilesystemVersion() *zfs.FilesystemVersion {
|
|||||||
CreateTXG: v.CreateTXG,
|
CreateTXG: v.CreateTXG,
|
||||||
Creation: ct,
|
Creation: ct,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1,4 +1,4 @@
|
|||||||
package replication
|
package pdu
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
19
cmd/replication/replication.go
Normal file
19
cmd/replication/replication.go
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
package replication
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"github.com/zrepl/zrepl/cmd/replication/common"
|
||||||
|
"github.com/zrepl/zrepl/cmd/replication/internal/mainfsm"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Report = mainfsm.Report
|
||||||
|
|
||||||
|
type Replication interface {
|
||||||
|
Drive(ctx context.Context, ep common.EndpointPair)
|
||||||
|
Report() *Report
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewReplication() Replication {
|
||||||
|
return mainfsm.NewReplication()
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user