extern/zrepl
1
0
Fork 0
mirror of https://github.com/zrepl/zrepl.git synced 2025-01-03 12:58:57 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

zfs + replication: migrate dead zfs/diff_test.go to replication/logic/diff

Browse Source 
(and remove the dead code from package zfs)
This commit is contained in:
Christian Schwarz 2019-03-13 16:39:10 +01:00
parent c87759affe
commit 8129ed91f1
6 changed files with 230 additions and 413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
replication/logic/diff/diff.go
View File

@ -1,4 +1,4 @@
package mainfsm
package diff
import (
"sort"

122
replication/logic/diff/diff_test.go Normal file
View File

@ -0,0 +1,122 @@
package diff
import (
"strconv"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
. "github.com/zrepl/zrepl/replication/logic/pdu"
)
func fsvlist(fsv ...string) (r []*FilesystemVersion) {
r = make([]*FilesystemVersion, len(fsv))
for i, f := range fsv {
// parse the id from fsvlist. it is used to derivce Guid,CreateTXG and Creation attrs
split := strings.Split(f, ",")
if len(split) != 2 {
panic("invalid fsv spec")
}
id, err := strconv.Atoi(split[1])
if err != nil {
panic(err)
}
creation := func(id int) string {
return FilesystemVersionCreation(time.Unix(0, 0).Add(time.Duration(id) * time.Second))
}
if strings.HasPrefix(f, "#") {
r[i] = &FilesystemVersion{
Name: strings.TrimPrefix(f, "#"),
Type: FilesystemVersion_Bookmark,
Guid: uint64(id),
CreateTXG: uint64(id),
Creation: creation(id),
}
} else if strings.HasPrefix(f, "@") {
r[i] = &FilesystemVersion{
Name: strings.TrimPrefix(f, "@"),
Type: FilesystemVersion_Snapshot,
Guid: uint64(id),
CreateTXG: uint64(id),
Creation: creation(id),
}
} else {
panic("invalid character")
}
}
return
}
func doTest(receiver, sender []*FilesystemVersion, validate func(incpath []*FilesystemVersion, conflict error)) {
p, err := IncrementalPath(receiver, sender)
validate(p, err)
}
func TestIncrementalPath_SnapshotsOnly(t *testing.T) {
l := fsvlist
// basic functionality
doTest(l("@a,1", "@b,2"), l("@a,1", "@b,2", "@c,3", "@d,4"), func(path []*FilesystemVersion, conflict error) {
assert.Equal(t, l("@b,2", "@c,3", "@d,4"), path)
})
// no common ancestor
doTest(l(), l("@a,1"), func(path []*FilesystemVersion, conflict error) {
assert.Nil(t, path)
ca, ok := conflict.(*ConflictNoCommonAncestor)
require.True(t, ok)
assert.Equal(t, l("@a,1"), ca.SortedSenderVersions)
})
doTest(l("@a,1", "@b,2"), l("@c,3", "@d,4"), func(path []*FilesystemVersion, conflict error) {
assert.Nil(t, path)
ca, ok := conflict.(*ConflictNoCommonAncestor)
require.True(t, ok)
assert.Equal(t, l("@a,1", "@b,2"), ca.SortedReceiverVersions)
assert.Equal(t, l("@c,3", "@d,4"), ca.SortedSenderVersions)
})
// divergence is detected
doTest(l("@a,1", "@b1,2"), l("@a,1", "@b2,3"), func(path []*FilesystemVersion, conflict error) {
assert.Nil(t, path)
cd, ok := conflict.(*ConflictDiverged)
require.True(t, ok)
assert.Equal(t, l("@a,1")[0], cd.CommonAncestor)
assert.Equal(t, l("@b1,2"), cd.ReceiverOnly)
assert.Equal(t, l("@b2,3"), cd.SenderOnly)
})
// gaps before most recent common ancestor do not matter
doTest(l("@a,1", "@b,2", "@c,3"), l("@a,1", "@c,3", "@d,4"), func(path []*FilesystemVersion, conflict error) {
assert.Equal(t, l("@c,3", "@d,4"), path)
})
}
func TestIncrementalPath_BookmarkSupport(t *testing.T) {
l := fsvlist
// bookmarks are used
doTest(l("@a,1"), l("#a,1", "@b,2"), func(path []*FilesystemVersion, conflict error) {
assert.Equal(t, l("#a,1", "@b,2"), path)
})
// boomarks are stripped from IncrementalPath (cannot send incrementally)
doTest(l("@a,1"), l("#a,1", "#b,2", "@c,3"), func(path []*FilesystemVersion, conflict error) {
assert.Equal(t, l("#a,1", "@c,3"), path)
})
// test that snapshots are preferred over bookmarks in IncrementalPath
doTest(l("@a,1"), l("#a,1", "@a,1", "@b,2"), func(path []*FilesystemVersion, conflict error) {
assert.Equal(t, l("@a,1", "@b,2"), path)
})
doTest(l("@a,1"), l("@a,1", "#a,1", "@b,2"), func(path []*FilesystemVersion, conflict error) {
assert.Equal(t, l("@a,1", "@b,2"), path)
})
}

16
zfs/conflict_string.go
View File

@ -1,16 +0,0 @@
// Code generated by "stringer -type=Conflict"; DO NOT EDIT.
package zfs
import "strconv"
const _Conflict_name = "ConflictIncrementalConflictAllRightConflictNoCommonAncestorConflictDiverged"
var _Conflict_index = [...]uint8{0, 19, 35, 59, 75}
func (i Conflict) String() string {
if i < 0 || i >= Conflict(len(_Conflict_index)-1) {
return "Conflict(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _Conflict_name[_Conflict_index[i]:_Conflict_index[i+1]]
}

284
zfs/diff.go
View File

@ -1,284 +0,0 @@
package zfs
import (
"bytes"
"crypto/sha512"
"encoding/hex"
"fmt"
"io"
"os/exec"
"sort"
)
type fsbyCreateTXG []FilesystemVersion
func (l fsbyCreateTXG) Len() int { return len(l) }
func (l fsbyCreateTXG) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
func (l fsbyCreateTXG) Less(i, j int) bool {
return l[i].CreateTXG < l[j].CreateTXG
}
//go:generate stringer -type=Conflict
type Conflict int
const (
ConflictIncremental Conflict = iota // no conflict, incremental repl possible
ConflictAllRight // no conflict, initial repl possible
ConflictNoCommonAncestor
ConflictDiverged
)
/* The receiver (left) wants to know if the sender (right) has more recent versions
Left : | C |
Right: | A | B | C | D | E |
=> : | C | D | E |
Left: | C |
Right: | D | E |
=> : <empty list>, no common ancestor
Left : | C | D | E |
Right: | A | B | C |
=> : <empty list>, the left has newer versions
Left : | A | B | C | | F |
Right: | C | D | E |
=> : | C | | F | => diverged => <empty list>
IMPORTANT: since ZFS currently does not export dataset UUIDs, the best heuristic to
identify a filesystem version is the tuple (name,creation)
*/
type FilesystemDiff struct {
// Which kind of conflict / "way forward" is possible.
// Check this first to determine the semantics of this struct's remaining members
Conflict Conflict
// Conflict = Incremental | AllRight
// The incremental steps required to get left up to right's most recent version
// 0th element is the common ancestor, ordered by birthtime, oldest first
// If len() < 2, left and right are at same most recent version
// Conflict = otherwise
// nil; there is no incremental path for left to get to right's most recent version
IncrementalPath []FilesystemVersion
// Conflict = Incremental | AllRight: nil
// Conflict = NoCommonAncestor: left as passed as input
// Conflict = Diverged: contains path from left most recent common ancestor (mrca) to most
// recent version on left
MRCAPathLeft []FilesystemVersion
// Conflict = Incremental | AllRight: nil
// Conflict = NoCommonAncestor: right as passed as input
// Conflict = Diverged: contains path from right most recent common ancestor (mrca)
// to most recent version on right
MRCAPathRight []FilesystemVersion
}
func (f FilesystemDiff) String() (str string) {
var b bytes.Buffer
fmt.Fprintf(&b, "%s, ", f.Conflict)
switch f.Conflict {
case ConflictIncremental:
fmt.Fprintf(&b, "incremental path length %v, common ancestor at %s", len(f.IncrementalPath)-1, f.IncrementalPath[0])
case ConflictAllRight:
fmt.Fprintf(&b, "%v versions, most recent is %s", len(f.MRCAPathRight)-1, f.MRCAPathRight[len(f.MRCAPathRight)-1])
case ConflictDiverged:
fmt.Fprintf(&b, "diverged at %s", f.MRCAPathRight[0]) // right always has at least one snap...?
case ConflictNoCommonAncestor:
fmt.Fprintf(&b, "no diff to show")
default:
fmt.Fprintf(&b, "unknown conflict type, likely a bug")
}
return b.String()
}
// we must assume left and right are ordered ascendingly by ZFS_PROP_CREATETXG and that
// names are unique (bas ZFS_PROP_GUID replacement)
func MakeFilesystemDiff(left, right []FilesystemVersion) (diff FilesystemDiff) {
if right == nil {
panic("right must not be nil")
}
if left == nil {
diff = FilesystemDiff{
IncrementalPath: nil,
Conflict: ConflictAllRight,
MRCAPathLeft: left,
MRCAPathRight: right,
}
return
}
// Assert both left and right are sorted by createtxg
{
var leftSorted, rightSorted fsbyCreateTXG
leftSorted = left
rightSorted = right
if !sort.IsSorted(leftSorted) {
panic("cannot make filesystem diff: unsorted left")
}
if !sort.IsSorted(rightSorted) {
panic("cannot make filesystem diff: unsorted right")
}
}
// Find most recent common ancestor by name, preferring snapshots over bookmarks
mrcaLeft := len(left) - 1
var mrcaRight int
outer:
for ; mrcaLeft >= 0; mrcaLeft-- {
for i := len(right) - 1; i >= 0; i-- {
if left[mrcaLeft].Guid == right[i].Guid {
mrcaRight = i
if i-1 >= 0 && right[i-1].Guid == right[i].Guid && right[i-1].Type == Snapshot {
// prefer snapshots over bookmarks
mrcaRight = i - 1
}
break outer
}
}
}
// no common ancestor?
if mrcaLeft == -1 {
diff = FilesystemDiff{
IncrementalPath: nil,
Conflict: ConflictNoCommonAncestor,
MRCAPathLeft: left,
MRCAPathRight: right,
}
return
}
// diverged?
if mrcaLeft != len(left)-1 {
diff = FilesystemDiff{
IncrementalPath: nil,
Conflict: ConflictDiverged,
MRCAPathLeft: left[mrcaLeft:],
MRCAPathRight: right[mrcaRight:],
}
return
}
if mrcaLeft != len(left)-1 {
panic("invariant violated: mrca on left must be the last item in the left list")
}
// incPath must not contain bookmarks except initial one,
// and only if that initial bookmark's snapshot is gone
incPath := make([]FilesystemVersion, 0, len(right))
incPath = append(incPath, right[mrcaRight])
// right[mrcaRight] may be a bookmark if there's no equally named snapshot
for i := mrcaRight + 1; i < len(right); i++ {
if right[i].Type != Bookmark {
incPath = append(incPath, right[i])
}
}
diff = FilesystemDiff{
IncrementalPath: incPath,
}
return
}
const ZREPL_PLACEHOLDER_PROPERTY_NAME string = "zrepl:placeholder"
type FilesystemState struct {
Placeholder bool
// TODO extend with resume token when that feature is finally added
}
// A somewhat efficient way to determine if a filesystem exists on this host.
// Particularly useful if exists is called more than once (will only fork exec once and cache the result)
func ZFSListFilesystemState() (localState map[string]FilesystemState, err error) {
var actual [][]string
if actual, err = ZFSList([]string{"name", ZREPL_PLACEHOLDER_PROPERTY_NAME}, "-t", "filesystem,volume"); err != nil {
return
}
localState = make(map[string]FilesystemState, len(actual))
for _, e := range actual {
dp, err := NewDatasetPath(e[0])
if err != nil {
return nil, fmt.Errorf("ZFS does not return parseable dataset path: %s", e[0])
}
placeholder, _ := IsPlaceholder(dp, e[1])
localState[e[0]] = FilesystemState{
placeholder,
}
}
return
}
// Computes the value for the ZREPL_PLACEHOLDER_PROPERTY_NAME ZFS user property
// to mark the given DatasetPath p as a placeholder
//
// We cannot simply use booleans here since user properties are always
// inherited.
//
// We hash the DatasetPath and use it to check for a given path if it is the
// one originally marked as placeholder.
//
// However, this prohibits moving datasets around via `zfs rename`. The
// placeholder attribute must be re-computed for the dataset path after the
// move.
//
// TODO better solution available?
func PlaceholderPropertyValue(p *DatasetPath) string {
ps := []byte(p.ToString())
sum := sha512.Sum512_256(ps)
return hex.EncodeToString(sum[:])
}
func IsPlaceholder(p *DatasetPath, placeholderPropertyValue string) (isPlaceholder bool, err error) {
expected := PlaceholderPropertyValue(p)
isPlaceholder = expected == placeholderPropertyValue
if !isPlaceholder {
err = fmt.Errorf("expected %s, has %s", expected, placeholderPropertyValue)
}
return
}
// for nonexistent FS, isPlaceholder == false && err == nil
func ZFSIsPlaceholderFilesystem(p *DatasetPath) (isPlaceholder bool, err error) {
props, err := zfsGet(p.ToString(), []string{ZREPL_PLACEHOLDER_PROPERTY_NAME}, sourceAny)
if err == io.ErrUnexpectedEOF {
// interpret this as an early exit of the zfs binary due to the fs not existing
return false, nil
} else if err != nil {
return false, err
}
isPlaceholder, _ = IsPlaceholder(p, props.Get(ZREPL_PLACEHOLDER_PROPERTY_NAME))
return
}
func ZFSCreatePlaceholderFilesystem(p *DatasetPath) (err error) {
v := PlaceholderPropertyValue(p)
cmd := exec.Command(ZFS_BINARY, "create",
"-o", fmt.Sprintf("%s=%s", ZREPL_PLACEHOLDER_PROPERTY_NAME, v),
"-o", "mountpoint=none",
p.ToString())
stderr := bytes.NewBuffer(make([]byte, 0, 1024))
cmd.Stderr = stderr
if err = cmd.Start(); err != nil {
return err
}
if err = cmd.Wait(); err != nil {
err = &ZFSError{
Stderr: stderr.Bytes(),
WaitErr: err,
}
}
return
}

112
zfs/diff_test.go
View File

@ -1,112 +0,0 @@
package zfs
import (
"github.com/stretchr/testify/assert"
"strconv"
"strings"
"testing"
"time"
)
func fsvlist(fsv ...string) (r []FilesystemVersion) {
r = make([]FilesystemVersion, len(fsv))
for i, f := range fsv {
// parse the id from fsvlist. it is used to derivce Guid,CreateTXG and Creation attrs
split := strings.Split(f, ",")
if len(split) != 2 {
panic("invalid fsv spec")
}
id, err := strconv.Atoi(split[1])
if err != nil {
panic(err)
}
if strings.HasPrefix(f, "#") {
r[i] = FilesystemVersion{
Name: strings.TrimPrefix(f, "#"),
Type: Bookmark,
Guid: uint64(id),
CreateTXG: uint64(id),
Creation: time.Unix(0, 0).Add(time.Duration(id) * time.Second),
}
} else if strings.HasPrefix(f, "@") {
r[i] = FilesystemVersion{
Name: strings.TrimPrefix(f, "@"),
Type: Snapshot,
Guid: uint64(id),
CreateTXG: uint64(id),
Creation: time.Unix(0, 0).Add(time.Duration(id) * time.Second),
}
} else {
panic("invalid character")
}
}
return
}
func doTest(left, right []FilesystemVersion, validate func(d FilesystemDiff)) {
var d FilesystemDiff
d = MakeFilesystemDiff(left, right)
validate(d)
}
func TestMakeFilesystemDiff_IncrementalSnapshots(t *testing.T) {
l := fsvlist
// basic functionality
doTest(l("@a,1", "@b,2"), l("@a,1", "@b,2", "@c,3", "@d,4"), func(d FilesystemDiff) {
assert.Equal(t, l("@b,2", "@c,3", "@d,4"), d.IncrementalPath)
})
// no common ancestor
doTest(l(), l("@a,1"), func(d FilesystemDiff) {
assert.Nil(t, d.IncrementalPath)
assert.EqualValues(t, d.Conflict, ConflictNoCommonAncestor)
assert.Equal(t, l("@a,1"), d.MRCAPathRight)
})
doTest(l("@a,1", "@b,2"), l("@c,3", "@d,4"), func(d FilesystemDiff) {
assert.Nil(t, d.IncrementalPath)
assert.EqualValues(t, d.Conflict, ConflictNoCommonAncestor)
assert.Equal(t, l("@c,3", "@d,4"), d.MRCAPathRight)
})
// divergence is detected
doTest(l("@a,1", "@b1,2"), l("@a,1", "@b2,3"), func(d FilesystemDiff) {
assert.Nil(t, d.IncrementalPath)
assert.EqualValues(t, d.Conflict, ConflictDiverged)
assert.Equal(t, l("@a,1", "@b1,2"), d.MRCAPathLeft)
assert.Equal(t, l("@a,1", "@b2,3"), d.MRCAPathRight)
})
// gaps before most recent common ancestor do not matter
doTest(l("@a,1", "@b,2", "@c,3"), l("@a,1", "@c,3", "@d,4"), func(d FilesystemDiff) {
assert.Equal(t, l("@c,3", "@d,4"), d.IncrementalPath)
})
}
func TestMakeFilesystemDiff_BookmarksSupport(t *testing.T) {
l := fsvlist
// bookmarks are used
doTest(l("@a,1"), l("#a,1", "@b,2"), func(d FilesystemDiff) {
assert.Equal(t, l("#a,1", "@b,2"), d.IncrementalPath)
})
// boomarks are stripped from IncrementalPath (cannot send incrementally)
doTest(l("@a,1"), l("#a,1", "#b,2", "@c,3"), func(d FilesystemDiff) {
assert.Equal(t, l("#a,1", "@c,3"), d.IncrementalPath)
})
// test that snapshots are preferred over bookmarks in IncrementalPath
doTest(l("@a,1"), l("#a,1", "@a,1", "@b,2"), func(d FilesystemDiff) {
assert.Equal(t, l("@a,1", "@b,2"), d.IncrementalPath)
})
doTest(l("@a,1"), l("@a,1", "#a,1", "@b,2"), func(d FilesystemDiff) {
assert.Equal(t, l("@a,1", "@b,2"), d.IncrementalPath)
})
}

107
zfs/placeholder.go Normal file
View File

@ -0,0 +1,107 @@
package zfs
import (
"bytes"
"crypto/sha512"
"encoding/hex"
"fmt"
"io"
"os/exec"
)
const ZREPL_PLACEHOLDER_PROPERTY_NAME string = "zrepl:placeholder"
type FilesystemState struct {
Placeholder bool
// TODO extend with resume token when that feature is finally added
}
// A somewhat efficient way to determine if a filesystem exists on this host.
// Particularly useful if exists is called more than once (will only fork exec once and cache the result)
func ZFSListFilesystemState() (localState map[string]FilesystemState, err error) {
var actual [][]string
if actual, err = ZFSList([]string{"name", ZREPL_PLACEHOLDER_PROPERTY_NAME}, "-t", "filesystem,volume"); err != nil {
return
}
localState = make(map[string]FilesystemState, len(actual))
for _, e := range actual {
dp, err := NewDatasetPath(e[0])
if err != nil {
return nil, fmt.Errorf("ZFS does not return parseable dataset path: %s", e[0])
}
placeholder, _ := IsPlaceholder(dp, e[1])
localState[e[0]] = FilesystemState{
placeholder,
}
}
return
}
// Computes the value for the ZREPL_PLACEHOLDER_PROPERTY_NAME ZFS user property
// to mark the given DatasetPath p as a placeholder
//
// We cannot simply use booleans here since user properties are always
// inherited.
//
// We hash the DatasetPath and use it to check for a given path if it is the
// one originally marked as placeholder.
//
// However, this prohibits moving datasets around via `zfs rename`. The
// placeholder attribute must be re-computed for the dataset path after the
// move.
//
// TODO better solution available?
func PlaceholderPropertyValue(p *DatasetPath) string {
ps := []byte(p.ToString())
sum := sha512.Sum512_256(ps)
return hex.EncodeToString(sum[:])
}
func IsPlaceholder(p *DatasetPath, placeholderPropertyValue string) (isPlaceholder bool, err error) {
expected := PlaceholderPropertyValue(p)
isPlaceholder = expected == placeholderPropertyValue
if !isPlaceholder {
err = fmt.Errorf("expected %s, has %s", expected, placeholderPropertyValue)
}
return
}
// for nonexistent FS, isPlaceholder == false && err == nil
func ZFSIsPlaceholderFilesystem(p *DatasetPath) (isPlaceholder bool, err error) {
props, err := zfsGet(p.ToString(), []string{ZREPL_PLACEHOLDER_PROPERTY_NAME}, sourceAny)
if err == io.ErrUnexpectedEOF {
// interpret this as an early exit of the zfs binary due to the fs not existing
return false, nil
} else if err != nil {
return false, err
}
isPlaceholder, _ = IsPlaceholder(p, props.Get(ZREPL_PLACEHOLDER_PROPERTY_NAME))
return
}
func ZFSCreatePlaceholderFilesystem(p *DatasetPath) (err error) {
v := PlaceholderPropertyValue(p)
cmd := exec.Command(ZFS_BINARY, "create",
"-o", fmt.Sprintf("%s=%s", ZREPL_PLACEHOLDER_PROPERTY_NAME, v),
"-o", "mountpoint=none",
p.ToString())
stderr := bytes.NewBuffer(make([]byte, 0, 1024))
cmd.Stderr = stderr
if err = cmd.Start(); err != nil {
return err
}
if err = cmd.Wait(); err != nil {
err = &ZFSError{
Stderr: stderr.Bytes(),
WaitErr: err,
}
}
return
}