add platformtest: infrastructure for ZFS compatiblity testing

daemon/logging/build_logging.go

@@ -37,7 +37,7 @@ func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error)
 	var syslogOutlets, stdoutOutlets int
 	for lei, le := range in {
 
-		outlet, minLevel, err := parseOutlet(le)
+		outlet, minLevel, err := ParseOutlet(le)
 		if err != nil {
 			return nil, errors.Wrapf(err, "cannot parse outlet #%d", lei)
 		}
@@ -123,7 +123,7 @@ func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
 
 }
 
-func parseOutlet(in config.LoggingOutletEnum) (o logger.Outlet, level logger.Level, err error) {
+func ParseOutlet(in config.LoggingOutletEnum) (o logger.Outlet, level logger.Level, err error) {
 
 	parseCommon := func(common config.LoggingOutletCommon) (logger.Level, EntryFormatter, error) {
 		if common.Level == "" || common.Format == "" {

platformtest/harness/harness.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/platformtest/tests"
+)
+
+func main() {
+
+	root := flag.String("root", "", "empty root filesystem under which we conduct the platform test")
+	flag.Parse()
+	if *root == "" {
+		panic(*root)
+	}
+
+	outlets := logger.NewOutlets()
+	outlet, level, err := logging.ParseOutlet(config.LoggingOutletEnum{Ret: &config.StdoutLoggingOutlet{
+		LoggingOutletCommon: config.LoggingOutletCommon{
+			Level:  "debug",
+			Format: "human",
+		},
+	}})
+	if err != nil {
+		panic(err)
+	}
+	outlets.Add(outlet, level)
+	logger := logger.NewLogger(outlets, 1*time.Second)
+
+	ctx := &platformtest.Context{
+		Context:     platformtest.WithLogger(context.Background(), logger),
+		RootDataset: *root,
+	}
+
+	bold := color.New(color.Bold)
+	for _, c := range tests.Cases {
+		bold.Printf("BEGIN TEST CASE %s\n", c)
+		c(ctx)
+		bold.Printf("DONE  TEST CASE %s\n", c)
+		fmt.Println()
+	}
+
+}

platformtest/harness/harness_test.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"os"
+	"strings"
+	"testing"
+)
+
+// Idea taken from
+// https://github.com/openSUSE/umoci/blob/v0.2.1/cmd/umoci/main_test.go
+//
+// How to generate coverage:
+// go test -c -covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/...
+// sudo ../logmockzfs/logzfsenv /tmp/zrepl_platform_test.log /usr/bin/zfs \
+//     ./harness.test -test.coverprofile=/tmp/harness.out \
+//     -test.v __DEVEL--i-heard-you-like-tests \
+//     -root rpool/zreplplayground
+//  go tool cover -html=/tmp/harness.out -o /tmp/harness.html
+//
+// Merge with existing coverage reports using gocovmerge:
+//  https://github.com/wadey/gocovmerge
+
+func TestMain(t *testing.T) {
+	var (
+		args []string
+		run  bool
+	)
+
+	for _, arg := range os.Args {
+		switch {
+		case arg == "__DEVEL--i-heard-you-like-tests":
+			run = true
+		case strings.HasPrefix(arg, "-test"):
+		case strings.HasPrefix(arg, "__DEVEL"):
+		default:
+			args = append(args, arg)
+		}
+	}
+	os.Args = args
+
+	if run {
+		main()
+	}
+}

platformtest/logmockzfs/logzfsenv

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -ue
+export ZREPL_MOCK_ZFS_COMMAND_LOG="$1"
+shift
+export ZREPL_MOCK_ZFS_PATH="$1"
+shift
+export PATH="$(dirname "${BASH_SOURCE[0]}" )":"$PATH"
+args=("$@")
+exec "${args[@]}"

platformtest/logmockzfs/zfs

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -eu
+args=("$@")
+
+if [ -n "$ZREPL_MOCK_ZFS_COMMAND_LOG" ]; then
+    (
+        flock -x 200
+        jq --compact-output -n --argjson args "$(printf '%s\0' "${args[@]}" | jq -Rsc 'split("\u0000")')" '{date: now|strflocaltime("%Y-%m-%dT%H:%M:%S"), command: $args}' >> "$ZREPL_MOCK_ZFS_COMMAND_LOG"
+    ) 200>"$ZREPL_MOCK_ZFS_COMMAND_LOG".lock
+fi
+
+exec "$ZREPL_MOCK_ZFS_PATH" "${args[@]}"

platformtest/platformtest.go

@@ -0,0 +1,24 @@
+package platformtest
+
+import (
+	"context"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type Context struct {
+	context.Context
+	RootDataset string
+}
+
+var _ assert.TestingT = (*Context)(nil)
+var _ require.TestingT = (*Context)(nil)
+
+func (c *Context) Errorf(format string, args ...interface{}) {
+	getLog(c).Printf(format, args...)
+}
+
+func (c *Context) FailNow() {
+	panic(nil)
+}

platformtest/platformtest_exec.go

@@ -0,0 +1,42 @@
+package platformtest
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+)
+
+type ex struct {
+	log Logger
+}
+
+func newEx(log Logger) *ex {
+	return &ex{log}
+}
+
+func (e *ex) RunExpectSuccessNoOutput(ctx context.Context, cmd string, args ...string) error {
+	return e.runNoOutput(true, ctx, cmd, args...)
+}
+
+func (e *ex) RunExpectFailureNoOutput(ctx context.Context, cmd string, args ...string) error {
+	return e.runNoOutput(false, ctx, cmd, args...)
+}
+
+func (e *ex) runNoOutput(expectSuccess bool, ctx context.Context, cmd string, args ...string) error {
+	log := e.log.WithField("command", fmt.Sprintf("%q %q", cmd, args))
+	log.Debug("begin executing")
+	defer log.Debug("done executing")
+	ecmd := exec.CommandContext(ctx, cmd, args...)
+	// use circlog and capture stdout
+	err := ecmd.Run()
+	ee, ok := err.(*exec.ExitError)
+	if err != nil && !ok {
+		panic(err)
+	}
+	if expectSuccess && err != nil {
+		return fmt.Errorf("expecting no error, got error: %s\n%s", err, ee.Stderr)
+	} else if !expectSuccess && err == nil {
+		return fmt.Errorf("expecting error, got no error")
+	}
+	return nil
+}

platformtest/platformtest_logging.go

@@ -0,0 +1,24 @@
+package platformtest
+
+import (
+	"context"
+
+	"github.com/zrepl/zrepl/logger"
+)
+
+type Logger = logger.Logger
+
+type contextKey int
+
+const (
+	contextKeyLogger contextKey = iota
+)
+
+func WithLogger(ctx context.Context, logger Logger) context.Context {
+	ctx = context.WithValue(ctx, contextKeyLogger, logger)
+	return ctx
+}
+
+func getLog(ctx context.Context) Logger {
+	return ctx.Value(contextKeyLogger).(Logger)
+}

platformtest/platformtest_ops.go

@@ -0,0 +1,275 @@
+package platformtest
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"unicode"
+)
+
+type Execer interface {
+	RunExpectSuccessNoOutput(ctx context.Context, cmd string, args ...string) error
+	RunExpectFailureNoOutput(ctx context.Context, cmd string, args ...string) error
+}
+
+type Stmt interface {
+	Run(context context.Context, e Execer) error
+}
+
+type Op string
+
+const (
+	AssertExists    Op = "!E"
+	AssertNotExists Op = "!N"
+	Add             Op = "+"
+	Del             Op = "-"
+	RunCmd          Op = "R"
+	DestroyRoot     Op = "DESTROYROOT"
+	CreateRoot      Op = "CREATEROOT"
+)
+
+type DestroyRootOp struct {
+	Path string
+}
+
+func (o *DestroyRootOp) Run(ctx context.Context, e Execer) error {
+	const magicName = "zreplplayground"
+	// sanity check: root must contain
+	if !strings.Contains(o.Path, magicName) {
+		panic("sanity check failed")
+	}
+	// early-exit if it doesn't exist
+	if err := e.RunExpectSuccessNoOutput(ctx, "zfs", "get", "-H", "name", o.Path); err != nil {
+		getLog(ctx).WithField("root_ds", o.Path).Info("assume root ds doesn't exist")
+		return nil
+	}
+	op, err := exec.CommandContext(ctx, "zfs", "destroy", "-nvrp", o.Path).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("cannot clean root dataset %q: %s\n%s", o.Path, err, op)
+	}
+	sc := bufio.NewScanner(bytes.NewReader(op))
+	for sc.Scan() {
+		if !strings.Contains(sc.Text(), magicName) {
+			panic("sanity check failed")
+		}
+	}
+	return e.RunExpectSuccessNoOutput(ctx, "zfs", "destroy", "-r", o.Path)
+}
+
+type FSOp struct {
+	Op   Op
+	Path string
+}
+
+func (o *FSOp) Run(ctx context.Context, e Execer) error {
+	switch o.Op {
+	case AssertExists:
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "get", "-H", "name", o.Path)
+	case AssertNotExists:
+		return e.RunExpectFailureNoOutput(ctx, "zfs", "get", "-H", "name", o.Path)
+	case Add:
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "create", o.Path)
+	case Del:
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "destroy", o.Path)
+	default:
+		panic(o.Op)
+	}
+}
+
+type SnapOp struct {
+	Op   Op
+	Path string
+}
+
+func (o *SnapOp) Run(ctx context.Context, e Execer) error {
+	switch o.Op {
+	case AssertExists:
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "get", "-H", "name", o.Path)
+	case AssertNotExists:
+		return e.RunExpectFailureNoOutput(ctx, "zfs", "get", "-H", "name", o.Path)
+	case Add:
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "snapshot", o.Path)
+	case Del:
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "destroy", o.Path)
+	default:
+		panic(o.Op)
+	}
+}
+
+type RunOp struct {
+	RootDS string
+	Script string
+}
+
+func (o *RunOp) Run(ctx context.Context, e Execer) error {
+	cmd := exec.CommandContext(ctx, "/usr/bin/env", "bash", "-c", o.Script)
+	cmd.Env = os.Environ()
+	cmd.Env = append(cmd.Env, fmt.Sprintf("ROOTDS=%s", o.RootDS))
+	log := getLog(ctx).WithField("script", o.Script)
+	log.Info("start script")
+	defer log.Info("script done")
+	output, err := cmd.CombinedOutput()
+	if _, ok := err.(*exec.ExitError); err != nil && !ok {
+		panic(err)
+	}
+	log.Printf("script output:\n%s", output)
+	return err
+}
+
+type LineError struct {
+	Line string
+	What string
+}
+
+func (e LineError) Error() string {
+	return fmt.Sprintf("%q: %s", e.Line, e.What)
+}
+
+type RunKind int
+
+const (
+	PanicErr RunKind = 1 << iota
+	RunAll
+)
+
+func Run(ctx context.Context, rk RunKind, rootds string, stmtsStr string) {
+	stmt, err := parseSequence(rootds, stmtsStr)
+	if err != nil {
+		panic(err)
+	}
+	execer := newEx(getLog(ctx))
+	for _, s := range stmt {
+		err := s.Run(ctx, execer)
+		if err == nil {
+			continue
+		}
+		if rk == PanicErr {
+			panic(err)
+		} else if rk == RunAll {
+			continue
+		} else {
+			panic(rk)
+		}
+	}
+}
+
+func isNoSpace(r rune) bool {
+	return !unicode.IsSpace(r)
+}
+
+func splitQuotedWords(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	begin := bytes.IndexFunc(data, isNoSpace)
+	if begin == -1 {
+		return len(data), nil, nil
+	}
+	if data[begin] == '"' {
+		end := begin + 1
+		for end < len(data) {
+			endCandidate := bytes.Index(data[end:], []byte(`"`))
+			if endCandidate == -1 {
+				return 0, nil, nil
+			}
+			end += endCandidate
+			if data[end-1] != '\\' {
+				// unescaped quote, end of this string
+				// remove backslash-escapes
+				withBackslash := data[begin+1 : end]
+				withoutBaskslash := bytes.Replace(withBackslash, []byte("\\\""), []byte("\""), -1)
+				return end + 1, withoutBaskslash, nil
+			} else {
+				// continue to next quote
+				end += 1
+			}
+		}
+	} else {
+		endOffset := bytes.IndexFunc(data[begin:], unicode.IsSpace)
+		var end int
+		if endOffset == -1 {
+			if !atEOF {
+				return 0, nil, nil
+			} else {
+				end = len(data)
+			}
+		} else {
+			end = begin + endOffset
+		}
+		return end, data[begin:end], nil
+	}
+	return 0, nil, fmt.Errorf("unexpected")
+}
+
+func parseSequence(rootds, stmtsStr string) (stmts []Stmt, err error) {
+	scan := bufio.NewScanner(strings.NewReader(stmtsStr))
+nextLine:
+	for scan.Scan() {
+		if len(bytes.TrimSpace(scan.Bytes())) == 0 {
+			continue
+		}
+		comps := bufio.NewScanner(bytes.NewReader(scan.Bytes()))
+		comps.Split(splitQuotedWords)
+
+		expectMoreTokens := func() error {
+			if !comps.Scan() {
+				return &LineError{scan.Text(), "unexpected EOL"}
+			}
+			return nil
+		}
+
+		// Op
+		if err := expectMoreTokens(); err != nil {
+			return nil, err
+		}
+		var op Op
+		switch comps.Text() {
+
+		case string(RunCmd):
+			script := strings.TrimPrefix(strings.TrimSpace(scan.Text()), string(RunCmd))
+			stmts = append(stmts, &RunOp{RootDS: rootds, Script: script})
+			continue nextLine
+
+		case string(DestroyRoot):
+			if comps.Scan() {
+				return nil, &LineError{scan.Text(), fmt.Sprintf("unexpected tokens at EOL")}
+			}
+			stmts = append(stmts, &DestroyRootOp{rootds})
+			continue nextLine
+
+		case string(CreateRoot):
+			if comps.Scan() {
+				return nil, &LineError{scan.Text(), fmt.Sprintf("unexpected tokens at EOL")}
+			}
+			stmts = append(stmts, &FSOp{Op: Add, Path: rootds})
+			continue nextLine
+
+		case string(Add):
+			op = Add
+		case string(Del):
+			op = Del
+		case string(AssertExists):
+			op = AssertExists
+		case string(AssertNotExists):
+			op = AssertNotExists
+		default:
+			return nil, &LineError{scan.Text(), fmt.Sprintf("invalid op %q", comps.Text())}
+		}
+
+		// FS / SNAP
+		if err := expectMoreTokens(); err != nil {
+			return nil, err
+		}
+		if strings.ContainsAny(comps.Text(), "@") {
+			stmts = append(stmts, &SnapOp{Op: op, Path: fmt.Sprintf("%s/%s", rootds, comps.Text())})
+		} else {
+			stmts = append(stmts, &FSOp{Op: op, Path: fmt.Sprintf("%s/%s", rootds, comps.Text())})
+		}
+
+		if comps.Scan() {
+			return nil, &LineError{scan.Text(), fmt.Sprintf("unexpected tokens at EOL")}
+		}
+	}
+	return stmts, nil
+}

platformtest/platformtest_parser_test.go

@@ -0,0 +1,33 @@
+package platformtest
+
+import (
+	"bufio"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSplitQuotedWords(t *testing.T) {
+	s := bufio.NewScanner(strings.NewReader(`
+	foo "bar baz" blah "foo \"with single escape" "blah baz" "\"foo" "foo\""
+	`))
+	s.Split(splitQuotedWords)
+	var words []string
+	for s.Scan() {
+		words = append(words, s.Text())
+	}
+	assert.Equal(
+		t,
+		[]string{
+			"foo",
+			"bar baz",
+			"blah",
+			"foo \"with single escape",
+			"blah baz",
+			"\"foo",
+			"foo\"",
+		},
+		words)
+
+}

platformtest/tests/batchDestroy.go

@@ -0,0 +1,54 @@
+package tests
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func BatchDestroy(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@1"
+		+  "foo bar@2"
+		+  "foo bar@3"
+		R  zfs hold zrepl_platformtest "${ROOTDS}/foo bar@2"
+	`)
+
+	reqs := []*zfs.DestroySnapOp{
+		&zfs.DestroySnapOp{
+			ErrOut:     new(error),
+			Filesystem: fmt.Sprintf("%s/foo bar", ctx.RootDataset),
+			Name:       "3",
+		},
+		&zfs.DestroySnapOp{
+			ErrOut:     new(error),
+			Filesystem: fmt.Sprintf("%s/foo bar", ctx.RootDataset),
+			Name:       "2",
+		},
+	}
+	zfs.ZFSDestroyFilesystemVersions(reqs)
+	if *reqs[0].ErrOut != nil {
+		panic("expecting no error")
+	}
+	err := (*reqs[1].ErrOut).Error()
+	if !strings.Contains(err, fmt.Sprintf("%s/foo bar@2", ctx.RootDataset)) {
+		panic(fmt.Sprintf("expecting error about being unable to destroy @2: %T\n%s", err, err))
+	}
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+	!N "foo bar@3"
+	!E "foo bar@1"
+	!E "foo bar@2"
+	R zfs release zrepl_platformtest "${ROOTDS}/foo bar@2"
+	-  "foo bar@2"
+	-  "foo bar@1"
+	-  "foo bar"
+	`)
+
+}

platformtest/tests/getNonexistent.go

@@ -0,0 +1,54 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func GetNonexistent(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@1"
+	`)
+	defer platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+	`)
+
+	// test raw
+	_, err := zfs.ZFSGetRawAnySource(fmt.Sprintf("%s/foo bar", ctx.RootDataset), []string{"name"})
+	if err != nil {
+		panic(err)
+	}
+
+	// test nonexistent filesystem
+	nonexistent := fmt.Sprintf("%s/nonexistent filesystem", ctx.RootDataset)
+	props, err := zfs.ZFSGetRawAnySource(nonexistent, []string{"name"})
+	if err == nil {
+		panic(props)
+	}
+	dsne, ok := err.(*zfs.DatasetDoesNotExist)
+	if !ok {
+		panic(err)
+	} else if dsne.Path != nonexistent {
+		panic(err)
+	}
+
+	// test nonexistent snapshot
+	nonexistent = fmt.Sprintf("%s/foo bar@non existent", ctx.RootDataset)
+	props, err = zfs.ZFSGetRawAnySource(nonexistent, []string{"name"})
+	if err == nil {
+		panic(props)
+	}
+	dsne, ok = err.(*zfs.DatasetDoesNotExist)
+	if !ok {
+		panic(err)
+	} else if dsne.Path != nonexistent {
+		panic(err)
+	}
+
+}

platformtest/tests/tests.go

@@ -0,0 +1,20 @@
+package tests
+
+import (
+	"reflect"
+	"runtime"
+
+	"github.com/zrepl/zrepl/platformtest"
+)
+
+type Case func(*platformtest.Context)
+
+func (c Case) String() string {
+	return runtime.FuncForPC(reflect.ValueOf(c).Pointer()).Name()
+}
+
+var Cases = []Case{
+	BatchDestroy,
+	UndestroyableSnapshotParsing,
+	GetNonexistent,
+}

platformtest/tests/undestroyableSnapshotParsing.go

@@ -0,0 +1,40 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func UndestroyableSnapshotParsing(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@1 2 3"
+		+  "foo bar@4 5 6"
+		+  "foo bar@7 8 9"
+		R  zfs hold zrepl_platformtest "${ROOTDS}/foo bar@4 5 6"
+	`)
+	defer platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+	R zfs release zrepl_platformtest "${ROOTDS}/foo bar@4 5 6"
+	DESTROYROOT
+	`)
+
+	err := zfs.ZFSDestroy(fmt.Sprintf("%s/foo bar@1 2 3,4 5 6,7 8 9", t.RootDataset))
+	if err == nil {
+		panic("expecting destroy error due to hold")
+	}
+	if dse, ok := err.(*zfs.DestroySnapshotsError); !ok {
+		panic(fmt.Sprintf("expecting *zfs.DestroySnapshotsError, got %T\n%v\n%s", err, err, err))
+	} else {
+		if dse.Filesystem != fmt.Sprintf("%s/foo bar", t.RootDataset) {
+			panic(dse.Filesystem)
+		}
+		require.Equal(t, []string{"4 5 6"}, dse.Undestroyable)
+		require.Equal(t, []string{"dataset is busy"}, dse.Reason)
+	}
+
+}

zfs/zfs.go

@@ -925,7 +925,11 @@ func ZFSGet(fs *DatasetPath, props []string) (*ZFSProperties, error) {
 	return zfsGet(fs.ToString(), props, sourceAny)
 }
 
-var zfsGetDatasetDoesNotExistRegexp = regexp.MustCompile(`^cannot open '([^)]+)': (dataset does not exist|no such pool or dataset)`)
+func ZFSGetRawAnySource(path string, props []string) (*ZFSProperties, error) {
+	return zfsGet(path, props, sourceAny)
+}
+
+var zfsGetDatasetDoesNotExistRegexp = regexp.MustCompile(`^cannot open '([^)]+)': (dataset does not exist|no such pool or dataset)`) // TODO verify this works
 
 type DatasetDoesNotExist struct {
 	Path string