Implement DatasetMapping + basic ZFS list functionality.

zfs/mapping.go

@@ -0,0 +1,141 @@
+package zfs
+
+import (
+	"errors"
+	"os/exec"
+	"io"
+	"bufio"
+	"fmt"
+)
+
+type DatasetMapping interface {
+	Map(source DatasetPath) (target DatasetPath, err error)
+}
+
+type GlobMapping struct {
+	PrefixPath DatasetPath
+	TargetRoot DatasetPath
+}
+
+var NoMatchError error = errors.New("no match found in mapping")
+
+func (m GlobMapping) Map(source DatasetPath) (target DatasetPath, err error) {
+
+	if len(source) < len(m.PrefixPath) {
+		err = NoMatchError
+		return
+	}
+
+	target = make([]string, 0, len(source) + len(m.TargetRoot))
+	target = append(target, m.TargetRoot...)
+
+
+	for si, sc := range source {
+		target = append(target, sc)
+		if si < len(m.PrefixPath) {
+			if sc != m.PrefixPath[si] {
+				err = NoMatchError
+				return
+			}
+			continue
+		}
+	}
+
+	return
+}
+
+type ComboMapping struct {
+	Mappings []DatasetMapping
+}
+
+func (m ComboMapping) Map(source DatasetPath) (target DatasetPath, err error) {
+	for _, sm := range m.Mappings {
+		target, err = sm.Map(source)
+		if err == nil {
+			return target, err
+		}
+	}
+	return nil, NoMatchError
+}
+
+type DirectMapping struct {
+	Source DatasetPath
+	Target DatasetPath
+}
+
+func (m DirectMapping) Map(source DatasetPath) (target DatasetPath, err error) {
+	if len(m.Source) != len(source) {
+		return nil, NoMatchError
+	}
+
+	for i, c := range source {
+		if c != m.Source[i] {
+			return nil, NoMatchError
+		}
+	}
+
+	return m.Target, nil
+}
+
+type ExecMapping struct {
+	Name string
+	Args []string
+}
+
+func NewExecMapping(name string, args... string) (m *ExecMapping) {
+	m = &ExecMapping{
+		Name: name,
+		Args: args,
+	}
+	return
+}
+
+func (m ExecMapping) Map(source DatasetPath) (target DatasetPath, err error) {
+
+	var stdin io.Writer
+	var stdout io.Reader
+
+
+	cmd := exec.Command(m.Name, m.Args...)
+
+	if stdin, err = cmd.StdinPipe(); err != nil {
+		return
+	}
+
+	if stdout, err = cmd.StdoutPipe(); err != nil {
+		return
+	}
+
+	resp := bufio.NewScanner(stdout)
+
+	if err = cmd.Start(); err != nil {
+		return
+	}
+
+	go func() {
+		err := cmd.Wait()
+		if err != nil {
+			fmt.Printf("error: %v\n", err) // TODO
+		}
+	}()
+
+	if _, err = io.WriteString(stdin, source.ToString() + "\n"); err != nil {
+		return
+	}
+
+	if !resp.Scan() {
+		err = errors.New(fmt.Sprintf("unexpected end of file: %v", resp.Err()))
+		return
+	}
+
+	t := resp.Text()
+
+	switch {
+		case t == "NOMAP":
+			return nil, NoMatchError
+	}
+
+	target = toDatasetPath(t) // TODO discover garbage?
+
+	return
+}

zfs/mapping_test.go

@@ -0,0 +1,99 @@
+package zfs
+
+import (
+	"testing"
+  	"github.com/stretchr/testify/assert"
+)
+
+func TestGlobMapping(t *testing.T) {
+
+	m := GlobMapping{
+		PrefixPath: toDatasetPath("tank/usr/home"),
+		TargetRoot: toDatasetPath("backups/share1"),
+	}
+
+	var r DatasetPath
+	var err error
+
+	r, err = m.Map(toDatasetPath("tank/usr/home"))
+	assert.Nil(t, err)
+	assert.Equal(t, toDatasetPath("backups/share1/tank/usr/home"), r)
+
+	r, err = m.Map(toDatasetPath("zroot"))
+	assert.Equal(t, NoMatchError, err, "prefix-only match is an error")
+
+	r, err = m.Map(toDatasetPath("zroot/notmapped"))
+	assert.Equal(t, NoMatchError, err, "non-prefix is an error")
+
+}
+
+func TestComboMapping(t *testing.T) {
+
+	m1 := GlobMapping{
+		PrefixPath: toDatasetPath("a/b"),
+		TargetRoot: toDatasetPath("c/d"),
+	}
+
+	m2 := GlobMapping{
+		PrefixPath: toDatasetPath("a/x"),
+		TargetRoot: toDatasetPath("c/y"),
+	}
+
+	c := ComboMapping{
+		Mappings: []DatasetMapping{m1,m2},
+	}
+
+	var r DatasetPath
+	var err error
+
+	p := toDatasetPath("a/b/q")
+
+	r,err = m2.Map(p)
+	assert.Equal(t, NoMatchError, err)
+
+	r, err = c.Map(p)
+	assert.Nil(t, err)
+	assert.Equal(t, toDatasetPath("c/d/a/b/q"), r)
+
+}
+
+func TestDirectMapping(t *testing.T) {
+
+	m := DirectMapping{
+		Source: toDatasetPath("a/b/c"),
+		Target: toDatasetPath("x/y/z"),
+	}
+
+	var r DatasetPath
+	var err error
+
+	r, err = m.Map(toDatasetPath("a/b/c"))
+	assert.Nil(t, err)
+	assert.Equal(t, m.Target, r)
+
+	r, err = m.Map(toDatasetPath("not/matching"))
+	assert.Equal(t, NoMatchError, err)
+
+	r, err = m.Map(toDatasetPath("a/b"))
+	assert.Equal(t, NoMatchError, err)
+
+}
+
+func TestExecMapping(t *testing.T) {
+
+	var err error
+
+	var m DatasetMapping
+	m = NewExecMapping("test_helpers/exec_mapping_good.sh", "nostop")
+	assert.NoError(t, err)
+
+	var p DatasetPath
+	p, err = m.Map(toDatasetPath("nomap/foobar"))
+
+	assert.Equal(t, NoMatchError, err)
+
+	p, err = m.Map(toDatasetPath("willmap/something"))
+	assert.Nil(t, err)
+	assert.Equal(t, toDatasetPath("didmap/willmap/something"), p)
+
+}

zfs/test_helpers/exec_mapping_good.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+
+
+while read line # we know this is not always correct...
+do
+    if [[ "$line" =~ nomap* ]]; then
+        echo "NOMAP"
+        continue
+    fi
+
+    echo "didmap/${line}"
+
+    if [ "$1" == "stop" ]; then
+        break
+    fi
+
+done

zfs/test_helpers/zfs_failer.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "error: this is a mock" 1>&2
+exit 1

zfs/zfs.go

@@ -1,13 +1,111 @@
 package zfs
 
-func InitialSend(snapshot string) (io.Read, error) {
+import (
+	"github.com/zrepl/zrepl/model"
+	"os/exec"
+	"bufio"
+	"strings"
+	"errors"
+	"io"
+	"fmt"
+	"io/ioutil"
+)
+
+func InitialSend(snapshot string) (io.Reader, error) {
+	return nil, nil
+}
+
+func IncrementalSend(from, to string) (io.Reader, error) {
+	return nil, nil
+}
+
+func FilesystemsAtRoot(root string) (fs model.Filesystem, err error) {
+
+	_, _ = zfsList("zroot", func(path DatasetPath) bool {
+		return true
+	})
+
+	return
 
 }
 
-func IncrementalSend(from, to string) (io.Read, error) {
+type DatasetPath []string
 
+func (p DatasetPath) ToString() string {
+	return strings.Join(p, "/")
 }
 
-func FilesystemsAtRoot(root string) (model.Filesystem, error) {
+func toDatasetPath(s string) DatasetPath {
+	return strings.Split(s, "/")
+}
+
+type DatasetFilter func(path DatasetPath) bool
+
+type ZFSError struct {
+	Stderr  []byte
+	WaitErr error
+}
+
+func (e ZFSError) Error() string {
+	return fmt.Sprintf("zfs exited with error: %s", e.WaitErr.Error())
+}
+
+var ZFS_BINARY string = "zfs"
+
+func zfsList(root string, filter DatasetFilter) (datasets []DatasetPath, err error) {
+
+	const ZFS_LIST_FIELD_COUNT = 1
+
+	cmd := exec.Command(ZFS_BINARY, "list", "-H", "-r",
+						"-t", "filesystem,volume",
+						"-o", "name",
+						root)
+
+	var stdout io.Reader
+	var stderr io.Reader
+
+	if stdout, err = cmd.StdoutPipe(); err != nil {
+		return
+	}
+
+	if stderr, err = cmd.StderrPipe(); err != nil {
+		return
+	}
+
+	if err = cmd.Start(); err != nil {
+		return
+	}
+
+	s := bufio.NewScanner(stdout)
+	buf := make([]byte, 1024)
+	s.Buffer(buf, 0)
+
+	datasets = make([]DatasetPath, 0)
+
+	for s.Scan() {
+		fields := strings.SplitN(s.Text(), "\t", ZFS_LIST_FIELD_COUNT)
+		if len(fields) != ZFS_LIST_FIELD_COUNT {
+			err = errors.New("unexpected output")
+			return
+		}
+
+		dp := toDatasetPath(fields[0])
+
+		if filter(dp) {
+			datasets = append(datasets, dp)
+		}
+	}
+
+	stderrOutput, err := ioutil.ReadAll(stderr)
+
+	if waitErr:= cmd.Wait(); waitErr != nil {
+		err := ZFSError{
+			Stderr: stderrOutput,
+			WaitErr: waitErr,
+		}
+		return nil, err
+	}
+
+	return
 
 }

zfs/zfs_test.go

@@ -0,0 +1,21 @@
+package zfs
+
+import (
+	"testing"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestZFSListHandlesProducesZFSErrorOnNonZeroExit(t *testing.T) {
+	var err error
+
+	ZFS_BINARY = "./test_helpers/zfs_failer.sh"
+
+	_, err = zfsList("nonexistent/dataset", func(p DatasetPath) bool {
+		return true
+	})
+
+	assert.Error(t, err)
+	zfsError, ok := err.(ZFSError)
+	assert.True(t, ok)
+	assert.Equal(t, "error: this is a mock\n", string(zfsError.Stderr))
+}