extern/zrepl
1
0
Fork 0
mirror of https://github.com/zrepl/zrepl.git synced 2024-12-31 19:39:28 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

rework filters & mappings

Browse Source 
config defines a single datastructure that can act both as a Map and as a Filter
(DatasetMapFilter)

Cleanup wildcard syntax along the way (also changes semantics).
This commit is contained in:
Christian Schwarz 2017-08-05 21:15:37 +02:00
parent 3fac6a67df
commit 2ce07c9342
13 changed files with 478 additions and 459 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
cmd/config.go
View File

@ -52,26 +52,21 @@ type SSHTransport struct {
type Push struct {
JobName string // for use with jobrun package
To *Remote
Filter zfs.DatasetMapping
Filter zfs.DatasetFilter
InitialReplPolicy rpc.InitialReplPolicy
RepeatStrategy jobrun.RepeatStrategy
}
type Pull struct {
JobName string // for use with jobrun package
From *Remote
Mapping zfs.DatasetMapping
Mapping DatasetMapFilter
InitialReplPolicy rpc.InitialReplPolicy
RepeatStrategy jobrun.RepeatStrategy
}
type ClientMapping struct {
From string
Mapping zfs.DatasetMapping
}
type Prune struct {
JobName string // for use with jobrun package
DatasetFilter zfs.DatasetMapping
DatasetFilter zfs.DatasetFilter
SnapshotFilter zfs.FilesystemVersionFilter
RetentionPolicy *RetentionGrid // TODO abstract interface to support future policies?
}
@ -80,17 +75,17 @@ type Autosnap struct {
JobName string // for use with jobrun package
Prefix string
Interval jobrun.RepeatStrategy
DatasetFilter zfs.DatasetMapping
DatasetFilter zfs.DatasetFilter
}
type Config struct {
Remotes map[string]*Remote
Pushs map[string]*Push // job name -> job
Pulls map[string]*Pull // job name -> job
Sinks map[string]*ClientMapping // client identity -> mapping
PullACLs map[string]*ClientMapping // client identity -> mapping
Prunes map[string]*Prune // job name -> job
Autosnaps map[string]*Autosnap // job name -> job
Pushs map[string]*Push // job name -> job
Pulls map[string]*Pull // job name -> job
Sinks map[string]DatasetMapFilter // client identity -> mapping
PullACLs map[string]DatasetMapFilter // client identity -> filter
Prunes map[string]*Prune // job name -> job
Autosnaps map[string]*Autosnap // job name -> job
}
func ParseConfig(path string) (config Config, err error) {
@ -129,10 +124,10 @@ func parseMain(root map[string]interface{}) (c Config, err error) {
if c.Pulls, err = parsePulls(root["pulls"], remoteLookup); err != nil {
return
}
if c.Sinks, err = parseClientMappings(root["sinks"]); err != nil {
if c.Sinks, err = parseSinks(root["sinks"]); err != nil {
return
}
if c.PullACLs, err = parseClientMappings(root["pull_acls"]); err != nil {
if c.PullACLs, err = parsePullACLs(root["pull_acls"]); err != nil {
return
}
if c.Prunes, err = parsePrunes(root["prune"]); err != nil {
@ -239,7 +234,7 @@ func parsePushs(v interface{}, rl remoteLookup) (p map[string]*Push, err error)
if push.JobName, err = fullJobName(JobSectionPush, name); err != nil {
return
}
if push.Filter, err = parseComboMapping(e.Filter); err != nil {
if push.Filter, err = parseDatasetMapFilter(e.Filter, true); err != nil {
return
}
@ -298,7 +293,7 @@ func parsePulls(v interface{}, rl remoteLookup) (p map[string]*Pull, err error)
if pull.JobName, err = fullJobName(JobSectionPull, name); err != nil {
return
}
if pull.Mapping, err = parseComboMapping(e.Mapping); err != nil {
if pull.Mapping, err = parseDatasetMapFilter(e.Mapping, false); err != nil {
return
}
if pull.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
@ -365,97 +360,78 @@ func expectList(v interface{}) (asList []interface{}, err error) {
return
}
func parseClientMappings(v interface{}) (cm map[string]*ClientMapping, err error) {
var asMap map[string]interface{}
func parseSinks(v interface{}) (m map[string]DatasetMapFilter, err error) {
var asMap map[string]map[string]interface{}
if err = mapstructure.Decode(v, &asMap); err != nil {
return
}
cm = make(map[string]*ClientMapping, len(asMap))
m = make(map[string]DatasetMapFilter, len(asMap))
for identity, e := range asMap {
var m *ClientMapping
if m, err = parseClientMapping(e, identity); err != nil {
for identity, entry := range asMap {
parseSink := func() (mapping DatasetMapFilter, err error) {
mappingMap, ok := entry["mapping"]
if !ok {
err = fmt.Errorf("no mapping specified")
return
}
mapping, err = parseDatasetMapFilter(mappingMap, false)
return
}
cm[identity] = m
mapping, sinkErr := parseSink()
if sinkErr != nil {
err = fmt.Errorf("cannot parse sink for identity '%s': %s", identity, sinkErr)
return
}
m[identity] = mapping
}
return
}
func parseClientMapping(v interface{}, identity string) (s *ClientMapping, err error) {
t := struct {
Mapping map[string]string
}{}
if err = mapstructure.Decode(v, &t); err != nil {
func parsePullACLs(v interface{}) (m map[string]DatasetMapFilter, err error) {
var asMap map[string]map[string]interface{}
if err = mapstructure.Decode(v, &asMap); err != nil {
return
}
s = &ClientMapping{
From: identity,
m = make(map[string]DatasetMapFilter, len(asMap))
for identity, entry := range asMap {
parsePullACL := func() (filter DatasetMapFilter, err error) {
filterMap, ok := entry["filter"]
if !ok {
err = fmt.Errorf("no filter specified")
return
}
filter, err = parseDatasetMapFilter(filterMap, true)
return
}
filter, filterErr := parsePullACL()
if filterErr != nil {
err = fmt.Errorf("cannot parse pull-ACL for identity '%s': %s", identity, filterErr)
return
}
m[identity] = filter
}
s.Mapping, err = parseComboMapping(t.Mapping)
return
}
func parseComboMapping(m map[string]string) (c zfs.ComboMapping, err error) {
c.Mappings = make([]zfs.DatasetMapping, 0, len(m))
for lhs, rhs := range m {
if lhs == "*" && strings.HasPrefix(rhs, "!") {
m := zfs.ExecMapping{}
fields := strings.Fields(strings.TrimPrefix(rhs, "!"))
if len(fields) < 1 {
err = errors.New("ExecMapping without acceptor path")
return
}
m.Name = fields[0]
m.Args = fields[1:]
c.Mappings = append(c.Mappings, m)
} else if strings.HasSuffix(lhs, "*") {
m := zfs.GlobMapping{}
m.PrefixPath, err = zfs.NewDatasetPath(strings.TrimSuffix(lhs, "*"))
if err != nil {
return
}
if m.TargetRoot, err = zfs.NewDatasetPath(rhs); err != nil {
return
}
c.Mappings = append(c.Mappings, m)
} else {
m := zfs.DirectMapping{}
if lhs == "|" {
m.Source = nil
} else {
if m.Source, err = zfs.NewDatasetPath(lhs); err != nil {
return
}
}
if m.Target, err = zfs.NewDatasetPath(rhs); err != nil {
return
}
c.Mappings = append(c.Mappings, m)
}
func parseDatasetMapFilter(mi interface{}, filterOnly bool) (f DatasetMapFilter, err error) {
var m map[string]string
if err = mapstructure.Decode(mi, &m); err != nil {
err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err)
return
}
f = NewDatasetMapFilter(len(m), filterOnly)
for pathPattern, mapping := range m {
if err = f.Add(pathPattern, mapping); err != nil {
err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
return
}
}
return
}
func (t SSHTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
@ -551,7 +527,7 @@ func parsePrune(e map[string]interface{}, name string) (prune *Prune, err error)
prune.RetentionPolicy = NewRetentionGrid(intervals)
// Parse filters
if prune.DatasetFilter, err = parseComboMapping(i.DatasetFilter); err != nil {
if prune.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
err = fmt.Errorf("cannot parse dataset filter: %s", err)
return
}
@ -746,7 +722,7 @@ func parseAutosnap(m interface{}, name string) (a *Autosnap, err error) {
err = fmt.Errorf("dataset_filter not specified")
return
}
if a.DatasetFilter, err = parseComboMapping(i.DatasetFilter); err != nil {
if a.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
err = fmt.Errorf("cannot parse dataset filter: %s", err)
}

164
cmd/config_mapfilter.go Normal file
View File

@ -0,0 +1,164 @@
package cmd
import (
"errors"
"fmt"
"strings"
"github.com/zrepl/zrepl/zfs"
)
type DatasetMapFilter struct {
entries []datasetMapFilterEntry
// if set, only valid filter entries can be added using Add()
// and Map() will always return an error
filterOnly bool
}
type datasetMapFilterEntry struct {
path zfs.DatasetPath
// the mapping. since this datastructure acts as both mapping and filter
// we have to convert it to the desired rep dynamically
mapping string
subtreeMatch bool
}
var NoMatchError error = errors.New("no match found in mapping")
func NewDatasetMapFilter(capacity int, filterOnly bool) DatasetMapFilter {
return DatasetMapFilter{
entries: make([]datasetMapFilterEntry, 0, capacity),
}
}
func (m *DatasetMapFilter) Add(pathPattern, mapping string) (err error) {
if m.filterOnly {
if _, err = parseDatasetFilterResult(mapping); err != nil {
return
}
}
// assert path glob adheres to spec
const SUBTREE_PATTERN string = "<"
patternCount := strings.Count(pathPattern, SUBTREE_PATTERN)
switch {
case patternCount > 1:
case patternCount == 1 && !strings.HasSuffix(pathPattern, SUBTREE_PATTERN):
err = fmt.Errorf("pattern invalid: only one '<' at end of string allowed")
return
}
var path zfs.DatasetPath
pathStr := strings.TrimSuffix(pathPattern, SUBTREE_PATTERN)
path, err = zfs.NewDatasetPath(pathStr)
if err != nil {
err = fmt.Errorf("pattern is not a dataset path: %s", err)
return
}
entry := datasetMapFilterEntry{
path: path,
mapping: mapping,
subtreeMatch: patternCount > 0,
}
m.entries = append(m.entries, entry)
return
}
// find the most specific prefix mapping we have
//
// longer prefix wins over shorter prefix, direct wins over glob
func (m DatasetMapFilter) mostSpecificPrefixMapping(path zfs.DatasetPath) (idx int, found bool) {
lcp, lcp_entry_idx := -1, -1
direct_idx := -1
for e := range m.entries {
entry := m.entries[e]
ep := m.entries[e].path
lep := ep.Length()
switch {
case !entry.subtreeMatch && ep.Equal(path):
direct_idx = e
continue
case entry.subtreeMatch && path.HasPrefix(ep) && lep > lcp:
lcp = lep
lcp_entry_idx = e
default:
continue
}
}
if lcp_entry_idx >= 0 || direct_idx >= 0 {
found = true
switch {
case direct_idx >= 0:
idx = direct_idx
case lcp_entry_idx >= 0:
idx = lcp_entry_idx
}
}
return
}
func (m DatasetMapFilter) Map(source zfs.DatasetPath) (target zfs.DatasetPath, err error) {
if m.filterOnly {
err = fmt.Errorf("using a filter for mapping simply does not work")
return
}
mi, hasMapping := m.mostSpecificPrefixMapping(source)
if !hasMapping {
err = NoMatchError
return
}
me := m.entries[mi]
target, err = zfs.NewDatasetPath(me.mapping)
if err != nil {
err = fmt.Errorf("mapping target is not a dataset path: %s", err)
return
}
if m.entries[mi].subtreeMatch {
// strip common prefix
extendComps := source.Copy()
if me.path.Empty() {
// special case: trying to map the root => strip first component
extendComps.TrimNPrefixComps(1)
} else {
extendComps.TrimPrefix(me.path)
}
target.Extend(extendComps)
}
return
}
func (m DatasetMapFilter) Filter(p zfs.DatasetPath) (pass bool, err error) {
mi, hasMapping := m.mostSpecificPrefixMapping(p)
if !hasMapping {
pass = false
return
}
me := m.entries[mi]
pass, err = parseDatasetFilterResult(me.mapping)
return
}
// Parse a dataset filter result
func parseDatasetFilterResult(result string) (pass bool, err error) {
l := strings.ToLower(result)
switch strings.ToLower(l) {
case "ok":
pass = true
return
case "omit":
return
default:
err = fmt.Errorf("'%s' is not a valid filter result", result)
return
}
return
}

80
cmd/config_test.go
View File

@ -1,10 +1,12 @@
package cmd
import (
"github.com/stretchr/testify/assert"
"github.com/zrepl/zrepl/util"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/zrepl/zrepl/util"
"github.com/zrepl/zrepl/zfs"
)
func TestSampleConfigFileIsParsedWithoutErrors(t *testing.T) {
@ -45,3 +47,77 @@ func TestParseRetentionGridStringParsing(t *testing.T) {
assert.EqualValues(t, util.RetentionGridKeepCountAll, intervals[0].KeepCount)
}
func TestDatasetMapFilter(t *testing.T) {
expectMapping := func(m map[string]string, from, to string) {
dmf, err := parseDatasetMapFilter(m, false)
if err != nil {
t.Logf("expect test map to be valid: %s", err)
t.FailNow()
}
fromPath, err := zfs.NewDatasetPath(from)
if err != nil {
t.Logf("expect test from path to be valid: %s", err)
t.FailNow()
}
toPath, err := zfs.NewDatasetPath(to)
if err != nil {
t.Logf("expect test to path to be valid: %s", err)
t.FailNow()
}
res, err := dmf.Map(fromPath)
t.Logf("%s => %s", fromPath.ToString(), res.ToString())
assert.Nil(t, err)
assert.True(t, res.Equal(toPath))
}
expectFilter := func(m map[string]string, path string, pass bool) {
dmf, err := parseDatasetMapFilter(m, true)
if err != nil {
t.Logf("expect test filter to be valid: %s", err)
t.FailNow()
}
p, err := zfs.NewDatasetPath(path)
if err != nil {
t.Logf("expect test path to be valid: %s", err)
t.FailNow()
}
res, err := dmf.Filter(p)
assert.Nil(t, err)
assert.Equal(t, pass, res)
}
map1 := map[string]string{
"a/b/c<": "root1",
"a/b<": "root2",
"<": "root3/b/c",
"q<": "root4/1/2",
}
expectMapping(map1, "a/b/c", "root1")
expectMapping(map1, "a/b/c/d", "root1/d")
expectMapping(map1, "a/b/e", "root2/e")
expectMapping(map1, "a/b", "root2")
expectMapping(map1, "x", "root3/b/c")
expectMapping(map1, "x/y", "root3/b/c/y")
expectMapping(map1, "q", "root4/1/2")
expectMapping(map1, "q/r", "root4/1/2/r")
filter1 := map[string]string{
"<": "omit",
"a<": "ok",
"a/b<": "omit",
}
expectFilter(filter1, "b", false)
expectFilter(filter1, "a", true)
expectFilter(filter1, "a/d", true)
expectFilter(filter1, "a/b", false)
expectFilter(filter1, "a/b/c", false)
filter2 := map[string]string{}
expectFilter(filter2, "foo", false) // default to omit
}

10
cmd/handler.go
View File

@ -7,10 +7,14 @@ import (
"io"
)
type DatasetMapping interface {
Map(source zfs.DatasetPath) (target zfs.DatasetPath, err error)
}
type Handler struct {
Logger Logger
PullACL zfs.DatasetMapping
SinkMappingFunc func(clientIdentity string) (mapping zfs.DatasetMapping, err error)
PullACL zfs.DatasetFilter
SinkMappingFunc func(clientIdentity string) (mapping DatasetMapping, err error)
}
func (h Handler) HandleFilesystemRequest(r rpc.FilesystemRequest) (roots []zfs.DatasetPath, err error) {
@ -90,7 +94,7 @@ func (h Handler) HandlePullMeRequest(r rpc.PullMeRequest, clientIdentity string,
h.Logger.Printf("handling PullMeRequest: %#v", r)
var sinkMapping zfs.DatasetMapping
var sinkMapping DatasetMapping
sinkMapping, err = h.SinkMappingFunc(clientIdentity)
if err != nil {
h.Logger.Printf("no sink mapping for client identity '%s', denying PullMeRequest", clientIdentity)

20
cmd/replication.go
View File

@ -141,12 +141,24 @@ func cmdRun(cmd *cobra.Command, args []string) {
}
type localPullACL struct{}
func (a localPullACL) Filter(p zfs.DatasetPath) (pass bool, err error) {
return true, nil
}
func jobPull(pull *Pull, log jobrun.Logger) (err error) {
if lt, ok := pull.From.Transport.(LocalTransport); ok {
lt.SetHandler(Handler{
Logger: log,
PullACL: pull.Mapping,
Logger: log,
// Allow access to any dataset since we control what mapping
// is passed to the pull routine.
// All local datasets will be passed to its Map() function,
// but only those for which a mapping exists will actually be pulled.
// We can pay this small performance penalty for now.
PullACL: localPullACL{},
})
pull.From.Transport = lt
log.Printf("fixing up local transport: %#v", pull.From.Transport)
@ -228,7 +240,7 @@ func closeRPCWithTimeout(log Logger, remote rpc.RPCRequester, timeout time.Durat
type PullContext struct {
Remote rpc.RPCRequester
Log Logger
Mapping zfs.DatasetMapping
Mapping DatasetMapping
InitialReplPolicy rpc.InitialReplPolicy
}
@ -258,7 +270,7 @@ func doPull(pull PullContext) (err error) {
var localFs zfs.DatasetPath
localFs, err = pull.Mapping.Map(remoteFilesystems[fs])
if err != nil {
if err != zfs.NoMatchError {
if err != NoMatchError {
log.Printf("error mapping %s: %#v\n", remoteFilesystems[fs], err)
return err
}

68
cmd/sampleconf/zrepl.yml
View File

@ -12,8 +12,9 @@ pushs:
offsite:
to: offsite_backups
filter: {
"tank/var/db*":"ok",
"tank/usr/home*":"ok"
# like in pull_acls
"tank/var/db<": ok,
"tank/usr/home<": ok,
}
pulls:
@ -36,59 +37,39 @@ pulls:
sinks:
# direct mapping
# 1:1 mapping of remote dataset to local dataset
# We will reject a push request which contains > 0 datasets that do not
# match a mapping
db1:
mapping: {
# direct mapping
"ssdpool/var/db/postgresql9.6":"zroot/backups/db1/pg_data"
}
# "|" non-recursive wildcard
# the remote must present excatly one dataset, mapped to the rhs
cdn_master:
mapping: {
"|":"tank/srv/cdn" # NOTE: | is currently an invalid character for a ZFS dataset
}
# "*" recursive wildcard
# the remote may present an arbitrary set of marks a recursive wildcard, i.e. map all remotes to a tree under rhs
mirror1:
mapping: {
"tank/foo/bar*":"zroot/backups/mirror1" # NOTE: * is currently an invalid character for a ZFS dataset
# "<" subtree wildcard matches the dataset left of < and all its children
"tank/foo/bar<":"zroot/backups/mirror1"
}
# "*":"!..." acceptor script
# shell out to an accceptor that receives the remote's offered datasets
# on stdin and, foreach line of this input, returns the corresponding
# local dataset (same order) or '!<space>optional reason' on stdout
# If the acceptor scripts exits with non-zero status code, the remote's
# request will be rejected
complex_host:
mapping: { #
"*":"!/path/to/acceptor" # we could just wire the path to the acceptor directly to the mapping
# but let's stick with the same type for the mapping field for now'
# NOTE: * and ! are currently invalid characters for a ZFS dataset
mirror2:
# more specific path patterns win over less specific ones
# direct mappings win over subtree wildcards
# detailed rule precedence: check unit tests & docs for exact behavior
# TODO subcommand to test a mapping & filter
mapping: {
"tank<": "zroot/backups/mirror1/tank1",
"tank/cdn/root<": "storage/cdn/root",
"tank/legacydb": "legacypool/backups/legacydb",
}
# Mixing the rules
# Mixing should be possible if there is a defined precedence (direct before *)
# and non-recursive wildcards are not allowed in multi-entry mapping objects
special_snowflake:
mapping: { # an explicit mapping mixed with a recursive wildcard
"sun/usr/home": backups/special_snowflake/homedirs,
"sun/var/db": backups/special_snowflake/database,
"*": backups/special_snowflake/remainingbackup
# NOTE: ^ alignment, should be possible, looks nicer
}
pull_acls:
# same synatx as in sinks, but the returned mapping does not matter
office_backup:
mapping: {
"tank/usr/home":"notnull"
filter: {
# valid filter results (right hand side): ok, omit
# default is to omit
# rule precedence is same as for mappings
"tank<": omit,
"tank/usr/home": ok,
}
@ -98,7 +79,8 @@ prune:
policy: grid
grid: 6x10m | 24x1h | 7x1d | 5 x 1w | 4 x 5w
dataset_filter: {
"tank/backups/*": ok
"tank/backups/legacyscript<": omit,
"tank/backups<": ok,
}
snapshot_filter: {
prefix: zrepl_
@ -108,7 +90,7 @@ prune:
policy: grid
grid: 1x1m(keep=all)
dataset_filter: {
"pool1*": ok
"pool1<": ok
}
snapshot_filter: {
prefix: zrepl_hfbak_
@ -120,7 +102,7 @@ autosnap:
prefix: zrepl_hfbak_
interval: 1s
dataset_filter: {
"pool1*": ok
"pool1<": ok
}
# prune: hfbak_prune
# future versions may inline the retention policy here, but for now,

17
cmd/stdinserver.go
View File

@ -5,7 +5,6 @@ import (
"github.com/spf13/cobra"
"github.com/zrepl/zrepl/rpc"
"github.com/zrepl/zrepl/sshbytestream"
"github.com/zrepl/zrepl/zfs"
"io"
golog "log"
"os"
@ -37,8 +36,8 @@ func cmdStdinServer(cmd *cobra.Command, args []string) {
}
identity := args[0]
pullACL := conf.PullACLs[identity]
if pullACL == nil {
pullACL, ok := conf.PullACLs[identity]
if !ok {
err = fmt.Errorf("could not find PullACL for identity '%s'", identity)
return
}
@ -48,19 +47,19 @@ func cmdStdinServer(cmd *cobra.Command, args []string) {
return
}
sinkMapping := func(identity string) (m zfs.DatasetMapping, err error) {
sink := conf.Sinks[identity]
if sink == nil {
return nil, fmt.Errorf("could not find sink for dataset")
sinkMapping := func(identity string) (m DatasetMapping, err error) {
sink, ok := conf.Sinks[identity]
if !ok {
return nil, fmt.Errorf("could not find sink for identity '%s'", identity)
}
return sink.Mapping, nil
return sink, nil
}
sinkLogger := golog.New(logOut, fmt.Sprintf("sink[%s] ", identity), logFlags)
handler := Handler{
Logger: sinkLogger,
SinkMappingFunc: sinkMapping,
PullACL: pullACL.Mapping,
PullACL: pullACL,
}
if err = rpc.ListenByteStreamRPC(sshByteStream, identity, handler, sinkLogger); err != nil {

22
zfs/datasetpath_visitor.go
View File

@ -11,20 +11,20 @@ func NewDatasetPathForest() *DatasetPathForest {
}
func (f *DatasetPathForest) Add(p DatasetPath) {
if len(p) <= 0 {
if len(p.comps) <= 0 {
panic("dataset path too short. must have length > 0")
}
// Find its root
var root *datasetPathTree
for _, r := range f.roots {
if r.Add(p) {
if r.Add(p.comps) {
root = r
break
}
}
if root == nil {
root = newDatasetPathTree(p)
root = newDatasetPathTree(p.comps)
f.roots = append(f.roots, root)
}
}
@ -57,7 +57,7 @@ type datasetPathTree struct {
Children []*datasetPathTree
}
func (t *datasetPathTree) Add(p DatasetPath) bool {
func (t *datasetPathTree) Add(p []string) bool {
if len(p) == 0 {
return true
@ -88,11 +88,15 @@ func (t *datasetPathTree) Add(p DatasetPath) bool {
}
func (t *datasetPathTree) WalkTopDown(parent DatasetPath, visitor DatasetPathsVisitor) {
func (t *datasetPathTree) WalkTopDown(parent []string, visitor DatasetPathsVisitor) {
this := append(parent, t.Component)
visitChildTree := visitor(DatasetPathVisit{this, t.FilledIn})
thisVisit := DatasetPathVisit{
DatasetPath{this},
t.FilledIn,
}
visitChildTree := visitor(thisVisit)
if visitChildTree {
for _, c := range t.Children {
@ -102,15 +106,15 @@ func (t *datasetPathTree) WalkTopDown(parent DatasetPath, visitor DatasetPathsVi
}
func newDatasetPathTree(initial DatasetPath) (t *datasetPathTree) {
func newDatasetPathTree(initialComps []string) (t *datasetPathTree) {
t = &datasetPathTree{}
var cur *datasetPathTree
cur = t
for i, comp := range initial {
for i, comp := range initialComps {
cur.Component = comp
cur.FilledIn = true
cur.Children = make([]*datasetPathTree, 0, 1)
if i == len(initial)-1 {
if i == len(initialComps)-1 {
cur.FilledIn = false // last component is not filled in
break
}

2
zfs/datasetpath_visitor_test.go
View File

@ -7,7 +7,7 @@ import (
func TestNewDatasetPathTree(t *testing.T) {
r := newDatasetPathTree(toDatasetPath("pool1/foo/bar"))
r := newDatasetPathTree(toDatasetPath("pool1/foo/bar").comps)
assert.Equal(t, "pool1", r.Component)
assert.True(t, len(r.Children) == 1)

163
zfs/mapping.go
View File

@ -1,21 +1,15 @@
package zfs
import (
"bufio"
"errors"
"fmt"
"io"
"os/exec"
)
import "fmt"
type DatasetMapping interface {
Map(source DatasetPath) (target DatasetPath, err error)
type DatasetFilter interface {
Filter(p DatasetPath) (pass bool, err error)
}
func ZFSListMapping(mapping DatasetMapping) (datasets []DatasetPath, err error) {
func ZFSListMapping(filter DatasetFilter) (datasets []DatasetPath, err error) {
if mapping == nil {
panic("mapping must not be nil")
if filter == nil {
panic("filter must not be nil")
}
var lines [][]string
@ -30,12 +24,11 @@ func ZFSListMapping(mapping DatasetMapping) (datasets []DatasetPath, err error)
return
}
_, mapErr := mapping.Map(path)
if mapErr != nil && mapErr != NoMatchError {
return nil, mapErr
pass, filterErr := filter.Filter(path)
if filterErr != nil {
return nil, fmt.Errorf("error calling filter: %s", filterErr)
}
if mapErr == nil {
if pass {
datasets = append(datasets, path)
}
@ -43,139 +36,3 @@ func ZFSListMapping(mapping DatasetMapping) (datasets []DatasetPath, err error)
return
}
type GlobMapping struct {
PrefixPath DatasetPath
TargetRoot DatasetPath
}
var NoMatchError error = errors.New("no match found in mapping")
func (m GlobMapping) Map(source DatasetPath) (target DatasetPath, err error) {
if len(source) < len(m.PrefixPath) {
err = NoMatchError
return
}
target = make([]string, 0, len(source)+len(m.TargetRoot))
target = append(target, m.TargetRoot...)
for si, sc := range source {
target = append(target, sc)
if si < len(m.PrefixPath) {
compsMatch := sc == m.PrefixPath[si]
endOfPrefixPath := si == len(m.PrefixPath)-1 && m.PrefixPath[si] == ""
if !(compsMatch || endOfPrefixPath) {
err = NoMatchError
return
}
continue
}
}
return
}
type ComboMapping struct {
Mappings []DatasetMapping
}
func (m ComboMapping) Map(source DatasetPath) (target DatasetPath, err error) {
for _, sm := range m.Mappings {
target, err = sm.Map(source)
if err == nil {
return target, err
}
}
return nil, NoMatchError
}
type DirectMapping struct {
Source DatasetPath
Target DatasetPath
}
func (m DirectMapping) Map(source DatasetPath) (target DatasetPath, err error) {
if m.Source == nil {
return m.Target, nil
}
if len(m.Source) != len(source) {
return nil, NoMatchError
}
for i, c := range source {
if c != m.Source[i] {
return nil, NoMatchError
}
}
return m.Target, nil
}
type ExecMapping struct {
Name string
Args []string
}
func NewExecMapping(name string, args ...string) (m *ExecMapping) {
m = &ExecMapping{
Name: name,
Args: args,
}
return
}
func (m ExecMapping) Map(source DatasetPath) (target DatasetPath, err error) {
var stdin io.Writer
var stdout io.Reader
cmd := exec.Command(m.Name, m.Args...)
if stdin, err = cmd.StdinPipe(); err != nil {
return
}
if stdout, err = cmd.StdoutPipe(); err != nil {
return
}
resp := bufio.NewScanner(stdout)
if err = cmd.Start(); err != nil {
return
}
go func() {
err := cmd.Wait()
if err != nil {
panic(err)
// fmt.Printf("error: %v\n", err) // TODO
}
}()
if _, err = io.WriteString(stdin, source.ToString()+"\n"); err != nil {
return
}
if !resp.Scan() {
err = errors.New(fmt.Sprintf("unexpected end of file: %v", resp.Err()))
return
}
t := resp.Text()
switch {
case t == "NOMAP":
return nil, NoMatchError
}
target = toDatasetPath(t) // TODO discover garbage?
return
}

134
zfs/mapping_test.go
View File

@ -1,134 +0,0 @@
package zfs
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestGlobMappingPrefixWildcard(t *testing.T) {
m := GlobMapping{
PrefixPath: toDatasetPath("a/b/c/"), // TRAILING empty component!
TargetRoot: toDatasetPath("x/y"),
}
t.Logf("PrefixPath: %#v", m.PrefixPath)
var r DatasetPath
var err error
r, err = m.Map(toDatasetPath("a/b/c"))
assert.NotNil(t, err)
r, err = m.Map(toDatasetPath("a/b/c/d"))
assert.Nil(t, err)
assert.Equal(t, toDatasetPath("x/y/a/b/c/d"), r)
}
func TestGlobMapping(t *testing.T) {
m := GlobMapping{
PrefixPath: toDatasetPath("tank/usr/home"),
TargetRoot: toDatasetPath("backups/share1"),
}
var r DatasetPath
var err error
r, err = m.Map(toDatasetPath("tank/usr/home"))
assert.Nil(t, err)
assert.Equal(t, toDatasetPath("backups/share1/tank/usr/home"), r)
r, err = m.Map(toDatasetPath("zroot"))
assert.Equal(t, NoMatchError, err, "prefix-only match is an error")
r, err = m.Map(toDatasetPath("zroot/notmapped"))
assert.Equal(t, NoMatchError, err, "non-prefix is an error")
}
func TestGlobMappingWildcard(t *testing.T) {
m := GlobMapping{
PrefixPath: EmptyDatasetPath,
TargetRoot: toDatasetPath("backups/share1"),
}
var r DatasetPath
var err error
r, err = m.Map(toDatasetPath("tank/usr/home"))
assert.Equal(t, toDatasetPath("backups/share1/tank/usr/home"), r)
assert.NoError(t, err)
}
func TestComboMapping(t *testing.T) {
m1 := GlobMapping{
PrefixPath: toDatasetPath("a/b"),
TargetRoot: toDatasetPath("c/d"),
}
m2 := GlobMapping{
PrefixPath: toDatasetPath("a/x"),
TargetRoot: toDatasetPath("c/y"),
}
c := ComboMapping{
Mappings: []DatasetMapping{m1, m2},
}
var r DatasetPath
var err error
p := toDatasetPath("a/b/q")
r, err = m2.Map(p)
assert.Equal(t, NoMatchError, err)
r, err = c.Map(p)
assert.Nil(t, err)
assert.Equal(t, toDatasetPath("c/d/a/b/q"), r)
}
func TestDirectMapping(t *testing.T) {
m := DirectMapping{
Source: toDatasetPath("a/b/c"),
Target: toDatasetPath("x/y/z"),
}
var r DatasetPath
var err error
r, err = m.Map(toDatasetPath("a/b/c"))
assert.Nil(t, err)
assert.Equal(t, m.Target, r)
r, err = m.Map(toDatasetPath("not/matching"))
assert.Equal(t, NoMatchError, err)
r, err = m.Map(toDatasetPath("a/b"))
assert.Equal(t, NoMatchError, err)
}
func TestExecMapping(t *testing.T) {
var err error
var m DatasetMapping
m = NewExecMapping("test_helpers/exec_mapping_good.sh", "nostop")
assert.NoError(t, err)
var p DatasetPath
p, err = m.Map(toDatasetPath("nomap/foobar"))
assert.Equal(t, NoMatchError, err)
p, err = m.Map(toDatasetPath("willmap/something"))
assert.Nil(t, err)
assert.Equal(t, toDatasetPath("didmap/willmap/something"), p)
}

88
zfs/zfs.go
View File

@ -11,27 +11,97 @@ import (
"strings"
)
type DatasetPath []string
type DatasetPath struct {
comps []string
}
func (p DatasetPath) ToString() string {
return strings.Join(p, "/")
return strings.Join(p.comps, "/")
}
func (p DatasetPath) Empty() bool {
return len(p) == 0
return len(p.comps) == 0
}
var EmptyDatasetPath DatasetPath = []string{}
func (p *DatasetPath) Extend(extend DatasetPath) {
p.comps = append(p.comps, extend.comps...)
}
func (p DatasetPath) HasPrefix(prefix DatasetPath) bool {
if len(prefix.comps) > len(p.comps) {
return false
}
for i := range prefix.comps {
if prefix.comps[i] != p.comps[i] {
return false
}
}
return true
}
func (p *DatasetPath) TrimPrefix(prefix DatasetPath) {
if !p.HasPrefix(prefix) {
return
}
prelen := len(prefix.comps)
newlen := len(p.comps) - prelen
oldcomps := p.comps
p.comps = make([]string, newlen)
for i := 0; i < newlen; i++ {
p.comps[i] = oldcomps[prelen+i]
}
return
}
func (p *DatasetPath) TrimNPrefixComps(n int) {
if len(p.comps) < n {
n = len(p.comps)
}
if n == 0 {
return
}
p.comps = p.comps[n:]
}
func (p DatasetPath) Equal(q DatasetPath) bool {
if len(p.comps) != len(q.comps) {
return false
}
for i := range p.comps {
if p.comps[i] != q.comps[i] {
return false
}
}
return true
}
func (p DatasetPath) Length() int {
return len(p.comps)
}
func (p DatasetPath) Copy() (c DatasetPath) {
c.comps = make([]string, len(p.comps))
copy(c.comps, p.comps)
return
}
func NewDatasetPath(s string) (p DatasetPath, err error) {
if s == "" {
return EmptyDatasetPath, nil // the empty dataset path
p.comps = make([]string, 0)
return p, nil // the empty dataset path
}
const FORBIDDEN = "@#|\t "
const FORBIDDEN = "@#|\t <>*"
if strings.ContainsAny(s, FORBIDDEN) { // TODO space may be a bit too restrictive...
return nil, errors.New(fmt.Sprintf("path '%s' contains forbidden characters (any of '%s')", s, FORBIDDEN))
err = fmt.Errorf("contains forbidden characters (any of '%s')", FORBIDDEN)
return
}
return strings.Split(s, "/"), nil
p.comps = strings.Split(s, "/")
if p.comps[len(p.comps)-1] == "" {
err = fmt.Errorf("must not end with a '/'")
return
}
return
}
func toDatasetPath(s string) DatasetPath {
@ -42,8 +112,6 @@ func toDatasetPath(s string) DatasetPath {
return p
}
type DatasetFilter func(path DatasetPath) bool
type ZFSError struct {
Stderr []byte
WaitErr error

11
zfs/zfs_test.go
View File

@ -17,3 +17,14 @@ func TestZFSListHandlesProducesZFSErrorOnNonZeroExit(t *testing.T) {
assert.True(t, ok)
assert.Equal(t, "error: this is a mock\n", string(zfsError.Stderr))
}
func TestDatasetPathTrimNPrefixComps(t *testing.T) {
p, err := NewDatasetPath("foo/bar/a/b")
assert.Nil(t, err)
p.TrimNPrefixComps(2)
assert.True(t, p.Equal(toDatasetPath("a/b")))
p.TrimNPrefixComps((2))
assert.True(t, p.Empty())
p.TrimNPrefixComps((1))
assert.True(t, p.Empty(), "empty trimming shouldn't do harm")
}