format source tree using goimports

This commit is contained in:
Christian Schwarz 2019-03-22 19:41:12 +01:00
parent 5324f29693
commit afed762774
93 changed files with 585 additions and 463 deletions

View File

@ -30,6 +30,9 @@ generate: #not part of the build, must do that manually
protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
go generate -x ./... go generate -x ./...
format:
goimports -srcdir . -local 'github.com/zrepl/zrepl' -w $(shell find . -type f -name '*.go' -not -path "./vendor/*")
build: build:
@echo "INFO: In case of missing dependencies, run 'make vendordeps'" @echo "INFO: In case of missing dependencies, run 'make vendordeps'"
$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl" $(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl"

View File

@ -11,7 +11,9 @@
package main package main
import ( import (
"fmt"
_ "fmt" _ "fmt"
_ "github.com/alvaroloes/enumer" _ "github.com/alvaroloes/enumer"
_ "github.com/golang/protobuf/protoc-gen-go" _ "github.com/golang/protobuf/protoc-gen-go"
_ "golang.org/x/tools/cmd/stringer" _ "golang.org/x/tools/cmd/stringer"

View File

@ -2,10 +2,12 @@ package cli
import ( import (
"fmt" "fmt"
"os"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/pflag" "github.com/spf13/pflag"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"os"
) )
var rootArgs struct { var rootArgs struct {
@ -40,15 +42,15 @@ func init() {
} }
type Subcommand struct { type Subcommand struct {
Use string Use string
Short string Short string
Example string Example string
NoRequireConfig bool NoRequireConfig bool
Run func(subcommand *Subcommand, args []string) error Run func(subcommand *Subcommand, args []string) error
SetupFlags func(f *pflag.FlagSet) SetupFlags func(f *pflag.FlagSet)
SetupSubcommands func() []*Subcommand SetupSubcommands func() []*Subcommand
config *config.Config config *config.Config
configErr error configErr error
} }
@ -93,8 +95,8 @@ func AddSubcommand(s *Subcommand) {
func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) { func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) {
cmd := cobra.Command{ cmd := cobra.Command{
Use: s.Use, Use: s.Use,
Short: s.Short, Short: s.Short,
Example: s.Example, Example: s.Example,
} }
if s.SetupSubcommands == nil { if s.SetupSubcommands == nil {
@ -110,7 +112,6 @@ func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) {
c.AddCommand(&cmd) c.AddCommand(&cmd)
} }
func Run() { func Run() {
if err := rootCmd.Execute(); err != nil { if err := rootCmd.Execute(); err != nil {
os.Exit(1) os.Exit(1)

View File

@ -3,33 +3,35 @@ package client
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"os"
"github.com/kr/pretty" "github.com/kr/pretty"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/spf13/pflag" "github.com/spf13/pflag"
"github.com/zrepl/yaml-config" "github.com/zrepl/yaml-config"
"github.com/zrepl/zrepl/cli" "github.com/zrepl/zrepl/cli"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/job" "github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/daemon/logging" "github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/logger"
"os"
) )
var configcheckArgs struct { var configcheckArgs struct {
format string format string
what string what string
} }
var ConfigcheckCmd = &cli.Subcommand{ var ConfigcheckCmd = &cli.Subcommand{
Use: "configcheck", Use: "configcheck",
Short: "check if config can be parsed without errors", Short: "check if config can be parsed without errors",
SetupFlags: func(f *pflag.FlagSet) { SetupFlags: func(f *pflag.FlagSet) {
f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]") f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]") f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
}, },
Run: func(subcommand *cli.Subcommand, args []string) error { Run: func(subcommand *cli.Subcommand, args []string) error {
formatMap := map[string]func(interface{}) { formatMap := map[string]func(interface{}){
"": func(i interface{}) {}, "": func(i interface{}) {},
"pretty": func(i interface{}) { pretty.Println(i) }, "pretty": func(i interface{}) { pretty.Println(i) },
"json": func(i interface{}) { "json": func(i interface{}) {
json.NewEncoder(os.Stdout).Encode(subcommand.Config()) json.NewEncoder(os.Stdout).Encode(subcommand.Config())
@ -71,12 +73,11 @@ var ConfigcheckCmd = &cli.Subcommand{
} }
} }
whatMap := map[string]func(){
whatMap := map[string]func() {
"all": func() { "all": func() {
o := struct { o := struct {
config *config.Config config *config.Config
jobs []job.Job jobs []job.Job
logging *logger.Outlets logging *logger.Outlets
}{ }{
subcommand.Config(), subcommand.Config(),
@ -109,4 +110,3 @@ var ConfigcheckCmd = &cli.Subcommand{
} }
}, },
} }

View File

@ -4,10 +4,11 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
"github.com/pkg/errors"
"io" "io"
"net" "net"
"net/http" "net/http"
"github.com/pkg/errors"
) )
func controlHttpClient(sockpath string) (client http.Client, err error) { func controlHttpClient(sockpath string) (client http.Client, err error) {

View File

@ -6,6 +6,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/spf13/pflag" "github.com/spf13/pflag"
"github.com/zrepl/zrepl/zfs" "github.com/zrepl/zrepl/zfs"
"github.com/zrepl/zrepl/cli" "github.com/zrepl/zrepl/cli"

View File

@ -2,11 +2,12 @@ package client
import ( import (
"errors" "errors"
"log"
"os"
"github.com/zrepl/zrepl/cli" "github.com/zrepl/zrepl/cli"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon" "github.com/zrepl/zrepl/daemon"
"log"
"os"
) )
var pprofArgs struct { var pprofArgs struct {

View File

@ -2,6 +2,7 @@ package client
import ( import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/zrepl/zrepl/cli" "github.com/zrepl/zrepl/cli"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon" "github.com/zrepl/zrepl/daemon"
@ -28,10 +29,10 @@ func runSignalCmd(config *config.Config, args []string) error {
err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal, err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal,
struct { struct {
Name string Name string
Op string Op string
}{ }{
Name: args[1], Name: args[1],
Op: args[0], Op: args[0],
}, },
struct{}{}, struct{}{},
) )

View File

@ -2,15 +2,6 @@ package client
import ( import (
"fmt" "fmt"
"github.com/gdamore/tcell/termbox"
"github.com/pkg/errors"
"github.com/spf13/pflag"
"github.com/zrepl/yaml-config"
"github.com/zrepl/zrepl/cli"
"github.com/zrepl/zrepl/daemon"
"github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/daemon/pruner"
"github.com/zrepl/zrepl/replication/report"
"io" "io"
"math" "math"
"net/http" "net/http"
@ -19,18 +10,29 @@ import (
"strings" "strings"
"sync" "sync"
"time" "time"
"github.com/gdamore/tcell/termbox"
"github.com/pkg/errors"
"github.com/spf13/pflag"
"github.com/zrepl/yaml-config"
"github.com/zrepl/zrepl/cli"
"github.com/zrepl/zrepl/daemon"
"github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/daemon/pruner"
"github.com/zrepl/zrepl/replication/report"
) )
type byteProgressMeasurement struct { type byteProgressMeasurement struct {
time time.Time time time.Time
val int64 val int64
} }
type bytesProgressHistory struct { type bytesProgressHistory struct {
last *byteProgressMeasurement // pointer as poor man's optional last *byteProgressMeasurement // pointer as poor man's optional
changeCount int changeCount int
lastChange time.Time lastChange time.Time
bpsAvg float64 bpsAvg float64
} }
func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64, changeCount int) { func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64, changeCount int) {
@ -38,7 +40,7 @@ func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64
if p.last == nil { if p.last == nil {
p.last = &byteProgressMeasurement{ p.last = &byteProgressMeasurement{
time: time.Now(), time: time.Now(),
val: currentVal, val: currentVal,
} }
return 0, 0 return 0, 0
} }
@ -48,18 +50,17 @@ func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64
p.lastChange = time.Now() p.lastChange = time.Now()
} }
if time.Now().Sub(p.lastChange) > 3 * time.Second { if time.Now().Sub(p.lastChange) > 3*time.Second {
p.last = nil p.last = nil
return 0, 0 return 0, 0
} }
deltaV := currentVal - p.last.val
deltaV := currentVal - p.last.val;
deltaT := time.Now().Sub(p.last.time) deltaT := time.Now().Sub(p.last.time)
rate := float64(deltaV) / deltaT.Seconds() rate := float64(deltaV) / deltaT.Seconds()
factor := 0.3 factor := 0.3
p.bpsAvg = (1-factor) * p.bpsAvg + factor * rate p.bpsAvg = (1-factor)*p.bpsAvg + factor*rate
p.last.time = time.Now() p.last.time = time.Now()
p.last.val = currentVal p.last.val = currentVal
@ -119,7 +120,7 @@ func wrap(s string, width int) string {
rem = len(s) rem = len(s)
} }
if idx := strings.IndexAny(s, "\n\r"); idx != -1 && idx < rem { if idx := strings.IndexAny(s, "\n\r"); idx != -1 && idx < rem {
rem = idx+1 rem = idx + 1
} }
untilNewline := strings.TrimRight(s[:rem], "\n\r") untilNewline := strings.TrimRight(s[:rem], "\n\r")
s = s[rem:] s = s[rem:]
@ -135,12 +136,12 @@ func wrap(s string, width int) string {
func (t *tui) printfDrawIndentedAndWrappedIfMultiline(format string, a ...interface{}) { func (t *tui) printfDrawIndentedAndWrappedIfMultiline(format string, a ...interface{}) {
whole := fmt.Sprintf(format, a...) whole := fmt.Sprintf(format, a...)
width, _ := termbox.Size() width, _ := termbox.Size()
if !strings.ContainsAny(whole, "\n\r") && t.x + len(whole) <= width { if !strings.ContainsAny(whole, "\n\r") && t.x+len(whole) <= width {
t.printf(format, a...) t.printf(format, a...)
} else { } else {
t.addIndent(1) t.addIndent(1)
t.newline() t.newline()
t.write(wrap(whole, width - INDENT_MULTIPLIER*t.indent)) t.write(wrap(whole, width-INDENT_MULTIPLIER*t.indent))
t.addIndent(-1) t.addIndent(-1)
} }
} }
@ -159,7 +160,6 @@ func (t *tui) addIndent(indent int) {
t.moveLine(0, 0) t.moveLine(0, 0)
} }
var statusFlags struct { var statusFlags struct {
Raw bool Raw bool
} }
@ -180,7 +180,7 @@ func runStatus(s *cli.Subcommand, args []string) error {
} }
if statusFlags.Raw { if statusFlags.Raw {
resp, err := httpc.Get("http://unix"+daemon.ControlJobEndpointStatus) resp, err := httpc.Get("http://unix" + daemon.ControlJobEndpointStatus)
if err != nil { if err != nil {
return err return err
} }
@ -390,7 +390,7 @@ func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgress
t.newline() t.newline()
t.addIndent(1) t.addIndent(1)
for i, a := range rep.Attempts[:len(rep.Attempts)-1] { for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
t.printfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)", i + 1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt)) t.printfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
t.newline() t.newline()
} }
t.addIndent(-1) t.addIndent(-1)
@ -462,7 +462,7 @@ func (t *tui) renderPrunerReport(r *pruner.Report) {
*pruner.FSReport *pruner.FSReport
completed bool completed bool
} }
all := make([]commonFS, 0, len(r.Pending) + len(r.Completed)) all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
for i := range r.Pending { for i := range r.Pending {
all = append(all, commonFS{&r.Pending[i], false}) all = append(all, commonFS{&r.Pending[i], false})
} }
@ -471,7 +471,8 @@ func (t *tui) renderPrunerReport(r *pruner.Report) {
} }
switch state { switch state {
case pruner.Plan: fallthrough case pruner.Plan:
fallthrough
case pruner.PlanErr: case pruner.PlanErr:
return return
} }
@ -499,7 +500,7 @@ func (t *tui) renderPrunerReport(r *pruner.Report) {
t.write("[") t.write("[")
t.write(times("=", progress)) t.write(times("=", progress))
t.write(">") t.write(">")
t.write(times("-", 80 - progress)) t.write(times("-", 80-progress))
t.write("]") t.write("]")
t.printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount) t.printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
t.newline() t.newline()
@ -519,9 +520,9 @@ func (t *tui) renderPrunerReport(r *pruner.Report) {
if fs.LastError != "" { if fs.LastError != "" {
if strings.ContainsAny(fs.LastError, "\r\n") { if strings.ContainsAny(fs.LastError, "\r\n") {
t.printf("ERROR:") t.printf("ERROR:")
t.printfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError) t.printfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
} else { } else {
t.printfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError) t.printfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
} }
t.newline() t.newline()
continue continue
@ -531,7 +532,7 @@ func (t *tui) renderPrunerReport(r *pruner.Report) {
len(fs.DestroyList), len(fs.SnapshotList)) len(fs.DestroyList), len(fs.SnapshotList))
if fs.completed { if fs.completed {
t.printf( "Completed %s\n", pruneRuleActionStr) t.printf("Completed %s\n", pruneRuleActionStr)
continue continue
} }
@ -560,7 +561,6 @@ func rightPad(str string, length int, pad string) string {
return str + times(pad, length-len(str)) return str + times(pad, length-len(str))
} }
func leftPad(str string, length int, pad string) string { func leftPad(str string, length int, pad string) string {
if len(str) > length { if len(str) > length {
return str[len(str)-length:] return str[len(str)-length:]
@ -584,7 +584,7 @@ func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
t.write("[") t.write("[")
t.write(times("=", completedLength)) t.write(times("=", completedLength))
t.write( string(arrowPositions[changeCount%len(arrowPositions)])) t.write(string(arrowPositions[changeCount%len(arrowPositions)]))
t.write(times("-", length-completedLength)) t.write(times("-", length-completedLength))
t.write("]") t.write("]")
} }

View File

@ -1,13 +1,15 @@
package client package client
import ( import (
"github.com/zrepl/zrepl/cli"
"os" "os"
"github.com/problame/go-netssh"
"github.com/zrepl/zrepl/cli"
"github.com/zrepl/zrepl/config"
"context" "context"
"errors" "errors"
"github.com/problame/go-netssh"
"github.com/zrepl/zrepl/config"
"log" "log"
"path" "path"
) )

View File

@ -2,15 +2,17 @@ package client
import ( import (
"fmt" "fmt"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/spf13/pflag" "github.com/spf13/pflag"
"github.com/zrepl/zrepl/cli" "github.com/zrepl/zrepl/cli"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters" "github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/zfs" "github.com/zrepl/zrepl/zfs"
) )
var TestCmd = &cli.Subcommand { var TestCmd = &cli.Subcommand{
Use: "test", Use: "test",
SetupSubcommands: func() []*cli.Subcommand { SetupSubcommands: func() []*cli.Subcommand {
return []*cli.Subcommand{testFilter, testPlaceholder} return []*cli.Subcommand{testFilter, testPlaceholder}
@ -18,13 +20,13 @@ var TestCmd = &cli.Subcommand {
} }
var testFilterArgs struct { var testFilterArgs struct {
job string job string
all bool all bool
input string input string
} }
var testFilter = &cli.Subcommand{ var testFilter = &cli.Subcommand{
Use: "filesystems --job JOB [--all | --input INPUT]", Use: "filesystems --job JOB [--all | --input INPUT]",
Short: "test filesystems filter specified in push or source job", Short: "test filesystems filter specified in push or source job",
SetupFlags: func(f *pflag.FlagSet) { SetupFlags: func(f *pflag.FlagSet) {
f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job") f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job")
@ -51,8 +53,10 @@ func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
return err return err
} }
switch j := job.Ret.(type) { switch j := job.Ret.(type) {
case *config.SourceJob: confFilter = j.Filesystems case *config.SourceJob:
case *config.PushJob: confFilter = j.Filesystems confFilter = j.Filesystems
case *config.PushJob:
confFilter = j.Filesystems
default: default:
return fmt.Errorf("job type %T does not have filesystems filter", j) return fmt.Errorf("job type %T does not have filesystems filter", j)
} }

View File

@ -2,23 +2,25 @@ package client
import ( import (
"fmt" "fmt"
"os"
"github.com/spf13/pflag" "github.com/spf13/pflag"
"github.com/zrepl/zrepl/cli" "github.com/zrepl/zrepl/cli"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon" "github.com/zrepl/zrepl/daemon"
"github.com/zrepl/zrepl/version" "github.com/zrepl/zrepl/version"
"os"
) )
var versionArgs struct { var versionArgs struct {
Show string Show string
Config *config.Config Config *config.Config
ConfigErr error ConfigErr error
} }
var VersionCmd = &cli.Subcommand{ var VersionCmd = &cli.Subcommand{
Use: "version", Use: "version",
Short: "print version of zrepl binary and running daemon", Short: "print version of zrepl binary and running daemon",
NoRequireConfig: true, NoRequireConfig: true,
SetupFlags: func(f *pflag.FlagSet) { SetupFlags: func(f *pflag.FlagSet) {
f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)") f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")

View File

@ -2,8 +2,6 @@ package config
import ( import (
"fmt" "fmt"
"github.com/pkg/errors"
"github.com/zrepl/yaml-config"
"io/ioutil" "io/ioutil"
"log/syslog" "log/syslog"
"os" "os"
@ -11,6 +9,9 @@ import (
"regexp" "regexp"
"strconv" "strconv"
"time" "time"
"github.com/pkg/errors"
"github.com/zrepl/yaml-config"
) )
type Config struct { type Config struct {
@ -34,11 +35,16 @@ type JobEnum struct {
func (j JobEnum) Name() string { func (j JobEnum) Name() string {
var name string var name string
switch v := j.Ret.(type) { switch v := j.Ret.(type) {
case *SnapJob: name = v.Name case *SnapJob:
case *PushJob: name = v.Name name = v.Name
case *SinkJob: name = v.Name case *PushJob:
case *PullJob: name = v.Name name = v.Name
case *SourceJob: name = v.Name case *SinkJob:
name = v.Name
case *PullJob:
name = v.Name
case *SourceJob:
name = v.Name
default: default:
panic(fmt.Sprintf("unknown job type %T", v)) panic(fmt.Sprintf("unknown job type %T", v))
} }
@ -46,38 +52,38 @@ func (j JobEnum) Name() string {
} }
type ActiveJob struct { type ActiveJob struct {
Type string `yaml:"type"` Type string `yaml:"type"`
Name string `yaml:"name"` Name string `yaml:"name"`
Connect ConnectEnum `yaml:"connect"` Connect ConnectEnum `yaml:"connect"`
Pruning PruningSenderReceiver `yaml:"pruning"` Pruning PruningSenderReceiver `yaml:"pruning"`
Debug JobDebugSettings `yaml:"debug,optional"` Debug JobDebugSettings `yaml:"debug,optional"`
} }
type PassiveJob struct { type PassiveJob struct {
Type string `yaml:"type"` Type string `yaml:"type"`
Name string `yaml:"name"` Name string `yaml:"name"`
Serve ServeEnum `yaml:"serve"` Serve ServeEnum `yaml:"serve"`
Debug JobDebugSettings `yaml:"debug,optional"` Debug JobDebugSettings `yaml:"debug,optional"`
} }
type SnapJob struct { type SnapJob struct {
Type string `yaml:"type"` Type string `yaml:"type"`
Name string `yaml:"name"` Name string `yaml:"name"`
Pruning PruningLocal `yaml:"pruning"` Pruning PruningLocal `yaml:"pruning"`
Debug JobDebugSettings `yaml:"debug,optional"` Debug JobDebugSettings `yaml:"debug,optional"`
Snapshotting SnapshottingEnum `yaml:"snapshotting"` Snapshotting SnapshottingEnum `yaml:"snapshotting"`
Filesystems FilesystemsFilter `yaml:"filesystems"` Filesystems FilesystemsFilter `yaml:"filesystems"`
} }
type PushJob struct { type PushJob struct {
ActiveJob `yaml:",inline"` ActiveJob `yaml:",inline"`
Snapshotting SnapshottingEnum `yaml:"snapshotting"` Snapshotting SnapshottingEnum `yaml:"snapshotting"`
Filesystems FilesystemsFilter `yaml:"filesystems"` Filesystems FilesystemsFilter `yaml:"filesystems"`
} }
type PullJob struct { type PullJob struct {
ActiveJob `yaml:",inline"` ActiveJob `yaml:",inline"`
RootFS string `yaml:"root_fs"` RootFS string `yaml:"root_fs"`
Interval PositiveDurationOrManual `yaml:"interval"` Interval PositiveDurationOrManual `yaml:"interval"`
} }
@ -118,9 +124,9 @@ type SinkJob struct {
} }
type SourceJob struct { type SourceJob struct {
PassiveJob `yaml:",inline"` PassiveJob `yaml:",inline"`
Snapshotting SnapshottingEnum `yaml:"snapshotting"` Snapshotting SnapshottingEnum `yaml:"snapshotting"`
Filesystems FilesystemsFilter `yaml:"filesystems"` Filesystems FilesystemsFilter `yaml:"filesystems"`
} }
type FilesystemsFilter map[string]bool type FilesystemsFilter map[string]bool
@ -130,8 +136,8 @@ type SnapshottingEnum struct {
} }
type SnapshottingPeriodic struct { type SnapshottingPeriodic struct {
Type string `yaml:"type"` Type string `yaml:"type"`
Prefix string `yaml:"prefix"` Prefix string `yaml:"prefix"`
Interval time.Duration `yaml:"interval,positive"` Interval time.Duration `yaml:"interval,positive"`
} }
@ -191,7 +197,7 @@ type ConnectEnum struct {
} }
type ConnectCommon struct { type ConnectCommon struct {
Type string `yaml:"type"` Type string `yaml:"type"`
} }
type TCPConnect struct { type TCPConnect struct {
@ -223,8 +229,8 @@ type SSHStdinserverConnect struct {
} }
type LocalConnect struct { type LocalConnect struct {
ConnectCommon `yaml:",inline"` ConnectCommon `yaml:",inline"`
ListenerName string `yaml:"listener_name"` ListenerName string `yaml:"listener_name"`
ClientIdentity string `yaml:"client_identity"` ClientIdentity string `yaml:"client_identity"`
} }
@ -233,7 +239,7 @@ type ServeEnum struct {
} }
type ServeCommon struct { type ServeCommon struct {
Type string `yaml:"type"` Type string `yaml:"type"`
} }
type TCPServe struct { type TCPServe struct {
@ -253,12 +259,12 @@ type TLSServe struct {
} }
type StdinserverServer struct { type StdinserverServer struct {
ServeCommon `yaml:",inline"` ServeCommon `yaml:",inline"`
ClientIdentities []string `yaml:"client_identities"` ClientIdentities []string `yaml:"client_identities"`
} }
type LocalServe struct { type LocalServe struct {
ServeCommon `yaml:",inline"` ServeCommon `yaml:",inline"`
ListenerName string `yaml:"listener_name"` ListenerName string `yaml:"listener_name"`
} }
@ -267,8 +273,8 @@ type PruningEnum struct {
} }
type PruneKeepNotReplicated struct { type PruneKeepNotReplicated struct {
Type string `yaml:"type"` Type string `yaml:"type"`
KeepSnapshotAtCursor bool `yaml:"keep_snapshot_at_cursor,optional,default=true"` KeepSnapshotAtCursor bool `yaml:"keep_snapshot_at_cursor,optional,default=true"`
} }
type PruneKeepLastN struct { type PruneKeepLastN struct {
@ -277,8 +283,8 @@ type PruneKeepLastN struct {
} }
type PruneKeepRegex struct { // FIXME rename to KeepRegex type PruneKeepRegex struct { // FIXME rename to KeepRegex
Type string `yaml:"type"` Type string `yaml:"type"`
Regex string `yaml:"regex"` Regex string `yaml:"regex"`
Negate bool `yaml:"negate,optional,default=false"` Negate bool `yaml:"negate,optional,default=false"`
} }
@ -301,7 +307,7 @@ type StdoutLoggingOutlet struct {
type SyslogLoggingOutlet struct { type SyslogLoggingOutlet struct {
LoggingOutletCommon `yaml:",inline"` LoggingOutletCommon `yaml:",inline"`
Facility *SyslogFacility `yaml:"facility,optional,fromdefaults"` Facility *SyslogFacility `yaml:"facility,optional,fromdefaults"`
RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"` RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"`
} }
type TCPLoggingOutlet struct { type TCPLoggingOutlet struct {
@ -392,7 +398,7 @@ func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error)
"tcp": &TCPConnect{}, "tcp": &TCPConnect{},
"tls": &TLSConnect{}, "tls": &TLSConnect{},
"ssh+stdinserver": &SSHStdinserverConnect{}, "ssh+stdinserver": &SSHStdinserverConnect{},
"local": &LocalConnect{}, "local": &LocalConnect{},
}) })
return return
} }
@ -402,7 +408,7 @@ func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
"tcp": &TCPServe{}, "tcp": &TCPServe{},
"tls": &TLSServe{}, "tls": &TLSServe{},
"stdinserver": &StdinserverServer{}, "stdinserver": &StdinserverServer{},
"local" : &LocalServe{}, "local": &LocalServe{},
}) })
return return
} }
@ -420,7 +426,7 @@ func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error)
func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{ t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"periodic": &SnapshottingPeriodic{}, "periodic": &SnapshottingPeriodic{},
"manual": &SnapshottingManual{}, "manual": &SnapshottingManual{},
}) })
return return
} }
@ -448,31 +454,51 @@ func (t *SyslogFacility) UnmarshalYAML(u func(interface{}, bool) error) (err err
} }
var level syslog.Priority var level syslog.Priority
switch s { switch s {
case "kern": level = syslog.LOG_KERN case "kern":
case "user": level = syslog.LOG_USER level = syslog.LOG_KERN
case "mail": level = syslog.LOG_MAIL case "user":
case "daemon": level = syslog.LOG_DAEMON level = syslog.LOG_USER
case "auth": level = syslog.LOG_AUTH case "mail":
case "syslog": level = syslog.LOG_SYSLOG level = syslog.LOG_MAIL
case "lpr": level = syslog.LOG_LPR case "daemon":
case "news": level = syslog.LOG_NEWS level = syslog.LOG_DAEMON
case "uucp": level = syslog.LOG_UUCP case "auth":
case "cron": level = syslog.LOG_CRON level = syslog.LOG_AUTH
case "authpriv": level = syslog.LOG_AUTHPRIV case "syslog":
case "ftp": level = syslog.LOG_FTP level = syslog.LOG_SYSLOG
case "local0": level = syslog.LOG_LOCAL0 case "lpr":
case "local1": level = syslog.LOG_LOCAL1 level = syslog.LOG_LPR
case "local2": level = syslog.LOG_LOCAL2 case "news":
case "local3": level = syslog.LOG_LOCAL3 level = syslog.LOG_NEWS
case "local4": level = syslog.LOG_LOCAL4 case "uucp":
case "local5": level = syslog.LOG_LOCAL5 level = syslog.LOG_UUCP
case "local6": level = syslog.LOG_LOCAL6 case "cron":
case "local7": level = syslog.LOG_LOCAL7 level = syslog.LOG_CRON
case "authpriv":
level = syslog.LOG_AUTHPRIV
case "ftp":
level = syslog.LOG_FTP
case "local0":
level = syslog.LOG_LOCAL0
case "local1":
level = syslog.LOG_LOCAL1
case "local2":
level = syslog.LOG_LOCAL2
case "local3":
level = syslog.LOG_LOCAL3
case "local4":
level = syslog.LOG_LOCAL4
case "local5":
level = syslog.LOG_LOCAL5
case "local6":
level = syslog.LOG_LOCAL6
case "local7":
level = syslog.LOG_LOCAL7
default: default:
return fmt.Errorf("invalid syslog level: %q", s) return fmt.Errorf("invalid syslog level: %q", s)
} }
*t = SyslogFacility(level) *t = SyslogFacility(level)
return nil return nil
} }
var ConfigFileDefaultLocations = []string{ var ConfigFileDefaultLocations = []string{

View File

@ -2,11 +2,12 @@ package config
import ( import (
"fmt" "fmt"
"log/syslog"
"testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/zrepl/yaml-config" "github.com/zrepl/yaml-config"
"log/syslog"
"testing"
) )
func testValidGlobalSection(t *testing.T, s string) *Config { func testValidGlobalSection(t *testing.T, s string) *Config {
@ -24,7 +25,7 @@ jobs:
` `
_, err := ParseConfigBytes([]byte(jobdef)) _, err := ParseConfigBytes([]byte(jobdef))
require.NoError(t, err) require.NoError(t, err)
return testValidConfig(t, s + jobdef) return testValidConfig(t, s+jobdef)
} }
func TestOutletTypes(t *testing.T) { func TestOutletTypes(t *testing.T) {
@ -71,7 +72,7 @@ global:
- type: prometheus - type: prometheus
listen: ':9091' listen: ':9091'
`) `)
assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen) assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
} }
func TestSyslogLoggingOutletFacility(t *testing.T) { func TestSyslogLoggingOutletFacility(t *testing.T) {

View File

@ -2,6 +2,7 @@ package config
import ( import (
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
@ -36,4 +37,4 @@ jobs:
- type: last_n - type: last_n
count: 1 count: 1
`) `)
} }

View File

@ -2,9 +2,10 @@ package config
import ( import (
"fmt" "fmt"
"github.com/stretchr/testify/assert"
"testing" "testing"
"time" "time"
"github.com/stretchr/testify/assert"
) )
func TestSnapshotting(t *testing.T) { func TestSnapshotting(t *testing.T) {
@ -37,7 +38,7 @@ jobs:
interval: 10m interval: 10m
` `
fillSnapshotting := func(s string) string {return fmt.Sprintf(tmpl, s)} fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
var c *Config var c *Config
t.Run("manual", func(t *testing.T) { t.Run("manual", func(t *testing.T) {
@ -51,7 +52,7 @@ jobs:
snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic) snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
assert.Equal(t, "periodic", snp.Type) assert.Equal(t, "periodic", snp.Type)
assert.Equal(t, 10*time.Minute, snp.Interval) assert.Equal(t, 10*time.Minute, snp.Interval)
assert.Equal(t, "zrepl_" , snp.Prefix) assert.Equal(t, "zrepl_", snp.Prefix)
}) })
} }

View File

@ -11,9 +11,9 @@ import (
type RetentionIntervalList []RetentionInterval type RetentionIntervalList []RetentionInterval
type PruneGrid struct { type PruneGrid struct {
Type string `yaml:"type"` Type string `yaml:"type"`
Grid RetentionIntervalList `yaml:"grid"` Grid RetentionIntervalList `yaml:"grid"`
Regex string `yaml:"regex"` Regex string `yaml:"regex"`
} }
type RetentionInterval struct { type RetentionInterval struct {

View File

@ -12,6 +12,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/daemon/job" "github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/daemon/nethelpers" "github.com/zrepl/zrepl/daemon/nethelpers"
"github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/logger"
@ -43,24 +44,24 @@ func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInt
func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false } func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
var promControl struct { var promControl struct {
requestBegin *prometheus.CounterVec requestBegin *prometheus.CounterVec
requestFinished *prometheus.HistogramVec requestFinished *prometheus.HistogramVec
} }
func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) { func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
promControl.requestBegin = prometheus.NewCounterVec(prometheus.CounterOpts{ promControl.requestBegin = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "zrepl", Namespace: "zrepl",
Subsystem: "control", Subsystem: "control",
Name: "request_begin", Name: "request_begin",
Help: "number of request we started to handle", Help: "number of request we started to handle",
}, []string{"endpoint"}) }, []string{"endpoint"})
promControl.requestFinished = prometheus.NewHistogramVec(prometheus.HistogramOpts{ promControl.requestFinished = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "zrepl", Namespace: "zrepl",
Subsystem: "control", Subsystem: "control",
Name: "request_finished", Name: "request_finished",
Help: "time it took a request to finih", Help: "time it took a request to finih",
Buckets: []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3,10e-3, 100e-3, 200e-3,400e-3,800e-3, 1, 10, 20}, Buckets: []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
}, []string{"endpoint"}) }, []string{"endpoint"})
registerer.MustRegister(promControl.requestBegin) registerer.MustRegister(promControl.requestBegin)
registerer.MustRegister(promControl.requestFinished) registerer.MustRegister(promControl.requestFinished)
@ -114,7 +115,7 @@ func (j *controlJob) Run(ctx context.Context) {
requestLogger{log: log, handler: jsonRequestResponder{func(decoder jsonDecoder) (interface{}, error) { requestLogger{log: log, handler: jsonRequestResponder{func(decoder jsonDecoder) (interface{}, error) {
type reqT struct { type reqT struct {
Name string Name string
Op string Op string
} }
var req reqT var req reqT
if decoder(&req) != nil { if decoder(&req) != nil {
@ -136,8 +137,8 @@ func (j *controlJob) Run(ctx context.Context) {
server := http.Server{ server := http.Server{
Handler: mux, Handler: mux,
// control socket is local, 1s timeout should be more than sufficient, even on a loaded system // control socket is local, 1s timeout should be more than sufficient, even on a loaded system
WriteTimeout: 1*time.Second, WriteTimeout: 1 * time.Second,
ReadTimeout: 1*time.Second, ReadTimeout: 1 * time.Second,
} }
outer: outer:

View File

@ -3,8 +3,16 @@ package daemon
import ( import (
"context" "context"
"fmt" "fmt"
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/job" "github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/daemon/job/reset" "github.com/zrepl/zrepl/daemon/job/reset"
@ -12,12 +20,6 @@ import (
"github.com/zrepl/zrepl/daemon/logging" "github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/version" "github.com/zrepl/zrepl/version"
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
) )
func Run(conf *config.Config) error { func Run(conf *config.Config) error {
@ -74,12 +76,11 @@ func Run(conf *config.Config) error {
return errors.Errorf("unknown monitoring job #%d (type %T)", i, v) return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
} }
if err != nil { if err != nil {
return errors.Wrapf(err,"cannot build monitorin gjob #%d", i) return errors.Wrapf(err, "cannot build monitorin gjob #%d", i)
} }
jobs.start(ctx, job, true) jobs.start(ctx, job, true)
} }
log.Info("starting daemon") log.Info("starting daemon")
// start regular jobs // start regular jobs
@ -103,7 +104,7 @@ type jobs struct {
// m protects all fields below it // m protects all fields below it
m sync.RWMutex m sync.RWMutex
wakeups map[string]wakeup.Func // by Job.Name wakeups map[string]wakeup.Func // by Job.Name
resets map[string]reset.Func // by Job.Name resets map[string]reset.Func // by Job.Name
jobs map[string]job.Job jobs map[string]job.Job
} }

View File

@ -2,10 +2,12 @@ package filters
import ( import (
"fmt" "fmt"
"strings"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/zrepl/zrepl/endpoint" "github.com/zrepl/zrepl/endpoint"
"github.com/zrepl/zrepl/zfs" "github.com/zrepl/zrepl/zfs"
"strings"
) )
type DatasetMapFilter struct { type DatasetMapFilter struct {

View File

@ -1,8 +1,9 @@
package filters package filters
import ( import (
"github.com/zrepl/zrepl/zfs"
"strings" "strings"
"github.com/zrepl/zrepl/zfs"
) )
type AnyFSVFilter struct{} type AnyFSVFilter struct{}
@ -17,7 +18,6 @@ func (AnyFSVFilter) Filter(t zfs.VersionType, name string) (accept bool, err err
return true, nil return true, nil
} }
type PrefixFilter struct { type PrefixFilter struct {
prefix string prefix string
fstype zfs.VersionType fstype zfs.VersionType

View File

@ -6,6 +6,7 @@ import (
"strings" "strings"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
) )

View File

@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/zfs" "github.com/zrepl/zrepl/zfs"
) )
@ -29,7 +30,6 @@ func WithLogger(ctx context.Context, l Logger) context.Context {
return context.WithValue(ctx, contextKeyLog, l) return context.WithValue(ctx, contextKeyLog, l)
} }
type Job interface { type Job interface {
Name() string Name() string
Run(ctx context.Context) Run(ctx context.Context)
@ -44,15 +44,15 @@ type Type string
const ( const (
TypeInternal Type = "internal" TypeInternal Type = "internal"
TypeSnap Type = "snap" TypeSnap Type = "snap"
TypePush Type = "push" TypePush Type = "push"
TypeSink Type = "sink" TypeSink Type = "sink"
TypePull Type = "pull" TypePull Type = "pull"
TypeSource Type = "source" TypeSource Type = "source"
) )
type Status struct { type Status struct {
Type Type Type Type
JobSpecific interface{} JobSpecific interface{}
} }
@ -65,8 +65,8 @@ func (s *Status) MarshalJSON() ([]byte, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
m := map[string]json.RawMessage { m := map[string]json.RawMessage{
"type": typeJson, "type": typeJson,
string(s.Type): jobJSON, string(s.Type): jobJSON,
} }
return json.Marshal(m) return json.Marshal(m)
@ -94,12 +94,14 @@ func (s *Status) UnmarshalJSON(in []byte) (err error) {
var st SnapJobStatus var st SnapJobStatus
err = json.Unmarshal(jobJSON, &st) err = json.Unmarshal(jobJSON, &st)
s.JobSpecific = &st s.JobSpecific = &st
case TypePull: fallthrough case TypePull:
fallthrough
case TypePush: case TypePush:
var st ActiveSideStatus var st ActiveSideStatus
err = json.Unmarshal(jobJSON, &st) err = json.Unmarshal(jobJSON, &st)
s.JobSpecific = &st s.JobSpecific = &st
case TypeSource: fallthrough case TypeSource:
fallthrough
case TypeSink: case TypeSink:
var st PassiveStatus var st PassiveStatus
err = json.Unmarshal(jobJSON, &st) err = json.Unmarshal(jobJSON, &st)

View File

@ -7,6 +7,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters" "github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/daemon/job/wakeup" "github.com/zrepl/zrepl/daemon/job/wakeup"

View File

@ -4,11 +4,13 @@ import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"fmt" "fmt"
"time"
"github.com/fatih/color" "github.com/fatih/color"
"github.com/go-logfmt/logfmt" "github.com/go-logfmt/logfmt"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/logger"
"time"
) )
const ( const (

View File

@ -4,12 +4,14 @@ import (
"bytes" "bytes"
"context" "context"
"crypto/tls" "crypto/tls"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/logger"
"io" "io"
"log/syslog" "log/syslog"
"net" "net"
"time" "time"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/logger"
) )
type EntryFormatter interface { type EntryFormatter interface {

View File

@ -7,7 +7,7 @@ import (
type Logger = logger.Logger type Logger = logger.Logger
var DaemonCmd = &cli.Subcommand { var DaemonCmd = &cli.Subcommand{
Use: "daemon", Use: "daemon",
Short: "run the zrepl daemon", Short: "run the zrepl daemon",
Run: func(subcommand *cli.Subcommand, args []string) error { Run: func(subcommand *cli.Subcommand, args []string) error {

View File

@ -1,10 +1,11 @@
package nethelpers package nethelpers
import ( import (
"github.com/pkg/errors"
"net" "net"
"os" "os"
"path/filepath" "path/filepath"
"github.com/pkg/errors"
) )
func PreparePrivateSockpath(sockpath string) error { func PreparePrivateSockpath(sockpath string) error {

View File

@ -2,15 +2,17 @@ package daemon
import ( import (
"context" "context"
"net"
"net/http"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/job" "github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/rpc/dataconn/frameconn" "github.com/zrepl/zrepl/rpc/dataconn/frameconn"
"github.com/zrepl/zrepl/zfs" "github.com/zrepl/zrepl/zfs"
"net"
"net/http"
) )
type prometheusJob struct { type prometheusJob struct {
@ -25,7 +27,7 @@ func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob
} }
var prom struct { var prom struct {
taskLogEntries *prometheus.CounterVec taskLogEntries *prometheus.CounterVec
} }
func init() { func init() {
@ -93,4 +95,3 @@ func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error {
prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc() prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc()
return nil return nil
} }

View File

@ -3,17 +3,19 @@ package pruner
import ( import (
"context" "context"
"fmt" "fmt"
"sort"
"strings"
"sync"
"time"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/pruning" "github.com/zrepl/zrepl/pruning"
"github.com/zrepl/zrepl/replication/logic/pdu" "github.com/zrepl/zrepl/replication/logic/pdu"
"github.com/zrepl/zrepl/util/envconst" "github.com/zrepl/zrepl/util/envconst"
"sort"
"strings"
"sync"
"time"
) )
// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint // Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
@ -53,7 +55,7 @@ type args struct {
rules []pruning.KeepRule rules []pruning.KeepRule
retryWait time.Duration retryWait time.Duration
considerSnapAtCursorReplicated bool considerSnapAtCursorReplicated bool
promPruneSecs prometheus.Observer promPruneSecs prometheus.Observer
} }
type Pruner struct { type Pruner struct {
@ -64,7 +66,7 @@ type Pruner struct {
state State state State
// State PlanErr // State PlanErr
err error err error
// State Exec // State Exec
execQueue *execQueue execQueue *execQueue
@ -75,7 +77,7 @@ type PrunerFactory struct {
receiverRules []pruning.KeepRule receiverRules []pruning.KeepRule
retryWait time.Duration retryWait time.Duration
considerSnapAtCursorReplicated bool considerSnapAtCursorReplicated bool
promPruneSecs *prometheus.HistogramVec promPruneSecs *prometheus.HistogramVec
} }
type LocalPrunerFactory struct { type LocalPrunerFactory struct {
@ -137,11 +139,11 @@ func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus
considerSnapAtCursorReplicated = considerSnapAtCursorReplicated || !knr.KeepSnapshotAtCursor considerSnapAtCursorReplicated = considerSnapAtCursorReplicated || !knr.KeepSnapshotAtCursor
} }
f := &PrunerFactory{ f := &PrunerFactory{
senderRules: keepRulesSender, senderRules: keepRulesSender,
receiverRules: keepRulesReceiver, receiverRules: keepRulesReceiver,
retryWait: envconst.Duration("ZREPL_PRUNER_RETRY_INTERVAL", 10 * time.Second), retryWait: envconst.Duration("ZREPL_PRUNER_RETRY_INTERVAL", 10*time.Second),
considerSnapAtCursorReplicated: considerSnapAtCursorReplicated, considerSnapAtCursorReplicated: considerSnapAtCursorReplicated,
promPruneSecs: promPruneSecs, promPruneSecs: promPruneSecs,
} }
return f, nil return f, nil
} }
@ -213,17 +215,17 @@ func (p *Pruner) Prune() {
func (p *Pruner) prune(args args) { func (p *Pruner) prune(args args) {
u := func(f func(*Pruner)) { u := func(f func(*Pruner)) {
p.mtx.Lock() p.mtx.Lock()
defer p.mtx.Unlock() defer p.mtx.Unlock()
f(p) f(p)
} }
// TODO support automatic retries // TODO support automatic retries
// It is advisable to merge this code with package replication/driver before // It is advisable to merge this code with package replication/driver before
// That will likely require re-modelling struct fs like replication/driver.attempt, // That will likely require re-modelling struct fs like replication/driver.attempt,
// including figuring out how to resume a plan after being interrupted by network errors // including figuring out how to resume a plan after being interrupted by network errors
// The non-retrying code in this package should move straight to replication/logic. // The non-retrying code in this package should move straight to replication/logic.
doOneAttempt(&args, u) doOneAttempt(&args, u)
} }
type Report struct { type Report struct {
State string State string
@ -239,9 +241,9 @@ type FSReport struct {
} }
type SnapshotReport struct { type SnapshotReport struct {
Name string Name string
Replicated bool Replicated bool
Date time.Time Date time.Time
} }
func (p *Pruner) Report() *Report { func (p *Pruner) Report() *Report {
@ -250,9 +252,9 @@ func (p *Pruner) Report() *Report {
r := Report{State: p.state.String()} r := Report{State: p.state.String()}
if p.err != nil { if p.err != nil {
r.Error = p.err.Error() r.Error = p.err.Error()
} }
if p.execQueue != nil { if p.execQueue != nil {
r.Pending, r.Completed = p.execQueue.Report() r.Pending, r.Completed = p.execQueue.Report()
@ -268,7 +270,7 @@ func (p *Pruner) State() State {
} }
type fs struct { type fs struct {
path string path string
// permanent error during planning // permanent error during planning
planErr error planErr error
@ -316,7 +318,7 @@ func (f *fs) Report() FSReport {
if f.planErr != nil { if f.planErr != nil {
r.LastError = f.planErr.Error() r.LastError = f.planErr.Error()
} else if f.execErrLast != nil { } else if f.execErrLast != nil {
r.LastError = f.execErrLast.Error() r.LastError = f.execErrLast.Error()
} }
@ -326,7 +328,7 @@ func (f *fs) Report() FSReport {
} }
r.DestroyList = make([]SnapshotReport, len(f.destroyList)) r.DestroyList = make([]SnapshotReport, len(f.destroyList))
for i, snap := range f.destroyList{ for i, snap := range f.destroyList {
r.DestroyList[i] = snap.(snapshot).Report() r.DestroyList[i] = snap.(snapshot).Report()
} }
@ -490,9 +492,9 @@ tfss_loop:
}) })
for { for {
var pfs *fs var pfs *fs
u(func(pruner *Pruner) { u(func(pruner *Pruner) {
pfs = pruner.execQueue.Pop() pfs = pruner.execQueue.Pop()
}) })
if pfs == nil { if pfs == nil {
break break
@ -516,16 +518,15 @@ tfss_loop:
hadErr := false hadErr := false
for _, fsr := range rep.Completed { for _, fsr := range rep.Completed {
hadErr = hadErr || fsr.SkipReason.NotSkipped() && fsr.LastError != "" hadErr = hadErr || fsr.SkipReason.NotSkipped() && fsr.LastError != ""
} }
if hadErr { if hadErr {
p.state = ExecErr p.state = ExecErr
} else { } else {
p.state = Done p.state = Done
} }
}) })
} }
// attempts to exec pfs, puts it back into the queue with the result // attempts to exec pfs, puts it back into the queue with the result
func doOneAttemptExec(a *args, u updater, pfs *fs) { func doOneAttemptExec(a *args, u updater, pfs *fs) {
@ -558,20 +559,20 @@ func doOneAttemptExec(a *args, u updater, pfs *fs) {
err = nil err = nil
destroyFails := make([]*pdu.DestroySnapshotRes, 0) destroyFails := make([]*pdu.DestroySnapshotRes, 0)
for _, reqDestroy := range destroyList { for _, reqDestroy := range destroyList {
res, ok := destroyResults[reqDestroy.Name] res, ok := destroyResults[reqDestroy.Name]
if !ok { if !ok {
err = fmt.Errorf("missing destroy-result for %s", reqDestroy.RelName()) err = fmt.Errorf("missing destroy-result for %s", reqDestroy.RelName())
break break
} else if res.Error != "" { } else if res.Error != "" {
destroyFails = append(destroyFails, res) destroyFails = append(destroyFails, res)
} }
} }
if err == nil && len(destroyFails) > 0 { if err == nil && len(destroyFails) > 0 {
names := make([]string, len(destroyFails)) names := make([]string, len(destroyFails))
pairs := make([]string, len(destroyFails)) pairs := make([]string, len(destroyFails))
allSame := true allSame := true
lastMsg := destroyFails[0].Error lastMsg := destroyFails[0].Error
for i := 0; i < len(destroyFails); i++{ for i := 0; i < len(destroyFails); i++ {
allSame = allSame && destroyFails[i].Error == lastMsg allSame = allSame && destroyFails[i].Error == lastMsg
relname := destroyFails[i].Snapshot.RelName() relname := destroyFails[i].Snapshot.RelName()
names[i] = relname names[i] = relname

View File

@ -7,13 +7,13 @@ import (
) )
type execQueue struct { type execQueue struct {
mtx sync.Mutex mtx sync.Mutex
pending, completed []*fs pending, completed []*fs
} }
func newExecQueue(cap int) *execQueue { func newExecQueue(cap int) *execQueue {
q := execQueue{ q := execQueue{
pending: make([]*fs, 0, cap), pending: make([]*fs, 0, cap),
completed: make([]*fs, 0, cap), completed: make([]*fs, 0, cap),
} }
return &q return &q
@ -55,7 +55,7 @@ func (q *execQueue) Pop() *fs {
return fs return fs
} }
func(q *execQueue) Put(fs *fs, err error, done bool) { func (q *execQueue) Put(fs *fs, err error, done bool) {
fs.mtx.Lock() fs.mtx.Lock()
fs.execErrLast = err fs.execErrLast = err
if done || err != nil { if done || err != nil {
@ -79,5 +79,4 @@ func(q *execQueue) Put(fs *fs, err error, done bool) {
}) })
q.mtx.Unlock() q.mtx.Unlock()
}
}

View File

@ -1,18 +1,19 @@
package snapper package snapper
import ( import (
"github.com/zrepl/zrepl/config"
"github.com/pkg/errors"
"time"
"context" "context"
"github.com/zrepl/zrepl/daemon/filters"
"fmt" "fmt"
"github.com/zrepl/zrepl/zfs"
"sort" "sort"
"github.com/zrepl/zrepl/logger"
"sync" "sync"
) "time"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/zfs"
)
//go:generate stringer -type=SnapState //go:generate stringer -type=SnapState
type SnapState uint type SnapState uint
@ -28,7 +29,7 @@ type snapProgress struct {
state SnapState state SnapState
// SnapStarted, SnapDone, SnapError // SnapStarted, SnapDone, SnapError
name string name string
startAt time.Time startAt time.Time
// SnapDone // SnapDone
@ -44,13 +45,13 @@ type args struct {
prefix string prefix string
interval time.Duration interval time.Duration
fsf *filters.DatasetMapFilter fsf *filters.DatasetMapFilter
snapshotsTaken chan<-struct{} snapshotsTaken chan<- struct{}
} }
type Snapper struct { type Snapper struct {
args args args args
mtx sync.Mutex mtx sync.Mutex
state State state State
// set in state Plan, used in Waiting // set in state Plan, used in Waiting
@ -70,7 +71,7 @@ type Snapper struct {
type State uint type State uint
const ( const (
SyncUp State = 1<<iota SyncUp State = 1 << iota
SyncUpErrWait SyncUpErrWait
Planning Planning
Snapshotting Snapshotting
@ -81,13 +82,13 @@ const (
func (s State) sf() state { func (s State) sf() state {
m := map[State]state{ m := map[State]state{
SyncUp: syncUp, SyncUp: syncUp,
SyncUpErrWait: wait, SyncUpErrWait: wait,
Planning: plan, Planning: plan,
Snapshotting: snapshot, Snapshotting: snapshot,
Waiting: wait, Waiting: wait,
ErrorWait: wait, ErrorWait: wait,
Stopped: nil, Stopped: nil,
} }
return m[s] return m[s]
} }
@ -123,9 +124,9 @@ func PeriodicFromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in *con
} }
args := args{ args := args{
prefix: in.Prefix, prefix: in.Prefix,
interval: in.Interval, interval: in.Interval,
fsf: fsf, fsf: fsf,
// ctx and log is set in Run() // ctx and log is set in Run()
} }
@ -199,7 +200,7 @@ func syncUp(a args, u updater) state {
if err != nil { if err != nil {
return onErr(err, u) return onErr(err, u)
} }
u(func(s *Snapper){ u(func(s *Snapper) {
s.sleepUntil = syncPoint s.sleepUntil = syncPoint
}) })
t := time.NewTimer(syncPoint.Sub(time.Now())) t := time.NewTimer(syncPoint.Sub(time.Now()))
@ -386,4 +387,3 @@ func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval t
return snaptimes[0].time, nil return snaptimes[0].time, nil
} }

View File

@ -3,6 +3,7 @@ package snapper
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters" "github.com/zrepl/zrepl/daemon/filters"
) )
@ -17,7 +18,7 @@ type PeriodicOrManual struct {
s *Snapper s *Snapper
} }
func (s *PeriodicOrManual) Run(ctx context.Context, wakeUpCommon chan <- struct{}) { func (s *PeriodicOrManual) Run(ctx context.Context, wakeUpCommon chan<- struct{}) {
if s.s != nil { if s.s != nil {
s.s.Run(ctx, wakeUpCommon) s.s.Run(ctx, wakeUpCommon)
} }

View File

@ -2,6 +2,7 @@ package endpoint
import ( import (
"context" "context"
"github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/logger"
) )

View File

@ -7,6 +7,7 @@ import (
"path" "path"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/zrepl/zrepl/replication/logic/pdu" "github.com/zrepl/zrepl/replication/logic/pdu"
"github.com/zrepl/zrepl/util/chainlock" "github.com/zrepl/zrepl/util/chainlock"
"github.com/zrepl/zrepl/zfs" "github.com/zrepl/zrepl/zfs"

View File

@ -4,10 +4,11 @@ import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"fmt" "fmt"
"github.com/fatih/color"
"github.com/pkg/errors"
"sync" "sync"
"time" "time"
"github.com/fatih/color"
"github.com/pkg/errors"
) )
type Level int type Level int

View File

@ -2,10 +2,12 @@ package logger_test
import ( import (
"fmt" "fmt"
"github.com/kr/pretty"
"github.com/zrepl/zrepl/logger"
"testing" "testing"
"time" "time"
"github.com/kr/pretty"
"github.com/zrepl/zrepl/logger"
) )
type TestOutlet struct { type TestOutlet struct {

View File

@ -9,7 +9,7 @@ type stderrLogger struct {
Logger Logger
} }
type stderrLoggerOutlet struct {} type stderrLoggerOutlet struct{}
func (stderrLoggerOutlet) WriteEntry(entry Entry) error { func (stderrLoggerOutlet) WriteEntry(entry Entry) error {
fmt.Fprintf(os.Stderr, "%#v\n", entry) fmt.Fprintf(os.Stderr, "%#v\n", entry)

View File

@ -2,12 +2,14 @@ package pruning
import ( import (
"fmt" "fmt"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/pruning/retentiongrid"
"regexp" "regexp"
"sort" "sort"
"time" "time"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/pruning/retentiongrid"
) )
// KeepGrid fits snapshots that match a given regex into a retentiongrid.Grid, // KeepGrid fits snapshots that match a given regex into a retentiongrid.Grid,
@ -15,7 +17,7 @@ import (
// and deletes all snapshots that do not fit the grid specification. // and deletes all snapshots that do not fit the grid specification.
type KeepGrid struct { type KeepGrid struct {
retentionGrid *retentiongrid.Grid retentionGrid *retentiongrid.Grid
re *regexp.Regexp re *regexp.Regexp
} }
func NewKeepGrid(in *config.PruneGrid) (p *KeepGrid, err error) { func NewKeepGrid(in *config.PruneGrid) (p *KeepGrid, err error) {

View File

@ -1,8 +1,9 @@
package pruning package pruning
import ( import (
"github.com/stretchr/testify/assert"
"testing" "testing"
"github.com/stretchr/testify/assert"
) )
func TestShallowCopySnapList(t *testing.T) { func TestShallowCopySnapList(t *testing.T) {

View File

@ -1,8 +1,9 @@
package pruning package pruning
import ( import (
"github.com/pkg/errors"
"sort" "sort"
"github.com/pkg/errors"
) )
type KeepLastN struct { type KeepLastN struct {

View File

@ -1,9 +1,10 @@
package pruning package pruning
import ( import (
"github.com/stretchr/testify/assert"
"testing" "testing"
"time" "time"
"github.com/stretchr/testify/assert"
) )
func TestKeepLastN(t *testing.T) { func TestKeepLastN(t *testing.T) {

View File

@ -5,7 +5,7 @@ import (
) )
type KeepRegex struct { type KeepRegex struct {
expr *regexp.Regexp expr *regexp.Regexp
negate bool negate bool
} }

View File

@ -2,9 +2,11 @@ package pruning
import ( import (
"fmt" "fmt"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"time" "time"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
) )
type KeepRule interface { type KeepRule interface {

View File

@ -2,11 +2,12 @@ package retentiongrid
import ( import (
"fmt" "fmt"
"github.com/stretchr/testify/assert"
"strconv" "strconv"
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/stretchr/testify/assert"
) )
type retentionIntervalStub struct { type retentionIntervalStub struct {

View File

@ -10,11 +10,12 @@ import (
"sync" "sync"
"time" "time"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/zrepl/zrepl/replication/report" "github.com/zrepl/zrepl/replication/report"
"github.com/zrepl/zrepl/util/chainlock" "github.com/zrepl/zrepl/util/chainlock"
"github.com/zrepl/zrepl/util/envconst" "github.com/zrepl/zrepl/util/envconst"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
) )
type interval struct { type interval struct {

View File

@ -26,4 +26,4 @@ func debugPrefix(prefixFormat string, prefixFormatArgs ...interface{}) debugFunc
return func(format string, args ...interface{}) { return func(format string, args ...interface{}) {
debug("%s: %s", prefix, fmt.Sprintf(format, args)) debug("%s: %s", prefix, fmt.Sprintf(format, args))
} }
} }

View File

@ -10,6 +10,7 @@ import (
"time" "time"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/replication/report" "github.com/zrepl/zrepl/replication/report"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"

View File

@ -97,7 +97,7 @@ func TestIncrementalPath_SnapshotsOnly(t *testing.T) {
}) })
// sender with earlier but also current version as sender is not a conflict // sender with earlier but also current version as sender is not a conflict
doTest(l("@c,3"), l("@a,1", "@b,2", "@c,3") , func(path []*FilesystemVersion, conflict error) { doTest(l("@c,3"), l("@a,1", "@b,2", "@c,3"), func(path []*FilesystemVersion, conflict error) {
t.Logf("path: %#v", path) t.Logf("path: %#v", path)
t.Logf("conflict: %#v", conflict) t.Logf("conflict: %#v", conflict)
assert.Empty(t, path) assert.Empty(t, path)

View File

@ -2,8 +2,9 @@ package pdu
import ( import (
"fmt" "fmt"
"github.com/zrepl/zrepl/zfs"
"time" "time"
"github.com/zrepl/zrepl/zfs"
) )
func (v *FilesystemVersion) RelName() string { func (v *FilesystemVersion) RelName() string {

View File

@ -1,9 +1,10 @@
package pdu package pdu
import ( import (
"github.com/stretchr/testify/assert"
"testing" "testing"
"time" "time"
"github.com/stretchr/testify/assert"
) )
func TestFilesystemVersion_RelName(t *testing.T) { func TestFilesystemVersion_RelName(t *testing.T) {
@ -18,24 +19,24 @@ func TestFilesystemVersion_RelName(t *testing.T) {
tcs := []TestCase{ tcs := []TestCase{
{ {
In: FilesystemVersion{ In: FilesystemVersion{
Type: FilesystemVersion_Snapshot, Type: FilesystemVersion_Snapshot,
Name: "foobar", Name: "foobar",
Creation: creat, Creation: creat,
}, },
Out: "@foobar", Out: "@foobar",
}, },
{ {
In: FilesystemVersion{ In: FilesystemVersion{
Type: FilesystemVersion_Bookmark, Type: FilesystemVersion_Bookmark,
Name: "foobar", Name: "foobar",
Creation: creat, Creation: creat,
}, },
Out: "#foobar", Out: "#foobar",
}, },
{ {
In: FilesystemVersion{ In: FilesystemVersion{
Type: 2342, Type: 2342,
Name: "foobar", Name: "foobar",
Creation: creat, Creation: creat,
}, },
Panic: true, Panic: true,
@ -58,7 +59,7 @@ func TestFilesystemVersion_RelName(t *testing.T) {
func TestFilesystemVersion_ZFSFilesystemVersion(t *testing.T) { func TestFilesystemVersion_ZFSFilesystemVersion(t *testing.T) {
empty := &FilesystemVersion{} empty := &FilesystemVersion{}
_, err:= empty.ZFSFilesystemVersion() _, err := empty.ZFSFilesystemVersion()
assert.Error(t, err) assert.Error(t, err)
dateInvalid := &FilesystemVersion{Creation: "foobar"} dateInvalid := &FilesystemVersion{Creation: "foobar"}

View File

@ -26,7 +26,7 @@ type Endpoint interface {
ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
ListFilesystemVersions(ctx context.Context, req *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error) ListFilesystemVersions(ctx context.Context, req *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error)
DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error)
WaitForConnectivity(ctx context.Context) (error) WaitForConnectivity(ctx context.Context) error
} }
type Sender interface { type Sender interface {
@ -107,7 +107,7 @@ type Filesystem struct {
sender Sender sender Sender
receiver Receiver receiver Receiver
Path string // compat Path string // compat
receiverFS *pdu.Filesystem receiverFS *pdu.Filesystem
promBytesReplicated prometheus.Counter // compat promBytesReplicated prometheus.Counter // compat
} }

View File

@ -7,6 +7,7 @@ import (
"strings" "strings"
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/zrepl/zrepl/replication/logic/pdu" "github.com/zrepl/zrepl/replication/logic/pdu"
"github.com/zrepl/zrepl/rpc/dataconn/stream" "github.com/zrepl/zrepl/rpc/dataconn/stream"
"github.com/zrepl/zrepl/transport" "github.com/zrepl/zrepl/transport"
@ -214,13 +215,12 @@ func (c *Client) ReqRecv(ctx context.Context, req *pdu.ReceiveReq, streamCopier
return res.res, cause return res.res, cause
} }
func (c *Client) ReqPing(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, error) { func (c *Client) ReqPing(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, error) {
conn, err := c.getWire(ctx) conn, err := c.getWire(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer c.putWire(conn) defer c.putWire(conn)
if err := c.send(ctx, conn, EndpointPing, req, nil); err != nil { if err := c.send(ctx, conn, EndpointPing, req, nil); err != nil {
return nil, err return nil, err
@ -232,4 +232,4 @@ func (c *Client) ReqPing(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, e
} }
return &res, nil return &res, nil
} }

View File

@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/replication/logic/pdu" "github.com/zrepl/zrepl/replication/logic/pdu"
"github.com/zrepl/zrepl/rpc/dataconn/stream" "github.com/zrepl/zrepl/rpc/dataconn/stream"

View File

@ -12,6 +12,7 @@ import (
"time" "time"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/rpc/dataconn/base2bufpool" "github.com/zrepl/zrepl/rpc/dataconn/base2bufpool"
"github.com/zrepl/zrepl/rpc/dataconn/timeoutconn" "github.com/zrepl/zrepl/rpc/dataconn/timeoutconn"
) )

View File

@ -3,8 +3,8 @@ package frameconn
import "sync" import "sync"
type shutdownFSM struct { type shutdownFSM struct {
mtx sync.Mutex mtx sync.Mutex
state shutdownFSMState state shutdownFSMState
} }
type shutdownFSMState uint32 type shutdownFSMState uint32
@ -16,7 +16,7 @@ const (
func newShutdownFSM() *shutdownFSM { func newShutdownFSM() *shutdownFSM {
fsm := &shutdownFSM{ fsm := &shutdownFSM{
state: shutdownStateOpen, state: shutdownStateOpen,
} }
return fsm return fsm
} }
@ -34,4 +34,3 @@ func (f *shutdownFSM) IsShuttingDown() bool {
defer f.mtx.Unlock() defer f.mtx.Unlock()
return f.state != shutdownStateOpen return f.state != shutdownStateOpen
} }

View File

@ -19,4 +19,3 @@ func TestIsPublicFrameType(t *testing.T) {
assert.True(t, IsPublicFrameType(255)) assert.True(t, IsPublicFrameType(255))
assert.False(t, IsPublicFrameType(rstFrameType)) assert.False(t, IsPublicFrameType(rstFrameType))
} }

View File

@ -5,6 +5,7 @@ import (
"time" "time"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/zrepl/zrepl/rpc/dataconn/frameconn" "github.com/zrepl/zrepl/rpc/dataconn/frameconn"
) )

View File

@ -11,6 +11,7 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/rpc/dataconn/heartbeatconn" "github.com/zrepl/zrepl/rpc/dataconn/heartbeatconn"
"github.com/zrepl/zrepl/util/socketpair" "github.com/zrepl/zrepl/util/socketpair"

View File

@ -11,6 +11,7 @@ import (
netssh "github.com/problame/go-netssh" netssh "github.com/problame/go-netssh"
"github.com/zrepl/yaml-config" "github.com/zrepl/yaml-config"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/transport" "github.com/zrepl/zrepl/transport"
transportconfig "github.com/zrepl/zrepl/transport/fromconfig" transportconfig "github.com/zrepl/zrepl/transport/fromconfig"

View File

@ -85,7 +85,7 @@ func (CloseWrite) receiver(wire transport.Wire) {
// consume half the test data, then detect an error, send it and CloseWrite // consume half the test data, then detect an error, send it and CloseWrite
r := io.LimitReader(wire, int64(5 * len(closeWriteTestSendData)/3)) r := io.LimitReader(wire, int64(5*len(closeWriteTestSendData)/3))
_, err := io.Copy(ioutil.Discard, r) _, err := io.Copy(ioutil.Discard, r)
noerror(err) noerror(err)
@ -103,7 +103,7 @@ func (CloseWrite) receiver(wire transport.Wire) {
// io.Copy masks io.EOF to nil, and we expect io.EOF from the client's Close() call // io.Copy masks io.EOF to nil, and we expect io.EOF from the client's Close() call
log.Panicf("unexpected error returned from reading conn: %s", err) log.Panicf("unexpected error returned from reading conn: %s", err)
} }
closeErr := wire.Close() closeErr := wire.Close()
log.Printf("closeErr=%T %s", closeErr, closeErr) log.Printf("closeErr=%T %s", closeErr, closeErr)

View File

@ -10,6 +10,7 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/util/socketpair" "github.com/zrepl/zrepl/util/socketpair"
) )

View File

@ -18,11 +18,12 @@ import (
"net" "net"
"time" "time"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/transport"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials"
"google.golang.org/grpc/peer" "google.golang.org/grpc/peer"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/transport"
) )
type Logger = logger.Logger type Logger = logger.Logger

View File

@ -5,10 +5,11 @@ package pdu
import ( import (
fmt "fmt" fmt "fmt"
math "math"
proto "github.com/golang/protobuf/proto" proto "github.com/golang/protobuf/proto"
context "golang.org/x/net/context" context "golang.org/x/net/context"
grpc "google.golang.org/grpc" grpc "google.golang.org/grpc"
math "math"
) )
// Reference imports to suppress errors if they are not otherwise used. // Reference imports to suppress errors if they are not otherwise used.

View File

@ -25,8 +25,9 @@ package netadaptor
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/zrepl/zrepl/logger"
"net" "net"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/transport" "github.com/zrepl/zrepl/transport"
) )

View File

@ -12,6 +12,7 @@ import (
"google.golang.org/grpc" "google.golang.org/grpc"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/zrepl/zrepl/replication/logic" "github.com/zrepl/zrepl/replication/logic"
"github.com/zrepl/zrepl/replication/logic/pdu" "github.com/zrepl/zrepl/replication/logic/pdu"
"github.com/zrepl/zrepl/rpc/dataconn" "github.com/zrepl/zrepl/rpc/dataconn"
@ -158,7 +159,7 @@ func (c *Client) WaitForConnectivity(ctx context.Context) error {
time.Sleep(envconst.Duration("ZREPL_RPC_DATACONN_PING_SLEEP", 1*time.Second)) time.Sleep(envconst.Duration("ZREPL_RPC_DATACONN_PING_SLEEP", 1*time.Second))
continue continue
} }
// it's not a dial timeout, // it's not a dial timeout,
checkRes(data, dataErr, loggers.Data, &dataOk) checkRes(data, dataErr, loggers.Data, &dataOk)
return return
} }

View File

@ -115,4 +115,3 @@ package rpc
// - remove the comments // // - remove the comments //
// - vim: set virtualedit+=all // - vim: set virtualedit+=all
// - vim: set ft=text // - vim: set ft=text

View File

@ -4,8 +4,8 @@ import (
"context" "context"
"time" "time"
"github.com/zrepl/zrepl/transport"
"github.com/zrepl/zrepl/rpc/transportmux" "github.com/zrepl/zrepl/rpc/transportmux"
"github.com/zrepl/zrepl/transport"
"github.com/zrepl/zrepl/util/envconst" "github.com/zrepl/zrepl/util/envconst"
) )

View File

@ -49,10 +49,10 @@ func (l *demuxListener) Accept(ctx context.Context) (*transport.AuthConn, error)
return res.conn, res.err return res.conn, res.err
} }
type demuxAddr struct {} type demuxAddr struct{}
func (demuxAddr) Network() string { return "demux" } func (demuxAddr) Network() string { return "demux" }
func (demuxAddr) String() string { return "demux" } func (demuxAddr) String() string { return "demux" }
func (l *demuxListener) Addr() net.Addr { func (l *demuxListener) Addr() net.Addr {
return demuxAddr{} return demuxAddr{}
@ -64,7 +64,7 @@ func (l *demuxListener) Close() error { return nil } // TODO
// This is a protocol constant, changing it breaks the wire protocol. // This is a protocol constant, changing it breaks the wire protocol.
const LabelLen = 64 const LabelLen = 64
func padLabel(out []byte, label string) (error) { func padLabel(out []byte, label string) error {
if len(label) > LabelLen { if len(label) > LabelLen {
return fmt.Errorf("label %q exceeds max length (is %d, max %d)", label, len(label), LabelLen) return fmt.Errorf("label %q exceeds max length (is %d, max %d)", label, len(label), LabelLen)
} }
@ -153,7 +153,7 @@ func Demux(ctx context.Context, rawListener transport.AuthenticatedListener, lab
type labeledConnecter struct { type labeledConnecter struct {
label []byte label []byte
transport.Connecter transport.Connecter
} }
func (c labeledConnecter) Connect(ctx context.Context) (transport.Wire, error) { func (c labeledConnecter) Connect(ctx context.Context) (transport.Wire, error) {
@ -202,4 +202,3 @@ func MuxConnecter(rawConnecter transport.Connecter, labels []string, timeout tim
} }
return ret, nil return ret, nil
} }

View File

@ -17,7 +17,7 @@ import (
type HandshakeMessage struct { type HandshakeMessage struct {
ProtocolVersion int ProtocolVersion int
Extensions []string Extensions []string
} }
// A HandshakeError describes what went wrong during the handshake. // A HandshakeError describes what went wrong during the handshake.
@ -25,7 +25,7 @@ type HandshakeMessage struct {
type HandshakeError struct { type HandshakeError struct {
msg string msg string
// If not nil, the underlying IO error that caused the handshake to fail. // If not nil, the underlying IO error that caused the handshake to fail.
IOError error IOError error
isAcceptError bool isAcceptError bool
} }
@ -36,10 +36,10 @@ func (e HandshakeError) Error() string { return e.msg }
// Like with net.OpErr (Go issue 6163), a client failing to handshake // Like with net.OpErr (Go issue 6163), a client failing to handshake
// should be a temporary Accept error toward the Listener . // should be a temporary Accept error toward the Listener .
func (e HandshakeError) Temporary() bool { func (e HandshakeError) Temporary() bool {
if e.isAcceptError { if e.isAcceptError {
return true return true
} }
te, ok := e.IOError.(interface{ Temporary() bool }); te, ok := e.IOError.(interface{ Temporary() bool })
return ok && te.Temporary() return ok && te.Temporary()
} }
@ -52,11 +52,11 @@ func (e HandshakeError) Timeout() bool {
return false return false
} }
func hsErr(format string, args... interface{}) *HandshakeError { func hsErr(format string, args ...interface{}) *HandshakeError {
return &HandshakeError{msg: fmt.Sprintf(format, args...)} return &HandshakeError{msg: fmt.Sprintf(format, args...)}
} }
func hsIOErr(err error, format string, args... interface{}) *HandshakeError { func hsIOErr(err error, format string, args ...interface{}) *HandshakeError {
return &HandshakeError{IOError: err, msg: fmt.Sprintf(format, args...)} return &HandshakeError{IOError: err, msg: fmt.Sprintf(format, args...)}
} }
@ -145,7 +145,7 @@ func (m *HandshakeMessage) DecodeReader(r io.Reader, maxLen int) error {
if exts[len(exts)-1] != "" { if exts[len(exts)-1] != "" {
return hsErr("unexpected data trailing after last extension newline") return hsErr("unexpected data trailing after last extension newline")
} }
m.Extensions = exts[0:len(exts)-1] m.Extensions = exts[0 : len(exts)-1]
return nil return nil
} }
@ -160,7 +160,7 @@ const HandshakeMessageMaxLen = 16 * 4096
func DoHandshakeVersion(conn net.Conn, deadline time.Time, version int) *HandshakeError { func DoHandshakeVersion(conn net.Conn, deadline time.Time, version int) *HandshakeError {
ours := HandshakeMessage{ ours := HandshakeMessage{
ProtocolVersion: version, ProtocolVersion: version,
Extensions: nil, Extensions: nil,
} }
hsb, err := ours.Encode() hsb, err := ours.Encode()
if err != nil { if err != nil {

View File

@ -3,13 +3,15 @@ package versionhandshake
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/util/socketpair"
"io" "io"
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/util/socketpair"
) )
func TestHandshakeMessage_Encode(t *testing.T) { func TestHandshakeMessage_Encode(t *testing.T) {
@ -23,8 +25,6 @@ func TestHandshakeMessage_Encode(t *testing.T) {
enc := string(encB) enc := string(encB)
t.Logf("enc: %s", enc) t.Logf("enc: %s", enc)
assert.False(t, strings.ContainsAny(enc[0:10], " ")) assert.False(t, strings.ContainsAny(enc[0:10], " "))
assert.True(t, enc[10] == ' ') assert.True(t, enc[10] == ' ')
@ -45,7 +45,7 @@ func TestHandshakeMessage_Encode(t *testing.T) {
func TestHandshakeMessage_Encode_InvalidProtocolVersion(t *testing.T) { func TestHandshakeMessage_Encode_InvalidProtocolVersion(t *testing.T) {
for _, pv := range []int{-1, 0, 10000, 10001} { for _, pv := range []int{-1, 0, 10000, 10001} {
t.Logf("testing invalid protocol version = %v", pv) t.Logf("testing invalid protocol version = %v", pv)
msg := HandshakeMessage{ msg := HandshakeMessage{
ProtocolVersion: pv, ProtocolVersion: pv,
@ -68,7 +68,7 @@ func TestHandshakeMessage_DecodeReader(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
out := HandshakeMessage{} out := HandshakeMessage{}
err = out.DecodeReader(bytes.NewReader([]byte(enc)), 4 * 4096) err = out.DecodeReader(bytes.NewReader([]byte(enc)), 4*4096)
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, 2342, out.ProtocolVersion) assert.Equal(t, 2342, out.ProtocolVersion)
assert.Equal(t, 2, len(out.Extensions)) assert.Equal(t, 2, len(out.Extensions))

View File

@ -4,12 +4,13 @@ import (
"context" "context"
"net" "net"
"time" "time"
"github.com/zrepl/zrepl/transport" "github.com/zrepl/zrepl/transport"
) )
type HandshakeConnecter struct { type HandshakeConnecter struct {
connecter transport.Connecter connecter transport.Connecter
timeout time.Duration timeout time.Duration
} }
func (c HandshakeConnecter) Connect(ctx context.Context) (transport.Wire, error) { func (c HandshakeConnecter) Connect(ctx context.Context) (transport.Wire, error) {
@ -31,17 +32,17 @@ func (c HandshakeConnecter) Connect(ctx context.Context) (transport.Wire, error)
func Connecter(connecter transport.Connecter, timeout time.Duration) HandshakeConnecter { func Connecter(connecter transport.Connecter, timeout time.Duration) HandshakeConnecter {
return HandshakeConnecter{ return HandshakeConnecter{
connecter: connecter, connecter: connecter,
timeout: timeout, timeout: timeout,
} }
} }
// wrapper type that performs a a protocol version handshake before returning the connection // wrapper type that performs a a protocol version handshake before returning the connection
type HandshakeListener struct { type HandshakeListener struct {
l transport.AuthenticatedListener l transport.AuthenticatedListener
timeout time.Duration timeout time.Duration
} }
func (l HandshakeListener) Addr() (net.Addr) { return l.l.Addr() } func (l HandshakeListener) Addr() net.Addr { return l.l.Addr() }
func (l HandshakeListener) Close() error { return l.l.Close() } func (l HandshakeListener) Close() error { return l.l.Close() }

View File

@ -4,7 +4,9 @@ package fromconfig
import ( import (
"fmt" "fmt"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/transport" "github.com/zrepl/zrepl/transport"
"github.com/zrepl/zrepl/transport/local" "github.com/zrepl/zrepl/transport/local"
@ -13,10 +15,10 @@ import (
"github.com/zrepl/zrepl/transport/tls" "github.com/zrepl/zrepl/transport/tls"
) )
func ListenerFactoryFromConfig(g *config.Global, in config.ServeEnum) (transport.AuthenticatedListenerFactory,error) { func ListenerFactoryFromConfig(g *config.Global, in config.ServeEnum) (transport.AuthenticatedListenerFactory, error) {
var ( var (
l transport.AuthenticatedListenerFactory l transport.AuthenticatedListenerFactory
err error err error
) )
switch v := in.Ret.(type) { switch v := in.Ret.(type) {
@ -35,7 +37,6 @@ func ListenerFactoryFromConfig(g *config.Global, in config.ServeEnum) (transport
return l, err return l, err
} }
func ConnecterFromConfig(g *config.Global, in config.ConnectEnum) (transport.Connecter, error) { func ConnecterFromConfig(g *config.Global, in config.ConnectEnum) (transport.Connecter, error) {
var ( var (
connecter transport.Connecter connecter transport.Connecter

View File

@ -3,12 +3,13 @@ package local
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/transport" "github.com/zrepl/zrepl/transport"
) )
type LocalConnecter struct { type LocalConnecter struct {
listenerName string listenerName string
clientIdentity string clientIdentity string
} }
@ -26,4 +27,3 @@ func (c *LocalConnecter) Connect(dialCtx context.Context) (transport.Wire, error
l := GetLocalListener(c.listenerName) l := GetLocalListener(c.listenerName)
return l.Connect(dialCtx, c.clientIdentity) return l.Connect(dialCtx, c.clientIdentity)
} }

View File

@ -3,20 +3,21 @@ package local
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/util/socketpair"
"net" "net"
"sync" "sync"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/transport" "github.com/zrepl/zrepl/transport"
"github.com/zrepl/zrepl/util/socketpair"
) )
var localListeners struct { var localListeners struct {
m map[string]*LocalListener // listenerName -> listener m map[string]*LocalListener // listenerName -> listener
init sync.Once init sync.Once
mtx sync.Mutex mtx sync.Mutex
} }
func GetLocalListener(listenerName string) (*LocalListener) { func GetLocalListener(listenerName string) *LocalListener {
localListeners.init.Do(func() { localListeners.init.Do(func() {
localListeners.m = make(map[string]*LocalListener) localListeners.m = make(map[string]*LocalListener)
@ -36,12 +37,12 @@ func GetLocalListener(listenerName string) (*LocalListener) {
type connectRequest struct { type connectRequest struct {
clientIdentity string clientIdentity string
callback chan connectResult callback chan connectResult
} }
type connectResult struct { type connectResult struct {
conn transport.Wire conn transport.Wire
err error err error
} }
type LocalListener struct { type LocalListener struct {
@ -60,7 +61,7 @@ func (l *LocalListener) Connect(dialCtx context.Context, clientIdentity string)
// place request // place request
req := connectRequest{ req := connectRequest{
clientIdentity: clientIdentity, clientIdentity: clientIdentity,
callback: make(chan connectResult), callback: make(chan connectResult),
} }
select { select {
case l.connects <- req: case l.connects <- req:
@ -70,7 +71,7 @@ func (l *LocalListener) Connect(dialCtx context.Context, clientIdentity string)
// wait for listener response // wait for listener response
select { select {
case connRes := <- req.callback: case connRes := <-req.callback:
conn, err = connRes.conn, connRes.err conn, err = connRes.conn, connRes.err
case <-dialCtx.Done(): case <-dialCtx.Done():
close(req.callback) // sending to the channel afterwards will panic, the listener has to catch this close(req.callback) // sending to the channel afterwards will panic, the listener has to catch this
@ -88,7 +89,7 @@ func (localAddr) Network() string { return "local" }
func (a localAddr) String() string { return a.S } func (a localAddr) String() string { return a.S }
func (l *LocalListener) Addr() (net.Addr) { return localAddr{"<listening>"} } func (l *LocalListener) Addr() net.Addr { return localAddr{"<listening>"} }
func (l *LocalListener) Accept(ctx context.Context) (*transport.AuthConn, error) { func (l *LocalListener) Accept(ctx context.Context) (*transport.AuthConn, error) {
respondToRequest := func(req connectRequest, res connectResult) (err error) { respondToRequest := func(req connectRequest, res connectResult) (err error) {
@ -163,12 +164,12 @@ func (l *LocalListener) Close() error {
return nil return nil
} }
func LocalListenerFactoryFromConfig(g *config.Global, in *config.LocalServe) (transport.AuthenticatedListenerFactory,error) { func LocalListenerFactoryFromConfig(g *config.Global, in *config.LocalServe) (transport.AuthenticatedListenerFactory, error) {
if in.ListenerName == "" { if in.ListenerName == "" {
return nil, fmt.Errorf("ListenerName must not be empty") return nil, fmt.Errorf("ListenerName must not be empty")
} }
listenerName := in.ListenerName listenerName := in.ListenerName
lf := func() (transport.AuthenticatedListener,error) { lf := func() (transport.AuthenticatedListener, error) {
return GetLocalListener(listenerName), nil return GetLocalListener(listenerName), nil
} }
return lf, nil return lf, nil

View File

@ -2,12 +2,14 @@ package ssh
import ( import (
"context" "context"
"time"
"github.com/jinzhu/copier" "github.com/jinzhu/copier"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/problame/go-netssh" "github.com/problame/go-netssh"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/transport" "github.com/zrepl/zrepl/transport"
"time"
) )
type SSHStdinserverConnecter struct { type SSHStdinserverConnecter struct {

View File

@ -1,19 +1,21 @@
package ssh package ssh
import ( import (
"github.com/problame/go-netssh" "context"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/nethelpers"
"github.com/zrepl/zrepl/transport"
"fmt" "fmt"
"net" "net"
"path" "path"
"context"
"github.com/pkg/errors"
"sync/atomic" "sync/atomic"
"github.com/pkg/errors"
"github.com/problame/go-netssh"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/nethelpers"
"github.com/zrepl/zrepl/transport"
) )
func MultiStdinserverListenerFactoryFromConfig(g *config.Global, in *config.StdinserverServer) (transport.AuthenticatedListenerFactory,error) { func MultiStdinserverListenerFactoryFromConfig(g *config.Global, in *config.StdinserverServer) (transport.AuthenticatedListenerFactory, error) {
for _, ci := range in.ClientIdentities { for _, ci := range in.ClientIdentities {
if err := transport.ValidateClientIdentity(ci); err != nil { if err := transport.ValidateClientIdentity(ci); err != nil {
@ -24,7 +26,7 @@ func MultiStdinserverListenerFactoryFromConfig(g *config.Global, in *config.Stdi
clientIdentities := in.ClientIdentities clientIdentities := in.ClientIdentities
sockdir := g.Serve.StdinServer.SockDir sockdir := g.Serve.StdinServer.SockDir
lf := func() (transport.AuthenticatedListener,error) { lf := func() (transport.AuthenticatedListener, error) {
return multiStdinserverListenerFromClientIdentities(sockdir, clientIdentities) return multiStdinserverListenerFromClientIdentities(sockdir, clientIdentities)
} }
@ -33,13 +35,13 @@ func MultiStdinserverListenerFactoryFromConfig(g *config.Global, in *config.Stdi
type multiStdinserverAcceptRes struct { type multiStdinserverAcceptRes struct {
conn *transport.AuthConn conn *transport.AuthConn
err error err error
} }
type MultiStdinserverListener struct { type MultiStdinserverListener struct {
listeners []*stdinserverListener listeners []*stdinserverListener
accepts chan multiStdinserverAcceptRes accepts chan multiStdinserverAcceptRes
closed int32 closed int32
} }
// client identities must be validated // client identities must be validated
@ -48,7 +50,7 @@ func multiStdinserverListenerFromClientIdentities(sockdir string, cis []string)
var err error var err error
for _, ci := range cis { for _, ci := range cis {
sockpath := path.Join(sockdir, ci) sockpath := path.Join(sockdir, ci)
l := &stdinserverListener{clientIdentity: ci} l := &stdinserverListener{clientIdentity: ci}
if err = nethelpers.PreparePrivateSockpath(sockpath); err != nil { if err = nethelpers.PreparePrivateSockpath(sockpath); err != nil {
break break
} }
@ -66,7 +68,7 @@ func multiStdinserverListenerFromClientIdentities(sockdir string, cis []string)
return &MultiStdinserverListener{listeners: listeners}, nil return &MultiStdinserverListener{listeners: listeners}, nil
} }
func (m *MultiStdinserverListener) Accept(ctx context.Context) (*transport.AuthConn, error){ func (m *MultiStdinserverListener) Accept(ctx context.Context) (*transport.AuthConn, error) {
if m.accepts == nil { if m.accepts == nil {
m.accepts = make(chan multiStdinserverAcceptRes, len(m.listeners)) m.accepts = make(chan multiStdinserverAcceptRes, len(m.listeners))
@ -80,7 +82,7 @@ func (m *MultiStdinserverListener) Accept(ctx context.Context) (*transport.AuthC
} }
} }
res := <- m.accepts res := <-m.accepts
return res.conn, res.err return res.conn, res.err
} }
@ -116,7 +118,7 @@ func (m *MultiStdinserverListener) Close() error {
// a single stdinserverListener (part of multiStinserverListener) // a single stdinserverListener (part of multiStinserverListener)
type stdinserverListener struct { type stdinserverListener struct {
l *netssh.Listener l *netssh.Listener
clientIdentity string clientIdentity string
} }

View File

@ -1,15 +1,17 @@
package tcp package tcp
import ( import (
"github.com/zrepl/zrepl/config"
"net"
"github.com/pkg/errors"
"context" "context"
"net"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/transport" "github.com/zrepl/zrepl/transport"
) )
type ipMapEntry struct { type ipMapEntry struct {
ip net.IP ip net.IP
ident string ident string
} }
@ -25,7 +27,7 @@ func ipMapFromConfig(clients map[string]string) (*ipMap, error) {
return nil, errors.Errorf("cannot parse client IP %q", clientIPString) return nil, errors.Errorf("cannot parse client IP %q", clientIPString)
} }
if err := transport.ValidateClientIdentity(clientIdent); err != nil { if err := transport.ValidateClientIdentity(clientIdent); err != nil {
return nil, errors.Wrapf(err,"invalid client identity for IP %q", clientIPString) return nil, errors.Wrapf(err, "invalid client identity for IP %q", clientIPString)
} }
entries = append(entries, ipMapEntry{clientIP, clientIdent}) entries = append(entries, ipMapEntry{clientIP, clientIdent})
} }
@ -79,4 +81,3 @@ func (f *TCPAuthListener) Accept(ctx context.Context) (*transport.AuthConn, erro
} }
return transport.NewAuthConn(nc, clientIdent), nil return transport.NewAuthConn(nc, clientIdent), nil
} }

View File

@ -6,6 +6,7 @@ import (
"net" "net"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/tlsconf" "github.com/zrepl/zrepl/tlsconf"
"github.com/zrepl/zrepl/transport" "github.com/zrepl/zrepl/transport"

View File

@ -1,16 +1,18 @@
package tls package tls
import ( import (
"context"
"crypto/tls" "crypto/tls"
"crypto/x509" "crypto/x509"
"fmt" "fmt"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/transport"
"github.com/zrepl/zrepl/tlsconf"
"net" "net"
"time" "time"
"context"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/tlsconf"
"github.com/zrepl/zrepl/transport"
) )
type TLSListenerFactory struct { type TLSListenerFactory struct {
@ -18,10 +20,10 @@ type TLSListenerFactory struct {
clientCA *x509.CertPool clientCA *x509.CertPool
serverCert tls.Certificate serverCert tls.Certificate
handshakeTimeout time.Duration handshakeTimeout time.Duration
clientCNs map[string]struct{} clientCNs map[string]struct{}
} }
func TLSListenerFactoryFromConfig(c *config.Global, in *config.TLSServe) (transport.AuthenticatedListenerFactory,error) { func TLSListenerFactoryFromConfig(c *config.Global, in *config.TLSServe) (transport.AuthenticatedListenerFactory, error) {
address := in.Listen address := in.Listen
handshakeTimeout := in.HandshakeTimeout handshakeTimeout := in.HandshakeTimeout
@ -85,5 +87,3 @@ func (l tlsAuthListener) Accept(ctx context.Context) (*transport.AuthConn, error
adaptor := newWireAdaptor(tlsConn, tcpConn) adaptor := newWireAdaptor(tlsConn, tcpConn)
return transport.NewAuthConn(adaptor, cn), nil return transport.NewAuthConn(adaptor, cn), nil
} }

View File

@ -5,6 +5,7 @@ import (
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/zrepl/zrepl/zfs" "github.com/zrepl/zrepl/zfs"
) )

View File

@ -39,4 +39,4 @@ func (l *L) NewCond() *sync.Cond {
func (l *L) DropWhile(f func()) { func (l *L) DropWhile(f func()) {
defer l.Unlock().Lock() defer l.Unlock().Lock()
f() f()
} }

View File

@ -3,11 +3,12 @@ package util
import ( import (
"bytes" "bytes"
"encoding/binary" "encoding/binary"
"github.com/stretchr/testify/assert"
"io" "io"
"reflect" "reflect"
"testing" "testing"
"testing/quick" "testing/quick"
"github.com/stretchr/testify/assert"
) )
func TestUnchunker(t *testing.T) { func TestUnchunker(t *testing.T) {

View File

@ -2,10 +2,11 @@ package util
import ( import (
"context" "context"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"testing" "testing"
"time" "time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func TestContextWithOptionalDeadline(t *testing.T) { func TestContextWithOptionalDeadline(t *testing.T) {

View File

@ -101,16 +101,16 @@ func (c *ChainedReader) Read(buf []byte) (n int, err error) {
} }
type ByteCounterReader struct { type ByteCounterReader struct {
reader io.ReadCloser reader io.ReadCloser
// called & accessed synchronously during Read, no external access // called & accessed synchronously during Read, no external access
cb func(full int64) cb func(full int64)
cbEvery time.Duration cbEvery time.Duration
lastCbAt time.Time lastCbAt time.Time
bytesSinceLastCb int64 bytesSinceLastCb int64
// set atomically because it may be read by multiple threads // set atomically because it may be read by multiple threads
bytes int64 bytes int64
} }
func NewByteCounterReader(reader io.ReadCloser) *ByteCounterReader { func NewByteCounterReader(reader io.ReadCloser) *ByteCounterReader {

View File

@ -4,18 +4,19 @@ import (
"bytes" "bytes"
"context" "context"
"fmt" "fmt"
"github.com/zrepl/zrepl/util/envconst"
"io" "io"
"os" "os"
"os/exec" "os/exec"
"syscall" "syscall"
"time" "time"
"github.com/zrepl/zrepl/util/envconst"
) )
// An IOCommand exposes a forked process's std(in|out|err) through the io.ReadWriteCloser interface. // An IOCommand exposes a forked process's std(in|out|err) through the io.ReadWriteCloser interface.
type IOCommand struct { type IOCommand struct {
Cmd *exec.Cmd Cmd *exec.Cmd
kill context.CancelFunc kill context.CancelFunc
Stdin io.WriteCloser Stdin io.WriteCloser
Stdout io.ReadCloser Stdout io.ReadCloser
StderrBuf *bytes.Buffer StderrBuf *bytes.Buffer

View File

@ -1,8 +1,9 @@
package zfs package zfs
import ( import (
"github.com/stretchr/testify/assert"
"testing" "testing"
"github.com/stretchr/testify/assert"
) )
func TestNewDatasetPathTree(t *testing.T) { func TestNewDatasetPathTree(t *testing.T) {

View File

@ -13,7 +13,8 @@ type DatasetFilter interface {
func NoFilter() DatasetFilter { func NoFilter() DatasetFilter {
return noFilter{} return noFilter{}
} }
type noFilter struct {}
type noFilter struct{}
var _ DatasetFilter = noFilter{} var _ DatasetFilter = noFilter{}

View File

@ -2,8 +2,9 @@ package zfs
import ( import (
"fmt" "fmt"
"github.com/pkg/errors"
"strconv" "strconv"
"github.com/pkg/errors"
) )
const ReplicationCursorBookmarkName = "zrepl_replication_cursor" const ReplicationCursorBookmarkName = "zrepl_replication_cursor"

View File

@ -2,9 +2,11 @@ package zfs_test
import ( import (
"context" "context"
"github.com/stretchr/testify/assert"
"github.com/zrepl/zrepl/zfs"
"testing" "testing"
"github.com/stretchr/testify/assert"
"github.com/zrepl/zrepl/zfs"
) )
type ResumeTokenTest struct { type ResumeTokenTest struct {

View File

@ -5,11 +5,12 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"github.com/prometheus/client_golang/prometheus"
"io" "io"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"github.com/prometheus/client_golang/prometheus"
) )
type VersionType string type VersionType string

View File

@ -15,9 +15,11 @@ import (
"time" "time"
"context" "context"
"github.com/prometheus/client_golang/prometheus"
"regexp" "regexp"
"strconv" "strconv"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/util/envconst" "github.com/zrepl/zrepl/util/envconst"
) )
@ -351,7 +353,7 @@ type readErrRecorder struct {
type sendStreamCopierError struct { type sendStreamCopierError struct {
isReadErr bool // if false, it's a write error isReadErr bool // if false, it's a write error
err error err error
} }
func (e sendStreamCopierError) Error() string { func (e sendStreamCopierError) Error() string {
@ -362,7 +364,7 @@ func (e sendStreamCopierError) Error() string {
} }
} }
func (e sendStreamCopierError) IsReadError() bool { return e.isReadErr } func (e sendStreamCopierError) IsReadError() bool { return e.isReadErr }
func (e sendStreamCopierError) IsWriteError() bool { return !e.isReadErr } func (e sendStreamCopierError) IsWriteError() bool { return !e.isReadErr }
func (r *readErrRecorder) Read(p []byte) (n int, err error) { func (r *readErrRecorder) Read(p []byte) (n int, err error) {
@ -410,13 +412,12 @@ func pipeWithCapacityHint(capacity int) (r, w *os.File, err error) {
} }
type sendStream struct { type sendStream struct {
cmd *exec.Cmd cmd *exec.Cmd
kill context.CancelFunc kill context.CancelFunc
closeMtx sync.Mutex closeMtx sync.Mutex
stdoutReader *os.File stdoutReader *os.File
opErr error opErr error
} }
func (s *sendStream) Read(p []byte) (n int, err error) { func (s *sendStream) Read(p []byte) (n int, err error) {
@ -484,7 +485,7 @@ func (s *sendStream) killAndWait(precedingReadErr error) error {
if closePipeErr == nil { if closePipeErr == nil {
// avoid double-closes in case anything below doesn't work // avoid double-closes in case anything below doesn't work
// and someone calls Close again // and someone calls Close again
s.stdoutReader = nil s.stdoutReader = nil
} else { } else {
return closePipeErr return closePipeErr
} }
@ -493,7 +494,7 @@ func (s *sendStream) killAndWait(precedingReadErr error) error {
// we managed to tear things down, no let's give the user some pretty *ZFSError // we managed to tear things down, no let's give the user some pretty *ZFSError
if exitErr != nil { if exitErr != nil {
s.opErr = &ZFSError{ s.opErr = &ZFSError{
Stderr: exitErr.Stderr, Stderr: exitErr.Stderr,
WaitErr: exitErr, WaitErr: exitErr,
} }
} else { } else {
@ -545,15 +546,14 @@ func ZFSSend(ctx context.Context, fs string, from, to string, token string) (str
stdoutWriter.Close() stdoutWriter.Close()
stream := &sendStream{ stream := &sendStream{
cmd: cmd, cmd: cmd,
kill: cancel, kill: cancel,
stdoutReader: stdoutReader, stdoutReader: stdoutReader,
} }
return newSendStreamCopier(stream), err return newSendStreamCopier(stream), err
} }
type DrySendType string type DrySendType string
const ( const (
@ -563,25 +563,27 @@ const (
func DrySendTypeFromString(s string) (DrySendType, error) { func DrySendTypeFromString(s string) (DrySendType, error) {
switch s { switch s {
case string(DrySendTypeFull): return DrySendTypeFull, nil case string(DrySendTypeFull):
case string(DrySendTypeIncremental): return DrySendTypeIncremental, nil return DrySendTypeFull, nil
case string(DrySendTypeIncremental):
return DrySendTypeIncremental, nil
default: default:
return "", fmt.Errorf("unknown dry send type %q", s) return "", fmt.Errorf("unknown dry send type %q", s)
} }
} }
type DrySendInfo struct { type DrySendInfo struct {
Type DrySendType Type DrySendType
Filesystem string // parsed from To field Filesystem string // parsed from To field
From, To string // direct copy from ZFS output From, To string // direct copy from ZFS output
SizeEstimate int64 // -1 if size estimate is not possible SizeEstimate int64 // -1 if size estimate is not possible
} }
var ( var (
// keep same number of capture groups for unmarshalInfoLine homogenity // keep same number of capture groups for unmarshalInfoLine homogenity
sendDryRunInfoLineRegexFull = regexp.MustCompile(`^(full)\t()([^\t]+@[^\t]+)\t([0-9]+)$`) sendDryRunInfoLineRegexFull = regexp.MustCompile(`^(full)\t()([^\t]+@[^\t]+)\t([0-9]+)$`)
// cannot enforce '[#@]' in incremental source, see test cases // cannot enforce '[#@]' in incremental source, see test cases
sendDryRunInfoLineRegexIncremental = regexp.MustCompile(`^(incremental)\t([^\t]+)\t([^\t]+@[^\t]+)\t([0-9]+)$`) sendDryRunInfoLineRegexIncremental = regexp.MustCompile(`^(incremental)\t([^\t]+)\t([^\t]+@[^\t]+)\t([0-9]+)$`)
) )
@ -602,7 +604,6 @@ func (s *DrySendInfo) unmarshalZFSOutput(output []byte) (err error) {
return fmt.Errorf("no match for info line (regex1 %s) (regex2 %s)", sendDryRunInfoLineRegexFull, sendDryRunInfoLineRegexIncremental) return fmt.Errorf("no match for info line (regex1 %s) (regex2 %s)", sendDryRunInfoLineRegexFull, sendDryRunInfoLineRegexIncremental)
} }
// unmarshal info line, looks like this: // unmarshal info line, looks like this:
// full zroot/test/a@1 5389768 // full zroot/test/a@1 5389768
// incremental zroot/test/a@1 zroot/test/a@2 5383936 // incremental zroot/test/a@1 zroot/test/a@2 5383936
@ -653,19 +654,19 @@ func ZFSSendDry(fs string, from, to string, token string) (_ *DrySendInfo, err e
* Redacted send & recv will bring this functionality, see * Redacted send & recv will bring this functionality, see
* https://github.com/openzfs/openzfs/pull/484 * https://github.com/openzfs/openzfs/pull/484
*/ */
fromAbs, err := absVersion(fs, from) fromAbs, err := absVersion(fs, from)
if err != nil { if err != nil {
return nil, fmt.Errorf("error building abs version for 'from': %s", err) return nil, fmt.Errorf("error building abs version for 'from': %s", err)
} }
toAbs, err := absVersion(fs, to) toAbs, err := absVersion(fs, to)
if err != nil { if err != nil {
return nil, fmt.Errorf("error building abs version for 'to': %s", err) return nil, fmt.Errorf("error building abs version for 'to': %s", err)
} }
return &DrySendInfo{ return &DrySendInfo{
Type: DrySendTypeIncremental, Type: DrySendTypeIncremental,
Filesystem: fs, Filesystem: fs,
From: fromAbs, From: fromAbs,
To: toAbs, To: toAbs,
SizeEstimate: -1}, nil SizeEstimate: -1}, nil
} }
@ -784,7 +785,7 @@ func ZFSRecv(ctx context.Context, fs string, streamCopier StreamCopier, opts Rec
if err != nil { if err != nil {
return err return err
} }
cmd.Stdin = stdin cmd.Stdin = stdin
if err = cmd.Start(); err != nil { if err = cmd.Start(); err != nil {
@ -794,7 +795,7 @@ func ZFSRecv(ctx context.Context, fs string, streamCopier StreamCopier, opts Rec
} }
stdin.Close() stdin.Close()
defer stdinWriter.Close() defer stdinWriter.Close()
pid := cmd.Process.Pid pid := cmd.Process.Pid
debug := func(format string, args ...interface{}) { debug := func(format string, args ...interface{}) {
debug("recv: pid=%v: %s", pid, fmt.Sprintf(format, args...)) debug("recv: pid=%v: %s", pid, fmt.Sprintf(format, args...))
@ -823,7 +824,7 @@ func ZFSRecv(ctx context.Context, fs string, streamCopier StreamCopier, opts Rec
copierErr := <-copierErrChan copierErr := <-copierErrChan
debug("copierErr: %T %s", copierErr, copierErr) debug("copierErr: %T %s", copierErr, copierErr)
if copierErr != nil { if copierErr != nil {
cancelCmd() cancelCmd()
} }
waitErr := <-waitErrChan waitErr := <-waitErrChan
@ -838,7 +839,7 @@ func ZFSRecv(ctx context.Context, fs string, streamCopier StreamCopier, opts Rec
type ClearResumeTokenError struct { type ClearResumeTokenError struct {
ZFSOutput []byte ZFSOutput []byte
CmdError error CmdError error
} }
func (e ClearResumeTokenError) Error() string { func (e ClearResumeTokenError) Error() string {
@ -947,13 +948,27 @@ const (
func (s zfsPropertySource) zfsGetSourceFieldPrefixes() []string { func (s zfsPropertySource) zfsGetSourceFieldPrefixes() []string {
prefixes := make([]string, 0, 7) prefixes := make([]string, 0, 7)
if s&sourceLocal != 0 {prefixes = append(prefixes, "local")} if s&sourceLocal != 0 {
if s&sourceDefault != 0 {prefixes = append(prefixes, "default")} prefixes = append(prefixes, "local")
if s&sourceInherited != 0 {prefixes = append(prefixes, "inherited")} }
if s&sourceNone != 0 {prefixes = append(prefixes, "-")} if s&sourceDefault != 0 {
if s&sourceTemporary != 0 { prefixes = append(prefixes, "temporary")} prefixes = append(prefixes, "default")
if s&sourceReceived != 0 { prefixes = append(prefixes, "received")} }
if s == sourceAny { prefixes = append(prefixes, "") } if s&sourceInherited != 0 {
prefixes = append(prefixes, "inherited")
}
if s&sourceNone != 0 {
prefixes = append(prefixes, "-")
}
if s&sourceTemporary != 0 {
prefixes = append(prefixes, "temporary")
}
if s&sourceReceived != 0 {
prefixes = append(prefixes, "received")
}
if s == sourceAny {
prefixes = append(prefixes, "")
}
return prefixes return prefixes
} }
@ -992,7 +1007,7 @@ func zfsGet(path string, props []string, allowedSources zfsPropertySource) (*ZFS
return nil, fmt.Errorf("zfs get did not return property,value,source tuples") return nil, fmt.Errorf("zfs get did not return property,value,source tuples")
} }
for _, p := range allowedPrefixes { for _, p := range allowedPrefixes {
if strings.HasPrefix(fields[2],p) { if strings.HasPrefix(fields[2], p) {
res.m[fields[0]] = fields[1] res.m[fields[0]] = fields[1]
break break
} }
@ -1010,8 +1025,10 @@ func ZFSDestroy(dataset string) (err error) {
filesystem = dataset filesystem = dataset
} else { } else {
switch dataset[idx] { switch dataset[idx] {
case '@': dstype = "snapshot" case '@':
case '#': dstype = "bookmark" dstype = "snapshot"
case '#':
dstype = "bookmark"
} }
filesystem = dataset[:idx] filesystem = dataset[:idx]
} }

View File

@ -1,8 +1,9 @@
package zfs package zfs
import ( import (
"github.com/stretchr/testify/assert"
"testing" "testing"
"github.com/stretchr/testify/assert"
) )
func TestZFSListHandlesProducesZFSErrorOnNonZeroExit(t *testing.T) { func TestZFSListHandlesProducesZFSErrorOnNonZeroExit(t *testing.T) {
@ -33,8 +34,8 @@ func TestDatasetPathTrimNPrefixComps(t *testing.T) {
func TestZFSPropertySource(t *testing.T) { func TestZFSPropertySource(t *testing.T) {
tcs := []struct{ tcs := []struct {
in zfsPropertySource in zfsPropertySource
exp []string exp []string
}{ }{
{ {
@ -43,11 +44,11 @@ func TestZFSPropertySource(t *testing.T) {
exp: []string{"local", "default", "inherited", "-", "temporary", "received", ""}, exp: []string{"local", "default", "inherited", "-", "temporary", "received", ""},
}, },
{ {
in: sourceTemporary, in: sourceTemporary,
exp: []string{"temporary"}, exp: []string{"temporary"},
}, },
{ {
in: sourceLocal|sourceInherited, in: sourceLocal | sourceInherited,
exp: []string{"local", "inherited"}, exp: []string{"local", "inherited"},
}, },
} }
@ -137,9 +138,9 @@ size 10518512
incrementalWithSpacesInIntermediateComponent := "\nincremental\tblaffoo\tpool1/otherjob/another ds with spaces/childfs@blaffoo2\t624\nsize\t624\n" incrementalWithSpacesInIntermediateComponent := "\nincremental\tblaffoo\tpool1/otherjob/another ds with spaces/childfs@blaffoo2\t624\nsize\t624\n"
type tc struct { type tc struct {
name string name string
in string in string
exp *DrySendInfo exp *DrySendInfo
expErr bool expErr bool
} }
@ -147,10 +148,10 @@ size 10518512
{ {
name: "fullSend", in: fullSend, name: "fullSend", in: fullSend,
exp: &DrySendInfo{ exp: &DrySendInfo{
Type: DrySendTypeFull, Type: DrySendTypeFull,
Filesystem: "zroot/test/a", Filesystem: "zroot/test/a",
From: "", From: "",
To: "zroot/test/a@1", To: "zroot/test/a@1",
SizeEstimate: 5389768, SizeEstimate: 5389768,
}, },
}, },
@ -158,7 +159,7 @@ size 10518512
name: "incSend", in: incSend, name: "incSend", in: incSend,
exp: &DrySendInfo{ exp: &DrySendInfo{
Type: DrySendTypeIncremental, Type: DrySendTypeIncremental,
Filesystem: "zroot/test/a", Filesystem: "zroot/test/a",
From: "zroot/test/a@1", From: "zroot/test/a@1",
To: "zroot/test/a@2", To: "zroot/test/a@2",
SizeEstimate: 5383936, SizeEstimate: 5383936,
@ -168,16 +169,16 @@ size 10518512
name: "incSendBookmark", in: incSendBookmark, name: "incSendBookmark", in: incSendBookmark,
exp: &DrySendInfo{ exp: &DrySendInfo{
Type: DrySendTypeIncremental, Type: DrySendTypeIncremental,
Filesystem: "zroot/test/a", Filesystem: "zroot/test/a",
From: "zroot/test/a#1", From: "zroot/test/a#1",
To: "zroot/test/a@2", To: "zroot/test/a@2",
SizeEstimate: 5383312, SizeEstimate: 5383312,
}, },
}, },
{ {
name: "incNoToken", in: incNoToken, name: "incNoToken", in: incNoToken,
exp: &DrySendInfo{ exp: &DrySendInfo{
Type: DrySendTypeIncremental, Type: DrySendTypeIncremental,
Filesystem: "zroot/test/a", Filesystem: "zroot/test/a",
// as can be seen in the string incNoToken, // as can be seen in the string incNoToken,
// we cannot infer whether the incremental source is a snapshot or bookmark // we cannot infer whether the incremental source is a snapshot or bookmark
@ -189,10 +190,10 @@ size 10518512
{ {
name: "fullNoToken", in: fullNoToken, name: "fullNoToken", in: fullNoToken,
exp: &DrySendInfo{ exp: &DrySendInfo{
Type: DrySendTypeFull, Type: DrySendTypeFull,
Filesystem: "zroot/test/a", Filesystem: "zroot/test/a",
From: "", From: "",
To: "zroot/test/a@3", To: "zroot/test/a@3",
SizeEstimate: 10518512, SizeEstimate: 10518512,
}, },
}, },