zfs: send: improve error reporting by capturing stderr

util/circlog/circlog.go

@@ -27,6 +27,14 @@ type CircularLog struct {
 	mtx sync.Mutex
 }
 
+func MustNewCircularLog(max int) *CircularLog {
+	log, err := NewCircularLog(max)
+	if err != nil {
+		panic(err)
+	}
+	return log
+}
+
 func NewCircularLog(max int) (*CircularLog, error) {
 	if max <= 0 {
 		return nil, fmt.Errorf("max must be positive")

zfs/zfs.go

@@ -17,10 +17,10 @@ import (
 	"regexp"
 	"strconv"
 
+	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 
-	"github.com/pkg/errors"
-
+	"github.com/zrepl/zrepl/util/circlog"
 	"github.com/zrepl/zrepl/util/envconst"
 )
 
@@ -418,6 +418,7 @@ type sendStream struct {
 
 	closeMtx     sync.Mutex
 	stdoutReader *os.File
+	stderrBuf    *circlog.CircularLog
 	opErr        error
 }
 
@@ -495,7 +496,7 @@ func (s *sendStream) killAndWait(precedingReadErr error) error {
 	// we managed to tear things down, no let's give the user some pretty *ZFSError
 	if exitErr != nil {
 		s.opErr = &ZFSError{
-			Stderr:  exitErr.Stderr,
+			Stderr:  []byte(s.stderrBuf.String()),
 			WaitErr: exitErr,
 		}
 	} else {
@@ -512,6 +513,8 @@ func (s *sendStream) killAndWait(precedingReadErr error) error {
 	return s.opErr
 }
 
+var zfsSendStderrCaptureMaxSize = envconst.Int("ZREPL_ZFS_SEND_STDERR_MAX_CAPTURE_SIZE", 1<<15)
+
 // if token != "", then send -t token is used
 // otherwise send [-i from] to is used
 // (if from is "" a full ZFS send is done)
@@ -538,11 +541,14 @@ func ZFSSend(ctx context.Context, fs string, from, to string, token string) (str
 
 	cmd.Stdout = stdoutWriter
 
+	stderrBuf := circlog.MustNewCircularLog(zfsSendStderrCaptureMaxSize)
+	cmd.Stderr = stderrBuf
+
 	if err := cmd.Start(); err != nil {
 		cancel()
 		stdoutWriter.Close()
 		stdoutReader.Close()
-		return nil, err
+		return nil, errors.Wrap(err, "cannot start zfs send command")
 	}
 	stdoutWriter.Close()
 
@@ -550,9 +556,10 @@ func ZFSSend(ctx context.Context, fs string, from, to string, token string) (str
 		cmd:          cmd,
 		kill:         cancel,
 		stdoutReader: stdoutReader,
+		stderrBuf:    stderrBuf,
 	}
 
-	return newSendStreamCopier(stream), err
+	return newSendStreamCopier(stream), nil
 }
 
 type DrySendType string