From fa586b493ca159ccb158efa76a149404b730e903 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Sun, 26 Jul 2020 11:54:28 +0200 Subject: [PATCH] [#348] fix crash on early-recv error * The SendStream.Close() was not called by dataconn.Server, which left the zfs send process dangling. * When the source job's ctx interceptor closed the task, the dangling zfs send was detect by the trace package and panicked. 020-07-25T19:54:41-04:00 [ERRO][latitude][rpc.data][cyZj$J3Ca$J3Ca.CJwB]: cannot write send stream err="frameconn: shutting down" panic: end task: 1 active child tasks: end task: task still has active child tasks goroutine 196966 [running]: github.com/zrepl/zrepl/daemon/logging/trace.WithTask.func1.1(0xc000320680, 0xcde000) /home/jeremy/go/src/github.com/zrepl/zrepl/daemon/logging/trace/trace.go:221 +0x2f7 github.com/zrepl/zrepl/daemon/logging/trace.WithTask.func1() /home/jeremy/go/src/github.com/zrepl/zrepl/daemon/logging/trace/trace.go:237 +0x38 github.com/zrepl/zrepl/daemon/logging/trace.WithTaskAndSpan.func1() /home/jeremy/go/src/github.com/zrepl/zrepl/daemon/logging/trace/trace_convenience.go:41 +0x37 github.com/zrepl/zrepl/daemon/job.(*PassiveSide).Run.func1(0xdcf780, 0xc0000a3560, 0xdc65a0, 0xc00035e620, 0xc0000a34d0) /home/jeremy/go/src/github.com/zrepl/zrepl/daemon/job/passive.go:194 +0x2e7 github.com/zrepl/zrepl/rpc.NewServer.func3(0xdcf780, 0xc0001ce4b0, 0xdc65e0, 0xc00035e600, 0xc0000a34d0) /home/jeremy/go/src/github.com/zrepl/zrepl/rpc/rpc_server.go:82 +0xd5 github.com/zrepl/zrepl/rpc/dataconn.(*Server).serveConn(0xc0000a2ba0, 0xc00018eca0) /home/jeremy/go/src/github.com/zrepl/zrepl/rpc/dataconn/dataconn_server.go:149 +0x3be github.com/zrepl/zrepl/rpc/dataconn.(*Server).Serve.func3(0xc0000b8180, 0xc0000a2ba0, 0xc00018eca0) /home/jeremy/go/src/github.com/zrepl/zrepl/rpc/dataconn/dataconn_server.go:108 +0x5d created by github.com/zrepl/zrepl/rpc/dataconn.(*Server).Serve /home/jeremy/go/src/github.com/zrepl/zrepl/rpc/dataconn/dataconn_server.go:106 +0x24a 2020-07-25T19:58:55-04:00 [ERRO][latitude][rpc.data][Pt4F$gCWT$gCWT.fzhc]: cannot write send stream err="frameconn: shutting down" panic: end task: 1 active child tasks: end task: task still has active child tasks fixes #348 --- rpc/dataconn/dataconn_server.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rpc/dataconn/dataconn_server.go b/rpc/dataconn/dataconn_server.go index bad2cd4..344c480 100644 --- a/rpc/dataconn/dataconn_server.go +++ b/rpc/dataconn/dataconn_server.go @@ -239,6 +239,10 @@ func (s *Server) serveConnRequest(ctx context.Context, endpoint string, c *strea if sendStream != nil { err := c.SendStream(ctx, sendStream, ZFSStream) + closeErr := sendStream.Close() + if closeErr != nil { + s.log.WithError(err).Error("cannot close send stream") + } if err != nil { s.log.WithError(err).Error("cannot write send stream") }