[#307] add package trace, integrate it with logging, and adopt it throughout zrepl

package trace:

- introduce the concept of tasks and spans, tracked as linked list within ctx
    - see package-level docs for an overview of the concepts
    - **main feature 1**: unique stack of task and span IDs
        - makes it easy to follow a series of log entries in concurrent code
    - **main feature 2**: ability to produce a chrome://tracing-compatible trace file
        - either via an env variable or a `zrepl pprof` subcommand
        - this is not a CPU profile, we already have go pprof for that
        - but it is very useful to visually inspect where the
          replication / snapshotter / pruner spends its time
          ( fixes #307 )

usage in package daemon/logging:

- goal: every log entry should have a trace field with the ID stack from package trace

- make `logging.GetLogger(ctx, Subsys)` the authoritative `logger.Logger` factory function
    - the context carries a linked list of injected fields which
      `logging.GetLogger` adds to the logger it returns
    - `logging.GetLogger` also uses package `trace` to get the
      task-and-span-stack and injects it into the returned logger's fields

rpc/rpc_client.go

@@ -10,6 +10,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 	"google.golang.org/grpc"
 
 	"github.com/google/uuid"
@@ -83,6 +84,9 @@ func (c *Client) Close() {
 // callers must ensure that the returned io.ReadCloser is closed
 // TODO expose dataClient interface to the outside world
 func (c *Client) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.Send")
+	defer endSpan()
+
 	// TODO the returned sendStream may return a read error created by the remote side
 	res, stream, err := c.dataClient.ReqSend(ctx, r)
 	if err != nil {
@@ -97,34 +101,58 @@ func (c *Client) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
 }
 
 func (c *Client) Receive(ctx context.Context, req *pdu.ReceiveReq, stream io.ReadCloser) (*pdu.ReceiveRes, error) {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.Receive")
+	defer endSpan()
+
 	return c.dataClient.ReqRecv(ctx, req, stream)
 }
 
 func (c *Client) ListFilesystems(ctx context.Context, in *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.ListFilesystems")
+	defer endSpan()
+
 	return c.controlClient.ListFilesystems(ctx, in)
 }
 
 func (c *Client) ListFilesystemVersions(ctx context.Context, in *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error) {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.ListFilesystemVersions")
+	defer endSpan()
+
 	return c.controlClient.ListFilesystemVersions(ctx, in)
 }
 
 func (c *Client) DestroySnapshots(ctx context.Context, in *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.DestroySnapshots")
+	defer endSpan()
+
 	return c.controlClient.DestroySnapshots(ctx, in)
 }
 
 func (c *Client) ReplicationCursor(ctx context.Context, in *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.ReplicationCursor")
+	defer endSpan()
+
 	return c.controlClient.ReplicationCursor(ctx, in)
 }
 
 func (c *Client) SendCompleted(ctx context.Context, in *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error) {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.SendCompleted")
+	defer endSpan()
+
 	return c.controlClient.SendCompleted(ctx, in)
 }
 
 func (c *Client) HintMostRecentCommonAncestor(ctx context.Context, in *pdu.HintMostRecentCommonAncestorReq) (*pdu.HintMostRecentCommonAncestorRes, error) {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.HintMostRecentCommonAncestor")
+	defer endSpan()
+
 	return c.controlClient.HintMostRecentCommonAncestor(ctx, in)
 }
 
 func (c *Client) WaitForConnectivity(ctx context.Context) error {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.WaitForConnectivity")
+	defer endSpan()
+
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	msg := uuid.New().String()