hotfix: limit concurrency of zfs send & recv commands

ATM, the replication logic sends all dry-run requests in parallel,
which might overwhelm the ZFS pool on the sending side.
Since we use rpc/dataconn for dry sends, this also opens one TCP
connection per dry-run request.

Use a sempahore to limit the degree of concurrency where we know it is a
problem ATM.
As indicated by the comments, the cleaner solution would involve some
kind of 'resource exhaustion' error code.

refs #161
refs #164

util/semaphore/semaphore.go

@@ -0,0 +1,38 @@
+package semaphore
+
+import (
+	"context"
+
+	wsemaphore "golang.org/x/sync/semaphore"
+)
+
+type S struct {
+	ws *wsemaphore.Weighted
+}
+
+func New(max int64) *S {
+	return &S{wsemaphore.NewWeighted(max)}
+}
+
+type AcquireGuard struct {
+	s        *S
+	released bool
+}
+
+// The returned AcquireGuard is not goroutine-safe.
+func (s *S) Acquire(ctx context.Context) (*AcquireGuard, error) {
+	if err := s.ws.Acquire(ctx, 1); err != nil {
+		return nil, err
+	} else if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	return &AcquireGuard{s, false}, nil
+}
+
+func (g *AcquireGuard) Release() {
+	if g == nil || g.released {
+		return
+	}
+	g.released = true
+	g.s.ws.Release(1)
+}

util/semaphore/semaphore_test.go

@@ -0,0 +1,49 @@
+package semaphore
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSemaphore(t *testing.T) {
+	const numGoroutines = 10
+	const concurrentSemaphore = 5
+	const sleepTime = 1 * time.Second
+
+	begin := time.Now()
+
+	sem := New(concurrentSemaphore)
+
+	var aquisitions struct {
+		beforeT, afterT uint32
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(numGoroutines)
+	for i := 0; i < numGoroutines; i++ {
+		go func() {
+			defer wg.Done()
+			res, err := sem.Acquire(context.Background())
+			require.NoError(t, err)
+			defer res.Release()
+			if time.Since(begin) > sleepTime {
+				atomic.AddUint32(&aquisitions.beforeT, 1)
+			} else {
+				atomic.AddUint32(&aquisitions.afterT, 1)
+			}
+			time.Sleep(sleepTime)
+		}()
+	}
+
+	wg.Wait()
+
+	assert.True(t, aquisitions.beforeT == concurrentSemaphore)
+	assert.True(t, aquisitions.afterT == numGoroutines-concurrentSemaphore)
+
+}