check: make check do --checkers files concurrently - fixes #4318

fs/operations/operations.go

@@ -756,6 +756,8 @@ type checkFn func(ctx context.Context, a, b fs.Object) (differ bool, noHash bool
 type checkMarch struct {
 	fdst, fsrc      fs.Fs
 	check           checkFn
+	wg              sync.WaitGroup
+	tokens          chan struct{}
 	oneway          bool
 	differences     int32
 	noHashes        int32
@@ -833,18 +835,26 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b
 			if SkipDestructive(ctx, src, "check") {
 				return false
 			}
-			differ, noHash := c.checkIdentical(ctx, dstX, srcX)
+			c.wg.Add(1)
-			if differ {
+			c.tokens <- struct{}{} // put a token to limit concurrency
-				atomic.AddInt32(&c.differences, 1)
+			go func() {
-			} else {
+				defer func() {
-				atomic.AddInt32(&c.matches, 1)
+					<-c.tokens // get the token back to free up a slot
-				if noHash {
+					c.wg.Done()
-					atomic.AddInt32(&c.noHashes, 1)
+				}()
-					fs.Debugf(dstX, "OK - could not check hash")
+				differ, noHash := c.checkIdentical(ctx, dstX, srcX)
+				if differ {
+					atomic.AddInt32(&c.differences, 1)
 				} else {
-					fs.Debugf(dstX, "OK")
+					atomic.AddInt32(&c.matches, 1)
+					if noHash {
+						atomic.AddInt32(&c.noHashes, 1)
+						fs.Debugf(dstX, "OK - could not check hash")
+					} else {
+						fs.Debugf(dstX, "OK")
+					}
 				}
-			}
+			}()
 		} else {
 			err := errors.Errorf("is file on %v but directory on %v", c.fsrc, c.fdst)
 			fs.Errorf(src, "%v", err)
@@ -883,6 +893,7 @@ func CheckFn(ctx context.Context, fdst, fsrc fs.Fs, check checkFn, oneway bool)
 		fsrc:   fsrc,
 		check:  check,
 		oneway: oneway,
+		tokens: make(chan struct{}, fs.Config.Checkers),
 	}
 
 	// set up a march over fdst and fsrc
@@ -895,6 +906,7 @@ func CheckFn(ctx context.Context, fdst, fsrc fs.Fs, check checkFn, oneway bool)
 	}
 	fs.Debugf(fdst, "Waiting for checks to finish")
 	err := m.Run()
+	c.wg.Wait() // wait for background go-routines
 
 	if c.dstFilesMissing > 0 {
 		fs.Logf(fdst, "%d files missing", c.dstFilesMissing)