extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-12-23 15:38:57 +01:00
Code Issues Packages Projects Releases Wiki Activity

check: make check do --checkers files concurrently - fixes #4318

Browse Source
This commit is contained in:
Nick Craig-Wood 2020-06-10 13:04:51 +01:00
parent 63cf0b1cdd
commit d55053098f
1 changed files with 22 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
fs/operations/operations.go
View File

@ -756,6 +756,8 @@ type checkFn func(ctx context.Context, a, b fs.Object) (differ bool, noHash bool
type checkMarch struct { type checkMarch struct {
fdst, fsrc fs.Fs fdst, fsrc fs.Fs
check checkFn check checkFn
wg sync.WaitGroup
tokens chan struct{}
oneway bool oneway bool
differences int32 differences int32
noHashes int32 noHashes int32
@ -833,6 +835,13 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b
if SkipDestructive(ctx, src, "check") { if SkipDestructive(ctx, src, "check") {
return false return false
} }
c.wg.Add(1)
c.tokens <- struct{}{} // put a token to limit concurrency
go func() {
defer func() {
<-c.tokens // get the token back to free up a slot
c.wg.Done()
}()
differ, noHash := c.checkIdentical(ctx, dstX, srcX) differ, noHash := c.checkIdentical(ctx, dstX, srcX)
if differ { if differ {
atomic.AddInt32(&c.differences, 1) atomic.AddInt32(&c.differences, 1)
@ -845,6 +854,7 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b
fs.Debugf(dstX, "OK") fs.Debugf(dstX, "OK")
} }
} }
}()
} else { } else {
err := errors.Errorf("is file on %v but directory on %v", c.fsrc, c.fdst) err := errors.Errorf("is file on %v but directory on %v", c.fsrc, c.fdst)
fs.Errorf(src, "%v", err) fs.Errorf(src, "%v", err)
@ -883,6 +893,7 @@ func CheckFn(ctx context.Context, fdst, fsrc fs.Fs, check checkFn, oneway bool)
fsrc: fsrc, fsrc: fsrc,
check: check, check: check,
oneway: oneway, oneway: oneway,
tokens: make(chan struct{}, fs.Config.Checkers),
} }
// set up a march over fdst and fsrc // set up a march over fdst and fsrc
@ -895,6 +906,7 @@ func CheckFn(ctx context.Context, fdst, fsrc fs.Fs, check checkFn, oneway bool)
} }
fs.Debugf(fdst, "Waiting for checks to finish") fs.Debugf(fdst, "Waiting for checks to finish")
err := m.Run() err := m.Run()
c.wg.Wait() // wait for background go-routines
if c.dstFilesMissing > 0 { if c.dstFilesMissing > 0 {
fs.Logf(fdst, "%d files missing", c.dstFilesMissing) fs.Logf(fdst, "%d files missing", c.dstFilesMissing)