mirror of
https://github.com/rclone/rclone.git
synced 2024-11-07 09:04:52 +01:00
march: Fix excessive parallelism when using --no-traverse
When using `--no-traverse` the march routines call NewObject on each potential object in the destination. The concurrency limiter was accidentally arranged so that there were `--checkers` * `--checkers` NewObject calls going on at once. This became obvious when using the sftp backend which used too many connections. Fixes #5824
This commit is contained in:
parent
c6755aa768
commit
88bd80c1fa
@ -35,6 +35,7 @@ type March struct {
|
||||
srcListDir listDirFn // function to call to list a directory in the src
|
||||
dstListDir listDirFn // function to call to list a directory in the dst
|
||||
transforms []matchTransformFn
|
||||
limiter chan struct{} // make sure we don't do too many operations at once
|
||||
}
|
||||
|
||||
// Marcher is called on each match
|
||||
@ -69,6 +70,8 @@ func (m *March) init(ctx context.Context) {
|
||||
if m.Fdst.Features().CaseInsensitive || ci.IgnoreCaseSync {
|
||||
m.transforms = append(m.transforms, strings.ToLower)
|
||||
}
|
||||
// Limit parallelism for operations
|
||||
m.limiter = make(chan struct{}, ci.Checkers)
|
||||
}
|
||||
|
||||
// list a directory into entries, err
|
||||
@ -429,13 +432,11 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
|
||||
|
||||
// If NoTraverse is set, then try to find a matching object
|
||||
// for each item in the srcList to head dst object
|
||||
ci := fs.GetConfig(m.Ctx)
|
||||
limiter := make(chan struct{}, ci.Checkers)
|
||||
if m.NoTraverse && !m.NoCheckDest {
|
||||
for _, src := range srcList {
|
||||
wg.Add(1)
|
||||
limiter <- struct{}{}
|
||||
go func(limiter chan struct{}, src fs.DirEntry) {
|
||||
m.limiter <- struct{}{}
|
||||
go func(src fs.DirEntry) {
|
||||
defer wg.Done()
|
||||
if srcObj, ok := src.(fs.Object); ok {
|
||||
leaf := path.Base(srcObj.Remote())
|
||||
@ -446,8 +447,8 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
|
||||
mu.Unlock()
|
||||
}
|
||||
}
|
||||
<-limiter
|
||||
}(limiter, src)
|
||||
<-m.limiter
|
||||
}(src)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user