march: Fix excessive parallelism when using --no-traverse

When using `--no-traverse` the march routines call NewObject on each
potential object in the destination.

The concurrency limiter was accidentally arranged so that there were
`--checkers` * `--checkers` NewObject calls going on at once.

This became obvious when using the sftp backend which used too many
connections.

Fixes #5824
This commit is contained in:
Nick Craig-Wood 2023-10-10 18:03:56 +01:00
parent c6755aa768
commit 88bd80c1fa

View File

@ -35,6 +35,7 @@ type March struct {
srcListDir listDirFn // function to call to list a directory in the src
dstListDir listDirFn // function to call to list a directory in the dst
transforms []matchTransformFn
limiter chan struct{} // make sure we don't do too many operations at once
}
// Marcher is called on each match
@ -69,6 +70,8 @@ func (m *March) init(ctx context.Context) {
if m.Fdst.Features().CaseInsensitive || ci.IgnoreCaseSync {
m.transforms = append(m.transforms, strings.ToLower)
}
// Limit parallelism for operations
m.limiter = make(chan struct{}, ci.Checkers)
}
// list a directory into entries, err
@ -429,13 +432,11 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
// If NoTraverse is set, then try to find a matching object
// for each item in the srcList to head dst object
ci := fs.GetConfig(m.Ctx)
limiter := make(chan struct{}, ci.Checkers)
if m.NoTraverse && !m.NoCheckDest {
for _, src := range srcList {
wg.Add(1)
limiter <- struct{}{}
go func(limiter chan struct{}, src fs.DirEntry) {
m.limiter <- struct{}{}
go func(src fs.DirEntry) {
defer wg.Done()
if srcObj, ok := src.(fs.Object); ok {
leaf := path.Base(srcObj.Remote())
@ -446,8 +447,8 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
mu.Unlock()
}
}
<-limiter
}(limiter, src)
<-m.limiter
}(src)
}
wg.Wait()
}