filter: fix deadlock with errors on --files-from

Before this change if doing a recursive directory listing with
`--files-from` if more than `--checkers` files errored (other than
file not found) then rclone would deadlock.

This fixes the problem by exiting on the first error.
This commit is contained in:
douchen 2023-06-10 22:53:08 +08:00 committed by GitHub
parent ae3ff50580
commit 7e2deffc62
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 17 additions and 3 deletions

View File

@ -433,13 +433,13 @@ func (f *Filter) MakeListR(ctx context.Context, NewObject func(ctx context.Conte
var ( var (
checkers = ci.Checkers checkers = ci.Checkers
remotes = make(chan string, checkers) remotes = make(chan string, checkers)
g errgroup.Group g, gCtx = errgroup.WithContext(ctx)
) )
for i := 0; i < checkers; i++ { for i := 0; i < checkers; i++ {
g.Go(func() (err error) { g.Go(func() (err error) {
var entries = make(fs.DirEntries, 1) var entries = make(fs.DirEntries, 1)
for remote := range remotes { for remote := range remotes {
entries[0], err = NewObject(ctx, remote) entries[0], err = NewObject(gCtx, remote)
if err == fs.ErrorObjectNotFound { if err == fs.ErrorObjectNotFound {
// Skip files that are not found // Skip files that are not found
} else if err != nil { } else if err != nil {
@ -454,8 +454,13 @@ func (f *Filter) MakeListR(ctx context.Context, NewObject func(ctx context.Conte
return nil return nil
}) })
} }
outer:
for remote := range f.files { for remote := range f.files {
remotes <- remote select {
case remotes <- remote:
case <-gCtx.Done():
break outer
}
} }
close(remotes) close(remotes)
return g.Wait() return g.Wait()

View File

@ -360,6 +360,15 @@ func TestNewFilterMakeListR(t *testing.T) {
require.NoError(t, f.AddFile("error")) require.NoError(t, f.AddFile("error"))
err = listR(context.Background(), "", listRcallback) err = listR(context.Background(), "", listRcallback)
require.EqualError(t, err, assert.AnError.Error()) require.EqualError(t, err, assert.AnError.Error())
// The checker will exit by the error above
ci := fs.GetConfig(context.Background())
ci.Checkers = 1
// Now check an error is returned from NewObject
require.NoError(t, f.AddFile("error"))
err = listR(context.Background(), "", listRcallback)
require.EqualError(t, err, assert.AnError.Error())
} }
func TestNewFilterMinSize(t *testing.T) { func TestNewFilterMinSize(t *testing.T) {