mirror of
https://github.com/rclone/rclone.git
synced 2024-11-07 09:04:52 +01:00
filter: fix deadlock with errors on --files-from
Before this change if doing a recursive directory listing with `--files-from` if more than `--checkers` files errored (other than file not found) then rclone would deadlock. This fixes the problem by exiting on the first error.
This commit is contained in:
parent
ae3ff50580
commit
7e2deffc62
@ -433,13 +433,13 @@ func (f *Filter) MakeListR(ctx context.Context, NewObject func(ctx context.Conte
|
|||||||
var (
|
var (
|
||||||
checkers = ci.Checkers
|
checkers = ci.Checkers
|
||||||
remotes = make(chan string, checkers)
|
remotes = make(chan string, checkers)
|
||||||
g errgroup.Group
|
g, gCtx = errgroup.WithContext(ctx)
|
||||||
)
|
)
|
||||||
for i := 0; i < checkers; i++ {
|
for i := 0; i < checkers; i++ {
|
||||||
g.Go(func() (err error) {
|
g.Go(func() (err error) {
|
||||||
var entries = make(fs.DirEntries, 1)
|
var entries = make(fs.DirEntries, 1)
|
||||||
for remote := range remotes {
|
for remote := range remotes {
|
||||||
entries[0], err = NewObject(ctx, remote)
|
entries[0], err = NewObject(gCtx, remote)
|
||||||
if err == fs.ErrorObjectNotFound {
|
if err == fs.ErrorObjectNotFound {
|
||||||
// Skip files that are not found
|
// Skip files that are not found
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
@ -454,8 +454,13 @@ func (f *Filter) MakeListR(ctx context.Context, NewObject func(ctx context.Conte
|
|||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
outer:
|
||||||
for remote := range f.files {
|
for remote := range f.files {
|
||||||
remotes <- remote
|
select {
|
||||||
|
case remotes <- remote:
|
||||||
|
case <-gCtx.Done():
|
||||||
|
break outer
|
||||||
|
}
|
||||||
}
|
}
|
||||||
close(remotes)
|
close(remotes)
|
||||||
return g.Wait()
|
return g.Wait()
|
||||||
|
@ -360,6 +360,15 @@ func TestNewFilterMakeListR(t *testing.T) {
|
|||||||
require.NoError(t, f.AddFile("error"))
|
require.NoError(t, f.AddFile("error"))
|
||||||
err = listR(context.Background(), "", listRcallback)
|
err = listR(context.Background(), "", listRcallback)
|
||||||
require.EqualError(t, err, assert.AnError.Error())
|
require.EqualError(t, err, assert.AnError.Error())
|
||||||
|
|
||||||
|
// The checker will exit by the error above
|
||||||
|
ci := fs.GetConfig(context.Background())
|
||||||
|
ci.Checkers = 1
|
||||||
|
|
||||||
|
// Now check an error is returned from NewObject
|
||||||
|
require.NoError(t, f.AddFile("error"))
|
||||||
|
err = listR(context.Background(), "", listRcallback)
|
||||||
|
require.EqualError(t, err, assert.AnError.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewFilterMinSize(t *testing.T) {
|
func TestNewFilterMinSize(t *testing.T) {
|
||||||
|
Loading…
Reference in New Issue
Block a user