diff --git a/backend/memory/memory.go b/backend/memory/memory.go index 11c02d78a..608f42292 100644 --- a/backend/memory/memory.go +++ b/backend/memory/memory.go @@ -38,8 +38,7 @@ func init() { } // Options defines the configuration for this backend -type Options struct { -} +type Options struct{} // Fs represents a remote memory server type Fs struct { @@ -385,10 +384,22 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { bucket, directory := f.split(dir) list := walk.NewListRHelper(callback) + entries := fs.DirEntries{} listR := func(bucket, directory, prefix string, addBucket bool) error { - return f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, entry fs.DirEntry, isDirectory bool) error { - return list.Add(entry) + err = f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, entry fs.DirEntry, isDirectory bool) error { + entries = append(entries, entry) // can't list.Add here -- could deadlock + return nil }) + if err != nil { + return err + } + for _, entry := range entries { + err = list.Add(entry) + if err != nil { + return err + } + } + return nil } if bucket == "" { entries, err := f.listBuckets(ctx) diff --git a/backend/memory/memory_internal_test.go b/backend/memory/memory_internal_test.go new file mode 100644 index 000000000..fe8db1f57 --- /dev/null +++ b/backend/memory/memory_internal_test.go @@ -0,0 +1,40 @@ +package memory + +import ( + "context" + "fmt" + "testing" + + _ "github.com/rclone/rclone/backend/local" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/fstest/fstests" + "github.com/stretchr/testify/require" +) + +var t1 = fstest.Time("2001-02-03T04:05:06.499999999Z") + +// InternalTest dispatches all internal tests +func (f *Fs) InternalTest(t *testing.T) { + t.Run("PurgeListDeadlock", func(t *testing.T) { + testPurgeListDeadlock(t) + }) +} + +// test that Purge fallback does not result in deadlock from concurrently listing and removing +func testPurgeListDeadlock(t *testing.T) { + ctx := context.Background() + r := fstest.NewRunIndividual(t) + r.Mkdir(ctx, r.Fremote) + r.Fremote.Features().Disable("Purge") // force fallback-purge + + // make a lot of files to prevent it from finishing too quickly + for i := 0; i < 100; i++ { + dst := "file" + fmt.Sprint(i) + ".txt" + r.WriteObject(ctx, dst, "hello", t1) + } + + require.NoError(t, operations.Purge(ctx, r.Fremote, "")) +} + +var _ fstests.InternalTester = (*Fs)(nil)