From 96afeb1435e1c4bc60b299d2a840ce26df0bb42e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 9 Dec 2024 15:08:36 +0000 Subject: [PATCH] walk: move NewListRHelper into list.Helper to avoid circular dependency It turns out that the list helpers were at the wrong level and needed to be pushed down into the fs/list for future work. --- backend/azureblob/azureblob.go | 4 ++-- backend/b2/b2.go | 4 ++-- backend/cache/cache.go | 5 +++-- backend/drive/drive.go | 4 ++-- backend/gofile/gofile.go | 6 +++--- .../googlecloudstorage/googlecloudstorage.go | 4 ++-- backend/jottacloud/jottacloud.go | 4 ++-- backend/memory/memory.go | 4 ++-- backend/netstorage/netstorage.go | 4 ++-- backend/onedrive/onedrive.go | 3 ++- .../oracleobjectstorage/oracleobjectstorage.go | 4 ++-- backend/pcloud/pcloud.go | 4 ++-- backend/qingstor/qingstor.go | 4 ++-- backend/s3/s3.go | 4 ++-- backend/swift/swift.go | 4 ++-- fs/{walk => list}/helpers.go | 18 +++++++++--------- fs/{walk => list}/helpers_test.go | 10 +++++----- fs/walk/walk.go | 2 +- 18 files changed, 47 insertions(+), 45 deletions(-) rename fs/{walk => list}/helpers.go (59%) rename fs/{walk => list}/helpers_test.go (93%) diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index c7e43ecb4..4dc1de73a 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -41,7 +41,7 @@ import ( "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" - "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/lib/bucket" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/env" @@ -1302,7 +1302,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // of listing recursively that doing a directory traversal. func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { containerName, directory := f.split(dir) - list := walk.NewListRHelper(callback) + list := list.NewHelper(callback) listR := func(containerName, directory, prefix string, addContainer bool) error { return f.list(ctx, containerName, directory, prefix, addContainer, true, int32(f.opt.ListChunkSize), func(remote string, object *container.BlobItem, isDirectory bool) error { entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory) diff --git a/backend/b2/b2.go b/backend/b2/b2.go index a71f9e43c..a407223ed 100644 --- a/backend/b2/b2.go +++ b/backend/b2/b2.go @@ -30,7 +30,7 @@ import ( "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" - "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/lib/bucket" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/multipart" @@ -922,7 +922,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // of listing recursively that doing a directory traversal. func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { bucket, directory := f.split(dir) - list := walk.NewListRHelper(callback) + list := list.NewHelper(callback) listR := func(bucket, directory, prefix string, addBucket bool) error { last := "" return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, f.opt.Versions, false, func(remote string, object *api.File, isDirectory bool) error { diff --git a/backend/cache/cache.go b/backend/cache/cache.go index cb4e0e62b..c04d91ca9 100644 --- a/backend/cache/cache.go +++ b/backend/cache/cache.go @@ -29,6 +29,7 @@ import ( "github.com/rclone/rclone/fs/config/obscure" "github.com/rclone/rclone/fs/fspath" "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/fs/rc" "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/lib/atexit" @@ -1086,7 +1087,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e return cachedEntries, nil } -func (f *Fs) recurse(ctx context.Context, dir string, list *walk.ListRHelper) error { +func (f *Fs) recurse(ctx context.Context, dir string, list *list.Helper) error { entries, err := f.List(ctx, dir) if err != nil { return err @@ -1138,7 +1139,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( } // if we're here, we're gonna do a standard recursive traversal and cache everything - list := walk.NewListRHelper(callback) + list := list.NewHelper(callback) err = f.recurse(ctx, dir, list) if err != nil { return err diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 8e878751a..2a6b0ca54 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -37,8 +37,8 @@ import ( "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/fspath" "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/fs/operations" - "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/lib/dircache" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/env" @@ -2202,7 +2202,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( wg := sync.WaitGroup{} in := make(chan listREntry, listRInputBuffer) out := make(chan error, f.ci.Checkers) - list := walk.NewListRHelper(callback) + list := list.NewHelper(callback) overflow := []listREntry{} listed := 0 diff --git a/backend/gofile/gofile.go b/backend/gofile/gofile.go index ca4091046..d8981b9c6 100644 --- a/backend/gofile/gofile.go +++ b/backend/gofile/gofile.go @@ -25,7 +25,7 @@ import ( "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" - "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/lib/dircache" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/pacer" @@ -734,7 +734,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e } // implementation of ListR -func (f *Fs) listR(ctx context.Context, dir string, list *walk.ListRHelper) (err error) { +func (f *Fs) listR(ctx context.Context, dir string, list *list.Helper) (err error) { directoryID, err := f.dirCache.FindDir(ctx, dir, false) if err != nil { return err @@ -820,7 +820,7 @@ func (f *Fs) listR(ctx context.Context, dir string, list *walk.ListRHelper) (err // Don't implement this unless you have a more efficient way // of listing recursively than doing a directory traversal. func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { - list := walk.NewListRHelper(callback) + list := list.NewHelper(callback) err = f.listR(ctx, dir, list) if err != nil { return err diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go index cd57edd53..7c7bec373 100644 --- a/backend/googlecloudstorage/googlecloudstorage.go +++ b/backend/googlecloudstorage/googlecloudstorage.go @@ -35,7 +35,7 @@ import ( "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" - "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/lib/bucket" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/env" @@ -845,7 +845,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // of listing recursively that doing a directory traversal. func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { bucket, directory := f.split(dir) - list := walk.NewListRHelper(callback) + list := list.NewHelper(callback) listR := func(bucket, directory, prefix string, addBucket bool) error { return f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, object *storage.Object, isDirectory bool) error { entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory) diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index 6482070ab..f074ceea2 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -31,7 +31,7 @@ import ( "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" - "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/oauthutil" "github.com/rclone/rclone/lib/pacer" @@ -1264,7 +1264,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( Parameters: url.Values{}, } opts.Parameters.Set("mode", "liststream") - list := walk.NewListRHelper(callback) + list := list.NewHelper(callback) var resp *http.Response err = f.pacer.Call(func() (bool, error) { diff --git a/backend/memory/memory.go b/backend/memory/memory.go index 48cc9678b..9842c6eb2 100644 --- a/backend/memory/memory.go +++ b/backend/memory/memory.go @@ -17,7 +17,7 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/hash" - "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/lib/bucket" ) @@ -383,7 +383,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // of listing recursively that doing a directory traversal. func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { bucket, directory := f.split(dir) - list := walk.NewListRHelper(callback) + list := list.NewHelper(callback) entries := fs.DirEntries{} listR := func(bucket, directory, prefix string, addBucket bool) error { err = f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, entry fs.DirEntry, isDirectory bool) error { diff --git a/backend/netstorage/netstorage.go b/backend/netstorage/netstorage.go index 8e0426171..72583d0ae 100755 --- a/backend/netstorage/netstorage.go +++ b/backend/netstorage/netstorage.go @@ -28,7 +28,7 @@ import ( "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" - "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/lib/pacer" "github.com/rclone/rclone/lib/rest" ) @@ -516,7 +516,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( return fs.ErrorDirNotFound } - list := walk.NewListRHelper(callback) + list := list.NewHelper(callback) for resumeStart := u.Path; resumeStart != ""; { var files []File files, resumeStart, err = f.netStorageListRequest(ctx, URL, u.Path) diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index 980eec8c9..4508c48f9 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -30,6 +30,7 @@ import ( "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/fs/log" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/walk" @@ -1396,7 +1397,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( // So we have to filter things outside of the root which is // inefficient. - list := walk.NewListRHelper(callback) + list := list.NewHelper(callback) // list a folder conventionally - used for shared folders var listFolder func(dir string) error diff --git a/backend/oracleobjectstorage/oracleobjectstorage.go b/backend/oracleobjectstorage/oracleobjectstorage.go index a4d0f00fa..2e8c06581 100644 --- a/backend/oracleobjectstorage/oracleobjectstorage.go +++ b/backend/oracleobjectstorage/oracleobjectstorage.go @@ -18,8 +18,8 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/fs/operations" - "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/lib/bucket" "github.com/rclone/rclone/lib/pacer" ) @@ -649,7 +649,7 @@ of listing recursively that doing a directory traversal. */ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { bucketName, directory := f.split(dir) - list := walk.NewListRHelper(callback) + list := list.NewHelper(callback) listR := func(bucket, directory, prefix string, addBucket bool) error { return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error { entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory) diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go index f198c5563..254bb63c9 100644 --- a/backend/pcloud/pcloud.go +++ b/backend/pcloud/pcloud.go @@ -27,7 +27,7 @@ import ( "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" - "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/lib/dircache" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/oauthutil" @@ -631,7 +631,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // ListR lists the objects and directories of the Fs starting // from dir recursively into out. func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { - list := walk.NewListRHelper(callback) + list := list.NewHelper(callback) err = f.listHelper(ctx, dir, true, func(o fs.DirEntry) error { return list.Add(o) }) diff --git a/backend/qingstor/qingstor.go b/backend/qingstor/qingstor.go index 3dfa36b15..39113d149 100644 --- a/backend/qingstor/qingstor.go +++ b/backend/qingstor/qingstor.go @@ -22,7 +22,7 @@ import ( "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" - "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/lib/bucket" "github.com/rclone/rclone/lib/encoder" qsConfig "github.com/yunify/qingstor-sdk-go/v3/config" @@ -704,7 +704,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // of listing recursively that doing a directory traversal. func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { bucket, directory := f.split(dir) - list := walk.NewListRHelper(callback) + list := list.NewHelper(callback) listR := func(bucket, directory, prefix string, addBucket bool) error { return f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, object *qs.KeyType, isDirectory bool) error { entry, err := f.itemToDirEntry(remote, object, isDirectory) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index e27e3a60a..25d3f6a67 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -47,8 +47,8 @@ import ( "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/fs/operations" - "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/lib/atexit" "github.com/rclone/rclone/lib/bucket" "github.com/rclone/rclone/lib/encoder" @@ -4438,7 +4438,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // of listing recursively than doing a directory traversal. func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { bucket, directory := f.split(dir) - list := walk.NewListRHelper(callback) + list := list.NewHelper(callback) listR := func(bucket, directory, prefix string, addBucket bool) error { return f.list(ctx, listOpt{ bucket: bucket, diff --git a/backend/swift/swift.go b/backend/swift/swift.go index 452d5f11a..68fe5cb09 100644 --- a/backend/swift/swift.go +++ b/backend/swift/swift.go @@ -23,8 +23,8 @@ import ( "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/fs/operations" - "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/lib/atexit" "github.com/rclone/rclone/lib/bucket" "github.com/rclone/rclone/lib/encoder" @@ -846,7 +846,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // of listing recursively than doing a directory traversal. func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { container, directory := f.split(dir) - list := walk.NewListRHelper(callback) + list := list.NewHelper(callback) listR := func(container, directory, prefix string, addContainer bool) error { return f.list(ctx, container, directory, prefix, addContainer, true, false, func(entry fs.DirEntry) error { return list.Add(entry) diff --git a/fs/walk/helpers.go b/fs/list/helpers.go similarity index 59% rename from fs/walk/helpers.go rename to fs/list/helpers.go index 6b14b8586..e9ec5d92a 100644 --- a/fs/walk/helpers.go +++ b/fs/list/helpers.go @@ -1,25 +1,25 @@ -package walk +package list import "github.com/rclone/rclone/fs" // Listing helpers used by backends -// ListRHelper is used in the implementation of ListR to accumulate DirEntries -type ListRHelper struct { +// Helper is used in the implementation of ListR to accumulate DirEntries +type Helper struct { callback fs.ListRCallback entries fs.DirEntries } -// NewListRHelper should be called from ListR with the callback passed in -func NewListRHelper(callback fs.ListRCallback) *ListRHelper { - return &ListRHelper{ +// NewHelper should be called from ListR with the callback passed in +func NewHelper(callback fs.ListRCallback) *Helper { + return &Helper{ callback: callback, } } // send sends the stored entries to the callback if there are >= max // entries. -func (lh *ListRHelper) send(max int) (err error) { +func (lh *Helper) send(max int) (err error) { if len(lh.entries) >= max { err = lh.callback(lh.entries) lh.entries = lh.entries[:0] @@ -29,7 +29,7 @@ func (lh *ListRHelper) send(max int) (err error) { // Add an entry to the stored entries and send them if there are more // than a certain amount -func (lh *ListRHelper) Add(entry fs.DirEntry) error { +func (lh *Helper) Add(entry fs.DirEntry) error { if entry == nil { return nil } @@ -38,6 +38,6 @@ func (lh *ListRHelper) Add(entry fs.DirEntry) error { } // Flush the stored entries (if any) sending them to the callback -func (lh *ListRHelper) Flush() error { +func (lh *Helper) Flush() error { return lh.send(1) } diff --git a/fs/walk/helpers_test.go b/fs/list/helpers_test.go similarity index 93% rename from fs/walk/helpers_test.go rename to fs/list/helpers_test.go index a028fcde3..5066c5449 100644 --- a/fs/walk/helpers_test.go +++ b/fs/list/helpers_test.go @@ -1,4 +1,4 @@ -package walk +package list import ( "fmt" @@ -18,7 +18,7 @@ func mockCallback(entries fs.DirEntries) error { func TestNewListRHelper(t *testing.T) { callback := mockCallback - helper := NewListRHelper(callback) + helper := NewHelper(callback) assert.NotNil(t, helper) assert.Equal(t, fmt.Sprintf("%p", callback), fmt.Sprintf("%p", helper.callback)) @@ -32,7 +32,7 @@ func TestListRHelperAdd(t *testing.T) { return nil } - helper := NewListRHelper(callback) + helper := NewHelper(callback) entry := mockobject.Object("A") require.NoError(t, helper.Add(entry)) @@ -57,7 +57,7 @@ func TestListRHelperSend(t *testing.T) { return nil } - helper := NewListRHelper(callback) + helper := NewHelper(callback) // Add 100 entries to force the callback to be invoked for i := 0; i < 100; i++ { @@ -80,7 +80,7 @@ func TestListRHelperFlush(t *testing.T) { return nil } - helper := NewListRHelper(callback) + helper := NewHelper(callback) require.NoError(t, helper.Add(entry)) assert.False(t, callbackInvoked, "Callback should not have been invoked yet") require.NoError(t, helper.Flush()) diff --git a/fs/walk/walk.go b/fs/walk/walk.go index bb2daa9a6..74da4f9c6 100644 --- a/fs/walk/walk.go +++ b/fs/walk/walk.go @@ -274,7 +274,7 @@ func (dm *dirMap) sendEntries(fn fs.ListRCallback) (err error) { sort.Strings(dirs) // Now convert to bulkier Dir in batches and send now := time.Now() - list := NewListRHelper(fn) + list := list.NewHelper(fn) for _, dir := range dirs { err = list.Add(fs.NewDir(dir, now)) if err != nil {