extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-01-03 04:49:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

list: add ListDirSortedFn for callback oriented directory listing

Browse Source 
This will be used for the out of memory sync
This commit is contained in:
Nick Craig-Wood 2024-11-28 17:12:19 +00:00
parent 3e0af30704
commit 56bcdc552b
2 changed files with 100 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
fs/list/list.go
View File

@ -38,8 +38,64 @@ func DirSorted(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entri
return filterAndSortDir(ctx, entries, includeAll, dir, fi.IncludeObject, fi.IncludeDirectory(ctx, f))
}
// filter (if required) and check the entries, then sort them
func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string,
// listP for every backend
func listP(ctx context.Context, f fs.Fs, dir string, callback fs.ListRCallback) error {
if doListP := f.Features().ListP; doListP != nil {
return doListP(ctx, dir, callback)
}
// Fallback to List
entries, err := f.List(ctx, dir)
if err != nil {
return err
}
return callback(entries)
}
// DirSortedFn reads Object and *Dir into entries for the given Fs.
//
// dir is the start directory, "" for root
//
// If includeAll is specified all files will be added, otherwise only
// files and directories passing the filter will be added.
//
// Files will be returned through callback in sorted order
func DirSortedFn(ctx context.Context, f fs.Fs, includeAll bool, dir string, callback fs.ListRCallback, keyFn KeyFn) (err error) {
stats := accounting.Stats(ctx)
fi := filter.GetConfig(ctx)
// Sort the entries, in or out of memory
sorter, err := NewSorter(ctx, callback, keyFn)
if err != nil {
return fmt.Errorf("failed to create directory sorter: %w", err)
}
defer sorter.CleanUp()
// Get unfiltered entries from the fs
err = listP(ctx, f, dir, func(entries fs.DirEntries) error {
stats.Listed(int64(len(entries)))
// This should happen only if exclude files lives in the
// starting directory, otherwise ListDirSorted should not be
// called.
if !includeAll && fi.ListContainsExcludeFile(entries) {
fs.Debugf(dir, "Excluded")
return nil
}
entries, err := filterDir(ctx, entries, includeAll, dir, fi.IncludeObject, fi.IncludeDirectory(ctx, f))
if err != nil {
return err
}
return sorter.Add(entries)
})
if err != nil {
return err
}
return sorter.Send()
}
// Filter the entries passed in
func filterDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string,
IncludeObject func(ctx context.Context, o fs.Object) bool,
IncludeDirectory func(remote string) (bool, error)) (newEntries fs.DirEntries, err error) {
newEntries = entries[:0] // in place filter
@ -92,7 +148,18 @@ func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll boo
newEntries = append(newEntries, entry)
}
}
entries = newEntries
return newEntries, nil
}
// filter and sort the entries
func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string,
IncludeObject func(ctx context.Context, o fs.Object) bool,
IncludeDirectory func(remote string) (bool, error)) (newEntries fs.DirEntries, err error) {
// Filter the directory entries (in place)
entries, err = filterDir(ctx, entries, includeAll, dir, IncludeObject, IncludeDirectory)
if err != nil {
return nil, err
}
// Sort the directory entries by Remote
//

40
fs/operations/listdirsorted_test.go
View File

@ -12,9 +12,9 @@ import (
"github.com/stretchr/testify/require"
)
// TestListDirSorted is integration testing code in fs/list/list.go
// testListDirSorted is integration testing code in fs/list/list.go
// which can't be tested there due to import loops.
func TestListDirSorted(t *testing.T) {
func testListDirSorted(t *testing.T, listFn func(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error)) {
r := fstest.NewRun(t)
ctx := context.Background()
@ -52,20 +52,20 @@ func TestListDirSorted(t *testing.T) {
return name
}
items, err = list.DirSorted(context.Background(), r.Fremote, true, "")
items, err = listFn(context.Background(), r.Fremote, true, "")
require.NoError(t, err)
require.Len(t, items, 3)
assert.Equal(t, "a.txt", str(0))
assert.Equal(t, "sub dir/", str(1))
assert.Equal(t, "zend.txt", str(2))
items, err = list.DirSorted(context.Background(), r.Fremote, false, "")
items, err = listFn(context.Background(), r.Fremote, false, "")
require.NoError(t, err)
require.Len(t, items, 2)
assert.Equal(t, "sub dir/", str(0))
assert.Equal(t, "zend.txt", str(1))
items, err = list.DirSorted(context.Background(), r.Fremote, true, "sub dir")
items, err = listFn(context.Background(), r.Fremote, true, "sub dir")
require.NoError(t, err)
require.Len(t, items, 4)
assert.Equal(t, "sub dir/hello world", str(0))
@ -73,7 +73,7 @@ func TestListDirSorted(t *testing.T) {
assert.Equal(t, "sub dir/ignore dir/", str(2))
assert.Equal(t, "sub dir/sub sub dir/", str(3))
items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir")
items, err = listFn(context.Background(), r.Fremote, false, "sub dir")
require.NoError(t, err)
require.Len(t, items, 2)
assert.Equal(t, "sub dir/ignore dir/", str(0))
@ -82,25 +82,45 @@ func TestListDirSorted(t *testing.T) {
// testing ignore file
fi.Opt.ExcludeFile = []string{".ignore"}
items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir")
items, err = listFn(context.Background(), r.Fremote, false, "sub dir")
require.NoError(t, err)
require.Len(t, items, 1)
assert.Equal(t, "sub dir/sub sub dir/", str(0))
items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir")
items, err = listFn(context.Background(), r.Fremote, false, "sub dir/ignore dir")
require.NoError(t, err)
require.Len(t, items, 0)
items, err = list.DirSorted(context.Background(), r.Fremote, true, "sub dir/ignore dir")
items, err = listFn(context.Background(), r.Fremote, true, "sub dir/ignore dir")
require.NoError(t, err)
require.Len(t, items, 2)
assert.Equal(t, "sub dir/ignore dir/.ignore", str(0))
assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1))
fi.Opt.ExcludeFile = nil
items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir")
items, err = listFn(context.Background(), r.Fremote, false, "sub dir/ignore dir")
require.NoError(t, err)
require.Len(t, items, 2)
assert.Equal(t, "sub dir/ignore dir/.ignore", str(0))
assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1))
}
// TestListDirSorted is integration testing code in fs/list/list.go
// which can't be tested there due to import loops.
func TestListDirSorted(t *testing.T) {
testListDirSorted(t, list.DirSorted)
}
// TestListDirSortedFn is integration testing code in fs/list/list.go
// which can't be tested there due to import loops.
func TestListDirSortedFn(t *testing.T) {
listFn := func(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
callback := func(newEntries fs.DirEntries) error {
entries = append(entries, newEntries...)
return nil
}
err = list.DirSortedFn(ctx, f, includeAll, dir, callback, nil)
return entries, err
}
testListDirSorted(t, listFn)
}