diff --git a/fs/walk/helpers.go b/fs/walk/helpers.go new file mode 100644 index 000000000..6b14b8586 --- /dev/null +++ b/fs/walk/helpers.go @@ -0,0 +1,43 @@ +package walk + +import "github.com/rclone/rclone/fs" + +// Listing helpers used by backends + +// ListRHelper is used in the implementation of ListR to accumulate DirEntries +type ListRHelper struct { + callback fs.ListRCallback + entries fs.DirEntries +} + +// NewListRHelper should be called from ListR with the callback passed in +func NewListRHelper(callback fs.ListRCallback) *ListRHelper { + return &ListRHelper{ + callback: callback, + } +} + +// send sends the stored entries to the callback if there are >= max +// entries. +func (lh *ListRHelper) send(max int) (err error) { + if len(lh.entries) >= max { + err = lh.callback(lh.entries) + lh.entries = lh.entries[:0] + } + return err +} + +// Add an entry to the stored entries and send them if there are more +// than a certain amount +func (lh *ListRHelper) Add(entry fs.DirEntry) error { + if entry == nil { + return nil + } + lh.entries = append(lh.entries, entry) + return lh.send(100) +} + +// Flush the stored entries (if any) sending them to the callback +func (lh *ListRHelper) Flush() error { + return lh.send(1) +} diff --git a/fs/walk/helpers_test.go b/fs/walk/helpers_test.go new file mode 100644 index 000000000..a028fcde3 --- /dev/null +++ b/fs/walk/helpers_test.go @@ -0,0 +1,90 @@ +package walk + +import ( + "fmt" + "testing" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fstest/mockobject" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Mock callback to collect the entries +func mockCallback(entries fs.DirEntries) error { + // Do nothing or log for testing purposes + return nil +} + +func TestNewListRHelper(t *testing.T) { + callback := mockCallback + helper := NewListRHelper(callback) + + assert.NotNil(t, helper) + assert.Equal(t, fmt.Sprintf("%p", callback), fmt.Sprintf("%p", helper.callback)) + assert.Empty(t, helper.entries) +} + +func TestListRHelperAdd(t *testing.T) { + callbackInvoked := false + callback := func(entries fs.DirEntries) error { + callbackInvoked = true + return nil + } + + helper := NewListRHelper(callback) + entry := mockobject.Object("A") + require.NoError(t, helper.Add(entry)) + + assert.Len(t, helper.entries, 1) + assert.False(t, callbackInvoked, "Callback should not be invoked before reaching 100 entries") + + // Check adding a nil entry doesn't change anything + require.NoError(t, helper.Add(nil)) + assert.Len(t, helper.entries, 1) + assert.False(t, callbackInvoked, "Callback should not be invoked before reaching 100 entries") +} + +func TestListRHelperSend(t *testing.T) { + entry := mockobject.Object("A") + callbackInvoked := false + callback := func(entries fs.DirEntries) error { + callbackInvoked = true + assert.Equal(t, 100, len(entries)) + for _, obj := range entries { + assert.Equal(t, entry, obj) + } + return nil + } + + helper := NewListRHelper(callback) + + // Add 100 entries to force the callback to be invoked + for i := 0; i < 100; i++ { + require.NoError(t, helper.Add(entry)) + } + + assert.Len(t, helper.entries, 0) + assert.True(t, callbackInvoked, "Callback should be invoked after 100 entries") +} + +func TestListRHelperFlush(t *testing.T) { + entry := mockobject.Object("A") + callbackInvoked := false + callback := func(entries fs.DirEntries) error { + callbackInvoked = true + assert.Equal(t, 1, len(entries)) + for _, obj := range entries { + assert.Equal(t, entry, obj) + } + return nil + } + + helper := NewListRHelper(callback) + require.NoError(t, helper.Add(entry)) + assert.False(t, callbackInvoked, "Callback should not have been invoked yet") + require.NoError(t, helper.Flush()) + + assert.True(t, callbackInvoked, "Callback should be invoked on flush") + assert.Len(t, helper.entries, 0, "Entries should be cleared after flush") +} diff --git a/fs/walk/walk.go b/fs/walk/walk.go index 51e567a4f..d1c8ab2c5 100644 --- a/fs/walk/walk.go +++ b/fs/walk/walk.go @@ -641,41 +641,3 @@ func GetAll(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel }) return } - -// ListRHelper is used in the implementation of ListR to accumulate DirEntries -type ListRHelper struct { - callback fs.ListRCallback - entries fs.DirEntries -} - -// NewListRHelper should be called from ListR with the callback passed in -func NewListRHelper(callback fs.ListRCallback) *ListRHelper { - return &ListRHelper{ - callback: callback, - } -} - -// send sends the stored entries to the callback if there are >= max -// entries. -func (lh *ListRHelper) send(max int) (err error) { - if len(lh.entries) >= max { - err = lh.callback(lh.entries) - lh.entries = lh.entries[:0] - } - return err -} - -// Add an entry to the stored entries and send them if there are more -// than a certain amount -func (lh *ListRHelper) Add(entry fs.DirEntry) error { - if entry == nil { - return nil - } - lh.entries = append(lh.entries, entry) - return lh.send(100) -} - -// Flush the stored entries (if any) sending them to the callback -func (lh *ListRHelper) Flush() error { - return lh.send(1) -}