extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-06-20 19:57:51 +02:00
Code Issues Packages Projects Releases Wiki Activity

march: split src and dst

Browse Source 
splits m.key into separate functions for src and dst to prepare for
lib/transform which will want to do transforms on the src side only.

Co-Authored-By: Nick Craig-Wood <nick@craig-wood.com>
This commit is contained in:
nielash 2025-05-04 02:50:47 -04:00 committed by Nick Craig-Wood
parent cf1f5a7af6
commit 41a407dcc9
2 changed files with 20 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
fs/march/march.go
View File

@ -60,9 +60,9 @@ type Marcher interface {
// Note: this will flag filter-aware backends on the source side // Note: this will flag filter-aware backends on the source side
func (m *March) init(ctx context.Context) { func (m *March) init(ctx context.Context) {
ci := fs.GetConfig(ctx) ci := fs.GetConfig(ctx)
m.srcListDir = m.makeListDir(ctx, m.Fsrc, m.SrcIncludeAll) m.srcListDir = m.makeListDir(ctx, m.Fsrc, m.SrcIncludeAll, m.srcKey)
if !m.NoTraverse { if !m.NoTraverse {
m.dstListDir = m.makeListDir(ctx, m.Fdst, m.DstIncludeAll) m.dstListDir = m.makeListDir(ctx, m.Fdst, m.DstIncludeAll, m.dstKey)
} }
// Now create the matching transform // Now create the matching transform
// ..normalise the UTF8 first // ..normalise the UTF8 first
@ -80,8 +80,8 @@ func (m *March) init(ctx context.Context) {
} }
} }
// key turns a directory entry into a sort key using the defined transforms. // srcKey turns a directory entry into a sort key using the defined transforms.
func (m *March) key(entry fs.DirEntry) string { func (m *March) srcKey(entry fs.DirEntry) string {
if entry == nil { if entry == nil {
return "" return ""
} }
@ -99,17 +99,22 @@ func (m *March) key(entry fs.DirEntry) string {
return name return name
} }
// dstKey turns a directory entry into a sort key using the defined transforms.
func (m *March) dstKey(entry fs.DirEntry) string {
return m.srcKey(entry) // FIXME actually do something different
}
// makeListDir makes constructs a listing function for the given fs // makeListDir makes constructs a listing function for the given fs
// and includeAll flags for marching through the file system. // and includeAll flags for marching through the file system.
// Note: this will optionally flag filter-aware backends! // Note: this will optionally flag filter-aware backends!
func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listDirFn { func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool, keyFn list.KeyFn) listDirFn {
ci := fs.GetConfig(ctx) ci := fs.GetConfig(ctx)
fi := filter.GetConfig(ctx) fi := filter.GetConfig(ctx)
if !(ci.UseListR && f.Features().ListR != nil) && // !--fast-list active and if !(ci.UseListR && f.Features().ListR != nil) && // !--fast-list active and
!(ci.NoTraverse && fi.HaveFilesFrom()) { // !(--files-from and --no-traverse) !(ci.NoTraverse && fi.HaveFilesFrom()) { // !(--files-from and --no-traverse)
return func(dir string, callback fs.ListRCallback) (err error) { return func(dir string, callback fs.ListRCallback) (err error) {
dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
return list.DirSortedFn(dirCtx, f, includeAll, dir, callback, m.key) return list.DirSortedFn(dirCtx, f, includeAll, dir, callback, keyFn)
} }
} }
@ -144,7 +149,7 @@ func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listD
// in syncing as it will use the first entry for the sync // in syncing as it will use the first entry for the sync
// comparison. // comparison.
slices.SortStableFunc(entries, func(a, b fs.DirEntry) int { slices.SortStableFunc(entries, func(a, b fs.DirEntry) int {
return cmp.Compare(m.key(a), m.key(b)) return cmp.Compare(keyFn(a), keyFn(b))
}) })
return callback(entries) return callback(entries)
} }
@ -297,11 +302,11 @@ func (m *March) matchListings(srcChan, dstChan <-chan fs.DirEntry, srcOnly, dstO
// Reload src and dst if needed - we set them to nil if used // Reload src and dst if needed - we set them to nil if used
if src == nil { if src == nil {
src = <-srcChan src = <-srcChan
srcName = m.key(src) srcName = m.srcKey(src)
} }
if dst == nil { if dst == nil {
dst = <-dstChan dst = <-dstChan
dstName = m.key(dst) dstName = m.dstKey(dst)
} }
if src == nil && dst == nil { if src == nil && dst == nil {
break break
@ -406,7 +411,7 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
if m.NoTraverse && !m.NoCheckDest { if m.NoTraverse && !m.NoCheckDest {
originalSrcChan := srcChan originalSrcChan := srcChan
srcChan = make(chan fs.DirEntry, 100) srcChan = make(chan fs.DirEntry, 100)
ls, err := list.NewSorter(m.Ctx, m.Fdst, list.SortToChan(dstChan), m.key) ls, err := list.NewSorter(m.Ctx, m.Fdst, list.SortToChan(dstChan), m.dstKey)
if err != nil { if err != nil {
return nil, err return nil, err
} }

6
fs/march/march_test.go
View File

@ -502,7 +502,11 @@ func TestMatchListings(t *testing.T) {
// Make a channel to send the source (0) or dest (1) using a list.Sorter // Make a channel to send the source (0) or dest (1) using a list.Sorter
makeChan := func(offset int) <-chan fs.DirEntry { makeChan := func(offset int) <-chan fs.DirEntry {
out := make(chan fs.DirEntry) out := make(chan fs.DirEntry)
ls, err := list.NewSorter(ctx, nil, list.SortToChan(out), m.key) key := m.dstKey
if offset == 0 {
key = m.srcKey
}
ls, err := list.NewSorter(ctx, nil, list.SortToChan(out), key)
require.NoError(t, err) require.NoError(t, err)
wg.Add(1) wg.Add(1)
go func() { go func() {