From 6fc88ff32e5f4fad26b3ab25ff0534296077aa6a Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 6 Jun 2017 23:04:01 +0100 Subject: [PATCH] Use --fast-list flag for sync/copy/move - fixes #1277 Redo test framework to take a -fast-list flag and test remotes with that flag. --- fs/operations_test.go | 2 + fs/sync.go | 44 +++++++++++++- fs/test_all.go | 129 +++++++++++++++++++++++++++++++++++------- 3 files changed, 152 insertions(+), 23 deletions(-) diff --git a/fs/operations_test.go b/fs/operations_test.go index 59cfa129b..c184a22cb 100644 --- a/fs/operations_test.go +++ b/fs/operations_test.go @@ -52,6 +52,7 @@ var ( DumpBodies = flag.Bool("dump-bodies", false, "Set to dump bodies (needs -verbose)") Individual = flag.Bool("individual", false, "Make individual bucket/container/directory for each test - much slower") LowLevelRetries = flag.Int("low-level-retries", 10, "Number of low level retries") + UseListR = flag.Bool("fast-list", false, "Use recursive list if available. Uses more memory but fewer transactions.") ) // Some times used in the tests @@ -113,6 +114,7 @@ func newRun() *Run { fs.Config.DumpHeaders = *DumpHeaders fs.Config.DumpBodies = *DumpBodies fs.Config.LowLevelRetries = *LowLevelRetries + fs.Config.UseListR = *UseListR var err error r.fremote, r.fremoteName, r.cleanRemote, err = fstest.RandomRemote(*RemoteName, *SubDir) if err != nil { diff --git a/fs/sync.go b/fs/sync.go index 754badd0a..50eef9476 100644 --- a/fs/sync.go +++ b/fs/sync.go @@ -49,6 +49,8 @@ type syncCopyMove struct { renameCheck []Object // accumulate files to check for rename here backupDir Fs // place to store overwrites/deletes suffix string // suffix to add to files placed in backupDir + srcListDir listDirFn // function to call to list a directory in the src + dstListDir listDirFn // function to call to list a directory in the dst } func newSyncCopyMove(fdst, fsrc Fs, deleteMode DeleteMode, DoMove bool) (*syncCopyMove, error) { @@ -117,9 +119,47 @@ func newSyncCopyMove(fdst, fsrc Fs, deleteMode DeleteMode, DoMove bool) (*syncCo } s.suffix = Config.Suffix } + s.srcListDir = s.makeListDir(fsrc, false) + s.dstListDir = s.makeListDir(fdst, Config.Filter.DeleteExcluded) return s, nil } +// list a directory into entries, err +type listDirFn func(dir string) (entries DirEntries, err error) + +// makeListDir makes a listing function for the given fs and includeAll flags +func (s *syncCopyMove) makeListDir(f Fs, includeAll bool) listDirFn { + if !Config.UseListR || f.Features().ListR == nil { + return func(dir string) (entries DirEntries, err error) { + return ListDirSorted(f, includeAll, dir) + } + } + var ( + mu sync.Mutex + started bool + dirs DirTree + dirsErr error + ) + return func(dir string) (entries DirEntries, err error) { + mu.Lock() + defer mu.Unlock() + if !started { + dirs, dirsErr = NewDirTree(f, s.dir, includeAll, Config.MaxDepth) + started = true + } + if dirsErr != nil { + return nil, dirsErr + } + entries, ok := dirs[dir] + if !ok { + err = ErrorDirNotFound + } else { + delete(dirs, dir) + } + return entries, err + } +} + // Check to see if have set the abort flag func (s *syncCopyMove) aborting() bool { select { @@ -985,14 +1025,14 @@ func (s *syncCopyMove) _runDirAtATime(job listDirJob) (jobs []listDirJob) { wg.Add(1) go func() { defer wg.Done() - srcList, srcListErr = ListDirSorted(s.fsrc, false, job.remote) + srcList, srcListErr = s.srcListDir(job.remote) }() } if !job.noDst { wg.Add(1) go func() { defer wg.Done() - dstList, dstListErr = ListDirSorted(s.fdst, Config.Filter.DeleteExcluded, job.remote) + dstList, dstListErr = s.dstListDir(job.remote) }() } diff --git a/fs/test_all.go b/fs/test_all.go index 0e1e02c9e..43d78f6b0 100644 --- a/fs/test_all.go +++ b/fs/test_all.go @@ -20,22 +20,84 @@ import ( "github.com/ncw/rclone/fstest" ) +type remoteConfig struct { + Name string + SubDir bool + FastList bool +} + var ( - remotes = []string{ - "TestAmazonCloudDrive:", - "TestB2:", - "TestCryptDrive:", - "TestCryptSwift:", - "TestDrive:", - "TestDropbox:", - "TestGoogleCloudStorage:", - "TestHubic:", - "TestOneDrive:", - "TestS3:", - "TestSftp:", - "TestSwift:", - "TestYandex:", - "TestFTP:", + remotes = []remoteConfig{ + { + Name: "TestAmazonCloudDrive:", + SubDir: false, + FastList: false, + }, + { + Name: "TestB2:", + SubDir: true, + FastList: true, + }, + { + Name: "TestCryptDrive:", + SubDir: false, + FastList: false, + }, + { + Name: "TestCryptSwift:", + SubDir: false, + FastList: false, + }, + { + Name: "TestDrive:", + SubDir: false, + FastList: false, + }, + { + Name: "TestDropbox:", + SubDir: false, + FastList: false, + }, + { + Name: "TestGoogleCloudStorage:", + SubDir: true, + FastList: true, + }, + { + Name: "TestHubic:", + SubDir: false, + FastList: false, + }, + { + Name: "TestOneDrive:", + SubDir: false, + FastList: false, + }, + { + Name: "TestS3:", + SubDir: true, + FastList: true, + }, + { + Name: "TestSftp:", + SubDir: false, + FastList: false, + }, + { + Name: "TestSwift:", + SubDir: true, + FastList: true, + }, + { + Name: "TestYandex:", + SubDir: false, + FastList: false, + }, + { + Name: "TestFTP:", + SubDir: false, + FastList: false, + }, } binary = "fs.test" // Flags @@ -60,7 +122,7 @@ type test struct { } // newTest creates a new test -func newTest(remote string, subdir bool) *test { +func newTest(remote string, subdir bool, fastlist bool) *test { t := &test{ remote: remote, subdir: subdir, @@ -77,6 +139,9 @@ func newTest(remote string, subdir bool) *test { if subdir { t.cmdLine = append(t.cmdLine, "-subdir") } + if fastlist { + t.cmdLine = append(t.cmdLine, "-fast-list") + } t.cmdString = strings.Join(t.cmdLine, " ") return t } @@ -223,9 +288,25 @@ func removeTestBinary() { func main() { flag.Parse() if *runTests != "" { - remotes = strings.Split(*runTests, ",") + newRemotes := []remoteConfig{} + for _, name := range strings.Split(*runTests, ",") { + for i := range remotes { + if remotes[i].Name == name { + newRemotes = append(newRemotes, remotes[i]) + goto found + } + } + log.Printf("Remote %q not found - inserting with default flags", name) + newRemotes = append(newRemotes, remoteConfig{Name: name}) + found: + } + remotes = newRemotes } - log.Printf("Testing remotes: %s", strings.Join(remotes, ", ")) + var names []string + for _, remote := range remotes { + names = append(names, remote.Name) + } + log.Printf("Testing remotes: %s", strings.Join(names, ", ")) start := time.Now() if *clean { @@ -239,9 +320,14 @@ func main() { results := make(chan *test, 8) awaiting := 0 for _, remote := range remotes { - awaiting += 2 - go newTest(remote, false).run(results) - go newTest(remote, true).run(results) + for _, subdir := range []bool{false, true} { + for _, fastlist := range []bool{false, true} { + if (!subdir || subdir && remote.SubDir) && (!fastlist || fastlist && remote.FastList) { + go newTest(remote.Name, subdir, fastlist).run(results) + awaiting++ + } + } + } } // Wait for the tests to finish @@ -261,6 +347,7 @@ func main() { log.Printf("FAIL: %d tests failed in %v", len(failed), duration) for _, t := range failed { log.Printf(" * %s", t.cmdString) + log.Printf(" * Failed tests: %v", t.failedTests) } os.Exit(1) }