extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-08-09 21:57:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

Re-implement sync routine to work a directory at a time

Browse Source 
Multiple directories (up to --checkers worth) are scanned at once.

This uses much less memory than the previous scheme - only the amount
of memory needed to hold an entire directory listing of objects.

For directory based remotes the speed is unchanged.

For bucket based remotes, instead of doing one API call to list the
whole bucket, it does multiple calls, one for each pseudo directory.
However these are done in parallel so in practice this seems to speed
up directory listings.

This replaces the existing sync method as it performs faster and uses
less memory.

The old sync method is available with the temporary --old-sync-method
flag.

Fixes #517
Fixes #439
Fixes #236
Fixes #1067
This commit is contained in:
Nick Craig-Wood
2017-01-25 19:59:53 +00:00
parent 7c6cd3a9e1
commit 48cdedc97b
2 changed files with 358 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
fs/sync_test.go
View File

@ -457,6 +457,23 @@ func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) {
fstest.CheckItems(t, r.fremote, file1, file3)
}
// Sync after removing a file and adding a file
func TestSyncAfterRemovingAFileAndAddingAFileSubDir(t *testing.T) {
r := NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1)
file2 := r.WriteObject("b/potato", "SMALLER BUT SAME DATE", t2)
file3 := r.WriteBoth("c/non empty space", "AhHa!", t2)
fstest.CheckItems(t, r.fremote, file2, file3)
fstest.CheckItems(t, r.flocal, file1, file3)
fs.Stats.ResetCounters()
err := fs.Sync(r.fremote, r.flocal)
require.NoError(t, err)
fstest.CheckItems(t, r.flocal, file1, file3)
fstest.CheckItems(t, r.fremote, file1, file3)
}
// Sync after removing a file and adding a file with IO Errors
func TestSyncAfterRemovingAFileAndAddingAFileWithErrors(t *testing.T) {
r := NewRun(t)