diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.go index 0b333b51d..c4896f65c 100644 --- a/cmd/bisync/bisync_test.go +++ b/cmd/bisync/bisync_test.go @@ -34,6 +34,7 @@ import ( "github.com/rclone/rclone/fs/object" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/sync" + "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/lib/atexit" "github.com/rclone/rclone/lib/encoder" @@ -58,6 +59,8 @@ const ( fixSlash = (runtime.GOOS == "windows") ) +var initDate = time.Date(2000, time.January, 1, 0, 0, 0, 0, bisync.TZ) + // logReplacements make modern test logs comparable with golden dir. // It is a string slice of even length with this structure: // @@ -87,9 +90,9 @@ var logReplacements = []string{ `^.*? for same-side diffs on .*?$`, dropMe, `^.*?Downloading hashes.*?$`, dropMe, // ignore timestamps in directory time updates - `^(INFO : .*?: Made directory with (metadata|modification time)).*$`, `$1`, + `^(INFO : .*?: (Made directory with|Set directory) (metadata|modification time)).*$`, dropMe, // ignore sizes in directory time updates - `^(NOTICE: .*?: Skipped set directory modification time as --dry-run is set).*$`, `$1`, + `^(NOTICE: .*?: Skipped set directory modification time as --dry-run is set).*$`, dropMe, } // Some dry-run messages differ depending on the particular remote. @@ -320,7 +323,6 @@ func (b *bisyncTest) runTestCase(ctx context.Context, t *testing.T, testCase str // For test stability, jam initial dates to a fixed past date. // Test cases that change files will touch specific files to fixed new dates. - initDate := time.Date(2000, time.January, 1, 0, 0, 0, 0, bisync.TZ) err = filepath.Walk(b.initDir, func(path string, info os.FileInfo, err error) error { if err == nil && !info.IsDir() { return os.Chtimes(path, initDate, initDate) @@ -868,6 +870,20 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) { } } + // set all dirs to a fixed date for test stability, as they are considered as of v1.66. + jamDirTimes := func(f fs.Fs) { + err := walk.ListR(ctx, f, "", true, -1, walk.ListDirs, func(entries fs.DirEntries) error { + var err error + entries.ForDir(func(dir fs.Directory) { + _, err = operations.SetDirModTime(ctx, f, dir, "", initDate) + }) + return err + }) + assert.NoError(b.t, err, "error jamming dirtimes") + } + jamDirTimes(fs1) + jamDirTimes(fs2) + output := bilib.CaptureOutput(func() { err = bisync.Bisync(octx, fs1, fs2, opt) }) @@ -1385,10 +1401,10 @@ func (b *bisyncTest) mangleListing(text string, golden bool, file string) string lines[i] = match[1] + "-" + match[3] + match[4] // replace it with "-" for comparison purposes (see #5679) } // account for modtime precision - var lineRegex = regexp.MustCompile(`^(\S) +(-?\d+) (\S+) (\S+) (\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{9}[+-]\d{4}) (".+")$`) + lineRegex := regexp.MustCompile(`^(\S) +(-?\d+) (\S+) (\S+) (\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{9}[+-]\d{4}) (".+")$`) const timeFormat = "2006-01-02T15:04:05.000000000-0700" const lineFormat = "%s %8d %s %s %s %q\n" - var TZ = time.UTC + TZ := time.UTC fields := lineRegex.FindStringSubmatch(strings.TrimSuffix(lines[i], "\n")) if fields != nil { sizeVal, sizeErr := strconv.ParseInt(fields[2], 10, 64) @@ -1509,7 +1525,8 @@ func (b *bisyncTest) listDir(dir string) (names []string) { ignoreIt := func(file string) bool { ignoreList := []string{ // ".lst-control", ".lst-dry-control", ".lst-old", ".lst-dry-old", - ".DS_Store"} + ".DS_Store", + } for _, s := range ignoreList { if strings.Contains(file, s) { return true diff --git a/cmd/bisync/testdata/test_all_changed/golden/test.log b/cmd/bisync/testdata/test_all_changed/golden/test.log index 39c281bcb..cb74904c9 100644 --- a/cmd/bisync/testdata/test_all_changed/golden/test.log +++ b/cmd/bisync/testdata/test_all_changed/golden/test.log @@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -63,8 +59,6 @@ INFO : - Path1 Queue copy to Path2 - { INFO : - Path1 Queue copy to Path2 - {path2/}file1.txt INFO : - Path1 Queue copy to Path2 - {path2/}subdir/file20.txt INFO : - Path1 Do queued copies to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -139,8 +133,6 @@ INFO : - Path1 Queue copy to Path2 - { INFO : - Path1 Queue copy to Path2 - {path2/}file1.txt INFO : - Path1 Queue copy to Path2 - {path2/}subdir/file20.txt INFO : - Path1 Do queued copies to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_basic/golden/test.log b/cmd/bisync/testdata/test_basic/golden/test.log index 912423c26..d3086e790 100644 --- a/cmd/bisync/testdata/test_basic/golden/test.log +++ b/cmd/bisync/testdata/test_basic/golden/test.log @@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -57,8 +53,6 @@ INFO : - Path1 Queue copy to Path2 - { INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt INFO : - Path2 Do queued copies to - Path1 INFO : - Path1 Do queued copies to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_check_access/golden/test.log b/cmd/bisync/testdata/test_check_access/golden/test.log index abc480241..4184a0f30 100644 --- a/cmd/bisync/testdata/test_check_access/golden/test.log +++ b/cmd/bisync/testdata/test_check_access/golden/test.log @@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -91,11 +87,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -188,11 +180,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_check_access_filters/golden/test.log b/cmd/bisync/testdata/test_check_access_filters/golden/test.log index 82cc20179..eeb347438 100644 --- a/cmd/bisync/testdata/test_check_access_filters/golden/test.log +++ b/cmd/bisync/testdata/test_check_access_filters/golden/test.log @@ -21,15 +21,7 @@ INFO : Using filters file {workdir/}exclude-other-filtersfile.txt INFO : Storing filters file hash to {workdir/}exclude-other-filtersfile.txt.md5 INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir/subdirB: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir/subdirB: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir/subdirB: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir/subdirB: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -144,23 +136,7 @@ INFO : Using filters file {workdir/}include-other-filtersfile.txt INFO : Storing filters file hash to {workdir/}include-other-filtersfile.txt.md5 INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdirX: Set directory modification time (using SetModTime) -INFO : subdirX/subdirX1: Set directory modification time (using SetModTime) -INFO : subdir/subdirB: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdirX: Set directory modification time (using SetModTime) -INFO : subdirX/subdirX1: Set directory modification time (using SetModTime) -INFO : subdir/subdirB: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdirX: Set directory modification time (using SetModTime) -INFO : subdir/subdirB: Set directory modification time (using SetModTime) -INFO : subdirX/subdirX1: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdirX: Set directory modification time (using SetModTime) -INFO : subdir/subdirB: Set directory modification time (using SetModTime) -INFO : subdirX/subdirX1: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_check_filename/golden/test.log b/cmd/bisync/testdata/test_check_filename/golden/test.log index 8315bfc32..573976ede 100644 --- a/cmd/bisync/testdata/test_check_filename/golden/test.log +++ b/cmd/bisync/testdata/test_check_filename/golden/test.log @@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -94,11 +90,7 @@ INFO : Copying Path2 files to Path1 INFO : Checking access health INFO : Found 2 matching ".chk_file" files on both paths INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log index 3cc3d7e0e..096c5f0c5 100644 --- a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log +++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log @@ -147,11 +147,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Made directory with metadata (mtime=2024-02-27T04:53:52.809861575-05:00) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_extended_char_paths/golden/test.log b/cmd/bisync/testdata/test_extended_char_paths/golden/test.log index db033110e..463e8e967 100644 --- a/cmd/bisync/testdata/test_extended_char_paths/golden/test.log +++ b/cmd/bisync/testdata/test_extended_char_paths/golden/test.log @@ -84,11 +84,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -133,11 +129,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -182,11 +174,7 @@ INFO : Using filters file {workdir/}測試_filtersfile.txt INFO : Storing filters file hash to {workdir/}測試_filtersfile.txt.md5 INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_extended_filenames/golden/test.log b/cmd/bisync/testdata/test_extended_filenames/golden/test.log index 37cae89d2..eda13848c 100644 --- a/cmd/bisync/testdata/test_extended_filenames/golden/test.log +++ b/cmd/bisync/testdata/test_extended_filenames/golden/test.log @@ -25,11 +25,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) -INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) -INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -116,15 +112,7 @@ INFO : - Path2 Queue copy to Path1 - " INFO : - Path1 Queue delete - {path1/}subdir_with_ࢺ_/filename_contains_ě_.txt INFO : - Path2 Queue copy to Path1 - {path1/}subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt INFO : - Path2 Do queued copies to - Path1 -INFO : subdir with␊white space.txt: Made directory with metadata (mtime=2024-02-27T04:53:52.913860529-05:00) -INFO : subdir_rawchars_␙__: Made directory with metadata (mtime=2024-02-27T04:53:52.913860529-05:00) -INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) -INFO : subdir with␊white space.txt: Set directory modification time (using SetModTime) -INFO : subdir_rawchars_␙__: Set directory modification time (using SetModTime) -INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) INFO : - Path1 Do queued copies to - Path2 -INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) -INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_filters/golden/test.log b/cmd/bisync/testdata/test_filters/golden/test.log index d74044ef1..9da9b2130 100644 --- a/cmd/bisync/testdata/test_filters/golden/test.log +++ b/cmd/bisync/testdata/test_filters/golden/test.log @@ -20,11 +20,7 @@ INFO : Using filters file {workdir/}filtersfile.flt INFO : Storing filters file hash to {workdir/}filtersfile.flt.md5 INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -58,8 +54,6 @@ INFO : Path2 checking for diffs INFO : Applying changes INFO : - Path1 Queue copy to Path2 - {path2/}subdir/fileZ.txt INFO : - Path1 Do queued copies to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log b/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log index 6a2583402..05c2abf23 100644 --- a/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log +++ b/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log @@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -85,11 +81,7 @@ INFO : Using filters file {workdir/}filtersfile.txt INFO : Storing filters file hash to {workdir/}filtersfile.txt.md5 INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -154,11 +146,7 @@ INFO : Using filters file {workdir/}filtersfile.txt INFO : Skipped storing filters file hash to {workdir/}filtersfile.txt.md5 as --dry-run is set INFO : Copying Path2 files to Path1 NOTICE: - Path2 Resync is copying files to - Path1 -NOTICE: subdir: Skipped set directory modification time as --dry-run is set -NOTICE: {path1String}: Skipped set directory modification time as --dry-run is set NOTICE: - Path1 Resync is copying files to - Path2 -NOTICE: subdir: Skipped set directory modification time as --dry-run is set -NOTICE: {path2String}: Skipped set directory modification time as --dry-run is set INFO : Resync updating listings INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log b/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log index c7f3c1bd7..f54cd7baf 100644 --- a/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log +++ b/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log @@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -37,11 +33,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -77,8 +69,6 @@ INFO : - Path1 Queue copy to Path2 - { INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt INFO : - Path2 Do queued copies to - Path1 INFO : - Path1 Do queued copies to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_normalization/golden/test.log b/cmd/bisync/testdata/test_normalization/golden/test.log index 98144aee1..5e4cbbccf 100644 --- a/cmd/bisync/testdata/test_normalization/golden/test.log +++ b/cmd/bisync/testdata/test_normalization/golden/test.log @@ -17,11 +17,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) -INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -74,12 +70,8 @@ INFO : - Path1 Queue copy to Path2 - " INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt INFO : - Path2 Do queued copies to - Path1 INFO : - Path1 Do queued copies to - Path2 -INFO : 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Made directory with metadata (mtime=2024-02-27T04:53:52.993859723-05:00) -INFO : folder: Set directory modification time (using SetModTime) INFO : folder/hello,WORLD!.txt: Fixed case by renaming to: folder/HeLlO,wOrLd!.txt INFO : folder/éééö.txt: Fixed case by renaming to: folder/éééö.txt -INFO : 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) -INFO : folder: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -158,11 +150,7 @@ INFO : - Path2 Queue copy to Path1 - { INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Руский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éö/測試_Руский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éö.txt" INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt INFO : - Path2 Do queued copies to - Path1 -INFO : folder: Set directory modification time (using SetModTime) -INFO : folder: Set directory modification time (using SetModTime) INFO : - Path1 Do queued copies to - Path2 -INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Made directory with metadata (mtime=2024-02-27T04:53:53.001859642-05:00) -INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -182,15 +170,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : folder: Set directory modification time (using SetModTime) -INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) -INFO : folder: Set directory modification time (using SetModTime) -INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : folder: Set directory modification time (using SetModTime) -INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) -INFO : folder: Set directory modification time (using SetModTime) -INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -230,10 +210,6 @@ INFO : - Path1 Queue copy to Path2 - " INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt INFO : - Path2 Do queued copies to - Path1 INFO : - Path1 Do queued copies to - Path2 -INFO : folder: Set directory modification time (using SetModTime) -INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) -INFO : folder: Set directory modification time (using SetModTime) -INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_rclone_args/golden/test.log b/cmd/bisync/testdata/test_rclone_args/golden/test.log index 4fbe6c385..73ede1e69 100644 --- a/cmd/bisync/testdata/test_rclone_args/golden/test.log +++ b/cmd/bisync/testdata/test_rclone_args/golden/test.log @@ -15,11 +15,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -111,11 +107,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -149,8 +141,6 @@ INFO : Applying changes INFO : - Path2 Queue copy to Path1 - {path1/}file2.txt INFO : - Path2 Queue copy to Path1 - {path1/}subdir/file21.txt INFO : - Path2 Do queued copies to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -168,11 +158,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -214,11 +200,7 @@ INFO : - Path1 Queue copy to Path2 - { INFO : - Path2 Queue copy to Path1 - {path1/}file2.txt INFO : - Path2 Queue copy to Path1 - {path1/}subdir/file21.txt INFO : - Path2 Do queued copies to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Do queued copies to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_rmdirs/golden/test.log b/cmd/bisync/testdata/test_rmdirs/golden/test.log index 899e9a5f3..4e6763801 100644 --- a/cmd/bisync/testdata/test_rmdirs/golden/test.log +++ b/cmd/bisync/testdata/test_rmdirs/golden/test.log @@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -49,8 +45,6 @@ INFO : Path2 checking for diffs INFO : Applying changes INFO : - Path2 Queue delete - {path2/}subdir/file20.txt INFO : - Path1 Do queued copies to - Path2 -INFO : subdir: Set directory modification time (using SetModTime) -INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index ce891f2b4..206888fc3 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -55,7 +55,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) { flags.StringVarP(flagSet, &tempDir, "temp-dir", "", os.TempDir(), "Directory rclone will use for temporary files", "Config") flags.BoolVarP(flagSet, &ci.CheckSum, "checksum", "c", ci.CheckSum, "Check for changes with size & checksum (if available, or fallback to size only).", "Copy") flags.BoolVarP(flagSet, &ci.SizeOnly, "size-only", "", ci.SizeOnly, "Skip based on size only, not modtime or checksum", "Copy") - flags.BoolVarP(flagSet, &ci.IgnoreTimes, "ignore-times", "I", ci.IgnoreTimes, "Don't skip files that match size and time - transfer all files", "Copy") + flags.BoolVarP(flagSet, &ci.IgnoreTimes, "ignore-times", "I", ci.IgnoreTimes, "Don't skip items that match size and time - transfer all unconditionally", "Copy") flags.BoolVarP(flagSet, &ci.IgnoreExisting, "ignore-existing", "", ci.IgnoreExisting, "Skip all files that exist on destination", "Copy") flags.BoolVarP(flagSet, &ci.IgnoreErrors, "ignore-errors", "", ci.IgnoreErrors, "Delete even if there are I/O errors", "Sync") flags.BoolVarP(flagSet, &ci.DryRun, "dry-run", "n", ci.DryRun, "Do a trial run with no permanent changes", "Config,Important") diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 125608fcf..b9eed93a1 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -141,6 +141,40 @@ func Equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool { return equal(ctx, src, dst, defaultEqualOpt(ctx)) } +// DirsEqual is like Equal but for dirs instead of objects. +// It returns true if two dirs should be considered "equal" for the purposes of syncCopyMove +// (in other words, true == "skip updating modtime/metadata for this dir".) +// Unlike Equal, it does not consider size or checksum, as these do not apply to directories. +func DirsEqual(ctx context.Context, src, dst fs.Directory, opt DirsEqualOpt) (equal bool) { + if dst == nil { + return false + } + ci := fs.GetConfig(ctx) + if ci.SizeOnly || ci.Immutable || ci.IgnoreExisting || opt.ModifyWindow == fs.ModTimeNotSupported { + return true + } + if ci.IgnoreTimes { + return false + } + if !(opt.SetDirModtime || opt.SetDirMetadata) { + return true + } + srcModTime, dstModTime := src.ModTime(ctx), dst.ModTime(ctx) + if srcModTime.IsZero() || dstModTime.IsZero() { + return false + } + dt := dstModTime.Sub(srcModTime) + if dt < opt.ModifyWindow && dt > -opt.ModifyWindow { + fs.Debugf(dst, "Directory modification time the same (differ by %s, within tolerance %s)", dt, opt.ModifyWindow) + return true + } + if ci.UpdateOlder && dt >= opt.ModifyWindow { + fs.Debugf(dst, "Destination directory is newer than source, skipping") + return true + } + return false +} + // sizeDiffers compare the size of src and dst taking into account the // various ways of ignoring sizes func sizeDiffers(ctx context.Context, src, dst fs.ObjectInfo) bool { @@ -172,6 +206,13 @@ func defaultEqualOpt(ctx context.Context) equalOpt { } } +// DirsEqualOpt represents options for DirsEqual function() +type DirsEqualOpt struct { + ModifyWindow time.Duration // Max time diff to be considered the same + SetDirModtime bool // whether to consider dir modtime + SetDirMetadata bool // whether to consider dir metadata +} + var modTimeUploadOnce sync.Once // emit a log if we are about to upload a file to set its modification time diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 15143ee92..4f4cf4cfd 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -1754,3 +1754,74 @@ func TestSetDirModTime(t *testing.T) { } fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2) } + +func TestDirsEqual(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + r := fstest.NewRun(t) + if !r.Fremote.Features().WriteDirMetadata && r.Fremote.Features().MkdirMetadata == nil { + t.Skip("Skipping test as remote does not support WriteDirMetadata or MkdirMetadata") + } + + opt := operations.DirsEqualOpt{ + ModifyWindow: fs.GetModifyWindow(ctx, r.Flocal, r.Fremote), + SetDirModtime: true, + SetDirMetadata: true, + } + + // Create a source local directory with metadata + src, err := operations.MkdirMetadata(ctx, r.Flocal, "dir with metadata to be copied", testMetadata) + require.NoError(t, err) + require.NotNil(t, src) + + // try with nil dst -- should be false + equal := operations.DirsEqual(ctx, src, nil, opt) + assert.False(t, equal) + + // make a dest with an equal modtime + dst, err := operations.MkdirModTime(ctx, r.Fremote, "dst", src.ModTime(ctx)) + require.NoError(t, err) + + // try with equal modtimes -- should be true + equal = operations.DirsEqual(ctx, src, dst, opt) + assert.True(t, equal) + + // try with unequal modtimes -- should be false + dst, err = operations.SetDirModTime(ctx, r.Fremote, dst, "", t2) + require.NoError(t, err) + equal = operations.DirsEqual(ctx, src, dst, opt) + assert.False(t, equal) + + // try with unequal modtimes that are within modify window -- should be true + halfWindow := opt.ModifyWindow / 2 + dst, err = operations.SetDirModTime(ctx, r.Fremote, dst, "", src.ModTime(ctx).Add(halfWindow)) + require.NoError(t, err) + equal = operations.DirsEqual(ctx, src, dst, opt) + assert.True(t, equal) + + // test ignoretimes -- should be false + ci.IgnoreTimes = true + equal = operations.DirsEqual(ctx, src, dst, opt) + assert.False(t, equal) + + // test immutable -- should be true + ci.IgnoreTimes = false + ci.Immutable = true + dst, err = operations.SetDirModTime(ctx, r.Fremote, dst, "", t3) + require.NoError(t, err) + equal = operations.DirsEqual(ctx, src, dst, opt) + assert.True(t, equal) + + // test dst newer than src with --update -- should be true + ci.Immutable = false + ci.UpdateOlder = true + equal = operations.DirsEqual(ctx, src, dst, opt) + assert.True(t, equal) + + // test no SetDirModtime or SetDirMetadata -- should be true + ci.UpdateOlder = false + opt.SetDirMetadata, opt.SetDirModtime = false, false + equal = operations.DirsEqual(ctx, src, dst, opt) + assert.True(t, equal) +} diff --git a/fs/sync/sync.go b/fs/sync/sync.go index 0ca8e1d27..9dac7c694 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -89,9 +89,10 @@ type syncCopyMove struct { setDirMetadata bool // if set we set the directory metadata setDirModTime bool // if set we set the directory modtimes setDirModTimeAfter bool // if set we set the directory modtimes at the end of the sync - setDirModTimeMu sync.Mutex // protect setDirModTimeMu + setDirModTimeMu sync.Mutex // protect setDirModTimes and modifiedDirs setDirModTimes []setDirModTime // directories that need their modtime set setDirModTimesMaxLevel int // max level of the directories to set + modifiedDirs map[string]struct{} // dirs with changed contents (if s.setDirModTimeAfter) } // For keeping track of delayed modtime sets @@ -155,6 +156,7 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete setDirMetadata: ci.Metadata && fsrc.Features().ReadDirMetadata && fdst.Features().WriteDirMetadata, setDirModTime: fdst.Features().WriteDirSetModTime || fdst.Features().MkdirMetadata != nil || fdst.Features().DirSetModTime != nil, setDirModTimeAfter: fdst.Features().DirModTimeUpdatesOnWrite, + modifiedDirs: make(map[string]struct{}), } s.logger, s.usingLogger = operations.GetLogger(ctx) @@ -399,6 +401,11 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.W fs.Errorf(pair.Dst, "Source and destination exist but do not match: %v", err) s.processError(err) } else { + if pair.Dst != nil { + s.markDirModifiedObject(pair.Dst) + } else { + s.markDirModifiedObject(src) + } // If destination already exists, then we must move it into --backup-dir if required if pair.Dst != nil && s.backupDir != nil { err := operations.MoveBackupDir(s.ctx, s.backupDir, pair.Dst) @@ -764,7 +771,6 @@ func (s *syncCopyMove) renameID(obj fs.Object, renamesStrategy trackRenamesStrat if renamesStrategy.hash() { var err error hash, err := obj.Hash(s.ctx, s.commonHash) - if err != nil { fs.Debugf(obj, "Hash failed: %v", err) return "" @@ -1079,6 +1085,26 @@ func (s *syncCopyMove) DstOnly(dst fs.DirEntry) (recurse bool) { return false } +// keeps track of dirs with changed contents, to avoid setting modtimes on dirs that haven't changed +func (s *syncCopyMove) markDirModified(dir string) { + if !s.setDirModTimeAfter { + return + } + s.setDirModTimeMu.Lock() + defer s.setDirModTimeMu.Unlock() + s.modifiedDirs[dir] = struct{}{} +} + +// like markDirModified, but accepts an Object instead of a string. +// the marked dir will be this object's parent. +func (s *syncCopyMove) markDirModifiedObject(o fs.Object) { + dir := path.Dir(o.Remote()) + if dir == "." { + dir = "" + } + s.markDirModified(dir) +} + // copyDirMetadata copies the src directory modTime or Metadata to dst // or f if nil. If dst is nil then it uses dir as the name of the new // directory. @@ -1087,17 +1113,25 @@ func (s *syncCopyMove) DstOnly(dst fs.DirEntry) (recurse bool) { // be nil. func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory) { var err error - if s.setDirMetadata { - newDst, err = operations.CopyDirMetadata(ctx, f, dst, dir, src) - } else if s.setDirModTime { - if dst == nil { - newDst, err = operations.MkdirModTime(ctx, f, dir, src.ModTime(ctx)) - } else { - newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx)) + equal := operations.DirsEqual(ctx, src, dst, operations.DirsEqualOpt{ModifyWindow: s.modifyWindow, SetDirModtime: s.setDirModTime, SetDirMetadata: s.setDirMetadata}) + if !s.setDirModTimeAfter && equal { + return nil + } + if s.setDirModTimeAfter && equal { + newDst = dst + } else { + if s.setDirMetadata { + newDst, err = operations.CopyDirMetadata(ctx, f, dst, dir, src) + } else if s.setDirModTime { + if dst == nil { + newDst, err = operations.MkdirModTime(ctx, f, dir, src.ModTime(ctx)) + } else { + newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx)) + } + } else if dst == nil { + // Create the directory if it doesn't exist + err = operations.Mkdir(ctx, f, dir) } - } else if dst == nil { - // Create the directory if it doesn't exist - err = operations.Mkdir(ctx, f, dir) } // If we need to set modtime after and we created a dir, then save it for later if s.setDirModTime && s.setDirModTimeAfter && err == nil { @@ -1138,7 +1172,7 @@ func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error { // Timestamp all directories at the same level in parallel, deepest first // We do this by iterating the slice multiple times to save memory // There could be a lot of directories in this slice. - var errCount = errcount.New() + errCount := errcount.New() for level := s.setDirModTimesMaxLevel; level >= 0; level-- { g, gCtx := errgroup.WithContext(ctx) g.SetLimit(s.ci.Checkers) @@ -1151,6 +1185,9 @@ func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error { break } item := item + if _, ok := s.modifiedDirs[item.dir]; !ok { + continue + } g.Go(func() error { _, err := operations.SetDirModTime(gCtx, s.fdst, item.dst, item.dir, item.modTime) if err != nil { @@ -1201,6 +1238,7 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) { if !NoNeedTransfer { // No need to check since doesn't exist fs.Debugf(src, "Need to transfer - File not found at Destination") + s.markDirModifiedObject(x) ok := s.toBeUploaded.Put(s.inCtx, fs.ObjectPair{Src: x, Dst: nil}) if !ok { return @@ -1217,6 +1255,7 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) { s.srcEmptyDirsMu.Unlock() // Create the directory and make sure the Metadata/ModTime is correct + s.markDirModified(x.Remote()) s.copyDirMetadata(s.ctx, s.fdst, nil, x.Remote(), x) return true default: diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index 1f21806fc..150051062 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -19,6 +19,7 @@ import ( mutex "sync" // renamed as "sync" already in use _ "github.com/rclone/rclone/backend/all" // import all backends + "github.com/rclone/rclone/cmd/bisync/bilib" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/filter" @@ -2499,6 +2500,74 @@ func TestSyncConcurrentTruncate(t *testing.T) { testSyncConcurrent(t, "truncate") } +// Tests that nothing is transferred when src and dst already match +// Run the same sync twice, ensure no action is taken the second time +func TestNothingToTransfer(t *testing.T) { + accounting.GlobalStats().ResetCounters() + ctx, _ := fs.AddConfig(context.Background()) + r := fstest.NewRun(t) + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + file2 := r.WriteFile("sub dir2/very/very/very/very/very/nested/subdir/hello world", "hello world", t1) + r.CheckLocalItems(t, file1, file2) + _, err := operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t2) + if err != nil && !errors.Is(err, fs.ErrorNotImplemented) { + require.NoError(t, err) + } + r.Mkdir(ctx, r.Fremote) + _, err = operations.MkdirModTime(ctx, r.Fremote, "sub dir", t3) + require.NoError(t, err) + + // set logging + // (this checks log output as DirModtime operations do not yet have stats, and r.CheckDirectoryModTimes also does not tell us what actions were taken) + oldLogLevel := fs.GetConfig(context.Background()).LogLevel + defer func() { fs.GetConfig(context.Background()).LogLevel = oldLogLevel }() // reset to old val after test + // need to do this as fs.Infof only respects the globalConfig + fs.GetConfig(context.Background()).LogLevel = fs.LogLevelInfo + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + output := bilib.CaptureOutput(func() { + err = CopyDir(ctx, r.Fremote, r.Flocal, true) + require.NoError(t, err) + }) + require.NotNil(t, output) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t, file1, file2) + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, "sub dir") + + // check that actions were taken + assert.True(t, strings.Contains(string(output), "Copied"), `expected to find at least one "Copied" log: `+string(output)) + if r.Fremote.Features().DirSetModTime != nil || r.Fremote.Features().MkdirMetadata != nil { + assert.True(t, strings.Contains(string(output), "Set directory modification time"), `expected to find at least one "Set directory modification time" log: `+string(output)) + } + assert.False(t, strings.Contains(string(output), "There was nothing to transfer"), `expected to find no "There was nothing to transfer" logs, but found one: `+string(output)) + assert.Equal(t, int64(2), accounting.GlobalStats().GetTransfers()) + + // run it again and make sure no actions were taken + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + output = bilib.CaptureOutput(func() { + err = CopyDir(ctx, r.Fremote, r.Flocal, true) + require.NoError(t, err) + }) + require.NotNil(t, output) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t, file1, file2) + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, "sub dir") + + // check that actions were NOT taken + assert.False(t, strings.Contains(string(output), "Copied"), `expected to find no "Copied" logs, but found one: `+string(output)) + if r.Fremote.Features().DirSetModTime != nil || r.Fremote.Features().MkdirMetadata != nil { + assert.False(t, strings.Contains(string(output), "Set directory modification time"), `expected to find no "Set directory modification time" logs, but found one: `+string(output)) + } + assert.True(t, strings.Contains(string(output), "There was nothing to transfer"), `expected to find a "There was nothing to transfer" log: `+string(output)) + assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers()) +} + // for testing logger: func predictDstFromLogger(ctx context.Context) context.Context { opt := operations.NewLoggerOpt()