From f5f86786b27290964d3b6b29790f1d245f66aa1d Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 6 Feb 2024 16:00:34 +0000 Subject: [PATCH] sync: implement directory sync for mod times and metadata Directory mod times are synced by default if the backend is capable and directory metadata is synced if the --metadata flag is provided and the backend is capable. This updates the bisync golden tests also which were affected by --dry-run setting of directory modtimes. Fixes #6685 --- cmd/bisync/bisync_test.go | 7 + .../testdata/test_all_changed/golden/test.log | 8 + .../testdata/test_basic/golden/test.log | 6 + .../test_check_access/golden/test.log | 12 ++ .../test_check_access_filters/golden/test.log | 24 +++ .../test_check_filename/golden/test.log | 8 + .../test_createemptysrcdirs/golden/test.log | 4 + .../test_extended_char_paths/golden/test.log | 12 ++ .../test_extended_filenames/golden/test.log | 12 ++ .../testdata/test_filters/golden/test.log | 6 + .../test_filtersfile_checks/golden/test.log | 12 ++ .../golden/test.log | 10 ++ .../test_normalization/golden/test.log | 24 +++ .../testdata/test_rclone_args/golden/test.log | 18 ++ .../testdata/test_resolve/golden/test.log | 2 +- .../testdata/test_rmdirs/golden/test.log | 6 + cmd/copy/copy.go | 9 + cmd/move/move.go | 8 + cmd/sync/sync.go | 8 + docs/content/docs.md | 19 +- fs/sync/sync.go | 122 ++++++++++++- fs/sync/sync_test.go | 165 +++++++++++++++++- fstest/run.go | 16 ++ 23 files changed, 504 insertions(+), 14 deletions(-) diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.go index e9307d9ce..0b333b51d 100644 --- a/cmd/bisync/bisync_test.go +++ b/cmd/bisync/bisync_test.go @@ -86,6 +86,10 @@ var logReplacements = []string{ `^.*?"SlowHashDetected":.*?$`, dropMe, `^.*? for same-side diffs on .*?$`, dropMe, `^.*?Downloading hashes.*?$`, dropMe, + // ignore timestamps in directory time updates + `^(INFO : .*?: Made directory with (metadata|modification time)).*$`, `$1`, + // ignore sizes in directory time updates + `^(NOTICE: .*?: Skipped set directory modification time as --dry-run is set).*$`, `$1`, } // Some dry-run messages differ depending on the particular remote. @@ -121,6 +125,9 @@ var logHoppers = []string{ // order of files re-checked prior to a conflict rename `ERROR : .*: md5 differ.*`, + + // Directory modification time setting can happen in any order + `INFO : .*: (Set directory modification time|Made directory with metadata).*`, } // Some log lines can contain Windows path separator that must be diff --git a/cmd/bisync/testdata/test_all_changed/golden/test.log b/cmd/bisync/testdata/test_all_changed/golden/test.log index cb74904c9..39c281bcb 100644 --- a/cmd/bisync/testdata/test_all_changed/golden/test.log +++ b/cmd/bisync/testdata/test_all_changed/golden/test.log @@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -59,6 +63,8 @@ INFO : - Path1 Queue copy to Path2 - { INFO : - Path1 Queue copy to Path2 - {path2/}file1.txt INFO : - Path1 Queue copy to Path2 - {path2/}subdir/file20.txt INFO : - Path1 Do queued copies to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -133,6 +139,8 @@ INFO : - Path1 Queue copy to Path2 - { INFO : - Path1 Queue copy to Path2 - {path2/}file1.txt INFO : - Path1 Queue copy to Path2 - {path2/}subdir/file20.txt INFO : - Path1 Do queued copies to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_basic/golden/test.log b/cmd/bisync/testdata/test_basic/golden/test.log index d3086e790..912423c26 100644 --- a/cmd/bisync/testdata/test_basic/golden/test.log +++ b/cmd/bisync/testdata/test_basic/golden/test.log @@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -53,6 +57,8 @@ INFO : - Path1 Queue copy to Path2 - { INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt INFO : - Path2 Do queued copies to - Path1 INFO : - Path1 Do queued copies to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_check_access/golden/test.log b/cmd/bisync/testdata/test_check_access/golden/test.log index 4184a0f30..abc480241 100644 --- a/cmd/bisync/testdata/test_check_access/golden/test.log +++ b/cmd/bisync/testdata/test_check_access/golden/test.log @@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -87,7 +91,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -180,7 +188,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_check_access_filters/golden/test.log b/cmd/bisync/testdata/test_check_access_filters/golden/test.log index eeb347438..82cc20179 100644 --- a/cmd/bisync/testdata/test_check_access_filters/golden/test.log +++ b/cmd/bisync/testdata/test_check_access_filters/golden/test.log @@ -21,7 +21,15 @@ INFO : Using filters file {workdir/}exclude-other-filtersfile.txt INFO : Storing filters file hash to {workdir/}exclude-other-filtersfile.txt.md5 INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir/subdirB: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir/subdirB: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir/subdirB: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir/subdirB: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -136,7 +144,23 @@ INFO : Using filters file {workdir/}include-other-filtersfile.txt INFO : Storing filters file hash to {workdir/}include-other-filtersfile.txt.md5 INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdirX: Set directory modification time (using SetModTime) +INFO : subdirX/subdirX1: Set directory modification time (using SetModTime) +INFO : subdir/subdirB: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdirX: Set directory modification time (using SetModTime) +INFO : subdirX/subdirX1: Set directory modification time (using SetModTime) +INFO : subdir/subdirB: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdirX: Set directory modification time (using SetModTime) +INFO : subdir/subdirB: Set directory modification time (using SetModTime) +INFO : subdirX/subdirX1: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdirX: Set directory modification time (using SetModTime) +INFO : subdir/subdirB: Set directory modification time (using SetModTime) +INFO : subdirX/subdirX1: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_check_filename/golden/test.log b/cmd/bisync/testdata/test_check_filename/golden/test.log index 573976ede..8315bfc32 100644 --- a/cmd/bisync/testdata/test_check_filename/golden/test.log +++ b/cmd/bisync/testdata/test_check_filename/golden/test.log @@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -90,7 +94,11 @@ INFO : Copying Path2 files to Path1 INFO : Checking access health INFO : Found 2 matching ".chk_file" files on both paths INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log index 096c5f0c5..3cc3d7e0e 100644 --- a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log +++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log @@ -147,7 +147,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Made directory with metadata (mtime=2024-02-27T04:53:52.809861575-05:00) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_extended_char_paths/golden/test.log b/cmd/bisync/testdata/test_extended_char_paths/golden/test.log index 463e8e967..db033110e 100644 --- a/cmd/bisync/testdata/test_extended_char_paths/golden/test.log +++ b/cmd/bisync/testdata/test_extended_char_paths/golden/test.log @@ -84,7 +84,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -129,7 +133,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -174,7 +182,11 @@ INFO : Using filters file {workdir/}測試_filtersfile.txt INFO : Storing filters file hash to {workdir/}測試_filtersfile.txt.md5 INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_extended_filenames/golden/test.log b/cmd/bisync/testdata/test_extended_filenames/golden/test.log index eda13848c..37cae89d2 100644 --- a/cmd/bisync/testdata/test_extended_filenames/golden/test.log +++ b/cmd/bisync/testdata/test_extended_filenames/golden/test.log @@ -25,7 +25,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) +INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) +INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -112,7 +116,15 @@ INFO : - Path2 Queue copy to Path1 - " INFO : - Path1 Queue delete - {path1/}subdir_with_ࢺ_/filename_contains_ě_.txt INFO : - Path2 Queue copy to Path1 - {path1/}subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt INFO : - Path2 Do queued copies to - Path1 +INFO : subdir with␊white space.txt: Made directory with metadata (mtime=2024-02-27T04:53:52.913860529-05:00) +INFO : subdir_rawchars_␙__: Made directory with metadata (mtime=2024-02-27T04:53:52.913860529-05:00) +INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) +INFO : subdir with␊white space.txt: Set directory modification time (using SetModTime) +INFO : subdir_rawchars_␙__: Set directory modification time (using SetModTime) +INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) INFO : - Path1 Do queued copies to - Path2 +INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) +INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_filters/golden/test.log b/cmd/bisync/testdata/test_filters/golden/test.log index 9da9b2130..d74044ef1 100644 --- a/cmd/bisync/testdata/test_filters/golden/test.log +++ b/cmd/bisync/testdata/test_filters/golden/test.log @@ -20,7 +20,11 @@ INFO : Using filters file {workdir/}filtersfile.flt INFO : Storing filters file hash to {workdir/}filtersfile.flt.md5 INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -54,6 +58,8 @@ INFO : Path2 checking for diffs INFO : Applying changes INFO : - Path1 Queue copy to Path2 - {path2/}subdir/fileZ.txt INFO : - Path1 Do queued copies to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log b/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log index a60aa5d49..28f4cfa89 100644 --- a/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log +++ b/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log @@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -81,7 +85,11 @@ INFO : Using filters file {workdir/}filtersfile.txt INFO : Storing filters file hash to {workdir/}filtersfile.txt.md5 INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -146,7 +154,11 @@ INFO : Using filters file {workdir/}filtersfile.txt INFO : Skipped storing filters file hash to {workdir/}filtersfile.txt.md5 as --dry-run is set INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +NOTICE: subdir: Skipped set directory modification time as --dry-run is set (size 4Ki) +NOTICE: {path1String}: Skipped set directory modification time as --dry-run is set INFO : - Path1 Resync is copying files to - Path2 +NOTICE: subdir: Skipped set directory modification time as --dry-run is set (size 4Ki) +NOTICE: {path2String}: Skipped set directory modification time as --dry-run is set INFO : Resync updating listings INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log b/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log index f54cd7baf..c7f3c1bd7 100644 --- a/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log +++ b/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log @@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -33,7 +37,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -69,6 +77,8 @@ INFO : - Path1 Queue copy to Path2 - { INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt INFO : - Path2 Do queued copies to - Path1 INFO : - Path1 Do queued copies to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_normalization/golden/test.log b/cmd/bisync/testdata/test_normalization/golden/test.log index 5e4cbbccf..98144aee1 100644 --- a/cmd/bisync/testdata/test_normalization/golden/test.log +++ b/cmd/bisync/testdata/test_normalization/golden/test.log @@ -17,7 +17,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) +INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -70,8 +74,12 @@ INFO : - Path1 Queue copy to Path2 - " INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt INFO : - Path2 Do queued copies to - Path1 INFO : - Path1 Do queued copies to - Path2 +INFO : 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Made directory with metadata (mtime=2024-02-27T04:53:52.993859723-05:00) +INFO : folder: Set directory modification time (using SetModTime) INFO : folder/hello,WORLD!.txt: Fixed case by renaming to: folder/HeLlO,wOrLd!.txt INFO : folder/éééö.txt: Fixed case by renaming to: folder/éééö.txt +INFO : 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) +INFO : folder: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -150,7 +158,11 @@ INFO : - Path2 Queue copy to Path1 - { INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Руский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éö/測試_Руский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éö.txt" INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt INFO : - Path2 Do queued copies to - Path1 +INFO : folder: Set directory modification time (using SetModTime) +INFO : folder: Set directory modification time (using SetModTime) INFO : - Path1 Do queued copies to - Path2 +INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Made directory with metadata (mtime=2024-02-27T04:53:53.001859642-05:00) +INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -170,7 +182,15 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : folder: Set directory modification time (using SetModTime) +INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) +INFO : folder: Set directory modification time (using SetModTime) +INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : folder: Set directory modification time (using SetModTime) +INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) +INFO : folder: Set directory modification time (using SetModTime) +INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -210,6 +230,10 @@ INFO : - Path1 Queue copy to Path2 - " INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt INFO : - Path2 Do queued copies to - Path1 INFO : - Path1 Do queued copies to - Path2 +INFO : folder: Set directory modification time (using SetModTime) +INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) +INFO : folder: Set directory modification time (using SetModTime) +INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_rclone_args/golden/test.log b/cmd/bisync/testdata/test_rclone_args/golden/test.log index 73ede1e69..4fbe6c385 100644 --- a/cmd/bisync/testdata/test_rclone_args/golden/test.log +++ b/cmd/bisync/testdata/test_rclone_args/golden/test.log @@ -15,7 +15,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -107,7 +111,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -141,6 +149,8 @@ INFO : Applying changes INFO : - Path2 Queue copy to Path1 - {path1/}file2.txt INFO : - Path2 Queue copy to Path1 - {path1/}subdir/file21.txt INFO : - Path2 Do queued copies to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -158,7 +168,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -200,7 +214,11 @@ INFO : - Path1 Queue copy to Path2 - { INFO : - Path2 Queue copy to Path1 - {path1/}file2.txt INFO : - Path2 Queue copy to Path1 - {path1/}subdir/file21.txt INFO : - Path2 Do queued copies to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Do queued copies to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_resolve/golden/test.log b/cmd/bisync/testdata/test_resolve/golden/test.log index 9223c3f6a..dfc303f07 100644 --- a/cmd/bisync/testdata/test_resolve/golden/test.log +++ b/cmd/bisync/testdata/test_resolve/golden/test.log @@ -283,8 +283,8 @@ INFO : Path2: 2 changes:  1 new,  1 modified,  INFO : (Modified:  1 newer,  0 older,  1 larger,  0 smaller) INFO : Applying changes INFO : Checking potential conflicts... -ERROR : file2.txt: md5 differ ERROR : file1.txt: md5 differ +ERROR : file2.txt: md5 differ NOTICE: {path2String}: 2 differences found NOTICE: {path2String}: 2 errors while checking INFO : Finished checking the potential conflicts. 2 differences found diff --git a/cmd/bisync/testdata/test_rmdirs/golden/test.log b/cmd/bisync/testdata/test_rmdirs/golden/test.log index 4e6763801..899e9a5f3 100644 --- a/cmd/bisync/testdata/test_rmdirs/golden/test.log +++ b/cmd/bisync/testdata/test_rmdirs/golden/test.log @@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings: INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Copying Path2 files to Path1 INFO : - Path2 Resync is copying files to - Path1 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : - Path1 Resync is copying files to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Resync updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful @@ -45,6 +49,8 @@ INFO : Path2 checking for diffs INFO : Applying changes INFO : - Path2 Queue delete - {path2/}subdir/file20.txt INFO : - Path1 Do queued copies to - Path2 +INFO : subdir: Set directory modification time (using SetModTime) +INFO : subdir: Set directory modification time (using SetModTime) INFO : Updating listings INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful diff --git a/cmd/copy/copy.go b/cmd/copy/copy.go index 745811de4..6a68a330c 100644 --- a/cmd/copy/copy.go +++ b/cmd/copy/copy.go @@ -79,6 +79,15 @@ recently very efficiently like this: rclone copy --max-age 24h --no-traverse /path/to/src remote: + +Rclone will sync the modification times of files and directories if +the backend supports it. If metadata syncing is required then use the +|--metadata| flag. + +Note that the modification time and metadata for the root directory +will **not** be synced. See https://github.com/rclone/rclone/issues/7652 +for more info. + **Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics. **Note**: Use the |--dry-run| or the |--interactive|/|-i| flag to test without copying anything. diff --git a/cmd/move/move.go b/cmd/move/move.go index ae7260430..ca3f448b7 100644 --- a/cmd/move/move.go +++ b/cmd/move/move.go @@ -55,6 +55,14 @@ whether rclone lists the destination directory or not. Supplying this option when moving a small number of files into a large destination can speed transfers up greatly. +Rclone will sync the modification times of files and directories if +the backend supports it. If metadata syncing is required then use the +|--metadata| flag. + +Note that the modification time and metadata for the root directory +will **not** be synced. See https://github.com/rclone/rclone/issues/7652 +for more info. + **Important**: Since this can cause data loss, test first with the |--dry-run| or the |--interactive|/|-i| flag. diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index 38076541e..ba35fa3e6 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -170,6 +170,14 @@ the destination from the sync with a filter rule or by putting an exclude-if-present file inside the destination directory and sync to a destination that is inside the source directory. +Rclone will sync the modification times of files and directories if +the backend supports it. If metadata syncing is required then use the +` + "`--metadata`" + ` flag. + +Note that the modification time and metadata for the root directory +will **not** be synced. See https://github.com/rclone/rclone/issues/7652 +for more info. + **Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics **Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors. diff --git a/docs/content/docs.md b/docs/content/docs.md index 821d6f4b0..3b3a50a63 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -446,18 +446,21 @@ This can be used when scripting to make aged backups efficiently, e.g. ## Metadata support {#metadata} -Metadata is data about a file which isn't the contents of the file. -Normally rclone only preserves the modification time and the content -(MIME) type where possible. +Metadata is data about a file (or directory) which isn't the contents +of the file (or directory). Normally rclone only preserves the +modification time and the content (MIME) type where possible. -Rclone supports preserving all the available metadata on files (not -directories) when using the `--metadata` or `-M` flag. +Rclone supports preserving all the available metadata on files and +directories when using the `--metadata` or `-M` flag. Exactly what metadata is supported and what that support means depends on the backend. Backends that support metadata have a metadata section in their docs and are listed in the [features table](/overview/#features) (Eg [local](/local/#metadata), [s3](/s3/#metadata)) +Some backends don't support metadata, some only support metadata on +files and some support metadata on both files and directories. + Rclone only supports a one-time sync of metadata. This means that metadata will be synced from the source object to the destination object only when the source object has changed and needs to be @@ -1560,10 +1563,10 @@ some context for the `Metadata` which may be important. - `SrcFsType` is the name of the source backend. - `DstFs` is the config string for the remote that the object is being copied to - `DstFsType` is the name of the destination backend. -- `Remote` is the path of the file relative to the root. -- `Size`, `MimeType`, `ModTime` are attributes of the file. +- `Remote` is the path of the object relative to the root. +- `Size`, `MimeType`, `ModTime` are attributes of the object. - `IsDir` is `true` if this is a directory (not yet implemented). -- `ID` is the source `ID` of the file if known. +- `ID` is the source `ID` of the object if known. - `Metadata` is the backend specific metadata as described in the backend docs. ```json diff --git a/fs/sync/sync.go b/fs/sync/sync.go index c464235c0..0ca8e1d27 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -18,6 +18,8 @@ import ( "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/march" "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/lib/errcount" + "golang.org/x/sync/errgroup" ) // ErrorMaxDurationReached defines error when transfer duration is reached @@ -84,6 +86,20 @@ type syncCopyMove struct { maxDurationEndTime time.Time // end time if --max-duration is set logger operations.LoggerFn // LoggerFn used to report the results of a sync (or bisync) to an io.Writer usingLogger bool // whether we are using logger + setDirMetadata bool // if set we set the directory metadata + setDirModTime bool // if set we set the directory modtimes + setDirModTimeAfter bool // if set we set the directory modtimes at the end of the sync + setDirModTimeMu sync.Mutex // protect setDirModTimeMu + setDirModTimes []setDirModTime // directories that need their modtime set + setDirModTimesMaxLevel int // max level of the directories to set +} + +// For keeping track of delayed modtime sets +type setDirModTime struct { + dst fs.Directory + dir string + modTime time.Time + level int // the level of the directory, 0 is root } type trackRenamesStrategy byte @@ -136,6 +152,9 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete modifyWindow: fs.GetModifyWindow(ctx, fsrc, fdst), trackRenamesCh: make(chan fs.Object, ci.Checkers), checkFirst: ci.CheckFirst, + setDirMetadata: ci.Metadata && fsrc.Features().ReadDirMetadata && fdst.Features().WriteDirMetadata, + setDirModTime: fdst.Features().WriteDirSetModTime || fdst.Features().MkdirMetadata != nil || fdst.Features().DirSetModTime != nil, + setDirModTimeAfter: fdst.Features().DirModTimeUpdatesOnWrite, } s.logger, s.usingLogger = operations.GetLogger(ctx) @@ -966,6 +985,11 @@ func (s *syncCopyMove) run() error { } } + // Update modtimes for directories if necessary + if s.setDirModTime && s.setDirModTimeAfter { + s.processError(s.setDelayedDirModTimes(s.ctx)) + } + // Prune empty directories if s.deleteMode != fs.DeleteModeOff { if s.currentError() != nil && !s.ci.IgnoreErrors { @@ -1055,6 +1079,96 @@ func (s *syncCopyMove) DstOnly(dst fs.DirEntry) (recurse bool) { return false } +// copyDirMetadata copies the src directory modTime or Metadata to dst +// or f if nil. If dst is nil then it uses dir as the name of the new +// directory. +// +// It returns the destination directory if possible. Note that this may +// be nil. +func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory) { + var err error + if s.setDirMetadata { + newDst, err = operations.CopyDirMetadata(ctx, f, dst, dir, src) + } else if s.setDirModTime { + if dst == nil { + newDst, err = operations.MkdirModTime(ctx, f, dir, src.ModTime(ctx)) + } else { + newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx)) + } + } else if dst == nil { + // Create the directory if it doesn't exist + err = operations.Mkdir(ctx, f, dir) + } + // If we need to set modtime after and we created a dir, then save it for later + if s.setDirModTime && s.setDirModTimeAfter && err == nil { + if newDst != nil { + dir = newDst.Remote() + } + level := strings.Count(dir, "/") + 1 + // The root directory "" is at the top level + if dir == "" { + level = 0 + } + s.setDirModTimeMu.Lock() + // Keep track of the maximum level inserted + if level > s.setDirModTimesMaxLevel { + s.setDirModTimesMaxLevel = level + } + s.setDirModTimes = append(s.setDirModTimes, setDirModTime{ + dst: newDst, + dir: dir, + modTime: src.ModTime(ctx), + level: level, + }) + s.setDirModTimeMu.Unlock() + fs.Debugf(nil, "Added delayed dir = %q, newDst=%v", dir, newDst) + } + s.processError(err) + if err != nil { + return nil + } + return newDst +} + +// Set the modtimes for directories +func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error { + s.setDirModTimeMu.Lock() + defer s.setDirModTimeMu.Unlock() + + // Timestamp all directories at the same level in parallel, deepest first + // We do this by iterating the slice multiple times to save memory + // There could be a lot of directories in this slice. + var errCount = errcount.New() + for level := s.setDirModTimesMaxLevel; level >= 0; level-- { + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(s.ci.Checkers) + for _, item := range s.setDirModTimes { + if item.level != level { + continue + } + // End early if error + if gCtx.Err() != nil { + break + } + item := item + g.Go(func() error { + _, err := operations.SetDirModTime(gCtx, s.fdst, item.dst, item.dir, item.modTime) + if err != nil { + err = fs.CountError(err) + fs.Errorf(item.dir, "Failed to timestamp directory: %v", err) + errCount.Add(err) + } + return nil // don't return errors, just count them + }) + } + err := g.Wait() + if err != nil { + return err + } + } + return errCount.Err("failed to set directory modtime") +} + // SrcOnly have an object which is in the source only func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) { if s.deleteMode == fs.DeleteModeOnly { @@ -1101,6 +1215,9 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) { s.srcEmptyDirs[src.Remote()] = src s.logger(s.ctx, operations.MissingOnDst, src, nil, fs.ErrorIsDir) s.srcEmptyDirsMu.Unlock() + + // Create the directory and make sure the Metadata/ModTime is correct + s.copyDirMetadata(s.ctx, s.fdst, nil, x.Remote(), x) return true default: panic("Bad object in DirEntries") @@ -1135,9 +1252,12 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse } case fs.Directory: // Do the same thing to the entire contents of the directory - _, ok := dst.(fs.Directory) + dstX, ok := dst.(fs.Directory) if ok { s.logger(s.ctx, operations.Match, src, dst, fs.ErrorIsDir) + // Create the directory and make sure the Metadata/ModTime is correct + s.copyDirMetadata(s.ctx, s.fdst, dstX, "", srcX) + // Only record matched (src & dst) empty dirs when performing move if s.DoMove { // Record the src directory for deletion diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index 7911be1d0..c0728cf06 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "io" "os" "os/exec" "runtime" @@ -65,15 +66,88 @@ func TestCopy(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + _, err := operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t2) + if err != nil && !errors.Is(err, fs.ErrorNotImplemented) { + require.NoError(t, err) + } r.Mkdir(ctx, r.Fremote) ctx = predictDstFromLogger(ctx) - err := CopyDir(ctx, r.Fremote, r.Flocal, false) + err = CopyDir(ctx, r.Fremote, r.Flocal, false) require.NoError(t, err) testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) r.CheckLocalItems(t, file1) r.CheckRemoteItems(t, file1) + + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, "sub dir") +} + +func TestCopyMetadata(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + r := fstest.NewRun(t) + features := r.Fremote.Features() + + if !features.ReadMetadata && !features.WriteMetadata && !features.UserMetadata && + !features.ReadDirMetadata && !features.WriteDirMetadata && !features.UserDirMetadata { + t.Skip("Skipping as metadata not supported") + } + + const content = "hello metadata world!" + const dirPath = "metadata sub dir" + const filePath = dirPath + "/hello metadata world" + + fileMetadata := fs.Metadata{ + // System metadata supported by all backends + "mtime": t1.Format(time.RFC3339Nano), + // User metadata + "potato": "jersey", + } + + dirMetadata := fs.Metadata{ + // System metadata supported by all backends + "mtime": t2.Format(time.RFC3339Nano), + // User metadata + "potato": "king edward", + } + + // Make the directory with metadata - may fall back to Mkdir + _, err := operations.MkdirMetadata(ctx, r.Flocal, dirPath, dirMetadata) + require.NoError(t, err) + + // Upload the file with metadata + in := io.NopCloser(bytes.NewBufferString(content)) + _, err = operations.Rcat(ctx, r.Flocal, filePath, in, t1, fileMetadata) + require.NoError(t, err) + file1 := fstest.NewItem(filePath, content, t1) + + // Reset the time of the directory + _, err = operations.SetDirModTime(ctx, r.Flocal, nil, dirPath, t2) + if err != nil && !errors.Is(err, fs.ErrorNotImplemented) { + require.NoError(t, err) + } + + ctx = predictDstFromLogger(ctx) + err = CopyDir(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) + + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, dirPath) + + // Check that the metadata on the directory and file is correct + if features.ReadMetadata { + fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewObject(ctx, t, r.Fremote, filePath), fileMetadata) + } + if features.ReadDirMetadata { + fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, dirPath), dirMetadata) + } } func TestCopyMissingDirectory(t *testing.T) { @@ -205,10 +279,15 @@ func TestCopyEmptyDirectories(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) file1 := r.WriteFile("sub dir/hello world", "hello world", t1) - err := operations.Mkdir(ctx, r.Flocal, "sub dir2") + _, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2) require.NoError(t, err) r.Mkdir(ctx, r.Fremote) + // Set the modtime on "sub dir" to something specific + // Without this it fails on the CI and in VirtualBox with variances of up to 10mS + _, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t1) + require.NoError(t, err) + ctx = predictDstFromLogger(ctx) err = CopyDir(ctx, r.Fremote, r.Flocal, true) require.NoError(t, err) @@ -224,6 +303,9 @@ func TestCopyEmptyDirectories(t *testing.T) { "sub dir2", }, ) + + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, "sub dir", "sub dir2") } // Test move empty directories @@ -231,8 +313,10 @@ func TestMoveEmptyDirectories(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) file1 := r.WriteFile("sub dir/hello world", "hello world", t1) - err := operations.Mkdir(ctx, r.Flocal, "sub dir2") + _, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2) require.NoError(t, err) + subDir := fstest.NewDirectory(ctx, t, r.Flocal, "sub dir") + subDirT := subDir.ModTime(ctx) r.Mkdir(ctx, r.Fremote) ctx = predictDstFromLogger(ctx) @@ -250,6 +334,14 @@ func TestMoveEmptyDirectories(t *testing.T) { "sub dir2", }, ) + + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, "sub dir2") + // Note that "sub dir" mod time is updated when file1 is deleted from it + // So check it more manually + got := fstest.NewDirectory(ctx, t, r.Fremote, "sub dir") + gotT := got.ModTime(ctx) + fstest.AssertTimeEqualWithPrecision(t, subDir.Remote(), subDirT, gotT, fs.GetModifyWindow(ctx, r.Fremote, r.Flocal)) } // Test sync empty directories @@ -257,8 +349,14 @@ func TestSyncEmptyDirectories(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) file1 := r.WriteFile("sub dir/hello world", "hello world", t1) - err := operations.Mkdir(ctx, r.Flocal, "sub dir2") + _, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2) require.NoError(t, err) + + // Set the modtime on "sub dir" to something specific + // Without this it fails on the CI and in VirtualBox with variances of up to 10mS + _, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t1) + require.NoError(t, err) + r.Mkdir(ctx, r.Fremote) ctx = predictDstFromLogger(ctx) @@ -276,6 +374,65 @@ func TestSyncEmptyDirectories(t *testing.T) { "sub dir2", }, ) + + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, "sub dir", "sub dir2") +} + +// Test delayed mod time setting +func TestSyncSetDelayedModTimes(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + + if !r.Fremote.Features().DirModTimeUpdatesOnWrite { + t.Skip("Backend doesn't have DirModTimeUpdatesOnWrite set") + } + + // Create directories without timestamps + require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d1/e1/f1")) + require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b2/c1/d1/e1/f1")) + require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d2/e1/f1")) + require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d2/e1/f2")) + + dirs := []string{ + "a1", + "a1/b1", + "a1/b1/c1", + "a1/b1/c1/d1", + "a1/b1/c1/d1/e1", + "a1/b1/c1/d1/e1/f1", + "a1/b1/c1/d2", + "a1/b1/c1/d2/e1", + "a1/b1/c1/d2/e1/f1", + "a1/b1/c1/d2/e1/f2", + "a1/b2", + "a1/b2/c1", + "a1/b2/c1/d1", + "a1/b2/c1/d1/e1", + "a1/b2/c1/d1/e1/f1", + } + r.CheckLocalListing(t, []fstest.Item{}, dirs) + + // Timestamp the directories in reverse order + ts := t1 + for i := len(dirs) - 1; i >= 0; i-- { + dir := dirs[i] + _, err := operations.SetDirModTime(ctx, r.Flocal, nil, dir, ts) + require.NoError(t, err) + ts = ts.Add(time.Minute) + } + + r.Mkdir(ctx, r.Fremote) + + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, true) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckRemoteListing(t, []fstest.Item{}, dirs) + + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, dirs...) } // Test a server-side copy if possible, or the backup path if not diff --git a/fstest/run.go b/fstest/run.go index d03d0ef83..dbb794337 100644 --- a/fstest/run.go +++ b/fstest/run.go @@ -360,6 +360,22 @@ func (r *Run) CheckRemoteListing(t *testing.T, items []Item, expectedDirs []stri CheckListingWithPrecision(t, r.Fremote, items, expectedDirs, r.Precision) } +// CheckDirectoryModTimes checks that the directory names in r.Flocal has the correct modtime compared to r.Fremote +func (r *Run) CheckDirectoryModTimes(t *testing.T, names ...string) { + if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil { + fs.Debugf(r.Fremote, "Skipping modtime test as remote does not support DirSetModTime or MkdirMetadata") + return + } + ctx := context.Background() + for _, name := range names { + wantT := NewDirectory(ctx, t, r.Flocal, name).ModTime(ctx) + got := NewDirectory(ctx, t, r.Fremote, name) + gotT := got.ModTime(ctx) + fs.Debugf(r.Fremote, "Testing directory mod time of %q: wantT=%v, gotT=%v", name, wantT, gotT) + AssertTimeEqualWithPrecision(t, got.Remote(), wantT, gotT, fs.GetModifyWindow(ctx, r.Fremote, r.Flocal)) + } +} + // Clean the temporary directory func (r *Run) cleanTempDir() { err := os.RemoveAll(r.LocalName)