extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-08-13 23:38:51 +02:00
Code Issues Packages Projects Releases Wiki Activity

sync: don't set dir modtimes if already set

Browse Source 
Before this change, directory modtimes (and metadata) were always synced from
src to dst, even if already in sync (i.e. their modtimes already matched.) This
potentially required excessive API calls, made logs noisy, and was potentially
problematic for backends that create "versions" or otherwise log activity
updates when modtime/metadata is updated.

After this change, a new DirsEqual function is added to check whether dirs are
equal based on a number of factors such as ModifyWindow and sync flags in use.
If the dirs are equal, the modtime/metadata update is skipped.

For backends that require setDirModTimeAfter, the "after" sync is performed only
for dirs that could have been changed by the sync (i.e. dirs containing files
that were created/updated.)

Note that dir metadata (other than modtime) is not currently considered by
DirsEqual, consistent with how object metadata is synced (only when objects are
unequal for reasons other than metadata).

To sync dir modtimes and metadata unconditionally (the previous behavior), use
--ignore-times.
This commit is contained in:
nielash
2024-02-28 19:29:38 -05:00
committed by Nick Craig-Wood
parent fd8faeb0e6
commit 8c69455c37
20 changed files with 257 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
cmd/bisync/bisync_test.go
View File

@ -34,6 +34,7 @@ import (
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/sync"
"github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/encoder"
@ -58,6 +59,8 @@ const (
fixSlash = (runtime.GOOS == "windows")
)
var initDate = time.Date(2000, time.January, 1, 0, 0, 0, 0, bisync.TZ)
// logReplacements make modern test logs comparable with golden dir.
// It is a string slice of even length with this structure:
//
@ -87,9 +90,9 @@ var logReplacements = []string{
`^.*? for same-side diffs on .*?$`, dropMe,
`^.*?Downloading hashes.*?$`, dropMe,
// ignore timestamps in directory time updates
`^(INFO : .*?: Made directory with (metadata|modification time)).*$`, `$1`,
`^(INFO : .*?: (Made directory with|Set directory) (metadata|modification time)).*$`, dropMe,
// ignore sizes in directory time updates
`^(NOTICE: .*?: Skipped set directory modification time as --dry-run is set).*$`, `$1`,
`^(NOTICE: .*?: Skipped set directory modification time as --dry-run is set).*$`, dropMe,
}
// Some dry-run messages differ depending on the particular remote.
@ -320,7 +323,6 @@ func (b *bisyncTest) runTestCase(ctx context.Context, t *testing.T, testCase str
// For test stability, jam initial dates to a fixed past date.
// Test cases that change files will touch specific files to fixed new dates.
initDate := time.Date(2000, time.January, 1, 0, 0, 0, 0, bisync.TZ)
err = filepath.Walk(b.initDir, func(path string, info os.FileInfo, err error) error {
if err == nil && !info.IsDir() {
return os.Chtimes(path, initDate, initDate)
@ -868,6 +870,20 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
}
}
// set all dirs to a fixed date for test stability, as they are considered as of v1.66.
jamDirTimes := func(f fs.Fs) {
err := walk.ListR(ctx, f, "", true, -1, walk.ListDirs, func(entries fs.DirEntries) error {
var err error
entries.ForDir(func(dir fs.Directory) {
_, err = operations.SetDirModTime(ctx, f, dir, "", initDate)
})
return err
})
assert.NoError(b.t, err, "error jamming dirtimes")
}
jamDirTimes(fs1)
jamDirTimes(fs2)
output := bilib.CaptureOutput(func() {
err = bisync.Bisync(octx, fs1, fs2, opt)
})
@ -1385,10 +1401,10 @@ func (b *bisyncTest) mangleListing(text string, golden bool, file string) string
lines[i] = match[1] + "-" + match[3] + match[4] // replace it with "-" for comparison purposes (see #5679)
}
// account for modtime precision
var lineRegex = regexp.MustCompile(`^(\S) +(-?\d+) (\S+) (\S+) (\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{9}[+-]\d{4}) (".+")$`)
lineRegex := regexp.MustCompile(`^(\S) +(-?\d+) (\S+) (\S+) (\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{9}[+-]\d{4}) (".+")$`)
const timeFormat = "2006-01-02T15:04:05.000000000-0700"
const lineFormat = "%s %8d %s %s %s %q\n"
var TZ = time.UTC
TZ := time.UTC
fields := lineRegex.FindStringSubmatch(strings.TrimSuffix(lines[i], "\n"))
if fields != nil {
sizeVal, sizeErr := strconv.ParseInt(fields[2], 10, 64)
@ -1509,7 +1525,8 @@ func (b *bisyncTest) listDir(dir string) (names []string) {
ignoreIt := func(file string) bool {
ignoreList := []string{
// ".lst-control", ".lst-dry-control", ".lst-old", ".lst-dry-old",
".DS_Store"}
".DS_Store",
}
for _, s := range ignoreList {
if strings.Contains(file, s) {
return true