sync: don't set dir modtimes if already set

Before this change, directory modtimes (and metadata) were always synced from
src to dst, even if already in sync (i.e. their modtimes already matched.) This
potentially required excessive API calls, made logs noisy, and was potentially
problematic for backends that create "versions" or otherwise log activity
updates when modtime/metadata is updated.

After this change, a new DirsEqual function is added to check whether dirs are
equal based on a number of factors such as ModifyWindow and sync flags in use.
If the dirs are equal, the modtime/metadata update is skipped.

For backends that require setDirModTimeAfter, the "after" sync is performed only
for dirs that could have been changed by the sync (i.e. dirs containing files
that were created/updated.)

Note that dir metadata (other than modtime) is not currently considered by
DirsEqual, consistent with how object metadata is synced (only when objects are
unequal for reasons other than metadata).

To sync dir modtimes and metadata unconditionally (the previous behavior), use
--ignore-times.
This commit is contained in:
nielash
2024-02-28 19:29:38 -05:00
committed by Nick Craig-Wood
parent fd8faeb0e6
commit 8c69455c37
20 changed files with 257 additions and 182 deletions

View File

@ -141,6 +141,40 @@ func Equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool {
return equal(ctx, src, dst, defaultEqualOpt(ctx))
}
// DirsEqual is like Equal but for dirs instead of objects.
// It returns true if two dirs should be considered "equal" for the purposes of syncCopyMove
// (in other words, true == "skip updating modtime/metadata for this dir".)
// Unlike Equal, it does not consider size or checksum, as these do not apply to directories.
func DirsEqual(ctx context.Context, src, dst fs.Directory, opt DirsEqualOpt) (equal bool) {
if dst == nil {
return false
}
ci := fs.GetConfig(ctx)
if ci.SizeOnly || ci.Immutable || ci.IgnoreExisting || opt.ModifyWindow == fs.ModTimeNotSupported {
return true
}
if ci.IgnoreTimes {
return false
}
if !(opt.SetDirModtime || opt.SetDirMetadata) {
return true
}
srcModTime, dstModTime := src.ModTime(ctx), dst.ModTime(ctx)
if srcModTime.IsZero() || dstModTime.IsZero() {
return false
}
dt := dstModTime.Sub(srcModTime)
if dt < opt.ModifyWindow && dt > -opt.ModifyWindow {
fs.Debugf(dst, "Directory modification time the same (differ by %s, within tolerance %s)", dt, opt.ModifyWindow)
return true
}
if ci.UpdateOlder && dt >= opt.ModifyWindow {
fs.Debugf(dst, "Destination directory is newer than source, skipping")
return true
}
return false
}
// sizeDiffers compare the size of src and dst taking into account the
// various ways of ignoring sizes
func sizeDiffers(ctx context.Context, src, dst fs.ObjectInfo) bool {
@ -172,6 +206,13 @@ func defaultEqualOpt(ctx context.Context) equalOpt {
}
}
// DirsEqualOpt represents options for DirsEqual function()
type DirsEqualOpt struct {
ModifyWindow time.Duration // Max time diff to be considered the same
SetDirModtime bool // whether to consider dir modtime
SetDirMetadata bool // whether to consider dir metadata
}
var modTimeUploadOnce sync.Once
// emit a log if we are about to upload a file to set its modification time