package bisync import ( "context" "errors" "fmt" "strings" mutex "sync" "time" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/lib/terminal" ) // CompareOpt describes the Compare options in force type CompareOpt = struct { Modtime bool Size bool Checksum bool HashType1 hash.Type HashType2 hash.Type NoSlowHash bool SlowHashSyncOnly bool SlowHashDetected bool DownloadHash bool } func (b *bisyncRun) setCompareDefaults(ctx context.Context) error { ci := fs.GetConfig(ctx) // defaults b.opt.Compare.Size = true b.opt.Compare.Modtime = true b.opt.Compare.Checksum = false if ci.SizeOnly { b.opt.Compare.Size = true b.opt.Compare.Modtime = false b.opt.Compare.Checksum = false } else if ci.CheckSum && !b.opt.IgnoreListingChecksum { b.opt.Compare.Size = true b.opt.Compare.Modtime = false b.opt.Compare.Checksum = true } if ci.IgnoreSize { b.opt.Compare.Size = false } err = b.setFromCompareFlag(ctx) if err != nil { return err } if b.fs1.Features().SlowHash || b.fs2.Features().SlowHash { b.opt.Compare.SlowHashDetected = true } if b.opt.Compare.Checksum && !b.opt.IgnoreListingChecksum { b.setHashType(ci) } // Checks and Warnings if b.opt.Compare.SlowHashSyncOnly && b.opt.Compare.SlowHashDetected && b.opt.Resync { fs.Logf(nil, Color(terminal.Dim, "Ignoring checksums during --resync as --slow-hash-sync-only is set.")) ci.CheckSum = false // note not setting b.opt.Compare.Checksum = false as we still want to build listings on the non-slow side, if any } else if b.opt.Compare.Checksum && !ci.CheckSum { fs.Logf(nil, Color(terminal.YellowFg, "WARNING: Checksums will be compared for deltas but not during sync as --checksum is not set.")) } if b.opt.Compare.Modtime && (b.fs1.Precision() == fs.ModTimeNotSupported || b.fs2.Precision() == fs.ModTimeNotSupported) { fs.Logf(nil, Color(terminal.YellowFg, "WARNING: Modtime compare was requested but at least one remote does not support it. It is recommended to use --checksum or --size-only instead.")) } if (ci.CheckSum || b.opt.Compare.Checksum) && b.opt.IgnoreListingChecksum { if (b.opt.Compare.HashType1 == hash.None || b.opt.Compare.HashType2 == hash.None) && !b.opt.Compare.DownloadHash { fs.Logf(nil, Color(terminal.YellowFg, `WARNING: Checksum compare was requested but at least one remote does not support checksums (or checksums are being ignored) and --ignore-listing-checksum is set. Ignoring Checksums globally and falling back to --compare modtime,size for sync. (Use --compare size or --size-only to ignore modtime). Path1 (%s): %s, Path2 (%s): %s`), b.fs1.String(), b.opt.Compare.HashType1.String(), b.fs2.String(), b.opt.Compare.HashType2.String()) b.opt.Compare.Modtime = true b.opt.Compare.Size = true ci.CheckSum = false b.opt.Compare.Checksum = false } else { fs.Logf(nil, Color(terminal.YellowFg, "WARNING: Ignoring checksum for deltas as --ignore-listing-checksum is set")) // note: --checksum will still affect the internal sync calls } } if !ci.CheckSum && !b.opt.Compare.Checksum && !b.opt.IgnoreListingChecksum { fs.Infof(nil, Color(terminal.Dim, "Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set.")) b.opt.IgnoreListingChecksum = true } if !b.opt.Compare.Size && !b.opt.Compare.Modtime && !b.opt.Compare.Checksum { return errors.New(Color(terminal.RedFg, "must set a Compare method. (size, modtime, and checksum can't all be false.)")) } notSupported := func(label string, value bool, opt *bool) { if value { fs.Logf(nil, Color(terminal.YellowFg, "WARNING: %s is set but bisync does not support it. It will be ignored."), label) *opt = false } } notSupported("--update", ci.UpdateOlder, &ci.UpdateOlder) notSupported("--no-check-dest", ci.NoCheckDest, &ci.NoCheckDest) notSupported("--no-traverse", ci.NoTraverse, &ci.NoTraverse) // TODO: thorough search for other flags that should be on this list... prettyprint(b.opt.Compare, "Bisyncing with Comparison Settings", fs.LogLevelInfo) return nil } // returns true if the sizes are definitely different. // returns false if equal, or if either is unknown. func sizeDiffers(a, b int64) bool { if a < 0 || b < 0 { return false } return a != b } // returns true if the hashes are definitely different. // returns false if equal, or if either is unknown. func hashDiffers(a, b string, ht1, ht2 hash.Type, size1, size2 int64) bool { if a == "" || b == "" { if ht1 != hash.None && ht2 != hash.None && !(size1 <= 0 || size2 <= 0) { fs.Logf(nil, Color(terminal.YellowFg, "WARNING: hash unexpectedly blank despite Fs support (%s, %s) (you may need to --resync!)"), a, b) } return false } if ht1 != ht2 { if !(downloadHash && ((ht1 == hash.MD5 && ht2 == hash.None) || (ht1 == hash.None && ht2 == hash.MD5))) { fs.Infof(nil, Color(terminal.YellowFg, "WARNING: Can't compare hashes of different types (%s, %s)"), ht1.String(), ht2.String()) return false } } return a != b } // chooses hash type, giving priority to types both sides have in common func (b *bisyncRun) setHashType(ci *fs.ConfigInfo) { downloadHash = b.opt.Compare.DownloadHash if b.opt.Compare.NoSlowHash && b.opt.Compare.SlowHashDetected { fs.Infof(nil, "Not checking for common hash as at least one slow hash detected.") } else { common := b.fs1.Hashes().Overlap(b.fs2.Hashes()) if common.Count() > 0 && common.GetOne() != hash.None { ht := common.GetOne() b.opt.Compare.HashType1 = ht b.opt.Compare.HashType2 = ht if !b.opt.Compare.SlowHashSyncOnly || !b.opt.Compare.SlowHashDetected { return } } else if b.opt.Compare.SlowHashSyncOnly && b.opt.Compare.SlowHashDetected { fs.Logf(b.fs2, Color(terminal.YellowFg, "Ignoring --slow-hash-sync-only and falling back to --no-slow-hash as Path1 and Path2 have no hashes in common.")) b.opt.Compare.SlowHashSyncOnly = false b.opt.Compare.NoSlowHash = true ci.CheckSum = false } } if !b.opt.Compare.DownloadHash && !b.opt.Compare.SlowHashSyncOnly { fs.Logf(b.fs2, Color(terminal.YellowFg, "--checksum is in use but Path1 and Path2 have no hashes in common; falling back to --compare modtime,size for sync. (Use --compare size or --size-only to ignore modtime)")) fs.Infof("Path1 hashes", "%v", b.fs1.Hashes().String()) fs.Infof("Path2 hashes", "%v", b.fs2.Hashes().String()) b.opt.Compare.Modtime = true b.opt.Compare.Size = true ci.CheckSum = false } if (b.opt.Compare.NoSlowHash || b.opt.Compare.SlowHashSyncOnly) && b.fs1.Features().SlowHash { fs.Infof(nil, Color(terminal.YellowFg, "Slow hash detected on Path1. Will ignore checksum due to slow-hash settings")) b.opt.Compare.HashType1 = hash.None } else { b.opt.Compare.HashType1 = b.fs1.Hashes().GetOne() if b.opt.Compare.HashType1 != hash.None { fs.Logf(b.fs1, Color(terminal.YellowFg, "will use %s for same-side diffs on Path1 only"), b.opt.Compare.HashType1) } } if (b.opt.Compare.NoSlowHash || b.opt.Compare.SlowHashSyncOnly) && b.fs2.Features().SlowHash { fs.Infof(nil, Color(terminal.YellowFg, "Slow hash detected on Path2. Will ignore checksum due to slow-hash settings")) b.opt.Compare.HashType1 = hash.None } else { b.opt.Compare.HashType2 = b.fs2.Hashes().GetOne() if b.opt.Compare.HashType2 != hash.None { fs.Logf(b.fs2, Color(terminal.YellowFg, "will use %s for same-side diffs on Path2 only"), b.opt.Compare.HashType2) } } if b.opt.Compare.HashType1 == hash.None && b.opt.Compare.HashType2 == hash.None && !b.opt.Compare.DownloadHash { fs.Logf(nil, Color(terminal.YellowFg, "WARNING: Ignoring checksums globally as hashes are ignored or unavailable on both sides.")) b.opt.Compare.Checksum = false ci.CheckSum = false b.opt.IgnoreListingChecksum = true } } // returns true if the times are definitely different (by more than the modify window). // returns false if equal, within modify window, or if either is unknown. // considers precision per-Fs. func timeDiffers(ctx context.Context, a, b time.Time, fsA, fsB fs.Info) bool { modifyWindow := fs.GetModifyWindow(ctx, fsA, fsB) if modifyWindow == fs.ModTimeNotSupported { return false } if a.IsZero() || b.IsZero() { fs.Logf(fsA, "Fs supports modtime, but modtime is missing") return false } dt := b.Sub(a) if dt < modifyWindow && dt > -modifyWindow { fs.Debugf(a, "modification time the same (differ by %s, within tolerance %s)", dt, modifyWindow) return false } fs.Debugf(a, "Modification times differ by %s: %v, %v", dt, a, b) return true } func (b *bisyncRun) setFromCompareFlag(ctx context.Context) error { if b.opt.CompareFlag == "" { return nil } var CompareFlag CompareOpt // for exlcusions opts := strings.Split(b.opt.CompareFlag, ",") for _, opt := range opts { switch strings.ToLower(strings.TrimSpace(opt)) { case "size": b.opt.Compare.Size = true CompareFlag.Size = true case "modtime": b.opt.Compare.Modtime = true CompareFlag.Modtime = true case "checksum": b.opt.Compare.Checksum = true CompareFlag.Checksum = true default: return fmt.Errorf(Color(terminal.RedFg, "unknown compare option: %s (must be size, modtime, or checksum)"), opt) } } // exclusions (override defaults, only if --compare != "") if !CompareFlag.Size { b.opt.Compare.Size = false } if !CompareFlag.Modtime { b.opt.Compare.Modtime = false } if !CompareFlag.Checksum { b.opt.Compare.Checksum = false } // override sync flags to match ci := fs.GetConfig(ctx) if b.opt.Compare.Checksum { ci.CheckSum = true } if b.opt.Compare.Modtime && !b.opt.Compare.Checksum { ci.CheckSum = false } if !b.opt.Compare.Size { ci.IgnoreSize = true } if !b.opt.Compare.Modtime { ci.UseServerModTime = true } if b.opt.Compare.Size && !b.opt.Compare.Modtime && !b.opt.Compare.Checksum { ci.SizeOnly = true } return nil } // downloadHash is true if we should attempt to compute hash by downloading when otherwise unavailable var downloadHash bool var downloadHashWarn mutex.Once var firstDownloadHash mutex.Once func tryDownloadHash(ctx context.Context, o fs.DirEntry, hashVal string) (string, error) { if hashVal != "" || !downloadHash { return hashVal, nil } obj, ok := o.(fs.Object) if !ok { fs.Infof(o, "failed to download hash -- not an fs.Object") return hashVal, fs.ErrorObjectNotFound } if o.Size() < 0 { downloadHashWarn.Do(func() { fs.Logf(o, Color(terminal.YellowFg, "Skipping hash download as checksum not reliable with files of unknown length.")) }) fs.Debugf(o, "Skipping hash download as checksum not reliable with files of unknown length.") return hashVal, hash.ErrUnsupported } firstDownloadHash.Do(func() { fs.Infof(obj.Fs().Name(), Color(terminal.Dim, "Downloading hashes...")) }) tr := accounting.Stats(ctx).NewCheckingTransfer(o, "computing hash with --download-hash") defer func() { tr.Done(ctx, nil) }() sum, err := operations.HashSum(ctx, hash.MD5, false, true, obj) if err != nil { fs.Infof(o, "DownloadHash -- hash: %v, err: %v", sum, err) } else { fs.Debugf(o, "DownloadHash -- hash: %v", sum) } return sum, err }