From 5aa98110845d436a85d7c6db81b6dde40f0edac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marvin=20R=C3=B6sch?= Date: Fri, 20 Jun 2025 17:55:00 +0200 Subject: [PATCH] copy,copyto,move,moveto: implement logger flags to store result of sync This enables the logger flags (`--combined`, `--missing-on-src` etc.) for the `rclone copy` and `move` commands (as well as their `copyto` and `moveto` variants) akin to `rclone sync`. Warnings for unsupported/wonky flag combinations are also printed, e.g. when the destination is not traversed but `--dest-after` is specified. - fs/operations: add reusable methods for operation logging - cmd/sync: use reusable methods for implementing logging in sync command - cmd: implement logging for copy/copyto/move/moveto commands - fs/operations/operationsflags: warn about logs in conjunction with --no-traverse - cmd: add logger docs to copy and move commands Fixes #8115 --- cmd/copy/copy.go | 26 ++- cmd/copyto/copyto.go | 29 ++- cmd/move/move.go | 25 ++- cmd/moveto/moveto.go | 29 ++- cmd/sync/sync.go | 174 ++---------------- fs/operations/logger.go | 38 ++++ .../operationsflags/operationsflags.go | 102 ++++++++++ .../operationsflags/operationsflags.md | 39 ++++ 8 files changed, 285 insertions(+), 177 deletions(-) create mode 100644 fs/operations/operationsflags/operationsflags.md diff --git a/cmd/copy/copy.go b/cmd/copy/copy.go index 44235ae19..e6b8c4596 100644 --- a/cmd/copy/copy.go +++ b/cmd/copy/copy.go @@ -8,18 +8,23 @@ import ( "github.com/rclone/rclone/cmd" "github.com/rclone/rclone/fs/config/flags" "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/operations/operationsflags" "github.com/rclone/rclone/fs/sync" "github.com/spf13/cobra" ) var ( createEmptySrcDirs = false + loggerOpt = operations.LoggerOpt{} + loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{} ) func init() { cmd.Root.AddCommand(commandDefinition) cmdFlags := commandDefinition.Flags() flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after copy", "") + operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt) + loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt) } var commandDefinition = &cobra.Command{ @@ -90,19 +95,30 @@ for more info. **Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics. **Note**: Use the |--dry-run| or the |--interactive|/|-i| flag to test without copying anything. -`, "|", "`"), + +`, "|", "`") + operationsflags.Help(), Annotations: map[string]string{ "groups": "Copy,Filter,Listing,Important", }, Run: func(command *cobra.Command, args []string) { - cmd.CheckArgs(2, 2, command, args) fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args) cmd.Run(true, true, command, func() error { - if srcFileName == "" { - return sync.CopyDir(context.Background(), fdst, fsrc, createEmptySrcDirs) + ctx := context.Background() + close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt) + if err != nil { + return err } - return operations.CopyFile(context.Background(), fdst, fsrc, srcFileName, srcFileName) + defer close() + + if loggerFlagsOpt.AnySet() { + ctx = operations.WithSyncLogger(ctx, loggerOpt) + } + + if srcFileName == "" { + return sync.CopyDir(ctx, fdst, fsrc, createEmptySrcDirs) + } + return operations.CopyFile(ctx, fdst, fsrc, srcFileName, srcFileName) }) }, } diff --git a/cmd/copyto/copyto.go b/cmd/copyto/copyto.go index edc793071..797654786 100644 --- a/cmd/copyto/copyto.go +++ b/cmd/copyto/copyto.go @@ -6,12 +6,21 @@ import ( "github.com/rclone/rclone/cmd" "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/operations/operationsflags" "github.com/rclone/rclone/fs/sync" "github.com/spf13/cobra" ) +var ( + loggerOpt = operations.LoggerOpt{} + loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{} +) + func init() { cmd.Root.AddCommand(commandDefinition) + cmdFlags := commandDefinition.Flags() + operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt) + loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt) } var commandDefinition = &cobra.Command{ @@ -46,7 +55,8 @@ the destination. *If you are looking to copy just a byte range of a file, please see 'rclone cat --offset X --count Y'* **Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics -`, + +` + operationsflags.Help(), Annotations: map[string]string{ "versionIntroduced": "v1.35", "groups": "Copy,Filter,Listing,Important", @@ -55,10 +65,21 @@ the destination. cmd.CheckArgs(2, 2, command, args) fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args) cmd.Run(true, true, command, func() error { - if srcFileName == "" { - return sync.CopyDir(context.Background(), fdst, fsrc, false) + ctx := context.Background() + close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt) + if err != nil { + return err } - return operations.CopyFile(context.Background(), fdst, fsrc, dstFileName, srcFileName) + defer close() + + if loggerFlagsOpt.AnySet() { + ctx = operations.WithSyncLogger(ctx, loggerOpt) + } + + if srcFileName == "" { + return sync.CopyDir(ctx, fdst, fsrc, false) + } + return operations.CopyFile(ctx, fdst, fsrc, dstFileName, srcFileName) }) }, } diff --git a/cmd/move/move.go b/cmd/move/move.go index e9b19d2a6..c2abd4ad3 100644 --- a/cmd/move/move.go +++ b/cmd/move/move.go @@ -8,6 +8,7 @@ import ( "github.com/rclone/rclone/cmd" "github.com/rclone/rclone/fs/config/flags" "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/operations/operationsflags" "github.com/rclone/rclone/fs/sync" "github.com/spf13/cobra" ) @@ -16,6 +17,8 @@ import ( var ( deleteEmptySrcDirs = false createEmptySrcDirs = false + loggerOpt = operations.LoggerOpt{} + loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{} ) func init() { @@ -23,6 +26,8 @@ func init() { cmdFlags := commandDefinition.Flags() flags.BoolVarP(cmdFlags, &deleteEmptySrcDirs, "delete-empty-src-dirs", "", deleteEmptySrcDirs, "Delete empty source dirs after move", "") flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after move", "") + operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt) + loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt) } var commandDefinition = &cobra.Command{ @@ -66,7 +71,8 @@ for more info. |--dry-run| or the |--interactive|/|-i| flag. **Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics. -`, "|", "`"), + +`, "|", "`") + operationsflags.Help(), Annotations: map[string]string{ "versionIntroduced": "v1.19", "groups": "Filter,Listing,Important,Copy", @@ -75,10 +81,21 @@ for more info. cmd.CheckArgs(2, 2, command, args) fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args) cmd.Run(true, true, command, func() error { - if srcFileName == "" { - return sync.MoveDir(context.Background(), fdst, fsrc, deleteEmptySrcDirs, createEmptySrcDirs) + ctx := context.Background() + close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt) + if err != nil { + return err } - return operations.MoveFile(context.Background(), fdst, fsrc, srcFileName, srcFileName) + defer close() + + if loggerFlagsOpt.AnySet() { + ctx = operations.WithSyncLogger(ctx, loggerOpt) + } + + if srcFileName == "" { + return sync.MoveDir(ctx, fdst, fsrc, deleteEmptySrcDirs, createEmptySrcDirs) + } + return operations.MoveFile(ctx, fdst, fsrc, srcFileName, srcFileName) }) }, } diff --git a/cmd/moveto/moveto.go b/cmd/moveto/moveto.go index 8444d845d..4bb2c4b8a 100644 --- a/cmd/moveto/moveto.go +++ b/cmd/moveto/moveto.go @@ -6,12 +6,21 @@ import ( "github.com/rclone/rclone/cmd" "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/operations/operationsflags" "github.com/rclone/rclone/fs/sync" "github.com/spf13/cobra" ) +var ( + loggerOpt = operations.LoggerOpt{} + loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{} +) + func init() { cmd.Root.AddCommand(commandDefinition) + cmdFlags := commandDefinition.Flags() + operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt) + loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt) } var commandDefinition = &cobra.Command{ @@ -47,7 +56,8 @@ successful transfer. ` + "`--dry-run` or the `--interactive`/`-i`" + ` flag. **Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics. -`, + +` + operationsflags.Help(), Annotations: map[string]string{ "versionIntroduced": "v1.35", "groups": "Filter,Listing,Important,Copy", @@ -57,10 +67,21 @@ successful transfer. fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args) cmd.Run(true, true, command, func() error { - if srcFileName == "" { - return sync.MoveDir(context.Background(), fdst, fsrc, false, false) + ctx := context.Background() + close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt) + if err != nil { + return err } - return operations.MoveFile(context.Background(), fdst, fsrc, dstFileName, srcFileName) + defer close() + + if loggerFlagsOpt.AnySet() { + ctx = operations.WithSyncLogger(ctx, loggerOpt) + } + + if srcFileName == "" { + return sync.MoveDir(ctx, fdst, fsrc, false, false) + } + return operations.MoveFile(ctx, fdst, fsrc, dstFileName, srcFileName) }) }, } diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index 157203e77..4222ec6a6 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -3,13 +3,9 @@ package sync import ( "context" - "io" - "os" - - mutex "sync" // renamed as "sync" already in use + "strings" "github.com/rclone/rclone/cmd" - "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config/flags" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/operations/operationsflags" @@ -19,7 +15,7 @@ import ( var ( createEmptySrcDirs = false - opt = operations.LoggerOpt{} + loggerOpt = operations.LoggerOpt{} loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{} ) @@ -27,119 +23,15 @@ func init() { cmd.Root.AddCommand(commandDefinition) cmdFlags := commandDefinition.Flags() flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after sync", "") - operationsflags.AddLoggerFlags(cmdFlags, &opt, &loggerFlagsOpt) - // TODO: add same flags to move and copy -} - -var lock mutex.Mutex - -func syncLoggerFn(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) { - lock.Lock() - defer lock.Unlock() - - if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil { - opt.PrintDestAfter(ctx, sigil, src, dst, err) - return - } - - _, srcOk := src.(fs.Object) - _, dstOk := dst.(fs.Object) - var filename string - if !srcOk && !dstOk { - return - } else if srcOk && !dstOk { - filename = src.String() - } else { - filename = dst.String() - } - - if sigil.Writer(opt) != nil { - operations.SyncFprintf(sigil.Writer(opt), "%s\n", filename) - } - if opt.Combined != nil { - operations.SyncFprintf(opt.Combined, "%c %s\n", sigil, filename) - fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename) - } - if opt.DestAfter != nil { - opt.PrintDestAfter(ctx, sigil, src, dst, err) - } -} - -// GetSyncLoggerOpt gets the options corresponding to the logger flags -func GetSyncLoggerOpt(ctx context.Context, fdst fs.Fs, command *cobra.Command) (operations.LoggerOpt, func(), error) { - closers := []io.Closer{} - - opt.LoggerFn = syncLoggerFn - if opt.TimeFormat == "max" { - opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision()) - } - opt.SetListFormat(ctx, command.Flags()) - opt.NewListJSON(ctx, fdst, "") - - open := func(name string, pout *io.Writer) error { - if name == "" { - return nil - } - if name == "-" { - *pout = os.Stdout - return nil - } - out, err := os.Create(name) - if err != nil { - return err - } - *pout = out - closers = append(closers, out) - return nil - } - - if err := open(loggerFlagsOpt.Combined, &opt.Combined); err != nil { - return opt, nil, err - } - if err := open(loggerFlagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil { - return opt, nil, err - } - if err := open(loggerFlagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil { - return opt, nil, err - } - if err := open(loggerFlagsOpt.Match, &opt.Match); err != nil { - return opt, nil, err - } - if err := open(loggerFlagsOpt.Differ, &opt.Differ); err != nil { - return opt, nil, err - } - if err := open(loggerFlagsOpt.ErrFile, &opt.Error); err != nil { - return opt, nil, err - } - if err := open(loggerFlagsOpt.DestAfter, &opt.DestAfter); err != nil { - return opt, nil, err - } - - close := func() { - for _, closer := range closers { - err := closer.Close() - if err != nil { - fs.Errorf(nil, "Failed to close report output: %v", err) - } - } - } - - return opt, close, nil -} - -func anyNotBlank(s ...string) bool { - for _, x := range s { - if x != "" { - return true - } - } - return false + operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt) + loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt) } var commandDefinition = &cobra.Command{ Use: "sync source:path dest:path", Short: `Make source and dest identical, modifying destination only.`, - Long: `Sync the source to the destination, changing the destination + // Warning! "|" will be replaced by backticks below + Long: strings.ReplaceAll(`Sync the source to the destination, changing the destination only. Doesn't transfer files that are identical on source and destination, testing by size and modification time or MD5SUM. Destination is updated to match source, including deleting files @@ -148,7 +40,7 @@ want to delete files from destination, use the [copy](/commands/rclone_copy/) command instead. **Important**: Since this can cause data loss, test first with the -` + "`--dry-run` or the `--interactive`/`-i`" + ` flag. +|--dry-run| or the |--interactive|/|i| flag. rclone sync --interactive SOURCE remote:DESTINATION @@ -171,55 +63,18 @@ destination that is inside the source directory. Rclone will sync the modification times of files and directories if the backend supports it. If metadata syncing is required then use the -` + "`--metadata`" + ` flag. +|--metadata| flag. Note that the modification time and metadata for the root directory will **not** be synced. See https://github.com/rclone/rclone/issues/7652 for more info. -**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics +**Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics -**Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors. +**Note**: Use the |rclone dedupe| command to deal with "Duplicate object/directory found in source/destination - ignoring" errors. See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info. -## Logger Flags - -The ` + "`--differ`" + `, ` + "`--missing-on-dst`" + `, ` + "`--missing-on-src`" + `, ` + - "`--match`" + ` and ` + "`--error`" + ` flags write paths, one per line, to the file name (or -stdout if it is ` + "`-`" + `) supplied. What they write is described in the -help below. For example ` + "`--differ`" + ` will write all paths which are -present on both the source and destination but different. - -The ` + "`--combined`" + ` flag will write a file (or stdout) which contains all -file paths with a symbol and then a space and then the path to tell -you what happened to it. These are reminiscent of diff files. - -- ` + "`= path`" + ` means path was found in source and destination and was identical -- ` + "`- path`" + ` means path was missing on the source, so only in the destination -- ` + "`+ path`" + ` means path was missing on the destination, so only in the source -- ` + "`* path`" + ` means path was present in source and destination but different. -- ` + "`! path`" + ` means there was an error reading or hashing the source or dest. - -The ` + "`--dest-after`" + ` flag writes a list file using the same format flags -as [` + "`lsf`" + `](/commands/rclone_lsf/#synopsis) (including [customizable options -for hash, modtime, etc.](/commands/rclone_lsf/#synopsis)) -Conceptually it is similar to rsync's ` + "`--itemize-changes`" + `, but not identical --- it should output an accurate list of what will be on the destination -after the sync. - -Note that these logger flags have a few limitations, and certain scenarios -are not currently supported: - -- ` + "`--max-duration`" + ` / ` + "`CutoffModeHard`" + ` -- ` + "`--compare-dest`" + ` / ` + "`--copy-dest`" + ` -- server-side moves of an entire dir at once -- High-level retries, because there would be duplicates (use ` + "`--retries 1`" + ` to disable) -- Possibly some unusual error scenarios - -Note also that each file is logged during the sync, as opposed to after, so it -is most useful as a predictor of what SHOULD happen to each file -(which may or may not match what actually DID.) -`, +`, "|", "`") + operationsflags.Help(), Annotations: map[string]string{ "groups": "Sync,Copy,Filter,Listing,Important", }, @@ -228,15 +83,14 @@ is most useful as a predictor of what SHOULD happen to each file fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args) cmd.Run(true, true, command, func() error { ctx := context.Background() - opt, close, err := GetSyncLoggerOpt(ctx, fdst, command) + close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt) if err != nil { return err } defer close() - if anyNotBlank(loggerFlagsOpt.Combined, loggerFlagsOpt.MissingOnSrc, loggerFlagsOpt.MissingOnDst, - loggerFlagsOpt.Match, loggerFlagsOpt.Differ, loggerFlagsOpt.ErrFile, loggerFlagsOpt.DestAfter) { - ctx = operations.WithSyncLogger(ctx, opt) + if loggerFlagsOpt.AnySet() { + ctx = operations.WithSyncLogger(ctx, loggerOpt) } if srcFileName == "" { diff --git a/fs/operations/logger.go b/fs/operations/logger.go index 0572aef93..0d482206f 100644 --- a/fs/operations/logger.go +++ b/fs/operations/logger.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + mutex "sync" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/hash" @@ -106,6 +107,43 @@ type LoggerOpt struct { Absolute bool } +// NewDefaultLoggerFn creates a logger function that writes the sigil and path to configured files that match the sigil +func NewDefaultLoggerFn(opt *LoggerOpt) LoggerFn { + var lock mutex.Mutex + + return func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) { + lock.Lock() + defer lock.Unlock() + + if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil { + opt.PrintDestAfter(ctx, sigil, src, dst, err) + return + } + + _, srcOk := src.(fs.Object) + _, dstOk := dst.(fs.Object) + var filename string + if !srcOk && !dstOk { + return + } else if srcOk && !dstOk { + filename = src.String() + } else { + filename = dst.String() + } + + if sigil.Writer(*opt) != nil { + SyncFprintf(sigil.Writer(*opt), "%s\n", filename) + } + if opt.Combined != nil { + SyncFprintf(opt.Combined, "%c %s\n", sigil, filename) + fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename) + } + if opt.DestAfter != nil { + opt.PrintDestAfter(ctx, sigil, src, dst, err) + } + } +} + // WithLogger stores logger in ctx and returns a copy of ctx in which loggerKey = logger func WithLogger(ctx context.Context, logger LoggerFn) context.Context { return context.WithValue(ctx, loggerKey, logger) diff --git a/fs/operations/operationsflags/operationsflags.go b/fs/operations/operationsflags/operationsflags.go index 80a6d77fb..fe28f064b 100644 --- a/fs/operations/operationsflags/operationsflags.go +++ b/fs/operations/operationsflags/operationsflags.go @@ -3,12 +3,28 @@ package operationsflags import ( + "context" + _ "embed" + "io" + "os" + "strings" + + "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config/flags" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" + "github.com/spf13/cobra" "github.com/spf13/pflag" ) +//go:embed operationsflags.md +var help string + +// Help returns the help string cleaned up to simplify appending +func Help() string { + return strings.TrimSpace(help) + "\n\n" +} + // AddLoggerFlagsOptions contains options for the Logger Flags type AddLoggerFlagsOptions struct { Combined string // a file with file names with leading sigils @@ -20,6 +36,20 @@ type AddLoggerFlagsOptions struct { DestAfter string // files that exist on the destination post-sync } +// AnySet checks if any of the logger flags have a non-blank value +func (o AddLoggerFlagsOptions) AnySet() bool { + return anyNotBlank(o.Combined, o.MissingOnSrc, o.MissingOnDst, o.Match, o.Differ, o.ErrFile, o.DestAfter) +} + +func anyNotBlank(s ...string) bool { + for _, x := range s { + if x != "" { + return true + } + } + return false +} + // AddLoggerFlags adds the logger flags to the cmdFlags command func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt *AddLoggerFlagsOptions) { flags.StringVarP(cmdFlags, &flagsOpt.Combined, "combined", "", flagsOpt.Combined, "Make a combined report of changes to this file", "Sync") @@ -43,3 +73,75 @@ func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync") // flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "") } + +// ConfigureLoggers verifies and sets up writers for log files requested via CLI flags +func ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, opt *operations.LoggerOpt, flagsOpt AddLoggerFlagsOptions) (func(), error) { + closers := []io.Closer{} + + if opt.TimeFormat == "max" { + opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision()) + } + opt.SetListFormat(ctx, command.Flags()) + opt.NewListJSON(ctx, fdst, "") + + open := func(name string, pout *io.Writer) error { + if name == "" { + return nil + } + if name == "-" { + *pout = os.Stdout + return nil + } + out, err := os.Create(name) + if err != nil { + return err + } + *pout = out + closers = append(closers, out) + return nil + } + + if err := open(flagsOpt.Combined, &opt.Combined); err != nil { + return nil, err + } + if err := open(flagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil { + return nil, err + } + if err := open(flagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil { + return nil, err + } + if err := open(flagsOpt.Match, &opt.Match); err != nil { + return nil, err + } + if err := open(flagsOpt.Differ, &opt.Differ); err != nil { + return nil, err + } + if err := open(flagsOpt.ErrFile, &opt.Error); err != nil { + return nil, err + } + if err := open(flagsOpt.DestAfter, &opt.DestAfter); err != nil { + return nil, err + } + + close := func() { + for _, closer := range closers { + err := closer.Close() + if err != nil { + fs.Errorf(nil, "Failed to close report output: %v", err) + } + } + } + + ci := fs.GetConfig(ctx) + if ci.NoTraverse && opt.Combined != nil { + fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n") + } + if ci.NoTraverse && opt.MissingOnSrc != nil { + fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n") + } + if ci.NoTraverse && opt.DestAfter != nil { + fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n") + } + + return close, nil +} diff --git a/fs/operations/operationsflags/operationsflags.md b/fs/operations/operationsflags/operationsflags.md new file mode 100644 index 000000000..e3ebfa201 --- /dev/null +++ b/fs/operations/operationsflags/operationsflags.md @@ -0,0 +1,39 @@ +## Logger Flags + +The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match` and `--error` flags write paths, +one per line, to the file name (or stdout if it is `-`) supplied. What they write is described +in the help below. For example `--differ` will write all paths which are present +on both the source and destination but different. + +The `--combined` flag will write a file (or stdout) which contains all +file paths with a symbol and then a space and then the path to tell +you what happened to it. These are reminiscent of diff files. + +- `= path` means path was found in source and destination and was identical +- `- path` means path was missing on the source, so only in the destination +- `+ path` means path was missing on the destination, so only in the source +- `* path` means path was present in source and destination but different. +- `! path` means there was an error reading or hashing the source or dest. + +The `--dest-after` flag writes a list file using the same format flags +as [`lsf`](/commands/rclone_lsf/#synopsis) (including [customizable options +for hash, modtime, etc.](/commands/rclone_lsf/#synopsis)) +Conceptually it is similar to rsync's `--itemize-changes`, but not identical +-- it should output an accurate list of what will be on the destination +after the command is finished. + +When the `--no-traverse` flag is set, all logs involving files that exist only +on the destination will be incomplete or completely missing. + +Note that these logger flags have a few limitations, and certain scenarios +are not currently supported: + +- `--max-duration` / `CutoffModeHard` +- `--compare-dest` / `--copy-dest` +- server-side moves of an entire dir at once +- High-level retries, because there would be duplicates (use `--retries 1` to disable) +- Possibly some unusual error scenarios + +Note also that each file is logged during execution, as opposed to after, so it +is most useful as a predictor of what SHOULD happen to each file +(which may or may not match what actually DID.)