mirror of
https://github.com/rclone/rclone.git
synced 2025-08-17 00:51:34 +02:00
copy,copyto,move,moveto: implement logger flags to store result of sync
This enables the logger flags (`--combined`, `--missing-on-src` etc.) for the `rclone copy` and `move` commands (as well as their `copyto` and `moveto` variants) akin to `rclone sync`. Warnings for unsupported/wonky flag combinations are also printed, e.g. when the destination is not traversed but `--dest-after` is specified. - fs/operations: add reusable methods for operation logging - cmd/sync: use reusable methods for implementing logging in sync command - cmd: implement logging for copy/copyto/move/moveto commands - fs/operations/operationsflags: warn about logs in conjunction with --no-traverse - cmd: add logger docs to copy and move commands Fixes #8115
This commit is contained in:
Marvin Rösch
GitHub
174
cmd/sync/sync.go
174
cmd/sync/sync.go
@ -3,13 +3,9 @@ package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
mutex "sync" // renamed as "sync" already in use
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
@ -19,7 +15,7 @@ import (
|
||||
|
||||
var (
|
||||
createEmptySrcDirs = false
|
||||
opt = operations.LoggerOpt{}
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
@ -27,119 +23,15 @@ func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after sync", "")
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &opt, &loggerFlagsOpt)
|
||||
// TODO: add same flags to move and copy
|
||||
}
|
||||
|
||||
var lock mutex.Mutex
|
||||
|
||||
func syncLoggerFn(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil {
|
||||
opt.PrintDestAfter(ctx, sigil, src, dst, err)
|
||||
return
|
||||
}
|
||||
|
||||
_, srcOk := src.(fs.Object)
|
||||
_, dstOk := dst.(fs.Object)
|
||||
var filename string
|
||||
if !srcOk && !dstOk {
|
||||
return
|
||||
} else if srcOk && !dstOk {
|
||||
filename = src.String()
|
||||
} else {
|
||||
filename = dst.String()
|
||||
}
|
||||
|
||||
if sigil.Writer(opt) != nil {
|
||||
operations.SyncFprintf(sigil.Writer(opt), "%s\n", filename)
|
||||
}
|
||||
if opt.Combined != nil {
|
||||
operations.SyncFprintf(opt.Combined, "%c %s\n", sigil, filename)
|
||||
fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename)
|
||||
}
|
||||
if opt.DestAfter != nil {
|
||||
opt.PrintDestAfter(ctx, sigil, src, dst, err)
|
||||
}
|
||||
}
|
||||
|
||||
// GetSyncLoggerOpt gets the options corresponding to the logger flags
|
||||
func GetSyncLoggerOpt(ctx context.Context, fdst fs.Fs, command *cobra.Command) (operations.LoggerOpt, func(), error) {
|
||||
closers := []io.Closer{}
|
||||
|
||||
opt.LoggerFn = syncLoggerFn
|
||||
if opt.TimeFormat == "max" {
|
||||
opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision())
|
||||
}
|
||||
opt.SetListFormat(ctx, command.Flags())
|
||||
opt.NewListJSON(ctx, fdst, "")
|
||||
|
||||
open := func(name string, pout *io.Writer) error {
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
if name == "-" {
|
||||
*pout = os.Stdout
|
||||
return nil
|
||||
}
|
||||
out, err := os.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*pout = out
|
||||
closers = append(closers, out)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := open(loggerFlagsOpt.Combined, &opt.Combined); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.Match, &opt.Match); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.Differ, &opt.Differ); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.ErrFile, &opt.Error); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.DestAfter, &opt.DestAfter); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
|
||||
close := func() {
|
||||
for _, closer := range closers {
|
||||
err := closer.Close()
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Failed to close report output: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return opt, close, nil
|
||||
}
|
||||
|
||||
func anyNotBlank(s ...string) bool {
|
||||
for _, x := range s {
|
||||
if x != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
Use: "sync source:path dest:path",
|
||||
Short: `Make source and dest identical, modifying destination only.`,
|
||||
Long: `Sync the source to the destination, changing the destination
|
||||
// Warning! "|" will be replaced by backticks below
|
||||
Long: strings.ReplaceAll(`Sync the source to the destination, changing the destination
|
||||
only. Doesn't transfer files that are identical on source and
|
||||
destination, testing by size and modification time or MD5SUM.
|
||||
Destination is updated to match source, including deleting files
|
||||
@ -148,7 +40,7 @@ want to delete files from destination, use the
|
||||
[copy](/commands/rclone_copy/) command instead.
|
||||
|
||||
**Important**: Since this can cause data loss, test first with the
|
||||
` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
|
||||
|--dry-run| or the |--interactive|/|i| flag.
|
||||
|
||||
rclone sync --interactive SOURCE remote:DESTINATION
|
||||
|
||||
@ -171,55 +63,18 @@ destination that is inside the source directory.
|
||||
|
||||
Rclone will sync the modification times of files and directories if
|
||||
the backend supports it. If metadata syncing is required then use the
|
||||
` + "`--metadata`" + ` flag.
|
||||
|--metadata| flag.
|
||||
|
||||
Note that the modification time and metadata for the root directory
|
||||
will **not** be synced. See https://github.com/rclone/rclone/issues/7652
|
||||
for more info.
|
||||
|
||||
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
|
||||
**Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics
|
||||
|
||||
**Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
|
||||
**Note**: Use the |rclone dedupe| command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
|
||||
See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info.
|
||||
|
||||
## Logger Flags
|
||||
|
||||
The ` + "`--differ`" + `, ` + "`--missing-on-dst`" + `, ` + "`--missing-on-src`" + `, ` +
|
||||
"`--match`" + ` and ` + "`--error`" + ` flags write paths, one per line, to the file name (or
|
||||
stdout if it is ` + "`-`" + `) supplied. What they write is described in the
|
||||
help below. For example ` + "`--differ`" + ` will write all paths which are
|
||||
present on both the source and destination but different.
|
||||
|
||||
The ` + "`--combined`" + ` flag will write a file (or stdout) which contains all
|
||||
file paths with a symbol and then a space and then the path to tell
|
||||
you what happened to it. These are reminiscent of diff files.
|
||||
|
||||
- ` + "`= path`" + ` means path was found in source and destination and was identical
|
||||
- ` + "`- path`" + ` means path was missing on the source, so only in the destination
|
||||
- ` + "`+ path`" + ` means path was missing on the destination, so only in the source
|
||||
- ` + "`* path`" + ` means path was present in source and destination but different.
|
||||
- ` + "`! path`" + ` means there was an error reading or hashing the source or dest.
|
||||
|
||||
The ` + "`--dest-after`" + ` flag writes a list file using the same format flags
|
||||
as [` + "`lsf`" + `](/commands/rclone_lsf/#synopsis) (including [customizable options
|
||||
for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
|
||||
Conceptually it is similar to rsync's ` + "`--itemize-changes`" + `, but not identical
|
||||
-- it should output an accurate list of what will be on the destination
|
||||
after the sync.
|
||||
|
||||
Note that these logger flags have a few limitations, and certain scenarios
|
||||
are not currently supported:
|
||||
|
||||
- ` + "`--max-duration`" + ` / ` + "`CutoffModeHard`" + `
|
||||
- ` + "`--compare-dest`" + ` / ` + "`--copy-dest`" + `
|
||||
- server-side moves of an entire dir at once
|
||||
- High-level retries, because there would be duplicates (use ` + "`--retries 1`" + ` to disable)
|
||||
- Possibly some unusual error scenarios
|
||||
|
||||
Note also that each file is logged during the sync, as opposed to after, so it
|
||||
is most useful as a predictor of what SHOULD happen to each file
|
||||
(which may or may not match what actually DID.)
|
||||
`,
|
||||
`, "|", "`") + operationsflags.Help(),
|
||||
Annotations: map[string]string{
|
||||
"groups": "Sync,Copy,Filter,Listing,Important",
|
||||
},
|
||||
@ -228,15 +83,14 @@ is most useful as a predictor of what SHOULD happen to each file
|
||||
fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
|
||||
cmd.Run(true, true, command, func() error {
|
||||
ctx := context.Background()
|
||||
opt, close, err := GetSyncLoggerOpt(ctx, fdst, command)
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer close()
|
||||
|
||||
if anyNotBlank(loggerFlagsOpt.Combined, loggerFlagsOpt.MissingOnSrc, loggerFlagsOpt.MissingOnDst,
|
||||
loggerFlagsOpt.Match, loggerFlagsOpt.Differ, loggerFlagsOpt.ErrFile, loggerFlagsOpt.DestAfter) {
|
||||
ctx = operations.WithSyncLogger(ctx, opt)
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
|
||||
Reference in New Issue
Block a user