copy,move: restore --no-traverse flag

The --no-traverse flag was not implemented when the new sync routines
(using the march package) was implemented.

This re-implements --no-traverse in march by trying to find a match
for each object with NewObject rather than from a directory listing.
This commit is contained in:
Nick Craig-Wood 2018-11-25 16:49:38 +00:00
parent e3c4ebd59a
commit 107293c80e
8 changed files with 103 additions and 10 deletions

View File

@ -51,6 +51,17 @@ written a trailing / - meaning "copy the contents of this directory".
This applies to all commands and whether you are talking about the This applies to all commands and whether you are talking about the
source or destination. source or destination.
See the [--no-traverse](/docs/#no-traverse) option for controlling
whether rclone lists the destination directory or not. Supplying this
option when copying a small number of files into a large destination
can speed transfers up greatly.
For example, if you have many files in /path/to/src but only a few of
them change every day, you can to copy all the files which have
changed recently very efficiently like this:
rclone copy --max-age 24h --no-traverse /path/to/src remote:
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics **Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
`, `,
Run: func(command *cobra.Command, args []string) { Run: func(command *cobra.Command, args []string) {

View File

@ -37,6 +37,11 @@ into ` + "`dest:path`" + ` then delete the original (if no errors on copy) in
If you want to delete empty source directories after move, use the --delete-empty-src-dirs flag. If you want to delete empty source directories after move, use the --delete-empty-src-dirs flag.
See the [--no-traverse](/docs/#no-traverse) option for controlling
whether rclone lists the destination directory or not. Supplying this
option when moving a small number of files into a large destination
can speed transfers up greatly.
**Important**: Since this can cause data loss, test first with the **Important**: Since this can cause data loss, test first with the
--dry-run flag. --dry-run flag.

View File

@ -842,8 +842,8 @@ will fall back to the default behaviour and log an error level message
to the console. Note: Encrypted destinations are not supported to the console. Note: Encrypted destinations are not supported
by `--track-renames`. by `--track-renames`.
Note that `--track-renames` uses extra memory to keep track of all Note that `--track-renames` is incompatible with `--no-traverse` and
the rename candidates. that it uses extra memory to keep track of all the rename candidates.
Note also that `--track-renames` is incompatible with Note also that `--track-renames` is incompatible with
`--delete-before` and will select `--delete-after` instead of `--delete-before` and will select `--delete-after` instead of
@ -1132,6 +1132,24 @@ This option defaults to `false`.
**This should be used only for testing.** **This should be used only for testing.**
### --no-traverse ###
The `--no-traverse` flag controls whether the destination file system
is traversed when using the `copy` or `move` commands.
`--no-traverse` is not compatible with `sync` and will be ignored if
you supply it with `sync`.
If you are only copying a small number of files (or are filtering most
of the files) and/or have a large number of files on the destination
then `--no-traverse` will stop rclone listing the destination and save
time.
However, if you are copying a large number of files, especially if you
are doing a copy where lots of the files under consideration haven't
changed and won't need copying then you shouldn't use `--no-traverse`.
See [rclone copy](/commands/rclone_copy/) for an example of how to use it.
Filtering Filtering
--------- ---------

View File

@ -62,6 +62,7 @@ type ConfigInfo struct {
MaxDepth int MaxDepth int
IgnoreSize bool IgnoreSize bool
IgnoreChecksum bool IgnoreChecksum bool
NoTraverse bool
NoUpdateModTime bool NoUpdateModTime bool
DataRateUnit string DataRateUnit string
BackupDir string BackupDir string

View File

@ -27,7 +27,6 @@ var (
deleteAfter bool deleteAfter bool
bindAddr string bindAddr string
disableFeatures string disableFeatures string
noTraverse bool
) )
// AddFlags adds the non filing system specific flags to the command // AddFlags adds the non filing system specific flags to the command
@ -65,7 +64,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
flags.IntVarP(flagSet, &fs.Config.MaxDepth, "max-depth", "", fs.Config.MaxDepth, "If set limits the recursion depth to this.") flags.IntVarP(flagSet, &fs.Config.MaxDepth, "max-depth", "", fs.Config.MaxDepth, "If set limits the recursion depth to this.")
flags.BoolVarP(flagSet, &fs.Config.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use mod-time or checksum.") flags.BoolVarP(flagSet, &fs.Config.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use mod-time or checksum.")
flags.BoolVarP(flagSet, &fs.Config.IgnoreChecksum, "ignore-checksum", "", fs.Config.IgnoreChecksum, "Skip post copy check of checksums.") flags.BoolVarP(flagSet, &fs.Config.IgnoreChecksum, "ignore-checksum", "", fs.Config.IgnoreChecksum, "Skip post copy check of checksums.")
flags.BoolVarP(flagSet, &noTraverse, "no-traverse", "", noTraverse, "Obsolete - does nothing.") flags.BoolVarP(flagSet, &fs.Config.NoTraverse, "no-traverse", "", fs.Config.NoTraverse, "Don't traverse destination file system on copy.")
flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.") flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.")
flags.StringVarP(flagSet, &fs.Config.BackupDir, "backup-dir", "", fs.Config.BackupDir, "Make backups into hierarchy based in DIR.") flags.StringVarP(flagSet, &fs.Config.BackupDir, "backup-dir", "", fs.Config.BackupDir, "Make backups into hierarchy based in DIR.")
flags.StringVarP(flagSet, &fs.Config.Suffix, "suffix", "", fs.Config.Suffix, "Suffix for use with --backup-dir.") flags.StringVarP(flagSet, &fs.Config.Suffix, "suffix", "", fs.Config.Suffix, "Suffix for use with --backup-dir.")
@ -113,10 +112,6 @@ func SetFlags() {
} }
} }
if noTraverse {
fs.Logf(nil, "--no-traverse is obsolete and no longer needed - please remove")
}
if dumpHeaders { if dumpHeaders {
fs.Config.Dump |= fs.DumpHeaders fs.Config.Dump |= fs.DumpHeaders
fs.Logf(nil, "--dump-headers is obsolete - please use --dump headers instead") fs.Logf(nil, "--dump-headers is obsolete - please use --dump headers instead")

View File

@ -23,6 +23,7 @@ type March struct {
Fdst fs.Fs // source Fs Fdst fs.Fs // source Fs
Fsrc fs.Fs // dest Fs Fsrc fs.Fs // dest Fs
Dir string // directory Dir string // directory
NoTraverse bool // don't traverse the destination
SrcIncludeAll bool // don't include all files in the src SrcIncludeAll bool // don't include all files in the src
DstIncludeAll bool // don't include all files in the destination DstIncludeAll bool // don't include all files in the destination
Callback Marcher // object to call with results Callback Marcher // object to call with results
@ -45,7 +46,9 @@ type Marcher interface {
// init sets up a march over opt.Fsrc, and opt.Fdst calling back callback for each match // init sets up a march over opt.Fsrc, and opt.Fdst calling back callback for each match
func (m *March) init() { func (m *March) init() {
m.srcListDir = m.makeListDir(m.Fsrc, m.SrcIncludeAll) m.srcListDir = m.makeListDir(m.Fsrc, m.SrcIncludeAll)
if !m.NoTraverse {
m.dstListDir = m.makeListDir(m.Fdst, m.DstIncludeAll) m.dstListDir = m.makeListDir(m.Fdst, m.DstIncludeAll)
}
// Now create the matching transform // Now create the matching transform
// ..normalise the UTF8 first // ..normalise the UTF8 first
m.transforms = append(m.transforms, norm.NFC.String) m.transforms = append(m.transforms, norm.NFC.String)
@ -344,7 +347,7 @@ func (m *March) processJob(job listDirJob) (jobs []listDirJob) {
srcList, srcListErr = m.srcListDir(job.srcRemote) srcList, srcListErr = m.srcListDir(job.srcRemote)
}() }()
} }
if !job.noDst { if !m.NoTraverse && !job.noDst {
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
@ -367,6 +370,20 @@ func (m *March) processJob(job listDirJob) (jobs []listDirJob) {
return nil return nil
} }
// If NoTraverse is set, then try to find a matching object
// for each item in the srcList
if m.NoTraverse {
for _, src := range srcList {
if srcObj, ok := src.(fs.Object); ok {
leaf := path.Base(srcObj.Remote())
dstObj, err := m.Fdst.NewObject(path.Join(job.dstRemote, leaf))
if err == nil {
dstList = append(dstList, dstObj)
}
}
}
}
// Work out what to do and do it // Work out what to do and do it
srcOnly, dstOnly, matches := matchListings(srcList, dstList, m.transforms) srcOnly, dstOnly, matches := matchListings(srcList, dstList, m.transforms)
for _, src := range srcOnly { for _, src := range srcOnly {

View File

@ -29,6 +29,7 @@ type syncCopyMove struct {
// internal state // internal state
ctx context.Context // internal context for controlling go-routines ctx context.Context // internal context for controlling go-routines
cancel func() // cancel the context cancel func() // cancel the context
noTraverse bool // if set don't traverse the dst
deletersWg sync.WaitGroup // for delete before go routine deletersWg sync.WaitGroup // for delete before go routine
deleteFilesCh chan fs.Object // channel to receive deletes if delete before deleteFilesCh chan fs.Object // channel to receive deletes if delete before
trackRenames bool // set if we should do server side renames trackRenames bool // set if we should do server side renames
@ -75,6 +76,7 @@ func newSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, de
dstFilesResult: make(chan error, 1), dstFilesResult: make(chan error, 1),
dstEmptyDirs: make(map[string]fs.DirEntry), dstEmptyDirs: make(map[string]fs.DirEntry),
srcEmptyDirs: make(map[string]fs.DirEntry), srcEmptyDirs: make(map[string]fs.DirEntry),
noTraverse: fs.Config.NoTraverse,
toBeChecked: newPipe(accounting.Stats.SetCheckQueue, fs.Config.MaxBacklog), toBeChecked: newPipe(accounting.Stats.SetCheckQueue, fs.Config.MaxBacklog),
toBeUploaded: newPipe(accounting.Stats.SetTransferQueue, fs.Config.MaxBacklog), toBeUploaded: newPipe(accounting.Stats.SetTransferQueue, fs.Config.MaxBacklog),
deleteFilesCh: make(chan fs.Object, fs.Config.Checkers), deleteFilesCh: make(chan fs.Object, fs.Config.Checkers),
@ -84,6 +86,10 @@ func newSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, de
trackRenamesCh: make(chan fs.Object, fs.Config.Checkers), trackRenamesCh: make(chan fs.Object, fs.Config.Checkers),
} }
s.ctx, s.cancel = context.WithCancel(context.Background()) s.ctx, s.cancel = context.WithCancel(context.Background())
if s.noTraverse && s.deleteMode != fs.DeleteModeOff {
fs.Errorf(nil, "Ignoring --no-traverse with sync")
s.noTraverse = false
}
if s.trackRenames { if s.trackRenames {
// Don't track renames for remotes without server-side move support. // Don't track renames for remotes without server-side move support.
if !operations.CanServerSideMove(fdst) { if !operations.CanServerSideMove(fdst) {
@ -104,6 +110,10 @@ func newSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, de
if s.deleteMode != fs.DeleteModeOff { if s.deleteMode != fs.DeleteModeOff {
s.deleteMode = fs.DeleteModeAfter s.deleteMode = fs.DeleteModeAfter
} }
if s.noTraverse {
fs.Errorf(nil, "Ignoring --no-traverse with --track-renames")
s.noTraverse = false
}
} }
// Make Fs for --backup-dir if required // Make Fs for --backup-dir if required
if fs.Config.BackupDir != "" { if fs.Config.BackupDir != "" {
@ -651,6 +661,7 @@ func (s *syncCopyMove) run() error {
Fdst: s.fdst, Fdst: s.fdst,
Fsrc: s.fsrc, Fsrc: s.fsrc,
Dir: s.dir, Dir: s.dir,
NoTraverse: s.noTraverse,
Callback: s, Callback: s,
DstIncludeAll: filter.Active.Opt.DeleteExcluded, DstIncludeAll: filter.Active.Opt.DeleteExcluded,
} }

View File

@ -61,6 +61,41 @@ func TestCopy(t *testing.T) {
fstest.CheckItems(t, r.Fremote, file1) fstest.CheckItems(t, r.Fremote, file1)
} }
// Now with --no-traverse
func TestCopyNoTraverse(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
fs.Config.NoTraverse = true
defer func() { fs.Config.NoTraverse = false }()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
err := CopyDir(r.Fremote, r.Flocal)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote, file1)
}
// Now with --no-traverse
func TestSyncNoTraverse(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
fs.Config.NoTraverse = true
defer func() { fs.Config.NoTraverse = false }()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote, file1)
}
// Test copy with depth // Test copy with depth
func TestCopyWithDepth(t *testing.T) { func TestCopyWithDepth(t *testing.T) {
r := fstest.NewRun(t) r := fstest.NewRun(t)