mirror of
https://github.com/rclone/rclone.git
synced 2024-12-23 07:29:35 +01:00
copy,move: restore --no-traverse flag
The --no-traverse flag was not implemented when the new sync routines (using the march package) was implemented. This re-implements --no-traverse in march by trying to find a match for each object with NewObject rather than from a directory listing.
This commit is contained in:
parent
e3c4ebd59a
commit
107293c80e
@ -51,6 +51,17 @@ written a trailing / - meaning "copy the contents of this directory".
|
|||||||
This applies to all commands and whether you are talking about the
|
This applies to all commands and whether you are talking about the
|
||||||
source or destination.
|
source or destination.
|
||||||
|
|
||||||
|
See the [--no-traverse](/docs/#no-traverse) option for controlling
|
||||||
|
whether rclone lists the destination directory or not. Supplying this
|
||||||
|
option when copying a small number of files into a large destination
|
||||||
|
can speed transfers up greatly.
|
||||||
|
|
||||||
|
For example, if you have many files in /path/to/src but only a few of
|
||||||
|
them change every day, you can to copy all the files which have
|
||||||
|
changed recently very efficiently like this:
|
||||||
|
|
||||||
|
rclone copy --max-age 24h --no-traverse /path/to/src remote:
|
||||||
|
|
||||||
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
|
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
|
||||||
`,
|
`,
|
||||||
Run: func(command *cobra.Command, args []string) {
|
Run: func(command *cobra.Command, args []string) {
|
||||||
|
@ -37,6 +37,11 @@ into ` + "`dest:path`" + ` then delete the original (if no errors on copy) in
|
|||||||
|
|
||||||
If you want to delete empty source directories after move, use the --delete-empty-src-dirs flag.
|
If you want to delete empty source directories after move, use the --delete-empty-src-dirs flag.
|
||||||
|
|
||||||
|
See the [--no-traverse](/docs/#no-traverse) option for controlling
|
||||||
|
whether rclone lists the destination directory or not. Supplying this
|
||||||
|
option when moving a small number of files into a large destination
|
||||||
|
can speed transfers up greatly.
|
||||||
|
|
||||||
**Important**: Since this can cause data loss, test first with the
|
**Important**: Since this can cause data loss, test first with the
|
||||||
--dry-run flag.
|
--dry-run flag.
|
||||||
|
|
||||||
|
@ -842,8 +842,8 @@ will fall back to the default behaviour and log an error level message
|
|||||||
to the console. Note: Encrypted destinations are not supported
|
to the console. Note: Encrypted destinations are not supported
|
||||||
by `--track-renames`.
|
by `--track-renames`.
|
||||||
|
|
||||||
Note that `--track-renames` uses extra memory to keep track of all
|
Note that `--track-renames` is incompatible with `--no-traverse` and
|
||||||
the rename candidates.
|
that it uses extra memory to keep track of all the rename candidates.
|
||||||
|
|
||||||
Note also that `--track-renames` is incompatible with
|
Note also that `--track-renames` is incompatible with
|
||||||
`--delete-before` and will select `--delete-after` instead of
|
`--delete-before` and will select `--delete-after` instead of
|
||||||
@ -1132,6 +1132,24 @@ This option defaults to `false`.
|
|||||||
|
|
||||||
**This should be used only for testing.**
|
**This should be used only for testing.**
|
||||||
|
|
||||||
|
### --no-traverse ###
|
||||||
|
|
||||||
|
The `--no-traverse` flag controls whether the destination file system
|
||||||
|
is traversed when using the `copy` or `move` commands.
|
||||||
|
`--no-traverse` is not compatible with `sync` and will be ignored if
|
||||||
|
you supply it with `sync`.
|
||||||
|
|
||||||
|
If you are only copying a small number of files (or are filtering most
|
||||||
|
of the files) and/or have a large number of files on the destination
|
||||||
|
then `--no-traverse` will stop rclone listing the destination and save
|
||||||
|
time.
|
||||||
|
|
||||||
|
However, if you are copying a large number of files, especially if you
|
||||||
|
are doing a copy where lots of the files under consideration haven't
|
||||||
|
changed and won't need copying then you shouldn't use `--no-traverse`.
|
||||||
|
|
||||||
|
See [rclone copy](/commands/rclone_copy/) for an example of how to use it.
|
||||||
|
|
||||||
Filtering
|
Filtering
|
||||||
---------
|
---------
|
||||||
|
|
||||||
|
@ -62,6 +62,7 @@ type ConfigInfo struct {
|
|||||||
MaxDepth int
|
MaxDepth int
|
||||||
IgnoreSize bool
|
IgnoreSize bool
|
||||||
IgnoreChecksum bool
|
IgnoreChecksum bool
|
||||||
|
NoTraverse bool
|
||||||
NoUpdateModTime bool
|
NoUpdateModTime bool
|
||||||
DataRateUnit string
|
DataRateUnit string
|
||||||
BackupDir string
|
BackupDir string
|
||||||
|
@ -27,7 +27,6 @@ var (
|
|||||||
deleteAfter bool
|
deleteAfter bool
|
||||||
bindAddr string
|
bindAddr string
|
||||||
disableFeatures string
|
disableFeatures string
|
||||||
noTraverse bool
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// AddFlags adds the non filing system specific flags to the command
|
// AddFlags adds the non filing system specific flags to the command
|
||||||
@ -65,7 +64,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
|
|||||||
flags.IntVarP(flagSet, &fs.Config.MaxDepth, "max-depth", "", fs.Config.MaxDepth, "If set limits the recursion depth to this.")
|
flags.IntVarP(flagSet, &fs.Config.MaxDepth, "max-depth", "", fs.Config.MaxDepth, "If set limits the recursion depth to this.")
|
||||||
flags.BoolVarP(flagSet, &fs.Config.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use mod-time or checksum.")
|
flags.BoolVarP(flagSet, &fs.Config.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use mod-time or checksum.")
|
||||||
flags.BoolVarP(flagSet, &fs.Config.IgnoreChecksum, "ignore-checksum", "", fs.Config.IgnoreChecksum, "Skip post copy check of checksums.")
|
flags.BoolVarP(flagSet, &fs.Config.IgnoreChecksum, "ignore-checksum", "", fs.Config.IgnoreChecksum, "Skip post copy check of checksums.")
|
||||||
flags.BoolVarP(flagSet, &noTraverse, "no-traverse", "", noTraverse, "Obsolete - does nothing.")
|
flags.BoolVarP(flagSet, &fs.Config.NoTraverse, "no-traverse", "", fs.Config.NoTraverse, "Don't traverse destination file system on copy.")
|
||||||
flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.")
|
flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.")
|
||||||
flags.StringVarP(flagSet, &fs.Config.BackupDir, "backup-dir", "", fs.Config.BackupDir, "Make backups into hierarchy based in DIR.")
|
flags.StringVarP(flagSet, &fs.Config.BackupDir, "backup-dir", "", fs.Config.BackupDir, "Make backups into hierarchy based in DIR.")
|
||||||
flags.StringVarP(flagSet, &fs.Config.Suffix, "suffix", "", fs.Config.Suffix, "Suffix for use with --backup-dir.")
|
flags.StringVarP(flagSet, &fs.Config.Suffix, "suffix", "", fs.Config.Suffix, "Suffix for use with --backup-dir.")
|
||||||
@ -113,10 +112,6 @@ func SetFlags() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if noTraverse {
|
|
||||||
fs.Logf(nil, "--no-traverse is obsolete and no longer needed - please remove")
|
|
||||||
}
|
|
||||||
|
|
||||||
if dumpHeaders {
|
if dumpHeaders {
|
||||||
fs.Config.Dump |= fs.DumpHeaders
|
fs.Config.Dump |= fs.DumpHeaders
|
||||||
fs.Logf(nil, "--dump-headers is obsolete - please use --dump headers instead")
|
fs.Logf(nil, "--dump-headers is obsolete - please use --dump headers instead")
|
||||||
|
@ -23,6 +23,7 @@ type March struct {
|
|||||||
Fdst fs.Fs // source Fs
|
Fdst fs.Fs // source Fs
|
||||||
Fsrc fs.Fs // dest Fs
|
Fsrc fs.Fs // dest Fs
|
||||||
Dir string // directory
|
Dir string // directory
|
||||||
|
NoTraverse bool // don't traverse the destination
|
||||||
SrcIncludeAll bool // don't include all files in the src
|
SrcIncludeAll bool // don't include all files in the src
|
||||||
DstIncludeAll bool // don't include all files in the destination
|
DstIncludeAll bool // don't include all files in the destination
|
||||||
Callback Marcher // object to call with results
|
Callback Marcher // object to call with results
|
||||||
@ -45,7 +46,9 @@ type Marcher interface {
|
|||||||
// init sets up a march over opt.Fsrc, and opt.Fdst calling back callback for each match
|
// init sets up a march over opt.Fsrc, and opt.Fdst calling back callback for each match
|
||||||
func (m *March) init() {
|
func (m *March) init() {
|
||||||
m.srcListDir = m.makeListDir(m.Fsrc, m.SrcIncludeAll)
|
m.srcListDir = m.makeListDir(m.Fsrc, m.SrcIncludeAll)
|
||||||
|
if !m.NoTraverse {
|
||||||
m.dstListDir = m.makeListDir(m.Fdst, m.DstIncludeAll)
|
m.dstListDir = m.makeListDir(m.Fdst, m.DstIncludeAll)
|
||||||
|
}
|
||||||
// Now create the matching transform
|
// Now create the matching transform
|
||||||
// ..normalise the UTF8 first
|
// ..normalise the UTF8 first
|
||||||
m.transforms = append(m.transforms, norm.NFC.String)
|
m.transforms = append(m.transforms, norm.NFC.String)
|
||||||
@ -344,7 +347,7 @@ func (m *March) processJob(job listDirJob) (jobs []listDirJob) {
|
|||||||
srcList, srcListErr = m.srcListDir(job.srcRemote)
|
srcList, srcListErr = m.srcListDir(job.srcRemote)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
if !job.noDst {
|
if !m.NoTraverse && !job.noDst {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
@ -367,6 +370,20 @@ func (m *March) processJob(job listDirJob) (jobs []listDirJob) {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If NoTraverse is set, then try to find a matching object
|
||||||
|
// for each item in the srcList
|
||||||
|
if m.NoTraverse {
|
||||||
|
for _, src := range srcList {
|
||||||
|
if srcObj, ok := src.(fs.Object); ok {
|
||||||
|
leaf := path.Base(srcObj.Remote())
|
||||||
|
dstObj, err := m.Fdst.NewObject(path.Join(job.dstRemote, leaf))
|
||||||
|
if err == nil {
|
||||||
|
dstList = append(dstList, dstObj)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Work out what to do and do it
|
// Work out what to do and do it
|
||||||
srcOnly, dstOnly, matches := matchListings(srcList, dstList, m.transforms)
|
srcOnly, dstOnly, matches := matchListings(srcList, dstList, m.transforms)
|
||||||
for _, src := range srcOnly {
|
for _, src := range srcOnly {
|
||||||
|
@ -29,6 +29,7 @@ type syncCopyMove struct {
|
|||||||
// internal state
|
// internal state
|
||||||
ctx context.Context // internal context for controlling go-routines
|
ctx context.Context // internal context for controlling go-routines
|
||||||
cancel func() // cancel the context
|
cancel func() // cancel the context
|
||||||
|
noTraverse bool // if set don't traverse the dst
|
||||||
deletersWg sync.WaitGroup // for delete before go routine
|
deletersWg sync.WaitGroup // for delete before go routine
|
||||||
deleteFilesCh chan fs.Object // channel to receive deletes if delete before
|
deleteFilesCh chan fs.Object // channel to receive deletes if delete before
|
||||||
trackRenames bool // set if we should do server side renames
|
trackRenames bool // set if we should do server side renames
|
||||||
@ -75,6 +76,7 @@ func newSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, de
|
|||||||
dstFilesResult: make(chan error, 1),
|
dstFilesResult: make(chan error, 1),
|
||||||
dstEmptyDirs: make(map[string]fs.DirEntry),
|
dstEmptyDirs: make(map[string]fs.DirEntry),
|
||||||
srcEmptyDirs: make(map[string]fs.DirEntry),
|
srcEmptyDirs: make(map[string]fs.DirEntry),
|
||||||
|
noTraverse: fs.Config.NoTraverse,
|
||||||
toBeChecked: newPipe(accounting.Stats.SetCheckQueue, fs.Config.MaxBacklog),
|
toBeChecked: newPipe(accounting.Stats.SetCheckQueue, fs.Config.MaxBacklog),
|
||||||
toBeUploaded: newPipe(accounting.Stats.SetTransferQueue, fs.Config.MaxBacklog),
|
toBeUploaded: newPipe(accounting.Stats.SetTransferQueue, fs.Config.MaxBacklog),
|
||||||
deleteFilesCh: make(chan fs.Object, fs.Config.Checkers),
|
deleteFilesCh: make(chan fs.Object, fs.Config.Checkers),
|
||||||
@ -84,6 +86,10 @@ func newSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, de
|
|||||||
trackRenamesCh: make(chan fs.Object, fs.Config.Checkers),
|
trackRenamesCh: make(chan fs.Object, fs.Config.Checkers),
|
||||||
}
|
}
|
||||||
s.ctx, s.cancel = context.WithCancel(context.Background())
|
s.ctx, s.cancel = context.WithCancel(context.Background())
|
||||||
|
if s.noTraverse && s.deleteMode != fs.DeleteModeOff {
|
||||||
|
fs.Errorf(nil, "Ignoring --no-traverse with sync")
|
||||||
|
s.noTraverse = false
|
||||||
|
}
|
||||||
if s.trackRenames {
|
if s.trackRenames {
|
||||||
// Don't track renames for remotes without server-side move support.
|
// Don't track renames for remotes without server-side move support.
|
||||||
if !operations.CanServerSideMove(fdst) {
|
if !operations.CanServerSideMove(fdst) {
|
||||||
@ -104,6 +110,10 @@ func newSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, de
|
|||||||
if s.deleteMode != fs.DeleteModeOff {
|
if s.deleteMode != fs.DeleteModeOff {
|
||||||
s.deleteMode = fs.DeleteModeAfter
|
s.deleteMode = fs.DeleteModeAfter
|
||||||
}
|
}
|
||||||
|
if s.noTraverse {
|
||||||
|
fs.Errorf(nil, "Ignoring --no-traverse with --track-renames")
|
||||||
|
s.noTraverse = false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Make Fs for --backup-dir if required
|
// Make Fs for --backup-dir if required
|
||||||
if fs.Config.BackupDir != "" {
|
if fs.Config.BackupDir != "" {
|
||||||
@ -651,6 +661,7 @@ func (s *syncCopyMove) run() error {
|
|||||||
Fdst: s.fdst,
|
Fdst: s.fdst,
|
||||||
Fsrc: s.fsrc,
|
Fsrc: s.fsrc,
|
||||||
Dir: s.dir,
|
Dir: s.dir,
|
||||||
|
NoTraverse: s.noTraverse,
|
||||||
Callback: s,
|
Callback: s,
|
||||||
DstIncludeAll: filter.Active.Opt.DeleteExcluded,
|
DstIncludeAll: filter.Active.Opt.DeleteExcluded,
|
||||||
}
|
}
|
||||||
|
@ -61,6 +61,41 @@ func TestCopy(t *testing.T) {
|
|||||||
fstest.CheckItems(t, r.Fremote, file1)
|
fstest.CheckItems(t, r.Fremote, file1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Now with --no-traverse
|
||||||
|
func TestCopyNoTraverse(t *testing.T) {
|
||||||
|
r := fstest.NewRun(t)
|
||||||
|
defer r.Finalise()
|
||||||
|
|
||||||
|
fs.Config.NoTraverse = true
|
||||||
|
defer func() { fs.Config.NoTraverse = false }()
|
||||||
|
|
||||||
|
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
|
||||||
|
|
||||||
|
err := CopyDir(r.Fremote, r.Flocal)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
fstest.CheckItems(t, r.Flocal, file1)
|
||||||
|
fstest.CheckItems(t, r.Fremote, file1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now with --no-traverse
|
||||||
|
func TestSyncNoTraverse(t *testing.T) {
|
||||||
|
r := fstest.NewRun(t)
|
||||||
|
defer r.Finalise()
|
||||||
|
|
||||||
|
fs.Config.NoTraverse = true
|
||||||
|
defer func() { fs.Config.NoTraverse = false }()
|
||||||
|
|
||||||
|
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
|
||||||
|
|
||||||
|
accounting.Stats.ResetCounters()
|
||||||
|
err := Sync(r.Fremote, r.Flocal)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
fstest.CheckItems(t, r.Flocal, file1)
|
||||||
|
fstest.CheckItems(t, r.Fremote, file1)
|
||||||
|
}
|
||||||
|
|
||||||
// Test copy with depth
|
// Test copy with depth
|
||||||
func TestCopyWithDepth(t *testing.T) {
|
func TestCopyWithDepth(t *testing.T) {
|
||||||
r := fstest.NewRun(t)
|
r := fstest.NewRun(t)
|
||||||
|
Loading…
Reference in New Issue
Block a user