mirror of
https://github.com/rclone/rclone.git
synced 2024-11-07 09:04:52 +01:00
move: if --check-first and --order-by are set then delete with perfect ordering
If using rclone move and --check-first and --order-by then rclone uses the transfer routine to delete files to ensure perfect ordering. This will cause the transfer stats to have a larger than expected number of items in it so we don't enable this by default. Fixes #6033
This commit is contained in:
parent
4edcd16f5f
commit
dd6e229327
@ -789,6 +789,12 @@ interfere with checking.
|
||||
It can also be useful to ensure perfect ordering when using
|
||||
`--order-by`.
|
||||
|
||||
If both `--check-first` and `--order-by` are set when doing `rclone move`
|
||||
then rclone will use the transfer thread to delete source files which
|
||||
don't need transferring. This will enable perfect ordering of the
|
||||
transfers and deletes but will cause the transfer stats to have more
|
||||
items in than expected.
|
||||
|
||||
Using this flag can use more memory as it effectively sets
|
||||
`--max-backlog` to infinite. This means that all the info on the
|
||||
objects to transfer is held in memory before the transfers start.
|
||||
|
@ -85,6 +85,8 @@ func (p *pipe) Pop() interface{} {
|
||||
// It returns ok = false if the context was cancelled
|
||||
//
|
||||
// It will panic if you call it after Close()
|
||||
//
|
||||
// Note that pairs where src==dst aren't counted for stats
|
||||
func (p *pipe) Put(ctx context.Context, pair fs.ObjectPair) (ok bool) {
|
||||
if ctx.Err() != nil {
|
||||
return false
|
||||
@ -97,7 +99,7 @@ func (p *pipe) Put(ctx context.Context, pair fs.ObjectPair) (ok bool) {
|
||||
deheap.Push(p, pair)
|
||||
}
|
||||
size := pair.Src.Size()
|
||||
if size > 0 {
|
||||
if size > 0 && pair.Src != pair.Dst {
|
||||
p.totalSize += size
|
||||
}
|
||||
p.stats(len(p.queue), p.totalSize)
|
||||
@ -141,7 +143,7 @@ func (p *pipe) GetMax(ctx context.Context, fraction int) (pair fs.ObjectPair, ok
|
||||
pair = deheap.PopMax(p).(fs.ObjectPair)
|
||||
}
|
||||
size := pair.Src.Size()
|
||||
if size > 0 {
|
||||
if size > 0 && pair.Src != pair.Dst {
|
||||
p.totalSize -= size
|
||||
}
|
||||
if p.totalSize < 0 {
|
||||
|
@ -42,12 +42,18 @@ func TestPipe(t *testing.T) {
|
||||
obj1 := mockobject.New("potato").WithContent([]byte("hello"), mockobject.SeekModeNone)
|
||||
|
||||
pair1 := fs.ObjectPair{Src: obj1, Dst: nil}
|
||||
pairD := fs.ObjectPair{Src: obj1, Dst: obj1} // this object should not count to the stats
|
||||
|
||||
// Put an object
|
||||
ok := p.Put(ctx, pair1)
|
||||
assert.Equal(t, true, ok)
|
||||
checkStats(1, 5)
|
||||
|
||||
// Put an object to be deleted
|
||||
ok = p.Put(ctx, pairD)
|
||||
assert.Equal(t, true, ok)
|
||||
checkStats(2, 5)
|
||||
|
||||
// Close the pipe showing reading on closed pipe is OK
|
||||
p.Close()
|
||||
|
||||
@ -55,6 +61,12 @@ func TestPipe(t *testing.T) {
|
||||
pair2, ok := p.Get(ctx)
|
||||
assert.Equal(t, pair1, pair2)
|
||||
assert.Equal(t, true, ok)
|
||||
checkStats(1, 0)
|
||||
|
||||
// Read from pipe
|
||||
pair2, ok = p.Get(ctx)
|
||||
assert.Equal(t, pairD, pair2)
|
||||
assert.Equal(t, true, ok)
|
||||
checkStats(0, 0)
|
||||
|
||||
// Check read on closed pipe
|
||||
|
@ -377,6 +377,14 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.W
|
||||
fs.Logf(src, "Not removing source file as it is the same file as the destination")
|
||||
} else if s.ci.IgnoreExisting {
|
||||
fs.Debugf(src, "Not removing source file as destination file exists and --ignore-existing is set")
|
||||
} else if s.checkFirst && s.ci.OrderBy != "" {
|
||||
// If we want perfect ordering then use the transfers to delete the file
|
||||
//
|
||||
// We send src == dst, to say we want the src deleted
|
||||
ok = out.Put(s.ctx, fs.ObjectPair{Src: src, Dst: src})
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
s.processError(operations.DeleteFile(s.ctx, src))
|
||||
}
|
||||
@ -417,10 +425,16 @@ func (s *syncCopyMove) pairCopyOrMove(ctx context.Context, in *pipe, fdst fs.Fs,
|
||||
return
|
||||
}
|
||||
src := pair.Src
|
||||
dst := pair.Dst
|
||||
if s.DoMove {
|
||||
_, err = operations.Move(ctx, fdst, pair.Dst, src.Remote(), src)
|
||||
if src != dst {
|
||||
_, err = operations.Move(ctx, fdst, dst, src.Remote(), src)
|
||||
} else {
|
||||
// src == dst signals delete the src
|
||||
err = operations.DeleteFile(ctx, src)
|
||||
}
|
||||
} else {
|
||||
_, err = operations.Copy(ctx, fdst, pair.Dst, src.Remote(), src)
|
||||
_, err = operations.Copy(ctx, fdst, dst, src.Remote(), src)
|
||||
}
|
||||
s.processError(err)
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user