extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-08-14 15:58:26 +02:00
Code Issues Packages Projects Releases Wiki Activity

operations: fix overwrite of destination when multi-thread transfer fails

Browse Source 
Before this change, if a multithread upload failed (let's say the
source became unavailable) rclone would finalise the file first before
aborting the transfer.

This caused the partial file to be written which would overwrite any
existing files.

This was fixed by making sure we Abort the transfer before Close-ing
it.

This updates the docs to encourage calling of Abort before Close and
updates writerAtChunkWriter to make sure that works properly.

This also reworks the tests to detect this and to make sure we upload
and download to each multi-thread capable backend (we were only
downloading before which isn't a full test).

Fixes #7071
This commit is contained in:
Nick Craig-Wood
2023-11-22 15:05:44 +00:00
parent 94ccc95515
commit d5d28a7513
3 changed files with 206 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
fs/operations/multithread.go
View File

@ -165,9 +165,10 @@ func multiThreadCopy(ctx context.Context, f fs.Fs, remote string, src fs.Object,
uploadCtx, cancel := context.WithCancel(ctx)
defer cancel()
uploadedOK := false
defer atexit.OnError(&err, func() {
cancel()
if info.LeavePartsOnError {
if info.LeavePartsOnError || uploadedOK {
return
}
fs.Debugf(src, "multi-thread copy: cancelling transfer on exit")
@ -226,13 +227,14 @@ func multiThreadCopy(ctx context.Context, f fs.Fs, remote string, src fs.Object,
}
err = g.Wait()
closeErr := chunkWriter.Close(ctx)
if err != nil {
return nil, err
}
if closeErr != nil {
return nil, fmt.Errorf("multi-thread copy: failed to close object after copy: %w", closeErr)
err = chunkWriter.Close(ctx)
if err != nil {
return nil, fmt.Errorf("multi-thread copy: failed to close object after copy: %w", err)
}
uploadedOK = true // file is definitely uploaded OK so no need to abort
obj, err := f.NewObject(ctx, remote)
if err != nil {
@ -282,10 +284,11 @@ type writerAtChunkWriter struct {
chunks int
writeBufferSize int64
f fs.Fs
closed bool
}
// WriteChunk writes chunkNumber from reader
func (w writerAtChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (int64, error) {
func (w *writerAtChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (int64, error) {
fs.Debugf(w.remote, "writing chunk %v", chunkNumber)
bytesToWrite := w.chunkSize
@ -316,12 +319,20 @@ func (w writerAtChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, re
}
// Close the chunk writing
func (w writerAtChunkWriter) Close(ctx context.Context) error {
func (w *writerAtChunkWriter) Close(ctx context.Context) error {
if w.closed {
return nil
}
w.closed = true
return w.writerAt.Close()
}
// Abort the chunk writing
func (w writerAtChunkWriter) Abort(ctx context.Context) error {
func (w *writerAtChunkWriter) Abort(ctx context.Context) error {
err := w.Close(ctx)
if err != nil {
fs.Errorf(w.remote, "multi-thread copy: failed to close file before aborting: %v", err)
}
obj, err := w.f.NewObject(ctx, w.remote)
if err != nil {
return fmt.Errorf("multi-thread copy: failed to find temp file when aborting chunk writer: %w", err)