From ddb3b17e968f4a01f929b1813becfbcdd13ed40a Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 22 Mar 2023 10:23:17 +0000 Subject: [PATCH] s3: fix hang on aborting multpart upload with iDrive e2 Apparently the abort multipart upload call doesn't return while multipart uploads are in progress on iDrive e2. This means that if we CTRL-C a multpart upload rclone hangs until the all parts uploading have completed. However since rclone is uploading multiple parts at once this doesn't happen until after the entire file is uploaded. This was fixed by cancelling the upload context which causes all the uploads to stop instantly. --- backend/s3/s3.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index f8dc5cd37..c7b09e631 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -5122,7 +5122,9 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si } uid := cout.UploadId + uploadCtx, cancel := context.WithCancel(ctx) defer atexit.OnError(&err, func() { + cancel() if o.fs.opt.LeavePartsOnError { return } @@ -5142,7 +5144,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si })() var ( - g, gCtx = errgroup.WithContext(ctx) + g, gCtx = errgroup.WithContext(uploadCtx) finished = false partsMu sync.Mutex // to protect parts parts []*s3.CompletedPart @@ -5224,7 +5226,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si uout, err := f.c.UploadPartWithContext(gCtx, uploadPartReq) if err != nil { if partNum <= int64(concurrency) { - return f.shouldRetry(ctx, err) + return f.shouldRetry(gCtx, err) } // retry all chunks once have done the first batch return true, err @@ -5256,7 +5258,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si var resp *s3.CompleteMultipartUploadOutput err = f.pacer.Call(func() (bool, error) { - resp, err = f.c.CompleteMultipartUploadWithContext(ctx, &s3.CompleteMultipartUploadInput{ + resp, err = f.c.CompleteMultipartUploadWithContext(uploadCtx, &s3.CompleteMultipartUploadInput{ Bucket: req.Bucket, Key: req.Key, MultipartUpload: &s3.CompletedMultipartUpload{ @@ -5265,7 +5267,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si RequestPayer: req.RequestPayer, UploadId: uid, }) - return f.shouldRetry(ctx, err) + return f.shouldRetry(uploadCtx, err) }) if err != nil { return wantETag, gotETag, nil, fmt.Errorf("multipart upload failed to finalise: %w", err)