From b1d8da484bd66989fb2b80fb25e2da200b5402f2 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 23 Apr 2020 19:19:45 +0100 Subject: [PATCH] azureblob: retry InvalidBlobOrBlock error as it may indicate block concurrency problems According to Microsoft support this error can be caused by > A timing/concurrency issue where the PUT operations are happening > about the same time for a single blob. The Put Block List operation > writes a blob by specifying the list of block IDs that make up the > blob. In order to be written as part of a blob, a block must have > been successfully written to the server in a prior Put Block > operation. > > Documentation reference: > > https://docs.microsoft.com/en-us/rest/api/storageservices/put-block > > This error can happen when doing concurrent upload commits after you > have started the upload but before you commit. In that case, the > upload fails. The application can retry this error or attempt some > other recovery action based on the required scenario. See: https://forum.rclone.org/t/error-while-syncing-with-azure-blob-storage-x-ms-error-code-invalidbloborblock/15561 --- backend/azureblob/azureblob.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index e639415bd..19b607745 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -257,6 +257,12 @@ var retryErrorCodes = []int{ func (f *Fs) shouldRetry(err error) (bool, error) { // FIXME interpret special errors - more to do here if storageErr, ok := err.(azblob.StorageError); ok { + switch storageErr.ServiceCode() { + case "InvalidBlobOrBlock": + // These errors happen sometimes in multipart uploads + // because of block concurrency issues + return true, err + } statusCode := storageErr.Response().StatusCode for _, e := range retryErrorCodes { if statusCode == e { @@ -1322,7 +1328,7 @@ outer: // Upload the block, with MD5 for check md5sum := md5.Sum(buf) transactionalMD5 := md5sum[:] - err = o.fs.pacer.Call(func() (bool, error) { + err := o.fs.pacer.Call(func() (bool, error) { bufferReader := bytes.NewReader(buf) wrappedReader := wrap(bufferReader) rs := readSeeker{wrappedReader, bufferReader}