extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-03-13 14:59:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

s3: experiment checking SSE multipart etags - FIXME WIP DO NOT MERGE

Browse Source 
This retrieves the ETags from the uploaded parts and combines those
together to make the overall ETag.

This is probably slightly less robust...

See: https://forum.rclone.org/t/in-some-cases-rclone-does-not-use-etag-to-verify-files/36095/8
This commit is contained in:
Nick Craig-Wood 2023-02-27 15:45:30 +00:00
parent fb4600f6f9
commit b4555d98f7
1 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
backend/s3/s3.go
View File

@ -5160,7 +5160,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
// create checksum of buffer for integrity checking // create checksum of buffer for integrity checking
md5sumBinary := md5.Sum(buf) md5sumBinary := md5.Sum(buf)
addMd5(&md5sumBinary, partNum-1) //addMd5(&md5sumBinary, partNum-1)
md5sum := base64.StdEncoding.EncodeToString(md5sumBinary[:]) md5sum := base64.StdEncoding.EncodeToString(md5sumBinary[:])
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
@ -5191,7 +5191,15 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
ETag: uout.ETag, ETag: uout.ETag,
}) })
partsMu.Unlock() partsMu.Unlock()
if uout.ETag != nil {
etag := strings.Trim(strings.ToLower(*uout.ETag), `"`)
etagBinary, err := hex.DecodeString(etag)
if err != nil || len(etagBinary) != md5.Size {
fs.Errorf(o, "Failed to decode ETag %q: %v", etag, err)
} else {
addMd5((*[md5.Size]byte)(etagBinary), partNum-1)
}
}
return false, nil return false, nil
}) })
if err != nil { if err != nil {
@ -5586,7 +5594,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
o.setMetaData(head) o.setMetaData(head)
// Check multipart upload ETag if required // Check multipart upload ETag if required
if o.fs.opt.UseMultipartEtag.Value && !o.fs.etagIsNotMD5 && wantETag != "" && head.ETag != nil && *head.ETag != "" { if o.fs.opt.UseMultipartEtag.Value /*&& !o.fs.etagIsNotMD5*/ && wantETag != "" && head.ETag != nil && *head.ETag != "" {
gotETag := strings.Trim(strings.ToLower(*head.ETag), `"`) gotETag := strings.Trim(strings.ToLower(*head.ETag), `"`)
if wantETag != gotETag { if wantETag != gotETag {
return fmt.Errorf("multipart upload corrupted: Etag differ: expecting %s but got %s", wantETag, gotETag) return fmt.Errorf("multipart upload corrupted: Etag differ: expecting %s but got %s", wantETag, gotETag)