s3: fix download of compressed files from Cloudflare R2 - fixes #8137

Before this change attempting to download a file with
`Content-Encoding: gzip` from Cloudflare R2 gave this error

    corrupted on transfer: sizes differ src 0 vs dst 999

This was caused by the SDK v2 overriding our attempt to set
`Accept-Encoding: gzip`.

This fixes the problem by disabling the middleware that does that
overriding.

backend/s3/s3.go

@@ -5866,6 +5866,25 @@ func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options
 	return resp.Body, err
 }
 
+// middleware to stop the SDK adding `Accept-Encoding: identity`
+func removeDisableGzip() func(*middleware.Stack) error {
+	return func(stack *middleware.Stack) error {
+		_, err := stack.Finalize.Remove("DisableAcceptEncodingGzip")
+		return err
+	}
+}
+
+// middleware to set Accept-Encoding to how we want it
+//
+// This make sure we download compressed files as-is from all platforms
+func (f *Fs) acceptEncoding() (APIOptions []func(*middleware.Stack) error) {
+	APIOptions = append(APIOptions, removeDisableGzip())
+	if f.opt.UseAcceptEncodingGzip.Value {
+		APIOptions = append(APIOptions, smithyhttp.AddHeaderValue("Accept-Encoding", "gzip"))
+	}
+	return APIOptions
+}
+
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	bucket, bucketPath := o.split()
@@ -5899,11 +5918,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 	var APIOptions []func(*middleware.Stack) error
 
-	// Override the automatic decompression in the transport to
+	// Set the SDK to always download compressed files as-is
-	// download compressed files as-is
+	APIOptions = append(APIOptions, o.fs.acceptEncoding()...)
-	if o.fs.opt.UseAcceptEncodingGzip.Value {
-		APIOptions = append(APIOptions, smithyhttp.AddHeaderValue("Accept-Encoding", "gzip"))
-	}
 
 	for _, option := range options {
 		switch option.(type) {

docs/content/s3.md

@@ -2503,6 +2503,9 @@ Note that Cloudflare decompresses files uploaded with
 does. If this is causing a problem then upload the files with
 `--header-upload "Cache-Control: no-transform"`
 
+A consequence of this is that `Content-Encoding: gzip` will never
+appear in the metadata on Cloudflare.
+
 ### Dreamhost
 
 Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is