From 6079cab0907c74bf331e06a0d80aa8e7c9d42f53 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Tue, 22 Oct 2024 20:12:18 +0100
Subject: [PATCH] s3: fix download of compressed files from Cloudflare R2 -
 fixes #8137

Before this change attempting to download a file with
`Content-Encoding: gzip` from Cloudflare R2 gave this error

    corrupted on transfer: sizes differ src 0 vs dst 999

This was caused by the SDK v2 overriding our attempt to set
`Accept-Encoding: gzip`.

This fixes the problem by disabling the middleware that does that
overriding.
---
 backend/s3/s3.go   | 26 +++++++++++++++++++++-----
 docs/content/s3.md |  3 +++
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 3c62dd3c5..8d13f9be9 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -5866,6 +5866,25 @@ func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options
 	return resp.Body, err
 }
 
+// middleware to stop the SDK adding `Accept-Encoding: identity`
+func removeDisableGzip() func(*middleware.Stack) error {
+	return func(stack *middleware.Stack) error {
+		_, err := stack.Finalize.Remove("DisableAcceptEncodingGzip")
+		return err
+	}
+}
+
+// middleware to set Accept-Encoding to how we want it
+//
+// This make sure we download compressed files as-is from all platforms
+func (f *Fs) acceptEncoding() (APIOptions []func(*middleware.Stack) error) {
+	APIOptions = append(APIOptions, removeDisableGzip())
+	if f.opt.UseAcceptEncodingGzip.Value {
+		APIOptions = append(APIOptions, smithyhttp.AddHeaderValue("Accept-Encoding", "gzip"))
+	}
+	return APIOptions
+}
+
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	bucket, bucketPath := o.split()
@@ -5899,11 +5918,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 	var APIOptions []func(*middleware.Stack) error
 
-	// Override the automatic decompression in the transport to
-	// download compressed files as-is
-	if o.fs.opt.UseAcceptEncodingGzip.Value {
-		APIOptions = append(APIOptions, smithyhttp.AddHeaderValue("Accept-Encoding", "gzip"))
-	}
+	// Set the SDK to always download compressed files as-is
+	APIOptions = append(APIOptions, o.fs.acceptEncoding()...)
 
 	for _, option := range options {
 		switch option.(type) {
diff --git a/docs/content/s3.md b/docs/content/s3.md
index 58b16b79a..ac62e61ed 100644
--- a/docs/content/s3.md
+++ b/docs/content/s3.md
@@ -2503,6 +2503,9 @@ Note that Cloudflare decompresses files uploaded with
 does. If this is causing a problem then upload the files with
 `--header-upload "Cache-Control: no-transform"`
 
+A consequence of this is that `Content-Encoding: gzip` will never
+appear in the metadata on Cloudflare.
+
 ### Dreamhost
 
 Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is