diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 3c62dd3c5..8d13f9be9 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -5866,6 +5866,25 @@ func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options return resp.Body, err } +// middleware to stop the SDK adding `Accept-Encoding: identity` +func removeDisableGzip() func(*middleware.Stack) error { + return func(stack *middleware.Stack) error { + _, err := stack.Finalize.Remove("DisableAcceptEncodingGzip") + return err + } +} + +// middleware to set Accept-Encoding to how we want it +// +// This make sure we download compressed files as-is from all platforms +func (f *Fs) acceptEncoding() (APIOptions []func(*middleware.Stack) error) { + APIOptions = append(APIOptions, removeDisableGzip()) + if f.opt.UseAcceptEncodingGzip.Value { + APIOptions = append(APIOptions, smithyhttp.AddHeaderValue("Accept-Encoding", "gzip")) + } + return APIOptions +} + // Open an object for read func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { bucket, bucketPath := o.split() @@ -5899,11 +5918,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read var APIOptions []func(*middleware.Stack) error - // Override the automatic decompression in the transport to - // download compressed files as-is - if o.fs.opt.UseAcceptEncodingGzip.Value { - APIOptions = append(APIOptions, smithyhttp.AddHeaderValue("Accept-Encoding", "gzip")) - } + // Set the SDK to always download compressed files as-is + APIOptions = append(APIOptions, o.fs.acceptEncoding()...) for _, option := range options { switch option.(type) { diff --git a/docs/content/s3.md b/docs/content/s3.md index 58b16b79a..ac62e61ed 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -2503,6 +2503,9 @@ Note that Cloudflare decompresses files uploaded with does. If this is causing a problem then upload the files with `--header-upload "Cache-Control: no-transform"` +A consequence of this is that `Content-Encoding: gzip` will never +appear in the metadata on Cloudflare. + ### Dreamhost Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is