s3: add --s3-decompress flag to download gzip-encoded files

Before this change, if an object compressed with "Content-Encoding:
gzip" was downloaded, a length and hash mismatch would occur since the
go runtime automatically decompressed the object on download.

If --s3-decompress is set, this change erases the length and hash on
compressed objects so they can be downloaded successfully, at the cost
of not being able to check the length or the hash of the downloaded
object.

If --s3-decompress is not set the compressed files will be downloaded
as-is providing compressed objects with intact size and hash
information.

See #2658

backend/s3/s3.go

@@ -2004,6 +2004,19 @@ See [the time option docs](/docs/#time-option) for valid formats.
 `,
 			Default:  fs.Time{},
 			Advanced: true,
+		}, {
+			Name: "decompress",
+			Help: `If set this will decompress gzip encoded objects.
+
+It is possible to upload objects to S3 with "Content-Encoding: gzip"
+set. Normally rclone will download these files files as compressed objects.
+
+If this flag is set then rclone will decompress these files with
+"Content-Encoding: gzip" as they are received. This means that rclone
+can't check the size and hash but the file contents will be decompressed.
+`,
+			Advanced: true,
+			Default:  false,
 		},
 		}})
 }
@@ -2128,6 +2141,7 @@ type Options struct {
 	UsePresignedRequest   bool                 `config:"use_presigned_request"`
 	Versions              bool                 `config:"versions"`
 	VersionAt             fs.Time              `config:"version_at"`
+	Decompress            bool                 `config:"decompress"`
 }
 
 // Fs represents a remote s3 server
@@ -2150,6 +2164,7 @@ type Fs struct {
 	etagIsNotMD5   bool             // if set ETags are not MD5s
 	versioningMu   sync.Mutex
 	versioning     fs.Tristate // if set bucket is using versions
+	warnCompressed sync.Once   // warn once about compressed files
 }
 
 // Object describes a s3 object
@@ -4318,6 +4333,10 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	if t != hash.MD5 {
 		return "", hash.ErrUnsupported
 	}
+	// If decompressing, erase the hash
+	if o.bytes < 0 {
+		return "", nil
+	}
 	// If we haven't got an MD5, then check the metadata
 	if o.md5 == "" {
 		err := o.readMetaData(ctx)
@@ -4439,6 +4458,12 @@ func (o *Object) setMetaData(resp *s3.HeadObjectOutput) {
 	o.contentDisposition = resp.ContentDisposition
 	o.contentEncoding = resp.ContentEncoding
 	o.contentLanguage = resp.ContentLanguage
+
+	// If decompressing then size and md5sum are unknown
+	if o.fs.opt.Decompress && aws.StringValue(o.contentEncoding) == "gzip" {
+		o.bytes = -1
+		o.md5 = ""
+	}
 }
 
 // ModTime returns the modification time of the object
@@ -4596,6 +4621,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	httpReq, resp := o.fs.c.GetObjectRequest(&req)
 	fs.FixRangeOption(options, o.bytes)
+
+	// Override the automatic decompression in the transport to
+	// download compressed files as-is
+	httpReq.HTTPRequest.Header.Set("Accept-Encoding", "gzip")
+
 	for _, option := range options {
 		switch option.(type) {
 		case *fs.RangeOption, *fs.SeekOption:
@@ -4646,6 +4676,17 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	setFrom_s3HeadObjectOutput_s3GetObjectOutput(&head, resp)
 	head.ContentLength = size
 	o.setMetaData(&head)
+
+	// Decompress body if necessary
+	if aws.StringValue(resp.ContentEncoding) == "gzip" {
+		if o.fs.opt.Decompress {
+			return readers.NewGzipReader(resp.Body)
+		}
+		o.fs.warnCompressed.Do(func() {
+			fs.Logf(o, "Not decompressing 'Content-Encoding: gzip' compressed file. Use --s3-decompress to override")
+		})
+	}
+
 	return resp.Body, nil
 }
 

backend/s3/s3_internal_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
+	"crypto/md5"
 	"fmt"
 	"testing"
 	"time"
@@ -11,6 +12,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/rclone/rclone/lib/random"
@@ -29,9 +31,15 @@ func gz(t *testing.T, s string) string {
 	return buf.String()
 }
 
+func md5sum(t *testing.T, s string) string {
+	hash := md5.Sum([]byte(s))
+	return fmt.Sprintf("%x", hash)
+}
+
 func (f *Fs) InternalTestMetadata(t *testing.T) {
 	ctx := context.Background()
-	contents := gz(t, random.String(1000))
+	original := random.String(1000)
+	contents := gz(t, original)
 
 	item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
 	btime := time.Now()
@@ -68,6 +76,31 @@ func (f *Fs) InternalTestMetadata(t *testing.T) {
 			assert.Equal(t, v, got, k)
 		}
 	}
+
+	t.Run("GzipEncoding", func(t *testing.T) {
+		// Test that the gziped file we uploaded can be
+		// downloaded with and without decompression
+		checkDownload := func(wantContents string, wantSize int64, wantHash string) {
+			gotContents := fstests.ReadObject(ctx, t, o, -1)
+			assert.Equal(t, wantContents, gotContents)
+			assert.Equal(t, wantSize, o.Size())
+			gotHash, err := o.Hash(ctx, hash.MD5)
+			require.NoError(t, err)
+			assert.Equal(t, wantHash, gotHash)
+		}
+
+		t.Run("NoDecompress", func(t *testing.T) {
+			checkDownload(contents, int64(len(contents)), md5sum(t, contents))
+		})
+		t.Run("Decompress", func(t *testing.T) {
+			f.opt.Decompress = true
+			defer func() {
+				f.opt.Decompress = false
+			}()
+			checkDownload(original, -1, "")
+		})
+
+	})
 }
 
 func (f *Fs) InternalTestNoHead(t *testing.T) {