From 54a9488e5981764d6cb7fb6ece6f9ecb07ccea8b Mon Sep 17 00:00:00 2001 From: Anthony Pessy Date: Mon, 9 Jan 2023 07:54:51 +0100 Subject: [PATCH] s3: add GCS to provider list --- backend/s3/s3.go | 59 +++++++++++++++++++++++++++++++++++++++------- docs/content/s3.md | 16 +++++++++++++ 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index d269b017c..4d4994302 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -66,7 +66,7 @@ import ( func init() { fs.Register(&fs.RegInfo{ Name: "s3", - Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi", + Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, GCS, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi", NewFs: NewFs, CommandHelp: commandHelp, Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { @@ -109,6 +109,9 @@ func init() { }, { Value: "Dreamhost", Help: "Dreamhost DreamObjects", + }, { + Value: "GCS", + Help: "Google Cloud Storage", }, { Value: "HuaweiOBS", Help: "Huawei Object Storage Service", @@ -934,6 +937,14 @@ func init() { Value: "s3.eu-central-1.stackpathstorage.com", Help: "EU Endpoint", }}, + }, { + Name: "endpoint", + Help: "Endpoint for Google Cloud Storage.", + Provider: "GCS", + Examples: []fs.OptionExample{{ + Value: "https://storage.googleapis.com", + Help: "Google Cloud Storage endpoint", + }}, }, { Name: "endpoint", Help: "Endpoint for Storj Gateway.", @@ -1098,7 +1109,7 @@ func init() { }, { Name: "endpoint", Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.", - Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,Liara,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu", + Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu", Examples: []fs.OptionExample{{ Value: "objects-us-east-1.dream.io", Help: "Dream Objects endpoint", @@ -2258,6 +2269,24 @@ will decompress the object on the fly. If this is set to unset (the default) then rclone will choose according to the provider setting what to apply, but you can override rclone's choice here. +`, "|", "`"), + Default: fs.Tristate{}, + Advanced: true, + }, { + Name: "use_accept_encoding_gzip", + Help: strings.ReplaceAll(`Whether to send |Accept-Encoding: gzip| header. + +By default, rclone will append |Accept-Encoding: gzip| to the request to download +compressed objects whenever possible. + +However some providers such as Google Cloud Storage may alter the HTTP headers, breaking +the signature of the request. + +A symptom of this would be receiving errors like + + SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. + +In this case, you might want to try disabling this option. `, "|", "`"), Default: fs.Tristate{}, Advanced: true, @@ -2399,6 +2428,7 @@ type Options struct { VersionAt fs.Time `config:"version_at"` Decompress bool `config:"decompress"` MightGzip fs.Tristate `config:"might_gzip"` + UseAcceptEncodingGzip fs.Tristate `config:"use_accept_encoding_gzip"` NoSystemMetadata bool `config:"no_system_metadata"` } @@ -2791,11 +2821,12 @@ func setEndpointValueForIDriveE2(m configmap.Mapper) (err error) { // These should be differences from AWS S3 func setQuirks(opt *Options) { var ( - listObjectsV2 = true - virtualHostStyle = true - urlEncodeListings = true - useMultipartEtag = true - mightGzip = true // assume all providers might gzip until proven otherwise + listObjectsV2 = true + virtualHostStyle = true + urlEncodeListings = true + useMultipartEtag = true + useAcceptEncodingGzip = true + mightGzip = true // assume all providers might gzip until proven otherwise ) switch opt.Provider { case "AWS": @@ -2880,6 +2911,10 @@ func setQuirks(opt *Options) { case "Qiniu": useMultipartEtag = false urlEncodeListings = false + case "GCS": + // Google break request Signature by mutating accept-encoding HTTP header + // https://github.com/rclone/rclone/issues/6670 + useAcceptEncodingGzip = false case "Other": listObjectsV2 = false virtualHostStyle = false @@ -2924,6 +2959,12 @@ func setQuirks(opt *Options) { opt.MightGzip.Valid = true opt.MightGzip.Value = mightGzip } + + // set UseAcceptEncodingGzip if not manually set + if !opt.UseAcceptEncodingGzip.Valid { + opt.UseAcceptEncodingGzip.Valid = true + opt.UseAcceptEncodingGzip.Value = useAcceptEncodingGzip + } } // setRoot changes the root of the Fs @@ -4962,7 +5003,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read // Override the automatic decompression in the transport to // download compressed files as-is - httpReq.HTTPRequest.Header.Set("Accept-Encoding", "gzip") + if o.fs.opt.UseAcceptEncodingGzip.Value { + httpReq.HTTPRequest.Header.Set("Accept-Encoding", "gzip") + } for _, option := range options { switch option.(type) { diff --git a/docs/content/s3.md b/docs/content/s3.md index c6c5f6edb..2cd0adc93 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -17,6 +17,7 @@ The S3 backend can be used with a number of different providers: {{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud" >}} {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}} {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}} +{{< provider name="GCS" home="https://cloud.google.com/storage/docs" config="/s3/#google-cloud-storage" >}} {{< provider name="Huawei OBS" home="https://www.huaweicloud.com/intl/en-us/product/obs.html" config="/s3/#huawei-obs" >}} {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}} {{< provider name="IDrive e2" home="https://www.idrive.com/e2/?refer=rclone" config="/s3/#idrive-e2" >}} @@ -3394,6 +3395,21 @@ server_side_encryption = storage_class = ``` +### Google Cloud Storage + +[GoogleCloudStorage](https://cloud.google.com/storage/docs) is an [S3-interoperable](https://cloud.google.com/storage/docs/interoperability) object storage service from Google Cloud Platform. + +To connect to Google Cloud Storage you will need an access key and secret key. These can be retrieved by creating an [HMAC key](https://cloud.google.com/storage/docs/authentication/managing-hmackeys). + +``` +[gs] +type = s3 +provider = GCS +access_key_id = your_access_key +secret_access_key = your_secret_key +endpoint = https://storage.googleapis.com +``` + ### DigitalOcean Spaces [Spaces](https://www.digitalocean.com/products/object-storage/) is an [S3-interoperable](https://developers.digitalocean.com/documentation/spaces/) object storage service from cloud provider DigitalOcean.