mirror of
https://github.com/rclone/rclone.git
synced 2024-11-22 08:23:47 +01:00
s3: add GCS to provider list
This commit is contained in:
Anthony Pessy
Nick Craig-Wood
parent
29fe0177bd
commit
54a9488e59
@ -66,7 +66,7 @@ import (
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "s3",
|
||||
Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi",
|
||||
Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, GCS, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi",
|
||||
NewFs: NewFs,
|
||||
CommandHelp: commandHelp,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
|
||||
@ -109,6 +109,9 @@ func init() {
|
||||
}, {
|
||||
Value: "Dreamhost",
|
||||
Help: "Dreamhost DreamObjects",
|
||||
}, {
|
||||
Value: "GCS",
|
||||
Help: "Google Cloud Storage",
|
||||
}, {
|
||||
Value: "HuaweiOBS",
|
||||
Help: "Huawei Object Storage Service",
|
||||
@ -934,6 +937,14 @@ func init() {
|
||||
Value: "s3.eu-central-1.stackpathstorage.com",
|
||||
Help: "EU Endpoint",
|
||||
}},
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for Google Cloud Storage.",
|
||||
Provider: "GCS",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "https://storage.googleapis.com",
|
||||
Help: "Google Cloud Storage endpoint",
|
||||
}},
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for Storj Gateway.",
|
||||
@ -1098,7 +1109,7 @@ func init() {
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
|
||||
Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,Liara,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu",
|
||||
Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "objects-us-east-1.dream.io",
|
||||
Help: "Dream Objects endpoint",
|
||||
@ -2258,6 +2269,24 @@ will decompress the object on the fly.
|
||||
If this is set to unset (the default) then rclone will choose
|
||||
according to the provider setting what to apply, but you can override
|
||||
rclone's choice here.
|
||||
`, "|", "`"),
|
||||
Default: fs.Tristate{},
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_accept_encoding_gzip",
|
||||
Help: strings.ReplaceAll(`Whether to send |Accept-Encoding: gzip| header.
|
||||
|
||||
By default, rclone will append |Accept-Encoding: gzip| to the request to download
|
||||
compressed objects whenever possible.
|
||||
|
||||
However some providers such as Google Cloud Storage may alter the HTTP headers, breaking
|
||||
the signature of the request.
|
||||
|
||||
A symptom of this would be receiving errors like
|
||||
|
||||
SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided.
|
||||
|
||||
In this case, you might want to try disabling this option.
|
||||
`, "|", "`"),
|
||||
Default: fs.Tristate{},
|
||||
Advanced: true,
|
||||
@ -2399,6 +2428,7 @@ type Options struct {
|
||||
VersionAt fs.Time `config:"version_at"`
|
||||
Decompress bool `config:"decompress"`
|
||||
MightGzip fs.Tristate `config:"might_gzip"`
|
||||
UseAcceptEncodingGzip fs.Tristate `config:"use_accept_encoding_gzip"`
|
||||
NoSystemMetadata bool `config:"no_system_metadata"`
|
||||
}
|
||||
|
||||
@ -2791,11 +2821,12 @@ func setEndpointValueForIDriveE2(m configmap.Mapper) (err error) {
|
||||
// These should be differences from AWS S3
|
||||
func setQuirks(opt *Options) {
|
||||
var (
|
||||
listObjectsV2 = true
|
||||
virtualHostStyle = true
|
||||
urlEncodeListings = true
|
||||
useMultipartEtag = true
|
||||
mightGzip = true // assume all providers might gzip until proven otherwise
|
||||
listObjectsV2 = true
|
||||
virtualHostStyle = true
|
||||
urlEncodeListings = true
|
||||
useMultipartEtag = true
|
||||
useAcceptEncodingGzip = true
|
||||
mightGzip = true // assume all providers might gzip until proven otherwise
|
||||
)
|
||||
switch opt.Provider {
|
||||
case "AWS":
|
||||
@ -2880,6 +2911,10 @@ func setQuirks(opt *Options) {
|
||||
case "Qiniu":
|
||||
useMultipartEtag = false
|
||||
urlEncodeListings = false
|
||||
case "GCS":
|
||||
// Google break request Signature by mutating accept-encoding HTTP header
|
||||
// https://github.com/rclone/rclone/issues/6670
|
||||
useAcceptEncodingGzip = false
|
||||
case "Other":
|
||||
listObjectsV2 = false
|
||||
virtualHostStyle = false
|
||||
@ -2924,6 +2959,12 @@ func setQuirks(opt *Options) {
|
||||
opt.MightGzip.Valid = true
|
||||
opt.MightGzip.Value = mightGzip
|
||||
}
|
||||
|
||||
// set UseAcceptEncodingGzip if not manually set
|
||||
if !opt.UseAcceptEncodingGzip.Valid {
|
||||
opt.UseAcceptEncodingGzip.Valid = true
|
||||
opt.UseAcceptEncodingGzip.Value = useAcceptEncodingGzip
|
||||
}
|
||||
}
|
||||
|
||||
// setRoot changes the root of the Fs
|
||||
@ -4962,7 +5003,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
|
||||
// Override the automatic decompression in the transport to
|
||||
// download compressed files as-is
|
||||
httpReq.HTTPRequest.Header.Set("Accept-Encoding", "gzip")
|
||||
if o.fs.opt.UseAcceptEncodingGzip.Value {
|
||||
httpReq.HTTPRequest.Header.Set("Accept-Encoding", "gzip")
|
||||
}
|
||||
|
||||
for _, option := range options {
|
||||
switch option.(type) {
|
||||
|
||||
@ -17,6 +17,7 @@ The S3 backend can be used with a number of different providers:
|
||||
{{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud" >}}
|
||||
{{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
|
||||
{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
|
||||
{{< provider name="GCS" home="https://cloud.google.com/storage/docs" config="/s3/#google-cloud-storage" >}}
|
||||
{{< provider name="Huawei OBS" home="https://www.huaweicloud.com/intl/en-us/product/obs.html" config="/s3/#huawei-obs" >}}
|
||||
{{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
|
||||
{{< provider name="IDrive e2" home="https://www.idrive.com/e2/?refer=rclone" config="/s3/#idrive-e2" >}}
|
||||
@ -3394,6 +3395,21 @@ server_side_encryption =
|
||||
storage_class =
|
||||
```
|
||||
|
||||
### Google Cloud Storage
|
||||
|
||||
[GoogleCloudStorage](https://cloud.google.com/storage/docs) is an [S3-interoperable](https://cloud.google.com/storage/docs/interoperability) object storage service from Google Cloud Platform.
|
||||
|
||||
To connect to Google Cloud Storage you will need an access key and secret key. These can be retrieved by creating an [HMAC key](https://cloud.google.com/storage/docs/authentication/managing-hmackeys).
|
||||
|
||||
```
|
||||
[gs]
|
||||
type = s3
|
||||
provider = GCS
|
||||
access_key_id = your_access_key
|
||||
secret_access_key = your_secret_key
|
||||
endpoint = https://storage.googleapis.com
|
||||
```
|
||||
|
||||
### DigitalOcean Spaces
|
||||
|
||||
[Spaces](https://www.digitalocean.com/products/object-storage/) is an [S3-interoperable](https://developers.digitalocean.com/documentation/spaces/) object storage service from cloud provider DigitalOcean.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user