From 22937e8982e2cebee1b7d955be4f7469d8d4df52 Mon Sep 17 00:00:00 2001 From: wjielai Date: Tue, 8 Sep 2020 23:34:25 +0800 Subject: [PATCH] docs: add Tencent COS to s3 provider list - fixes #4468 * add Tencent COS to s3 provider list. Co-authored-by: wjielai --- README.md | 1 + backend/s3/s3.go | 104 ++++++++++++++++++++++++++++--- docs/content/_index.md | 1 + docs/content/s3.md | 137 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 234 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c5e3907bf..8cd853701 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath) * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/) * Tardigrade [:page_facing_up:](https://rclone.org/tardigrade/) + * Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos) * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi) * WebDAV [:page_facing_up:](https://rclone.org/webdav/) * Yandex Disk [:page_facing_up:](https://rclone.org/yandex/) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 64f09b267..b65f198cc 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -58,7 +58,7 @@ import ( func init() { fs.Register(&fs.RegInfo{ Name: "s3", - Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)", + Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, Tencent COS, etc)", NewFs: NewFs, CommandHelp: commandHelp, Options: []fs.Option{{ @@ -94,6 +94,9 @@ func init() { }, { Value: "StackPath", Help: "StackPath Object Storage", + }, { + Value: "TencentCOS", + Help: "Tencent Cloud Object Storage (COS)", }, { Value: "Wasabi", Help: "Wasabi Object Storage", @@ -185,7 +188,7 @@ func init() { }, { Name: "region", Help: "Region to connect to.\nLeave blank if you are using an S3 clone and you don't have a region.", - Provider: "!AWS,Alibaba,Scaleway", + Provider: "!AWS,Alibaba,Scaleway,TencentCOS", Examples: []fs.OptionExample{{ Value: "", Help: "Use this if unsure. Will use v4 signatures and an empty region.", @@ -476,10 +479,73 @@ func init() { Value: "s3.eu-central-1.stackpathstorage.com", Help: "EU Endpoint", }}, + }, { + // cos endpoints: https://intl.cloud.tencent.com/document/product/436/6224 + Name: "endpoint", + Help: "Endpoint for Tencent COS API.", + Provider: "TencentCOS", + Examples: []fs.OptionExample{{ + Value: "cos.ap-beijing.myqcloud.com", + Help: "Beijing Region.", + }, { + Value: "cos.ap-nanjing.myqcloud.com", + Help: "Nanjing Region.", + }, { + Value: "cos.ap-shanghai.myqcloud.com", + Help: "Shanghai Region.", + }, { + Value: "cos.ap-guangzhou.myqcloud.com", + Help: "Guangzhou Region.", + }, { + Value: "cos.ap-nanjing.myqcloud.com", + Help: "Nanjing Region.", + }, { + Value: "cos.ap-chengdu.myqcloud.com", + Help: "Chengdu Region.", + }, { + Value: "cos.ap-chongqing.myqcloud.com", + Help: "Chongqing Region.", + }, { + Value: "cos.ap-hongkong.myqcloud.com", + Help: "Hong Kong (China) Region.", + }, { + Value: "cos.ap-singapore.myqcloud.com", + Help: "Singapore Region.", + }, { + Value: "cos.ap-mumbai.myqcloud.com", + Help: "Mumbai Region.", + }, { + Value: "cos.ap-seoul.myqcloud.com", + Help: "Seoul Region.", + }, { + Value: "cos.ap-bangkok.myqcloud.com", + Help: "Bangkok Region.", + }, { + Value: "cos.ap-tokyo.myqcloud.com", + Help: "Tokyo Region.", + }, { + Value: "cos.na-siliconvalley.myqcloud.com", + Help: "Silicon Valley Region.", + }, { + Value: "cos.na-ashburn.myqcloud.com", + Help: "Virginia Region.", + }, { + Value: "cos.na-toronto.myqcloud.com", + Help: "Toronto Region.", + }, { + Value: "cos.eu-frankfurt.myqcloud.com", + Help: "Frankfurt Region.", + }, { + Value: "cos.eu-moscow.myqcloud.com", + Help: "Moscow Region.", + }, { + Value: "cos.accelerate.myqcloud.com", + Help: "Use Tencent COS Accelerate Endpoint.", + }}, }, { Name: "endpoint", Help: "Endpoint for S3 API.\nRequired when using an S3 clone.", - Provider: "!AWS,IBMCOS,Alibaba,Scaleway,StackPath", + Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,Scaleway,StackPath", Examples: []fs.OptionExample{{ Value: "objects-us-east-1.dream.io", Help: "Dream Objects endpoint", @@ -666,7 +732,7 @@ func init() { }, { Name: "location_constraint", Help: "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.", - Provider: "!AWS,IBMCOS,Alibaba,Scaleway,StackPath", + Provider: "!AWS,IBMCOS,Alibaba,Scaleway,StackPath,TencentCOS", }, { Name: "acl", Help: `Canned ACL used when creating buckets and storing or copying objects. @@ -678,9 +744,13 @@ For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview Note that this ACL is applied when server side copying objects as S3 doesn't copy the ACL from the source but rather writes a fresh one.`, Examples: []fs.OptionExample{{ + Value: "default", + Help: "Owner gets Full_CONTROL. No one else has access rights (default).", + Provider: "TencentCOS", + }, { Value: "private", Help: "Owner gets FULL_CONTROL. No one else has access rights (default).", - Provider: "!IBMCOS", + Provider: "!IBMCOS,TencentCOS", }, { Value: "public-read", Help: "Owner gets FULL_CONTROL. The AllUsers group gets READ access.", @@ -842,6 +912,24 @@ isn't set then "acl" is used instead.`, Value: "STANDARD_IA", Help: "Infrequent access storage mode.", }}, + }, { + // Mapping from here: https://intl.cloud.tencent.com/document/product/436/30925 + Name: "storage_class", + Help: "The storage class to use when storing new objects in Tencent COS.", + Provider: "TencentCOS", + Examples: []fs.OptionExample{{ + Value: "", + Help: "Default", + }, { + Value: "STANDARD", + Help: "Standard storage class", + }, { + Value: "ARCHIVE", + Help: "Archive storage mode.", + }, { + Value: "STANDARD_IA", + Help: "Infrequent access storage mode.", + }}, }, { // Mapping from here: https://www.scaleway.com/en/docs/object-storage-glacier/#-Scaleway-Storage-Classes Name: "storage_class", @@ -975,7 +1063,7 @@ if false then rclone will use virtual path style. See [the AWS S3 docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro) for more info. -Some providers (eg AWS, Aliyun OSS or Netease COS) require this set to +Some providers (eg AWS, Aliyun OSS, Netease COS or Tencent COS) require this set to false - rclone will do this automatically based on the provider setting.`, Default: true, @@ -1305,7 +1393,7 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) { if opt.Region == "" { opt.Region = "us-east-1" } - if opt.Provider == "AWS" || opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.Provider == "Scaleway" || opt.UseAccelerateEndpoint { + if opt.Provider == "AWS" || opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.Provider == "Scaleway" || opt.Provider == "TencentCOS" || opt.UseAccelerateEndpoint { opt.ForcePathStyle = false } if opt.Provider == "Scaleway" && opt.MaxUploadParts > 1000 { @@ -1587,7 +1675,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck // // So we enable only on providers we know supports it properly, all others can retry when a // XML Syntax error is detected. - var urlEncodeListings = (f.opt.Provider == "AWS" || f.opt.Provider == "Wasabi" || f.opt.Provider == "Alibaba" || f.opt.Provider == "Minio") + var urlEncodeListings = (f.opt.Provider == "AWS" || f.opt.Provider == "Wasabi" || f.opt.Provider == "Alibaba" || f.opt.Provider == "Minio" || f.opt.Provider == "TencentCOS") for { // FIXME need to implement ALL loop req := s3.ListObjectsInput{ diff --git a/docs/content/_index.md b/docs/content/_index.md index 51af844d4..d24a3fd57 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -148,6 +148,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}} {{< provider name="SugarSync" home="https://sugarsync.com/" config="/sugarsync/" >}} {{< provider name="Tardigrade" home="https://tardigrade.io/" config="/tardigrade/" >}} +{{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}} {{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" >}} {{< provider name="WebDAV" home="https://en.wikipedia.org/wiki/WebDAV" config="/webdav/" >}} {{< provider name="Yandex Disk" home="https://disk.yandex.com/" config="/yandex/" >}} diff --git a/docs/content/s3.md b/docs/content/s3.md index 7c3d6cf6d..877093d41 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -18,6 +18,7 @@ The S3 backend can be used with a number of different providers: {{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}} {{< provider name="Scaleway" home="https://www.scaleway.com/en/object-storage/" config="/s3/#scaleway" >}} {{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}} +{{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}} {{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" end="true" >}} {{< /provider_list >}} @@ -488,6 +489,8 @@ Choose your S3 provider. - StackPath Object Storage - "Wasabi" - Wasabi Object Storage + - "TencentCOS" + - Tencent Cloud Object Storage (COS) - "Other" - Any other S3 compatible provider @@ -1122,7 +1125,7 @@ The storage class to use when storing new objects in S3. ### Advanced Options -Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)). +Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, Tencent COS, etc)). #### --s3-bucket-acl @@ -2212,6 +2215,138 @@ d) Delete this remote y/e/d> y ``` +### Tencent COS {#tencent-cos} + +[Tencent Cloud Object Storage (COS)](https://intl.cloud.tencent.com/product/cos) is a distributed storage service offered by Tencent Cloud for unstructured data. It is secure, stable, massive, convenient, low-delay and low-cost. + +To configure access to Tencent COS, follow the steps below: + +1. Run `rclone config` and select `n` for a new remote. + +``` +rclone config +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +``` + +2. Give the name of the configuration. For example, name it 'cos'. + +``` +name> cos +``` + +3. Select `s3` storage. + +``` +Choose a number from below, or type in your own value +1 / 1Fichier + \ "fichier" + 2 / Alias for an existing remote + \ "alias" + 3 / Amazon Drive + \ "amazon cloud drive" + 4 / Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, Tencent COS, etc) + \ "s3" +[snip] +Storage> s3 +``` + +4. Select `TencentCOS` provider. +``` +Choose a number from below, or type in your own value +1 / Amazon Web Services (AWS) S3 + \ "AWS" +[snip] +11 / Tencent Cloud Object Storage (COS) + \ "TencentCOS" +[snip] +provider> TencentCOS +``` + +5. Enter your SecretId and SecretKey of Tencent Cloud. + +``` +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Enter a boolean value (true or false). Press Enter for the default ("false"). +Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \ "false" + 2 / Get AWS credentials from the environment (env vars or IAM) + \ "true" +env_auth> 1 +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a string value. Press Enter for the default (""). +access_key_id> AKIDxxxxxxxxxx +AWS Secret Access Key (password) +Leave blank for anonymous access or runtime credentials. +Enter a string value. Press Enter for the default (""). +secret_access_key> xxxxxxxxxxx +``` + +6. Select endpoint for Tencent COS. This is the standard endpoint for different region. + +``` + 1 / Beijing Region. + \ "cos.ap-beijing.myqcloud.com" + 2 / Nanjing Region. + \ "cos.ap-nanjing.myqcloud.com" + 3 / Shanghai Region. + \ "cos.ap-shanghai.myqcloud.com" + 4 / Guangzhou Region. + \ "cos.ap-guangzhou.myqcloud.com" +[snip] +endpoint> 4 +``` + +7. Choose acl and storage class. + +``` +Note that this ACL is applied when server side copying objects as S3 +doesn't copy the ACL from the source but rather writes a fresh one. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + 1 / Owner gets Full_CONTROL. No one else has access rights (default). + \ "default" +[snip] +acl> 1 +The storage class to use when storing new objects in Tencent COS. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + 1 / Default + \ "" +[snip] +storage_class> 1 +Edit advanced config? (y/n) +y) Yes +n) No (default) +y/n> n +Remote config +-------------------- +[cos] +type = s3 +provider = TencentCOS +env_auth = false +access_key_id = xxx +secret_access_key = xxx +endpoint = cos.ap-guangzhou.myqcloud.com +acl = default +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +Current remotes: + +Name Type +==== ==== +cos s3 +``` + ### Netease NOS ### For Netease NOS configure as per the configurator `rclone config`