s3: Support Alibaba Cloud (Aliyun) OSS

The existing s3 backend passed all integration tests with OSS provided
`force_path_style = false`.

This makes sure that is so and adds documentation and configuration
for OSS.

Thanks to @luolibin for their work on the OSS backend which we ended
up not needing.

Fixes #1641
Fixes #1237
This commit is contained in:
Nick Craig-Wood 2019-01-12 16:46:45 +00:00
parent 8d16a5693c
commit ff0b8e10af
5 changed files with 230 additions and 65 deletions

View File

@ -372,4 +372,3 @@ Add your fs to the docs - you'll need to pick an icon for it from [fontawesome](
* `docs/content/about.md` - front page of rclone.org
* `docs/layouts/chrome/navbar.html` - add it to the website navigation
* `bin/make_manual.py` - add the page to the `docs` constant
* `cmd/cmd.go` - the main help for rclone

View File

@ -20,6 +20,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
## Storage providers
* Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
* Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
* Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
* Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)

View File

@ -53,7 +53,7 @@ import (
func init() {
fs.Register(&fs.RegInfo{
Name: "s3",
Description: "Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)",
Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)",
NewFs: NewFs,
Options: []fs.Option{{
Name: fs.ConfigProvider,
@ -61,6 +61,9 @@ func init() {
Examples: []fs.OptionExample{{
Value: "AWS",
Help: "Amazon Web Services (AWS) S3",
}, {
Value: "Alibaba",
Help: "Alibaba Cloud Object Storage System (OSS) formerly Aliyun",
}, {
Value: "Ceph",
Help: "Ceph Object Storage",
@ -76,6 +79,9 @@ func init() {
}, {
Value: "Minio",
Help: "Minio Object Storage",
}, {
Value: "Netease",
Help: "Netease Object Storage (NOS)",
}, {
Value: "Wasabi",
Help: "Wasabi Object Storage",
@ -150,7 +156,7 @@ func init() {
}, {
Name: "region",
Help: "Region to connect to.\nLeave blank if you are using an S3 clone and you don't have a region.",
Provider: "!AWS",
Provider: "!AWS,Alibaba",
Examples: []fs.OptionExample{{
Value: "",
Help: "Use this if unsure. Will use v4 signatures and an empty region.",
@ -269,10 +275,73 @@ func init() {
Value: "s3.tor01.objectstorage.service.networklayer.com",
Help: "Toronto Single Site Private Endpoint",
}},
}, {
// oss endpoints: https://help.aliyun.com/document_detail/31837.html
Name: "endpoint",
Help: "Endpoint for OSS API.",
Provider: "Alibaba",
Examples: []fs.OptionExample{{
Value: "oss-cn-hangzhou.aliyuncs.com",
Help: "East China 1 (Hangzhou)",
}, {
Value: "oss-cn-shanghai.aliyuncs.com",
Help: "East China 2 (Shanghai)",
}, {
Value: "oss-cn-qingdao.aliyuncs.com",
Help: "North China 1 (Qingdao)",
}, {
Value: "oss-cn-beijing.aliyuncs.com",
Help: "North China 2 (Beijing)",
}, {
Value: "oss-cn-zhangjiakou.aliyuncs.com",
Help: "North China 3 (Zhangjiakou)",
}, {
Value: "oss-cn-huhehaote.aliyuncs.com",
Help: "North China 5 (Huhehaote)",
}, {
Value: "oss-cn-shenzhen.aliyuncs.com",
Help: "South China 1 (Shenzhen)",
}, {
Value: "oss-cn-hongkong.aliyuncs.com",
Help: "Hong Kong (Hong Kong)",
}, {
Value: "oss-us-west-1.aliyuncs.com",
Help: "US West 1 (Silicon Valley)",
}, {
Value: "oss-us-east-1.aliyuncs.com",
Help: "US East 1 (Virginia)",
}, {
Value: "oss-ap-southeast-1.aliyuncs.com",
Help: "Southeast Asia Southeast 1 (Singapore)",
}, {
Value: "oss-ap-southeast-2.aliyuncs.com",
Help: "Asia Pacific Southeast 2 (Sydney)",
}, {
Value: "oss-ap-southeast-3.aliyuncs.com",
Help: "Southeast Asia Southeast 3 (Kuala Lumpur)",
}, {
Value: "oss-ap-southeast-5.aliyuncs.com",
Help: "Asia Pacific Southeast 5 (Jakarta)",
}, {
Value: "oss-ap-northeast-1.aliyuncs.com",
Help: "Asia Pacific Northeast 1 (Japan)",
}, {
Value: "oss-ap-south-1.aliyuncs.com",
Help: "Asia Pacific South 1 (Mumbai)",
}, {
Value: "oss-eu-central-1.aliyuncs.com",
Help: "Central Europe 1 (Frankfurt)",
}, {
Value: "oss-eu-west-1.aliyuncs.com",
Help: "West Europe (London)",
}, {
Value: "oss-me-east-1.aliyuncs.com",
Help: "Middle East 1 (Dubai)",
}},
}, {
Name: "endpoint",
Help: "Endpoint for S3 API.\nRequired when using an S3 clone.",
Provider: "!AWS,IBMCOS",
Provider: "!AWS,IBMCOS,Alibaba",
Examples: []fs.OptionExample{{
Value: "objects-us-west-1.dream.io",
Help: "Dream Objects endpoint",
@ -449,7 +518,7 @@ func init() {
}, {
Name: "location_constraint",
Help: "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.",
Provider: "!AWS,IBMCOS",
Provider: "!AWS,IBMCOS,Alibaba",
}, {
Name: "acl",
Help: `Canned ACL used when creating buckets and storing or copying objects.
@ -547,6 +616,20 @@ doesn't copy the ACL from the source but rather writes a fresh one.`,
Value: "GLACIER",
Help: "Glacier storage class",
}},
}, {
Name: "storage_class",
Help: "The storage class to use when storing new objects in OSS.",
Provider: "Alibaba",
Examples: []fs.OptionExample{{
Value: "Standard",
Help: "Standard storage class",
}, {
Value: "Archive",
Help: "Archive storage mode.",
}, {
Value: "IA",
Help: "Infrequent access storage mode.",
}},
}, {
Name: "upload_cutoff",
Help: `Cutoff for switching to chunked upload
@ -714,7 +797,7 @@ func (f *Fs) Features() *fs.Features {
// retryErrorCodes is a slice of error codes that we will retry
// See: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
var retryErrorCodes = []int{
409, // Conflict - various states that could be resolved on a retry
// 409, // Conflict - various states that could be resolved on a retry
503, // Service Unavailable/Slow Down - "Reduce your request rate"
}
@ -722,14 +805,13 @@ var retryErrorCodes = []int{
// as it should notice closed connections and timeouts which are the most likely
// sort of failure modes
func shouldRetry(err error) (bool, error) {
// If this is an awserr object, try and extract more useful information to determine if we should retry
if awsError, ok := err.(awserr.Error); ok {
// Simple case, check the original embedded error in case it's generically retriable
if fserrors.ShouldRetry(awsError.OrigErr()) {
return true, err
}
//Failing that, if it's a RequestFailure it's probably got an http status code we can check
// Failing that, if it's a RequestFailure it's probably got an http status code we can check
if reqErr, ok := err.(awserr.RequestFailure); ok {
for _, e := range retryErrorCodes {
if reqErr.StatusCode() == e {
@ -738,7 +820,7 @@ func shouldRetry(err error) (bool, error) {
}
}
}
//Ok, not an awserr, check for generic failure conditions
// Ok, not an awserr, check for generic failure conditions
return fserrors.ShouldRetry(err), err
}
@ -815,6 +897,9 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
if opt.Region == "" {
opt.Region = "us-east-1"
}
if opt.Provider == "Alibaba" || opt.Provider == "Netease" {
opt.ForcePathStyle = false
}
awsConfig := aws.NewConfig().
WithRegion(opt.Region).
WithMaxRetries(maxRetries).

View File

@ -13,6 +13,7 @@ Rclone
Rclone is a command line program to sync files and directories to and from:
* {{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
* {{< provider name="Amazon Drive" home="https://www.amazon.com/clouddrive" config="/amazonclouddrive/" >}} ([See note](/amazonclouddrive/#status))
* {{< provider name="Amazon S3" home="https://aws.amazon.com/s3/" config="/s3/" >}}
* {{< provider name="Backblaze B2" home="https://www.backblaze.com/b2/cloud-storage.html" config="/b2/" >}}

View File

@ -10,6 +10,7 @@ date: "2016-07-11"
The S3 backend can be used with a number of different providers:
* {{< provider name="AWS S3" home="https://aws.amazon.com/s3/" config="/s3/#amazon-s3" >}}
* {{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
* {{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
* {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
* {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
@ -400,7 +401,7 @@ the object(s) in question before using rclone.
<!--- autogenerated options start - DO NOT EDIT, instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs -->
### Standard Options
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)).
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)).
#### --s3-provider
@ -413,6 +414,8 @@ Choose your S3 provider.
- Examples:
- "AWS"
- Amazon Web Services (AWS) S3
- "Alibaba"
- Alibaba Cloud Object Storage System (OSS) formerly Aliyun
- "Ceph"
- Ceph Object Storage
- "DigitalOcean"
@ -423,6 +426,8 @@ Choose your S3 provider.
- IBM COS S3
- "Minio"
- Minio Object Storage
- "Netease"
- Netease Object Storage (NOS)
- "Wasabi"
- Wasabi Object Storage
- "Other"
@ -622,6 +627,54 @@ Specify if using an IBM COS On Premise.
#### --s3-endpoint
Endpoint for OSS API.
- Config: endpoint
- Env Var: RCLONE_S3_ENDPOINT
- Type: string
- Default: ""
- Examples:
- "oss-cn-hangzhou.aliyuncs.com"
- East China 1 (Hangzhou)
- "oss-cn-shanghai.aliyuncs.com"
- East China 2 (Shanghai)
- "oss-cn-qingdao.aliyuncs.com"
- North China 1 (Qingdao)
- "oss-cn-beijing.aliyuncs.com"
- North China 2 (Beijing)
- "oss-cn-zhangjiakou.aliyuncs.com"
- North China 3 (Zhangjiakou)
- "oss-cn-huhehaote.aliyuncs.com"
- North China 5 (Huhehaote)
- "oss-cn-shenzhen.aliyuncs.com"
- South China 1 (Shenzhen)
- "oss-cn-hongkong.aliyuncs.com"
- Hong Kong (Hong Kong)
- "oss-us-west-1.aliyuncs.com"
- US West 1 (Silicon Valley)
- "oss-us-east-1.aliyuncs.com"
- US East 1 (Virginia)
- "oss-ap-southeast-1.aliyuncs.com"
- Southeast Asia Southeast 1 (Singapore)
- "oss-ap-southeast-2.aliyuncs.com"
- Asia Pacific Southeast 2 (Sydney)
- "oss-ap-southeast-3.aliyuncs.com"
- Southeast Asia Southeast 3 (Kuala Lumpur)
- "oss-ap-southeast-5.aliyuncs.com"
- Asia Pacific Southeast 5 (Jakarta)
- "oss-ap-northeast-1.aliyuncs.com"
- Asia Pacific Northeast 1 (Japan)
- "oss-ap-south-1.aliyuncs.com"
- Asia Pacific South 1 (Mumbai)
- "oss-eu-central-1.aliyuncs.com"
- Central Europe 1 (Frankfurt)
- "oss-eu-west-1.aliyuncs.com"
- West Europe (London)
- "oss-me-east-1.aliyuncs.com"
- Middle East 1 (Dubai)
#### --s3-endpoint
Endpoint for S3 API.
Required when using an S3 clone.
@ -855,11 +908,27 @@ The storage class to use when storing new objects in S3.
- "ONEZONE_IA"
- One Zone Infrequent Access storage class
- "GLACIER"
- GLACIER storage class
- Glacier storage class
#### --s3-storage-class
The storage class to use when storing new objects in OSS.
- Config: storage_class
- Env Var: RCLONE_S3_STORAGE_CLASS
- Type: string
- Default: ""
- Examples:
- "Standard"
- Standard storage class
- "Archive"
- Archive storage mode.
- "IA"
- Infrequent access storage mode.
### Advanced Options
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)).
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)).
#### --s3-upload-cutoff
@ -1458,30 +1527,41 @@ server_side_encryption =
storage_class =
```
### Aliyun OSS / Netease NOS ###
### Alibaba OSS {#alibaba-oss}
This describes how to set up Aliyun OSS - Netease NOS is the same
except for different endpoints.
Here is an example of making an [Alibaba Cloud (Aliyun) OSS](https://www.alibabacloud.com/product/oss/)
configuration. First run:
Note this is a pretty standard S3 setup, except for the setting of
`force_path_style = false` in the advanced config.
rclone config
This will guide you through an interactive setup process.
```
# rclone config
e/n/d/r/c/s/q> n
No remotes found - make a new one
n) New remote
s) Set configuration password
q) Quit config
n/s/q> n
name> oss
Type of storage to configure.
Enter a string value. Press Enter for the default ("").
Choose a number from below, or type in your own value
3 / Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)
[snip]
4 / Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)
\ "s3"
[snip]
Storage> s3
Choose your S3 provider.
Enter a string value. Press Enter for the default ("").
Choose a number from below, or type in your own value
8 / Any other S3 compatible provider
\ "Other"
provider> other
1 / Amazon Web Services (AWS) S3
\ "AWS"
2 / Alibaba Cloud Object Storage System (OSS) formerly Aliyun
\ "Alibaba"
3 / Ceph Object Storage
\ "Ceph"
[snip]
provider> Alibaba
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
Only applies if access_key_id and secret_access_key is blank.
Enter a boolean value (true or false). Press Enter for the default ("false").
@ -1494,70 +1574,69 @@ env_auth> 1
AWS Access Key ID.
Leave blank for anonymous access or runtime credentials.
Enter a string value. Press Enter for the default ("").
access_key_id> xxxxxxxxxxxx
access_key_id> accesskeyid
AWS Secret Access Key (password)
Leave blank for anonymous access or runtime credentials.
Enter a string value. Press Enter for the default ("").
secret_access_key> xxxxxxxxxxxxxxxxx
Region to connect to.
Leave blank if you are using an S3 clone and you don't have a region.
secret_access_key> secretaccesskey
Endpoint for OSS API.
Enter a string value. Press Enter for the default ("").
Choose a number from below, or type in your own value
1 / Use this if unsure. Will use v4 signatures and an empty region.
\ ""
2 / Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.
\ "other-v2-signature"
region> 1
Endpoint for S3 API.
Required when using an S3 clone.
Enter a string value. Press Enter for the default ("").
Choose a number from below, or type in your own value
endpoint> oss-cn-shenzhen.aliyuncs.com
Location constraint - must be set to match the Region.
Leave blank if not sure. Used when creating buckets only.
Enter a string value. Press Enter for the default ("").
location_constraint>
Canned ACL used when creating buckets and/or storing objects in S3.
For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
1 / East China 1 (Hangzhou)
\ "oss-cn-hangzhou.aliyuncs.com"
2 / East China 2 (Shanghai)
\ "oss-cn-shanghai.aliyuncs.com"
3 / North China 1 (Qingdao)
\ "oss-cn-qingdao.aliyuncs.com"
[snip]
endpoint> 1
Canned ACL used when creating buckets and storing or copying objects.
Note that this ACL is applied when server side copying objects as S3
doesn't copy the ACL from the source but rather writes a fresh one.
Enter a string value. Press Enter for the default ("").
Choose a number from below, or type in your own value
1 / Owner gets FULL_CONTROL. No one else has access rights (default).
\ "private"
2 / Owner gets FULL_CONTROL. The AllUsers group gets READ access.
\ "public-read"
/ Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access.
[snip]
acl> 1
The storage class to use when storing new objects in OSS.
Enter a string value. Press Enter for the default ("").
Choose a number from below, or type in your own value
1 / Standard storage class
\ "Standard"
2 / Archive storage mode.
\ "Archive"
3 / Infrequent access storage mode.
\ "IA"
storage_class> 1
Edit advanced config? (y/n)
y) Yes
n) No
y/n> y
Chunk size to use for uploading
Enter a size with suffix k,M,G,T. Press Enter for the default ("5M").
chunk_size>
Don't store MD5 checksum with object metadata
Enter a boolean value (true or false). Press Enter for the default ("false").
disable_checksum>
An AWS session token
Enter a string value. Press Enter for the default ("").
session_token>
Concurrency for multipart uploads.
Enter a signed integer. Press Enter for the default ("2").
upload_concurrency>
If true use path style access if false use virtual hosted style.
Some providers (eg Aliyun OSS or Netease COS) require this.
Enter a boolean value (true or false). Press Enter for the default ("true").
force_path_style> false
y/n> n
Remote config
--------------------
[oss]
type = s3
provider = Other
provider = Alibaba
env_auth = false
access_key_id = xxxxxxxxx
secret_access_key = xxxxxxxxxxxxx
endpoint = oss-cn-shenzhen.aliyuncs.com
access_key_id = accesskeyid
secret_access_key = secretaccesskey
endpoint = oss-cn-hangzhou.aliyuncs.com
acl = private
force_path_style = false
storage_class = Standard
--------------------
y) Yes this is OK
e) Edit this remote
d) Delete this remote
y/e/d> y
```
### Netease NOS ###
For Netease NOS configure as per the configurator `rclone config`
setting the provider `Netease`. This will automatically set
`force_path_style = false` which is necessary for it to run properly.