diff --git a/docs/content/s3.md b/docs/content/s3.md index 49f6c5c9b..401f9f647 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -24,17 +24,29 @@ n/q> n name> remote What type of source is it? Choose a number from below - 1) swift - 2) s3 - 3) local - 4) google cloud storage - 5) dropbox - 6) drive -type> 2 -AWS Access Key ID. -access_key_id> accesskey -AWS Secret Access Key (password). -secret_access_key> secretaccesskey + 1) amazon cloud drive + 2) b2 + 3) drive + 4) dropbox + 5) google cloud storage + 6) swift + 7) hubic + 8) local + 9) onedrive +10) s3 +11) yandex +type> 10 +Get AWS credentials from runtime (environment variables or EC2 meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own value + * Enter AWS credentials in the next step + 1) false + * Get AWS credentials from the environment (env vars or IAM) + 2) true +env_auth> 2 +AWS Access Key ID - leave blank for anonymous access or runtime credentials. +access_key_id> +AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. +secret_access_key> Region to connect to. Choose a number from below, or type in your own value * The default endpoint - a good choice if you are unsure. @@ -44,7 +56,10 @@ Choose a number from below, or type in your own value * US West (Oregon) Region * Needs location constraint us-west-2. 2) us-west-2 -[snip] + * US West (Northern California) Region + * Needs location constraint us-west-1. +[..snip..] + 8) ap-northeast-1 * South America (Sao Paulo) Region * Needs location constraint sa-east-1. 9) sa-east-1 @@ -52,31 +67,32 @@ Choose a number from below, or type in your own value 10) other-v2-signature * If using an S3 clone that understands v4 signatures set this and make sure you set the endpoint. 11) other-v4-signature -region> 1 +region> 3 Endpoint for S3 API. Leave blank if using AWS to use the default endpoint for the region. Specify if using an S3 clone such as Ceph. -endpoint> +endpoint> Location constraint - must be set to match the Region. Used when creating buckets only. Choose a number from below, or type in your own value * Empty for US Region, Northern Virginia or Pacific Northwest. - 1) + 1) * US West (Oregon) Region. 2) us-west-2 * US West (Northern California) Region. - 3) us-west-1 - * EU (Ireland) Region. - 4) eu-west-1 -[snip] -location_constraint> 1 +[..snip..] + 8) ap-northeast-1 + * South America (Sao Paulo) Region. + 9) sa-east-1 +location_constraint> 3 Remote config -------------------- [remote] -access_key_id = accesskey -secret_access_key = secretaccesskey -region = us-east-1 -endpoint = -location_constraint = +env_auth = true +access_key_id = +secret_access_key = +region = us-west-1 +endpoint = +location_constraint = us-west-1 -------------------- y) Yes this is OK e) Edit this remote @@ -92,7 +108,7 @@ e) Edit existing remote n) New remote d) Delete remote q) Quit config -e/n/d/q> q +e/n/d/q> ``` This remote is called `remote` and can now be used like this @@ -133,36 +149,57 @@ created in. If you attempt to access a bucket from the wrong region, you will get an error, `incorrect region, the bucket is not in 'XXX' region`. +### Authentication ### +There are two ways to supply `rclone` with a set of AWS +credentials. In order of precedence: + + - Directly in the rclone configuration file (as configured by `rclone config`) + - set `access_key_id` and `secret_access_key` + - Runtime configuration: + - set `env_auth` to `true` in the config file + - Exporting `AWS_ACCESS_KEY` and `AWS_SECRET_KEY` while running `rclone` + - Running `rclone` on an EC2 instance with an IAM role + +If none of these option actually end up providing `rclone` with AWS +credentials then S3 interaction will be non-authenticated (see below). + ### Anonymous access to public buckets ### If you want to use rclone to access a public bucket, configure with a blank `access_key_id` and `secret_access_key`. Eg ``` -e) Edit existing remote +No remotes found - make a new one n) New remote -d) Delete remote q) Quit config -e/n/d/q> n +n/q> n name> anons3 What type of source is it? Choose a number from below 1) amazon cloud drive - 2) drive - 3) dropbox - 4) google cloud storage - 5) local - 6) s3 - 7) swift -type> 6 -AWS Access Key ID - leave blank for anonymous access. -access_key_id> -AWS Secret Access Key (password) - leave blank for anonymous access. -secret_access_key> -Region to connect to. -region> 1 -endpoint> -location_constraint> + 2) b2 + 3) drive + 4) dropbox + 5) google cloud storage + 6) swift + 7) hubic + 8) local + 9) onedrive +10) s3 +11) yandex +type> 10 +Get AWS credentials from runtime (environment variables or EC2 meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own value + * Enter AWS credentials in the next step + 1) false + * Get AWS credentials from the environment (env vars or IAM) + 2) true +env_auth> 1 +AWS Access Key ID - leave blank for anonymous access or runtime credentials. +access_key_id> +AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. +secret_access_key> +... ``` Then use it as normal with the name of the public bucket, eg diff --git a/s3/s3.go b/s3/s3.go index 7134f4027..aed1c4f8e 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -17,6 +17,7 @@ import ( "errors" "fmt" "io" + "net/http" "net/url" "path" "regexp" @@ -27,6 +28,8 @@ import ( "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/corehandlers" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" + "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" @@ -42,11 +45,23 @@ func init() { NewFs: NewFs, // AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region Options: []fs.Option{{ + Name: "env_auth", + Help: "Get AWS credentials from runtime (environment variables or EC2 meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.", + Examples: []fs.OptionExample{ + { + Value: "false", + Help: "Enter AWS credentials in the next step", + }, { + Value: "true", + Help: "Get AWS credentials from the environment (env vars or IAM)", + }, + }, + }, { Name: "access_key_id", - Help: "AWS Access Key ID - leave blank for anonymous access.", + Help: "AWS Access Key ID - leave blank for anonymous access or runtime credentials.", }, { Name: "secret_access_key", - Help: "AWS Secret Access Key (password) - leave blank for anonymous access.", + Help: "AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.", }, { Name: "region", Help: "Region to connect to.", @@ -195,19 +210,38 @@ func s3ParsePath(path string) (bucket, directory string, err error) { // s3Connection makes a connection to s3 func s3Connection(name string) (*s3.S3, *session.Session, error) { // Make the auth - accessKeyID := fs.ConfigFile.MustValue(name, "access_key_id") - secretAccessKey := fs.ConfigFile.MustValue(name, "secret_access_key") - var auth *credentials.Credentials + v := credentials.Value{ + AccessKeyID: fs.ConfigFile.MustValue(name, "access_key_id"), + SecretAccessKey: fs.ConfigFile.MustValue(name, "secret_access_key"), + } + + // first provider to supply a credential set "wins" + providers := []credentials.Provider{ + // use static credentials if they're present (checked by provider) + &credentials.StaticProvider{Value: v}, + + // * Access Key ID: AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY + // * Secret Access Key: AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY + &credentials.EnvProvider{}, + + // Pick up IAM role in case we're on EC2 + &ec2rolecreds.EC2RoleProvider{ + Client: ec2metadata.New(session.New(), &aws.Config{ + HTTPClient: &http.Client{Timeout: 1 * time.Second}, // low timeout to ec2 metadata service + }), + ExpiryWindow: 3, + }, + } + cred := credentials.NewChainCredentials(providers) + switch { - case accessKeyID == "" && secretAccessKey == "": - fs.Debug(name, "Using anonymous access for S3") - auth = credentials.AnonymousCredentials - case accessKeyID == "": + case fs.ConfigFile.MustBool(name, "env_auth", false) && v.AccessKeyID == "" && v.SecretAccessKey == "": + // if no access key/secret and iam is explicitly disabled then fall back to anon interaction + cred = credentials.AnonymousCredentials + case v.AccessKeyID == "": return nil, nil, errors.New("access_key_id not found") - case secretAccessKey == "": + case v.SecretAccessKey == "": return nil, nil, errors.New("secret_access_key not found") - default: - auth = credentials.NewStaticCredentials(accessKeyID, secretAccessKey, "") } endpoint := fs.ConfigFile.MustValue(name, "endpoint") @@ -221,7 +255,7 @@ func s3Connection(name string) (*s3.S3, *session.Session, error) { awsConfig := aws.NewConfig(). WithRegion(region). WithMaxRetries(maxRetries). - WithCredentials(auth). + WithCredentials(cred). WithEndpoint(endpoint). WithHTTPClient(fs.Config.Client()). WithS3ForcePathStyle(true) @@ -235,7 +269,7 @@ func s3Connection(name string) (*s3.S3, *session.Session, error) { if req.Config.Credentials == credentials.AnonymousCredentials { return } - sign(accessKeyID, secretAccessKey, req.HTTPRequest) + sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest) } c.Handlers.Sign.Clear() c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)