Add IAM role and Env credentials

This will make the s3 provider authentaction logic

  - Configured credentials if both key and secret available
  - Anonymous if key and secret missing and env_auth not set
  - if env_auth is set to truthy (https://golang.org/pkg/strconv/#ParseBool)
    - AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY environment variables
    - IAM role credentials as fallback

docs/content/s3.md

@@ -24,17 +24,29 @@ n/q> n
 name> remote
 What type of source is it?
 Choose a number from below
- 1) swift
+ 1) amazon cloud drive
- 2) s3
+ 2) b2
- 3) local
+ 3) drive
- 4) google cloud storage
+ 4) dropbox
- 5) dropbox
+ 5) google cloud storage
- 6) drive
+ 6) swift
-type> 2
+ 7) hubic
-AWS Access Key ID.
+ 8) local
-access_key_id> accesskey
+ 9) onedrive
-AWS Secret Access Key (password). 
+10) s3
-secret_access_key> secretaccesskey
+11) yandex
+type> 10
+Get AWS credentials from runtime (environment variables or EC2 meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own value
+ * Enter AWS credentials in the next step
+ 1) false
+ * Get AWS credentials from the environment (env vars or IAM)
+ 2) true
+env_auth> 2
+AWS Access Key ID - leave blank for anonymous access or runtime credentials.
+access_key_id>
+AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.
+secret_access_key>
 Region to connect to.
 Choose a number from below, or type in your own value
  * The default endpoint - a good choice if you are unsure.
@@ -44,7 +56,10 @@ Choose a number from below, or type in your own value
  * US West (Oregon) Region
  * Needs location constraint us-west-2.
  2) us-west-2
-[snip]
+ * US West (Northern California) Region
+ * Needs location constraint us-west-1.
+[..snip..]
+ 8) ap-northeast-1
  * South America (Sao Paulo) Region
  * Needs location constraint sa-east-1.
  9) sa-east-1
@@ -52,31 +67,32 @@ Choose a number from below, or type in your own value
 10) other-v2-signature
  * If using an S3 clone that understands v4 signatures set this and make sure you set the endpoint.
 11) other-v4-signature
-region> 1
+region> 3
 Endpoint for S3 API.
 Leave blank if using AWS to use the default endpoint for the region.
 Specify if using an S3 clone such as Ceph.
-endpoint> 
+endpoint>
 Location constraint - must be set to match the Region. Used when creating buckets only.
 Choose a number from below, or type in your own value
  * Empty for US Region, Northern Virginia or Pacific Northwest.
- 1) 
+ 1)
  * US West (Oregon) Region.
  2) us-west-2
  * US West (Northern California) Region.
- 3) us-west-1
+[..snip..]
- * EU (Ireland) Region.
+ 8) ap-northeast-1
- 4) eu-west-1
+ * South America (Sao Paulo) Region.
-[snip]
+ 9) sa-east-1
-location_constraint> 1
+location_constraint> 3
 Remote config
 --------------------
 [remote]
-access_key_id = accesskey
+env_auth = true
-secret_access_key = secretaccesskey
+access_key_id =
-region = us-east-1
+secret_access_key =
-endpoint = 
+region = us-west-1
-location_constraint = 
+endpoint =
+location_constraint = us-west-1
 --------------------
 y) Yes this is OK
 e) Edit this remote
@@ -92,7 +108,7 @@ e) Edit existing remote
 n) New remote
 d) Delete remote
 q) Quit config
-e/n/d/q> q
+e/n/d/q>
 ```
 
 This remote is called `remote` and can now be used like this
@@ -133,36 +149,57 @@ created in.  If you attempt to access a bucket from the wrong region,
 you will get an error, `incorrect region, the bucket is not in 'XXX'
 region`.
 
+### Authentication ###
+There are two ways to supply `rclone` with a set of AWS
+credentials. In order of precedence:
+
+ - Directly in the rclone configuration file (as configured by `rclone config`)
+   - set `access_key_id` and `secret_access_key`
+ - Runtime configuration:
+   - set `env_auth` to `true` in the config file
+   - Exporting `AWS_ACCESS_KEY` and `AWS_SECRET_KEY` while running `rclone`
+   - Running `rclone` on an EC2 instance with an IAM role
+
+If none of these option actually end up providing `rclone` with AWS
+credentials then S3 interaction will be non-authenticated (see below).
+
 ### Anonymous access to public buckets ###
 
 If you want to use rclone to access a public bucket, configure with a
 blank `access_key_id` and `secret_access_key`.  Eg
 
 ```
-e) Edit existing remote
+No remotes found - make a new one
 n) New remote
-d) Delete remote
 q) Quit config
-e/n/d/q> n
+n/q> n
 name> anons3
 What type of source is it?
 Choose a number from below
  1) amazon cloud drive
- 2) drive
+ 2) b2
- 3) dropbox
+ 3) drive
- 4) google cloud storage
+ 4) dropbox
- 5) local
+ 5) google cloud storage
- 6) s3
+ 6) swift
- 7) swift
+ 7) hubic
-type> 6
+ 8) local
-AWS Access Key ID - leave blank for anonymous access.
+ 9) onedrive
-access_key_id> 
+10) s3
-AWS Secret Access Key (password) - leave blank for anonymous access.
+11) yandex
-secret_access_key> 
+type> 10
-Region to connect to.
+Get AWS credentials from runtime (environment variables or EC2 meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.
-region> 1
+Choose a number from below, or type in your own value
-endpoint> 
+ * Enter AWS credentials in the next step
-location_constraint> 
+ 1) false
+ * Get AWS credentials from the environment (env vars or IAM)
+ 2) true
+env_auth> 1
+AWS Access Key ID - leave blank for anonymous access or runtime credentials.
+access_key_id>
+AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.
+secret_access_key>
+...
 ```
 
 Then use it as normal with the name of the public bucket, eg

s3/s3.go

@@ -17,6 +17,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"net/url"
 	"path"
 	"regexp"
@@ -27,6 +28,8 @@ import (
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/corehandlers"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
+	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
@@ -42,11 +45,23 @@ func init() {
 		NewFs: NewFs,
 		// AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
 		Options: []fs.Option{{
+			Name: "env_auth",
+			Help: "Get AWS credentials from runtime (environment variables or EC2 meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.",
+			Examples: []fs.OptionExample{
+				{
+					Value: "false",
+					Help:  "Enter AWS credentials in the next step",
+				}, {
+					Value: "true",
+					Help:  "Get AWS credentials from the environment (env vars or IAM)",
+				},
+			},
+		}, {
 			Name: "access_key_id",
-			Help: "AWS Access Key ID - leave blank for anonymous access.",
+			Help: "AWS Access Key ID - leave blank for anonymous access or runtime credentials.",
 		}, {
 			Name: "secret_access_key",
-			Help: "AWS Secret Access Key (password) - leave blank for anonymous access.",
+			Help: "AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.",
 		}, {
 			Name: "region",
 			Help: "Region to connect to.",
@@ -195,19 +210,38 @@ func s3ParsePath(path string) (bucket, directory string, err error) {
 // s3Connection makes a connection to s3
 func s3Connection(name string) (*s3.S3, *session.Session, error) {
 	// Make the auth
-	accessKeyID := fs.ConfigFile.MustValue(name, "access_key_id")
+	v := credentials.Value{
-	secretAccessKey := fs.ConfigFile.MustValue(name, "secret_access_key")
+		AccessKeyID:     fs.ConfigFile.MustValue(name, "access_key_id"),
-	var auth *credentials.Credentials
+		SecretAccessKey: fs.ConfigFile.MustValue(name, "secret_access_key"),
+	}
+
+	// first provider to supply a credential set "wins"
+	providers := []credentials.Provider{
+		// use static credentials if they're present (checked by provider)
+		&credentials.StaticProvider{Value: v},
+
+		// * Access Key ID:     AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY
+		// * Secret Access Key: AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY
+		&credentials.EnvProvider{},
+
+		// Pick up IAM role in case we're on EC2
+		&ec2rolecreds.EC2RoleProvider{
+			Client: ec2metadata.New(session.New(), &aws.Config{
+				HTTPClient: &http.Client{Timeout: 1 * time.Second}, // low timeout to ec2 metadata service
+			}),
+			ExpiryWindow: 3,
+		},
+	}
+	cred := credentials.NewChainCredentials(providers)
+
 	switch {
-	case accessKeyID == "" && secretAccessKey == "":
+	case fs.ConfigFile.MustBool(name, "env_auth", false) && v.AccessKeyID == "" && v.SecretAccessKey == "":
-		fs.Debug(name, "Using anonymous access for S3")
+		// if no access key/secret and iam is explicitly disabled then fall back to anon interaction
-		auth = credentials.AnonymousCredentials
+		cred = credentials.AnonymousCredentials
-	case accessKeyID == "":
+	case v.AccessKeyID == "":
 		return nil, nil, errors.New("access_key_id not found")
-	case secretAccessKey == "":
+	case v.SecretAccessKey == "":
 		return nil, nil, errors.New("secret_access_key not found")
-	default:
-		auth = credentials.NewStaticCredentials(accessKeyID, secretAccessKey, "")
 	}
 
 	endpoint := fs.ConfigFile.MustValue(name, "endpoint")
@@ -221,7 +255,7 @@ func s3Connection(name string) (*s3.S3, *session.Session, error) {
 	awsConfig := aws.NewConfig().
 		WithRegion(region).
 		WithMaxRetries(maxRetries).
-		WithCredentials(auth).
+		WithCredentials(cred).
 		WithEndpoint(endpoint).
 		WithHTTPClient(fs.Config.Client()).
 		WithS3ForcePathStyle(true)
@@ -235,7 +269,7 @@ func s3Connection(name string) (*s3.S3, *session.Session, error) {
 			if req.Config.Credentials == credentials.AnonymousCredentials {
 				return
 			}
-			sign(accessKeyID, secretAccessKey, req.HTTPRequest)
+			sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
 		}
 		c.Handlers.Sign.Clear()
 		c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)