From 022ab4516db76b8b57b60e9e81eeb6be5c491184 Mon Sep 17 00:00:00 2001 From: Michal Witkowski Date: Wed, 20 Apr 2016 15:40:40 +0100 Subject: [PATCH] Add service account support for GCS --- docs/content/googlecloudstorage.md | 37 ++++++++++++++++++++++++ googlecloudstorage/googlecloudstorage.go | 35 ++++++++++++++++++++-- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/docs/content/googlecloudstorage.md b/docs/content/googlecloudstorage.md index 439ae876b..c97c349f2 100644 --- a/docs/content/googlecloudstorage.md +++ b/docs/content/googlecloudstorage.md @@ -57,6 +57,8 @@ Google Application Client Secret - leave blank normally. client_secret> Project number optional - needed only for list/create/delete buckets - see your developer console. project_number> 12345678 +Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login. +service_account_file> Access Control List for new objects. Choose a number from below, or type in your own value * Object owner gets OWNER access, and all Authenticated Users get READER access. @@ -139,6 +141,41 @@ files in the bucket. rclone sync /home/local/directory remote:bucket +### Service Account support ### + +You can set up rclone with Google Cloud Storage in an unattended mode, +i.e. not tied to a specific end-user Google account. This is useful +when you want to synchronise files onto machines that don't have +actively logged-in users, for example build machines. + +To get credentials for Google Cloud Platform +[IAM Service Accounts](https://cloud.google.com/iam/docs/service-accounts), +please head to the +[Service Account](https://console.cloud.google.com/permissions/serviceaccounts) +section of the Google Developer Console. Service Accounts behave just +like normal `User` permissions in +[Google Cloud Storage ACLs](https://cloud.google.com/storage/docs/access-control), +so you can limit their access (e.g. make them read only). After +creating an account, a JSON file containing the Service Account's +credentials will be downloaded onto your machines. These credentials +are what rclone will use for authentication. + +To use a Service Account instead of OAuth2 token flow, replace the +`token` section of your `.rclone.conf` with a `service_account_file` +pointing to the JSON credentials. + +For example, here's an example `.rclone.conf` that sets up read only +access using a service account: + +``` +[readonly-sync] +type = google cloud storage +project_number = 123456789 +service_account_file = $HOME/.rclone-service_account.json +object_acl = authenticatedRead +bucket_acl = authenticatedRead +``` + ### Modified time ### Google google cloud storage stores md5sums natively and rclone stores diff --git a/googlecloudstorage/googlecloudstorage.go b/googlecloudstorage/googlecloudstorage.go index a2f9f7b47..c29f2e297 100644 --- a/googlecloudstorage/googlecloudstorage.go +++ b/googlecloudstorage/googlecloudstorage.go @@ -17,8 +17,10 @@ import ( "encoding/hex" "fmt" "io" + "io/ioutil" "log" "net/http" + "os" "path" "regexp" "strings" @@ -74,6 +76,9 @@ func init() { }, { Name: "project_number", Help: "Project number optional - needed only for list/create/delete buckets - see your developer console.", + }, { + Name: "service_account_file", + Help: "Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login.", }, { Name: "object_acl", Help: "Access Control List for new objects.", @@ -181,11 +186,35 @@ func parsePath(path string) (bucket, directory string, err error) { return } +func getServiceAccountClient(keyJsonfilePath string) (*http.Client, error) { + data, err := ioutil.ReadFile(os.ExpandEnv(keyJsonfilePath)) + if err != nil { + return nil, fmt.Errorf("error opening credentials file: %v", err) + } + conf, err := google.JWTConfigFromJSON(data, storageConfig.Scopes...) + if err != nil { + return nil, fmt.Errorf("error processing credentials: %v", err) + } + ctxWithSpecialClient := oauthutil.Context() + return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil +} + // NewFs contstructs an Fs from the path, bucket:path func NewFs(name, root string) (fs.Fs, error) { - oAuthClient, err := oauthutil.NewClient(name, storageConfig) - if err != nil { - log.Fatalf("Failed to configure Google Cloud Storage: %v", err) + var oAuthClient *http.Client + var err error + + serviceAccountPath := fs.ConfigFile.MustValue(name, "service_account_file") + if serviceAccountPath != "" { + oAuthClient, err = getServiceAccountClient(serviceAccountPath) + if err != nil { + log.Fatalf("Failed configuring Google Cloud Storage Service Account: %v", err) + } + } else { + oAuthClient, err = oauthutil.NewClient(name, storageConfig) + if err != nil { + log.Fatalf("Failed to configure Google Cloud Storage: %v", err) + } } bucket, directory, err := parsePath(root)