2015-09-22 19:47:16 +02:00
// Package googlecloudstorage provides an interface to Google Cloud Storage
2014-07-13 18:54:03 +02:00
package googlecloudstorage
/ *
2014-07-15 00:35:41 +02:00
Notes
2014-07-13 18:54:03 +02:00
2014-07-15 00:35:41 +02:00
Can ' t set Updated but can set Metadata on object creation
2014-07-13 18:54:03 +02:00
2014-07-15 00:35:41 +02:00
Patch needs full_control not just read_write
FIXME Patch / Delete / Get isn ' t working with files with spaces in - giving 404 error
- https : //code.google.com/p/google-api-go-client/issues/detail?id=64
2014-07-13 18:54:03 +02:00
* /
import (
2019-03-01 18:05:31 +01:00
"context"
2014-07-13 18:54:03 +02:00
"encoding/base64"
"encoding/hex"
"fmt"
"io"
2016-04-20 16:40:40 +02:00
"io/ioutil"
2014-07-13 18:54:03 +02:00
"net/http"
2014-07-14 11:45:28 +02:00
"path"
2021-05-21 11:11:43 +02:00
"strconv"
2014-07-13 18:54:03 +02:00
"strings"
"time"
2016-06-12 16:06:02 +02:00
"github.com/pkg/errors"
2019-07-28 19:47:38 +02:00
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/walk"
2019-08-15 17:26:16 +02:00
"github.com/rclone/rclone/lib/bucket"
2020-01-14 18:33:35 +01:00
"github.com/rclone/rclone/lib/encoder"
2020-06-02 12:54:52 +02:00
"github.com/rclone/rclone/lib/env"
2019-07-28 19:47:38 +02:00
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
2015-08-18 09:55:09 +02:00
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
2014-12-12 21:02:08 +01:00
"google.golang.org/api/googleapi"
2019-03-01 18:05:31 +01:00
// NOTE: This API is deprecated
2017-09-16 22:46:02 +02:00
storage "google.golang.org/api/storage/v1"
2014-07-13 18:54:03 +02:00
)
const (
2016-02-28 20:57:19 +01:00
rcloneClientID = "202264815644.apps.googleusercontent.com"
2016-08-14 13:04:43 +02:00
rcloneEncryptedClientSecret = "Uj7C9jGfb9gmeaV70Lh058cNkWvepr-Es9sBm0zdgil7JaOWF1VySw"
2021-05-21 17:01:32 +02:00
timeFormat = time . RFC3339Nano
2021-05-21 11:11:43 +02:00
metaMtime = "mtime" // key to store mtime in metadata
metaMtimeGsutil = "goog-reserved-file-mtime" // key used by GSUtil to store mtime in metadata
listChunks = 1000 // chunk size to read directory listings
2018-05-09 15:27:21 +02:00
minSleep = 10 * time . Millisecond
2014-07-13 18:54:03 +02:00
)
var (
// Description of how to auth for this app
2015-08-18 09:55:09 +02:00
storageConfig = & oauth2 . Config {
2019-06-27 12:53:25 +02:00
Scopes : [ ] string { storage . DevstorageReadWriteScope } ,
2015-08-18 09:55:09 +02:00
Endpoint : google . Endpoint ,
ClientID : rcloneClientID ,
2018-01-18 21:19:55 +01:00
ClientSecret : obscure . MustReveal ( rcloneEncryptedClientSecret ) ,
2015-08-18 09:55:09 +02:00
RedirectURL : oauthutil . TitleBarRedirectURL ,
2014-07-13 18:54:03 +02:00
}
)
// Register with Fs
func init ( ) {
2016-02-18 12:35:25 +01:00
fs . Register ( & fs . RegInfo {
2016-02-15 19:11:53 +01:00
Name : "google cloud storage" ,
2018-05-14 19:06:57 +02:00
Prefix : "gcs" ,
2016-02-15 19:11:53 +01:00
Description : "Google Cloud Storage (this is not Google Drive)" ,
NewFs : NewFs ,
2021-04-29 10:28:18 +02:00
Config : func ( ctx context . Context , name string , m configmap . Mapper , config fs . ConfigIn ) ( * fs . ConfigOut , error ) {
2018-05-14 19:06:57 +02:00
saFile , _ := m . Get ( "service_account_file" )
saCreds , _ := m . Get ( "service_account_credentials" )
2020-06-30 17:01:02 +02:00
anonymous , _ := m . Get ( "anonymous" )
if saFile != "" || saCreds != "" || anonymous == "true" {
2021-04-29 10:28:18 +02:00
return nil , nil
2016-04-22 20:58:52 +02:00
}
2021-04-29 10:28:18 +02:00
return oauthutil . ConfigOut ( "" , & oauthutil . Options {
OAuth2Config : storageConfig ,
} )
2014-07-13 18:54:03 +02:00
} ,
2020-08-02 01:32:21 +02:00
Options : append ( oauthutil . SharedOptions , [ ] fs . Option { {
2014-07-15 00:35:41 +02:00
Name : "project_number" ,
2018-05-14 19:06:57 +02:00
Help : "Project number.\nOptional - needed only for list/create/delete buckets - see your developer console." ,
2016-04-20 16:40:40 +02:00
} , {
Name : "service_account_file" ,
2020-06-02 12:54:52 +02:00
Help : "Service Account Credentials JSON file path\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env . ShellExpandHelp ,
2018-05-14 19:06:57 +02:00
} , {
Name : "service_account_credentials" ,
Help : "Service Account Credentials JSON blob\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." ,
Hide : fs . OptionHideBoth ,
2020-06-30 17:01:02 +02:00
} , {
Name : "anonymous" ,
Help : "Access public buckets and objects without credentials\nSet to 'true' if you just want to download files and don't configure credentials." ,
Default : false ,
2014-07-15 00:35:41 +02:00
} , {
Name : "object_acl" ,
Help : "Access Control List for new objects." ,
Examples : [ ] fs . OptionExample { {
Value : "authenticatedRead" ,
Help : "Object owner gets OWNER access, and all Authenticated Users get READER access." ,
} , {
Value : "bucketOwnerFullControl" ,
Help : "Object owner gets OWNER access, and project team owners get OWNER access." ,
} , {
Value : "bucketOwnerRead" ,
Help : "Object owner gets OWNER access, and project team owners get READER access." ,
} , {
Value : "private" ,
Help : "Object owner gets OWNER access [default if left blank]." ,
} , {
Value : "projectPrivate" ,
Help : "Object owner gets OWNER access, and project team members get access according to their roles." ,
} , {
Value : "publicRead" ,
Help : "Object owner gets OWNER access, and all Users get READER access." ,
} } ,
} , {
Name : "bucket_acl" ,
Help : "Access Control List for new buckets." ,
Examples : [ ] fs . OptionExample { {
Value : "authenticatedRead" ,
Help : "Project team owners get OWNER access, and all Authenticated Users get READER access." ,
} , {
Value : "private" ,
Help : "Project team owners get OWNER access [default if left blank]." ,
} , {
Value : "projectPrivate" ,
Help : "Project team members get access according to their roles." ,
} , {
Value : "publicRead" ,
Help : "Project team owners get OWNER access, and all Users get READER access." ,
} , {
Value : "publicReadWrite" ,
Help : "Project team owners get OWNER access, and all Users get WRITER access." ,
} } ,
2019-03-04 15:52:54 +01:00
} , {
Name : "bucket_policy_only" ,
Help : ` Access checks should use bucket - level IAM policies .
If you want to upload objects to a bucket with Bucket Policy Only set
then you will need to set this .
When it is set , rclone :
- ignores ACLs set on buckets
- ignores ACLs set on objects
- creates buckets with Bucket Policy Only set
Docs : https : //cloud.google.com/storage/docs/bucket-policy-only
` ,
Default : false ,
2017-07-18 16:15:29 +02:00
} , {
Name : "location" ,
Help : "Location for the newly created buckets." ,
Examples : [ ] fs . OptionExample { {
Value : "" ,
Help : "Empty for default location (US)." ,
} , {
Value : "asia" ,
Help : "Multi-regional location for Asia." ,
} , {
Value : "eu" ,
Help : "Multi-regional location for Europe." ,
} , {
Value : "us" ,
Help : "Multi-regional location for United States." ,
} , {
Value : "asia-east1" ,
Help : "Taiwan." ,
2019-02-02 23:08:30 +01:00
} , {
Value : "asia-east2" ,
Help : "Hong Kong." ,
2017-07-18 16:15:29 +02:00
} , {
Value : "asia-northeast1" ,
Help : "Tokyo." ,
2019-02-02 23:08:30 +01:00
} , {
Value : "asia-south1" ,
Help : "Mumbai." ,
2017-07-18 16:15:29 +02:00
} , {
Value : "asia-southeast1" ,
Help : "Singapore." ,
} , {
Value : "australia-southeast1" ,
Help : "Sydney." ,
2019-02-02 23:08:30 +01:00
} , {
Value : "europe-north1" ,
Help : "Finland." ,
2017-07-18 16:15:29 +02:00
} , {
Value : "europe-west1" ,
Help : "Belgium." ,
} , {
Value : "europe-west2" ,
Help : "London." ,
2019-02-02 23:08:30 +01:00
} , {
Value : "europe-west3" ,
Help : "Frankfurt." ,
} , {
Value : "europe-west4" ,
Help : "Netherlands." ,
2017-07-18 16:15:29 +02:00
} , {
Value : "us-central1" ,
Help : "Iowa." ,
} , {
Value : "us-east1" ,
Help : "South Carolina." ,
} , {
Value : "us-east4" ,
Help : "Northern Virginia." ,
} , {
Value : "us-west1" ,
Help : "Oregon." ,
2019-02-02 23:08:30 +01:00
} , {
Value : "us-west2" ,
Help : "California." ,
2017-07-18 16:15:29 +02:00
} } ,
} , {
Name : "storage_class" ,
Help : "The storage class to use when storing objects in Google Cloud Storage." ,
Examples : [ ] fs . OptionExample { {
Value : "" ,
Help : "Default" ,
} , {
Value : "MULTI_REGIONAL" ,
Help : "Multi-regional storage class" ,
} , {
Value : "REGIONAL" ,
Help : "Regional storage class" ,
} , {
Value : "NEARLINE" ,
Help : "Nearline storage class" ,
} , {
Value : "COLDLINE" ,
Help : "Coldline storage class" ,
2020-04-27 02:16:57 +02:00
} , {
Value : "ARCHIVE" ,
Help : "Archive storage class" ,
2017-07-18 16:15:29 +02:00
} , {
Value : "DURABLE_REDUCED_AVAILABILITY" ,
Help : "Durable reduced availability storage class" ,
} } ,
2020-01-14 18:33:35 +01:00
} , {
Name : config . ConfigEncoding ,
Help : config . ConfigEncodingHelp ,
Advanced : true ,
2020-01-14 22:51:49 +01:00
Default : ( encoder . Base |
encoder . EncodeCrLf |
encoder . EncodeInvalidUtf8 ) ,
2020-08-02 01:32:21 +02:00
} } ... ) ,
2014-07-13 18:54:03 +02:00
} )
}
2018-05-14 19:06:57 +02:00
// Options defines the configuration for this backend
type Options struct {
2020-01-14 18:33:35 +01:00
ProjectNumber string ` config:"project_number" `
ServiceAccountFile string ` config:"service_account_file" `
ServiceAccountCredentials string ` config:"service_account_credentials" `
2020-06-30 17:01:02 +02:00
Anonymous bool ` config:"anonymous" `
2020-01-14 18:33:35 +01:00
ObjectACL string ` config:"object_acl" `
BucketACL string ` config:"bucket_acl" `
BucketPolicyOnly bool ` config:"bucket_policy_only" `
Location string ` config:"location" `
StorageClass string ` config:"storage_class" `
Enc encoder . MultiEncoder ` config:"encoding" `
2018-05-14 19:06:57 +02:00
}
2015-11-07 12:14:46 +01:00
// Fs represents a remote storage server
type Fs struct {
2019-08-15 17:26:16 +02:00
name string // name of this remote
root string // the path we are working on if any
opt Options // parsed options
features * fs . Features // optional features
svc * storage . Service // the connection to the storage server
client * http . Client // authorized client
rootBucket string // bucket part of root (if any)
rootDirectory string // directory part of root (if any)
cache * bucket . Cache // cache of bucket status
pacer * fs . Pacer // To pace the API calls
2014-07-13 18:54:03 +02:00
}
2015-11-07 12:14:46 +01:00
// Object describes a storage object
2014-07-13 18:54:03 +02:00
//
// Will definitely have info but maybe not meta
2015-11-07 12:14:46 +01:00
type Object struct {
2016-09-21 23:13:24 +02:00
fs * Fs // what this object is part of
remote string // The remote path
url string // download path
md5sum string // The MD5Sum of the object
bytes int64 // Bytes in the object
modTime time . Time // Modified time of the object
mimeType string
2014-07-13 18:54:03 +02:00
}
// ------------------------------------------------------------
2015-09-22 19:47:16 +02:00
// Name of the remote (as passed into NewFs)
2015-11-07 12:14:46 +01:00
func ( f * Fs ) Name ( ) string {
2015-08-22 17:53:11 +02:00
return f . name
}
2015-09-22 19:47:16 +02:00
// Root of the remote (as passed into NewFs)
2015-11-07 12:14:46 +01:00
func ( f * Fs ) Root ( ) string {
2019-08-15 17:26:16 +02:00
return f . root
2015-09-01 21:45:27 +02:00
}
2015-11-07 12:14:46 +01:00
// String converts this Fs to a string
func ( f * Fs ) String ( ) string {
2019-08-15 17:26:16 +02:00
if f . rootBucket == "" {
return fmt . Sprintf ( "GCS root" )
}
if f . rootDirectory == "" {
return fmt . Sprintf ( "GCS bucket %s" , f . rootBucket )
2014-07-13 18:54:03 +02:00
}
2019-08-15 17:26:16 +02:00
return fmt . Sprintf ( "GCS bucket %s path %s" , f . rootBucket , f . rootDirectory )
2014-07-13 18:54:03 +02:00
}
2017-01-13 18:21:47 +01:00
// Features returns the optional features of this Fs
func ( f * Fs ) Features ( ) * fs . Features {
return f . features
}
2019-02-07 18:41:17 +01:00
// shouldRetry determines whether a given err rates being retried
2021-03-11 15:44:01 +01:00
func shouldRetry ( ctx context . Context , err error ) ( again bool , errOut error ) {
if fserrors . ContextError ( ctx , & err ) {
return false , err
}
2018-05-09 15:27:21 +02:00
again = false
if err != nil {
if fserrors . ShouldRetry ( err ) {
again = true
} else {
switch gerr := err . ( type ) {
case * googleapi . Error :
if gerr . Code >= 500 && gerr . Code < 600 {
// All 5xx errors should be retried
again = true
} else if len ( gerr . Errors ) > 0 {
reason := gerr . Errors [ 0 ] . Reason
if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" {
again = true
}
}
}
}
}
return again , err
}
2019-08-15 17:26:16 +02:00
// parsePath parses a remote 'url'
func parsePath ( path string ) ( root string ) {
root = strings . Trim ( path , "/" )
2014-07-13 18:54:03 +02:00
return
}
2019-08-15 17:26:16 +02:00
// split returns bucket and bucketPath from the rootRelativePath
// relative to f.root
func ( f * Fs ) split ( rootRelativePath string ) ( bucketName , bucketPath string ) {
2019-05-19 17:54:46 +02:00
bucketName , bucketPath = bucket . Split ( path . Join ( f . root , rootRelativePath ) )
2020-01-14 18:33:35 +01:00
return f . opt . Enc . FromStandardName ( bucketName ) , f . opt . Enc . FromStandardPath ( bucketPath )
2019-08-15 17:26:16 +02:00
}
// split returns bucket and bucketPath from the object
func ( o * Object ) split ( ) ( bucket , bucketPath string ) {
return o . fs . split ( o . remote )
}
2020-11-05 19:02:26 +01:00
func getServiceAccountClient ( ctx context . Context , credentialsData [ ] byte ) ( * http . Client , error ) {
2018-04-27 17:07:37 +02:00
conf , err := google . JWTConfigFromJSON ( credentialsData , storageConfig . Scopes ... )
2016-04-20 16:40:40 +02:00
if err != nil {
2016-06-12 16:06:02 +02:00
return nil , errors . Wrap ( err , "error processing credentials" )
2016-04-20 16:40:40 +02:00
}
2020-11-13 16:24:43 +01:00
ctxWithSpecialClient := oauthutil . Context ( ctx , fshttp . NewClient ( ctx ) )
2016-04-20 16:40:40 +02:00
return oauth2 . NewClient ( ctxWithSpecialClient , conf . TokenSource ( ctxWithSpecialClient ) ) , nil
}
2019-08-15 17:26:16 +02:00
// setRoot changes the root of the Fs
func ( f * Fs ) setRoot ( root string ) {
f . root = parsePath ( root )
f . rootBucket , f . rootDirectory = bucket . Split ( f . root )
}
2019-02-07 18:41:17 +01:00
// NewFs constructs an Fs from the path, bucket:path
2020-11-05 16:18:51 +01:00
func NewFs ( ctx context . Context , name , root string , m configmap . Mapper ) ( fs . Fs , error ) {
2016-04-20 16:40:40 +02:00
var oAuthClient * http . Client
2018-05-14 19:06:57 +02:00
// Parse config into Options struct
opt := new ( Options )
err := configstruct . Set ( m , opt )
if err != nil {
return nil , err
}
if opt . ObjectACL == "" {
opt . ObjectACL = "private"
}
if opt . BucketACL == "" {
opt . BucketACL = "private"
}
2016-04-20 16:40:40 +02:00
2018-04-27 17:07:37 +02:00
// try loading service account credentials from env variable, then from a file
2018-09-04 12:28:45 +02:00
if opt . ServiceAccountCredentials == "" && opt . ServiceAccountFile != "" {
2020-06-02 12:54:52 +02:00
loadedCreds , err := ioutil . ReadFile ( env . ShellExpand ( opt . ServiceAccountFile ) )
2018-04-27 17:07:37 +02:00
if err != nil {
return nil , errors . Wrap ( err , "error opening service account credentials file" )
}
2018-05-14 19:06:57 +02:00
opt . ServiceAccountCredentials = string ( loadedCreds )
2018-04-27 17:07:37 +02:00
}
2020-06-30 17:01:02 +02:00
if opt . Anonymous {
2021-01-19 16:10:35 +01:00
oAuthClient = fshttp . NewClient ( ctx )
2020-06-30 17:01:02 +02:00
} else if opt . ServiceAccountCredentials != "" {
2020-11-05 19:02:26 +01:00
oAuthClient , err = getServiceAccountClient ( ctx , [ ] byte ( opt . ServiceAccountCredentials ) )
2016-04-20 16:40:40 +02:00
if err != nil {
2018-04-27 17:07:37 +02:00
return nil , errors . Wrap ( err , "failed configuring Google Cloud Storage Service Account" )
2016-04-20 16:40:40 +02:00
}
} else {
2020-11-05 19:02:26 +01:00
oAuthClient , _ , err = oauthutil . NewClient ( ctx , name , m , storageConfig )
2016-04-20 16:40:40 +02:00
if err != nil {
2019-03-01 18:05:31 +01:00
ctx := context . Background ( )
oAuthClient , err = google . DefaultClient ( ctx , storage . DevstorageFullControlScope )
if err != nil {
return nil , errors . Wrap ( err , "failed to configure Google Cloud Storage" )
}
2016-04-20 16:40:40 +02:00
}
2014-07-13 18:54:03 +02:00
}
2015-11-07 12:14:46 +01:00
f := & Fs {
2019-08-15 17:26:16 +02:00
name : name ,
root : root ,
opt : * opt ,
2020-11-05 12:33:32 +01:00
pacer : fs . NewPacer ( ctx , pacer . NewGoogleDrive ( pacer . MinSleep ( minSleep ) ) ) ,
2019-08-15 17:26:16 +02:00
cache : bucket . NewCache ( ) ,
2014-07-15 00:35:41 +02:00
}
2019-08-15 17:26:16 +02:00
f . setRoot ( root )
2017-08-09 16:27:43 +02:00
f . features = ( & fs . Features {
2019-08-15 17:26:16 +02:00
ReadMimeType : true ,
WriteMimeType : true ,
BucketBased : true ,
BucketBasedRootOK : true ,
2020-11-05 17:00:40 +01:00
} ) . Fill ( ctx , f )
2014-07-13 18:54:03 +02:00
// Create a new authorized Drive client.
2015-08-18 09:55:09 +02:00
f . client = oAuthClient
2014-07-13 18:54:03 +02:00
f . svc , err = storage . New ( f . client )
if err != nil {
2016-06-12 16:06:02 +02:00
return nil , errors . Wrap ( err , "couldn't create Google Cloud Storage client" )
2014-07-13 18:54:03 +02:00
}
2019-08-15 17:26:16 +02:00
if f . rootBucket != "" && f . rootDirectory != "" {
2014-07-13 18:54:03 +02:00
// Check to see if the object exists
2020-01-14 18:33:35 +01:00
encodedDirectory := f . opt . Enc . FromStandardPath ( f . rootDirectory )
2018-05-09 15:27:21 +02:00
err = f . pacer . Call ( func ( ) ( bool , error ) {
2019-05-19 17:54:46 +02:00
_ , err = f . svc . Objects . Get ( f . rootBucket , encodedDirectory ) . Context ( ctx ) . Do ( )
2021-03-11 15:44:01 +01:00
return shouldRetry ( ctx , err )
2018-05-09 15:27:21 +02:00
} )
2014-07-14 11:45:28 +02:00
if err == nil {
2019-08-15 17:26:16 +02:00
newRoot := path . Dir ( f . root )
if newRoot == "." {
newRoot = ""
2014-07-14 11:45:28 +02:00
}
2019-08-15 17:26:16 +02:00
f . setRoot ( newRoot )
2016-06-21 19:01:53 +02:00
// return an error with an fs which points to the parent
return f , fs . ErrorIsFile
2014-07-14 11:45:28 +02:00
}
2014-07-13 18:54:03 +02:00
}
return f , nil
}
2016-06-25 22:58:34 +02:00
// Return an Object from a path
2014-07-13 18:54:03 +02:00
//
2016-06-25 22:23:20 +02:00
// If it can't be found it returns the error fs.ErrorObjectNotFound.
2019-09-06 14:50:36 +02:00
func ( f * Fs ) newObjectWithInfo ( ctx context . Context , remote string , info * storage . Object ) ( fs . Object , error ) {
2015-11-07 12:14:46 +01:00
o := & Object {
fs : f ,
remote : remote ,
2014-07-13 18:54:03 +02:00
}
if info != nil {
o . setMetaData ( info )
} else {
2019-09-06 14:50:36 +02:00
err := o . readMetaData ( ctx ) // reads info and meta, returning an error
2014-07-13 18:54:03 +02:00
if err != nil {
2016-06-25 22:23:20 +02:00
return nil , err
2014-07-13 18:54:03 +02:00
}
}
2016-06-25 22:23:20 +02:00
return o , nil
2014-07-13 18:54:03 +02:00
}
2016-06-25 22:23:20 +02:00
// NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound.
2019-06-17 10:34:30 +02:00
func ( f * Fs ) NewObject ( ctx context . Context , remote string ) ( fs . Object , error ) {
2019-09-06 14:50:36 +02:00
return f . newObjectWithInfo ( ctx , remote , nil )
2014-07-13 18:54:03 +02:00
}
2016-04-21 21:06:21 +02:00
// listFn is called from list to handle an object.
type listFn func ( remote string , object * storage . Object , isDirectory bool ) error
2014-07-13 18:54:03 +02:00
// list the objects into the function supplied
//
2016-04-23 22:46:52 +02:00
// dir is the starting directory, "" for root
//
2017-06-11 23:43:31 +02:00
// Set recurse to read sub directories
2019-08-15 17:26:16 +02:00
//
// The remote has prefix removed from it and if addBucket is set
// then it adds the bucket to the start.
func ( f * Fs ) list ( ctx context . Context , bucket , directory , prefix string , addBucket bool , recurse bool , fn listFn ) ( err error ) {
if prefix != "" {
prefix += "/"
}
if directory != "" {
directory += "/"
2016-04-23 22:46:52 +02:00
}
2019-08-15 17:26:16 +02:00
list := f . svc . Objects . List ( bucket ) . Prefix ( directory ) . MaxResults ( listChunks )
2017-06-11 23:43:31 +02:00
if ! recurse {
2014-07-13 18:54:03 +02:00
list = list . Delimiter ( "/" )
}
for {
2018-05-09 15:27:21 +02:00
var objects * storage . Objects
err = f . pacer . Call ( func ( ) ( bool , error ) {
2019-09-06 14:50:36 +02:00
objects , err = list . Context ( ctx ) . Do ( )
2021-03-11 15:44:01 +01:00
return shouldRetry ( ctx , err )
2018-05-09 15:27:21 +02:00
} )
2014-07-13 18:54:03 +02:00
if err != nil {
2017-06-11 23:43:31 +02:00
if gErr , ok := err . ( * googleapi . Error ) ; ok {
if gErr . Code == http . StatusNotFound {
err = fs . ErrorDirNotFound
}
}
2016-04-21 21:06:21 +02:00
return err
2014-07-13 18:54:03 +02:00
}
2017-06-11 23:43:31 +02:00
if ! recurse {
2014-07-28 19:04:52 +02:00
var object storage . Object
2019-08-15 17:26:16 +02:00
for _ , remote := range objects . Prefixes {
if ! strings . HasSuffix ( remote , "/" ) {
2014-07-28 19:04:52 +02:00
continue
}
2020-01-14 18:33:35 +01:00
remote = f . opt . Enc . ToStandardPath ( remote )
2019-08-15 17:26:16 +02:00
if ! strings . HasPrefix ( remote , prefix ) {
fs . Logf ( f , "Odd name received %q" , remote )
continue
}
remote = remote [ len ( prefix ) : len ( remote ) - 1 ]
if addBucket {
remote = path . Join ( bucket , remote )
}
err = fn ( remote , & object , true )
2016-04-21 21:06:21 +02:00
if err != nil {
return err
}
}
}
for _ , object := range objects . Items {
2020-01-14 18:33:35 +01:00
remote := f . opt . Enc . ToStandardPath ( object . Name )
2019-05-19 17:54:46 +02:00
if ! strings . HasPrefix ( remote , prefix ) {
2017-02-09 12:01:20 +01:00
fs . Logf ( f , "Odd name received %q" , object . Name )
2016-04-21 21:06:21 +02:00
continue
}
2019-05-19 17:54:46 +02:00
remote = remote [ len ( prefix ) : ]
2020-03-31 12:44:24 +02:00
isDirectory := remote == "" || strings . HasSuffix ( remote , "/" )
2019-08-15 17:26:16 +02:00
if addBucket {
remote = path . Join ( bucket , remote )
}
2018-03-19 18:42:27 +01:00
// is this a directory marker?
2021-01-19 16:13:52 +01:00
if isDirectory {
2018-03-19 18:42:27 +01:00
continue // skip directory marker
}
2016-04-21 21:06:21 +02:00
err = fn ( remote , object , false )
if err != nil {
return err
2014-07-13 18:54:03 +02:00
}
}
if objects . NextPageToken == "" {
break
}
list . PageToken ( objects . NextPageToken )
}
2016-04-21 21:06:21 +02:00
return nil
2014-07-13 18:54:03 +02:00
}
2017-06-30 11:54:14 +02:00
// Convert a list item into a DirEntry
2019-09-06 14:50:36 +02:00
func ( f * Fs ) itemToDirEntry ( ctx context . Context , remote string , object * storage . Object , isDirectory bool ) ( fs . DirEntry , error ) {
2017-06-11 23:43:31 +02:00
if isDirectory {
2017-06-30 14:37:29 +02:00
d := fs . NewDir ( remote , time . Time { } ) . SetSize ( int64 ( object . Size ) )
2017-06-11 23:43:31 +02:00
return d , nil
2016-04-21 21:06:21 +02:00
}
2019-09-06 14:50:36 +02:00
o , err := f . newObjectWithInfo ( ctx , remote , object )
2017-06-11 23:43:31 +02:00
if err != nil {
return nil , err
}
return o , nil
}
// listDir lists a single directory
2019-08-15 17:26:16 +02:00
func ( f * Fs ) listDir ( ctx context . Context , bucket , directory , prefix string , addBucket bool ) ( entries fs . DirEntries , err error ) {
2016-04-21 21:06:21 +02:00
// List the objects
2019-08-15 17:26:16 +02:00
err = f . list ( ctx , bucket , directory , prefix , addBucket , false , func ( remote string , object * storage . Object , isDirectory bool ) error {
2019-09-06 14:50:36 +02:00
entry , err := f . itemToDirEntry ( ctx , remote , object , isDirectory )
2017-06-11 23:43:31 +02:00
if err != nil {
return err
}
if entry != nil {
entries = append ( entries , entry )
2016-04-21 21:06:21 +02:00
}
return nil
} )
if err != nil {
2017-06-11 23:43:31 +02:00
return nil , err
2014-07-13 18:54:03 +02:00
}
2018-03-01 13:11:34 +01:00
// bucket must be present if listing succeeded
2019-08-15 17:26:16 +02:00
f . cache . MarkOK ( bucket )
2017-06-11 23:43:31 +02:00
return entries , err
2014-07-13 18:54:03 +02:00
}
2017-06-11 23:43:31 +02:00
// listBuckets lists the buckets
2019-08-22 22:30:55 +02:00
func ( f * Fs ) listBuckets ( ctx context . Context ) ( entries fs . DirEntries , err error ) {
2018-05-14 19:06:57 +02:00
if f . opt . ProjectNumber == "" {
2017-06-11 23:43:31 +02:00
return nil , errors . New ( "can't list buckets without project number" )
2016-04-21 21:06:21 +02:00
}
2018-05-14 19:06:57 +02:00
listBuckets := f . svc . Buckets . List ( f . opt . ProjectNumber ) . MaxResults ( listChunks )
2016-04-21 21:06:21 +02:00
for {
2018-05-09 15:27:21 +02:00
var buckets * storage . Buckets
err = f . pacer . Call ( func ( ) ( bool , error ) {
2019-09-06 14:50:36 +02:00
buckets , err = listBuckets . Context ( ctx ) . Do ( )
2021-03-11 15:44:01 +01:00
return shouldRetry ( ctx , err )
2018-05-09 15:27:21 +02:00
} )
2016-04-21 21:06:21 +02:00
if err != nil {
2017-06-11 23:43:31 +02:00
return nil , err
2016-04-21 21:06:21 +02:00
}
for _ , bucket := range buckets . Items {
2020-01-14 18:33:35 +01:00
d := fs . NewDir ( f . opt . Enc . ToStandardName ( bucket . Name ) , time . Time { } )
2017-06-11 23:43:31 +02:00
entries = append ( entries , d )
2016-04-21 21:06:21 +02:00
}
if buckets . NextPageToken == "" {
break
}
listBuckets . PageToken ( buckets . NextPageToken )
}
2017-06-11 23:43:31 +02:00
return entries , nil
2016-04-21 21:06:21 +02:00
}
2017-06-11 23:43:31 +02:00
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
2019-06-17 10:34:30 +02:00
func ( f * Fs ) List ( ctx context . Context , dir string ) ( entries fs . DirEntries , err error ) {
2019-08-15 17:26:16 +02:00
bucket , directory := f . split ( dir )
if bucket == "" {
2019-08-22 22:30:55 +02:00
if directory != "" {
return nil , fs . ErrorListBucketRequired
}
return f . listBuckets ( ctx )
2014-07-13 18:54:03 +02:00
}
2019-08-15 17:26:16 +02:00
return f . listDir ( ctx , bucket , directory , f . rootDirectory , f . rootBucket == "" )
2014-07-13 18:54:03 +02:00
}
2017-06-05 17:14:24 +02:00
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
2017-06-11 23:43:31 +02:00
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
2019-06-17 10:34:30 +02:00
func ( f * Fs ) ListR ( ctx context . Context , dir string , callback fs . ListRCallback ) ( err error ) {
2019-08-15 17:26:16 +02:00
bucket , directory := f . split ( dir )
2018-01-12 17:30:54 +01:00
list := walk . NewListRHelper ( callback )
2019-08-15 17:26:16 +02:00
listR := func ( bucket , directory , prefix string , addBucket bool ) error {
return f . list ( ctx , bucket , directory , prefix , addBucket , true , func ( remote string , object * storage . Object , isDirectory bool ) error {
2019-09-06 14:50:36 +02:00
entry , err := f . itemToDirEntry ( ctx , remote , object , isDirectory )
2019-08-15 17:26:16 +02:00
if err != nil {
return err
}
return list . Add ( entry )
} )
}
if bucket == "" {
2019-08-22 22:30:55 +02:00
entries , err := f . listBuckets ( ctx )
2019-08-15 17:26:16 +02:00
if err != nil {
return err
}
for _ , entry := range entries {
err = list . Add ( entry )
if err != nil {
return err
}
bucket := entry . Remote ( )
err = listR ( bucket , "" , f . rootDirectory , true )
if err != nil {
return err
}
2019-08-22 22:30:55 +02:00
// bucket must be present if listing succeeded
f . cache . MarkOK ( bucket )
2019-08-15 17:26:16 +02:00
}
} else {
err = listR ( bucket , directory , f . rootDirectory , f . rootBucket == "" )
2017-06-11 23:43:31 +02:00
if err != nil {
return err
}
2019-08-22 22:30:55 +02:00
// bucket must be present if listing succeeded
f . cache . MarkOK ( bucket )
2017-06-11 23:43:31 +02:00
}
return list . Flush ( )
2017-06-05 17:14:24 +02:00
}
2014-07-13 18:54:03 +02:00
// Put the object into the bucket
//
// Copy the reader in to the new object which is returned
//
// The new object may have been created if an error is returned
2019-06-17 10:34:30 +02:00
func ( f * Fs ) Put ( ctx context . Context , in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) ( fs . Object , error ) {
2015-11-07 12:14:46 +01:00
// Temporary Object under construction
o := & Object {
fs : f ,
2016-02-18 12:35:25 +01:00
remote : src . Remote ( ) ,
2015-11-07 12:14:46 +01:00
}
2019-06-17 10:34:30 +02:00
return o , o . Update ( ctx , in , src , options ... )
2014-07-13 18:54:03 +02:00
}
2017-09-16 22:46:02 +02:00
// PutStream uploads to the remote path with the modTime given of indeterminate size
2019-06-17 10:34:30 +02:00
func ( f * Fs ) PutStream ( ctx context . Context , in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) ( fs . Object , error ) {
return f . Put ( ctx , in , src , options ... )
2017-09-16 22:46:02 +02:00
}
2014-07-13 18:54:03 +02:00
// Mkdir creates the bucket if it doesn't exist
2019-06-17 10:34:30 +02:00
func ( f * Fs ) Mkdir ( ctx context . Context , dir string ) ( err error ) {
2019-08-15 17:26:16 +02:00
bucket , _ := f . split ( dir )
2019-08-22 22:30:55 +02:00
return f . makeBucket ( ctx , bucket )
}
// makeBucket creates the bucket if it doesn't exist
func ( f * Fs ) makeBucket ( ctx context . Context , bucket string ) ( err error ) {
2019-08-15 17:26:16 +02:00
return f . cache . Create ( bucket , func ( ) error {
// List something from the bucket to see if it exists. Doing it like this enables the use of a
// service account that only has the "Storage Object Admin" role. See #2193 for details.
err = f . pacer . Call ( func ( ) ( bool , error ) {
2019-09-06 14:50:36 +02:00
_ , err = f . svc . Objects . List ( bucket ) . MaxResults ( 1 ) . Context ( ctx ) . Do ( )
2021-03-11 15:44:01 +01:00
return shouldRetry ( ctx , err )
2019-08-15 17:26:16 +02:00
} )
if err == nil {
// Bucket already exists
return nil
} else if gErr , ok := err . ( * googleapi . Error ) ; ok {
if gErr . Code != http . StatusNotFound {
return errors . Wrap ( err , "failed to get bucket" )
}
} else {
2017-08-10 11:29:21 +02:00
return errors . Wrap ( err , "failed to get bucket" )
}
2014-07-13 18:54:03 +02:00
2019-08-15 17:26:16 +02:00
if f . opt . ProjectNumber == "" {
return errors . New ( "can't make bucket without project number" )
}
2014-07-13 18:54:03 +02:00
2019-08-15 17:26:16 +02:00
bucket := storage . Bucket {
Name : bucket ,
Location : f . opt . Location ,
StorageClass : f . opt . StorageClass ,
2019-03-04 15:52:54 +01:00
}
2019-08-15 17:26:16 +02:00
if f . opt . BucketPolicyOnly {
bucket . IamConfiguration = & storage . BucketIamConfiguration {
BucketPolicyOnly : & storage . BucketIamConfigurationBucketPolicyOnly {
Enabled : true ,
} ,
}
2019-03-04 15:52:54 +01:00
}
2019-08-15 17:26:16 +02:00
return f . pacer . Call ( func ( ) ( bool , error ) {
insertBucket := f . svc . Buckets . Insert ( f . opt . ProjectNumber , & bucket )
if ! f . opt . BucketPolicyOnly {
insertBucket . PredefinedAcl ( f . opt . BucketACL )
}
2019-09-06 14:50:36 +02:00
_ , err = insertBucket . Context ( ctx ) . Do ( )
2021-03-11 15:44:01 +01:00
return shouldRetry ( ctx , err )
2019-08-15 17:26:16 +02:00
} )
} , nil )
2014-07-13 18:54:03 +02:00
}
2015-11-07 16:31:04 +01:00
// Rmdir deletes the bucket if the fs is at the root
2014-07-13 18:54:03 +02:00
//
// Returns an error if it isn't empty: Error 409: The bucket you tried
// to delete was not empty.
2019-06-17 10:34:30 +02:00
func ( f * Fs ) Rmdir ( ctx context . Context , dir string ) ( err error ) {
2019-08-15 17:26:16 +02:00
bucket , directory := f . split ( dir )
if bucket == "" || directory != "" {
2015-11-07 16:31:04 +01:00
return nil
}
2019-08-15 17:26:16 +02:00
return f . cache . Remove ( bucket , func ( ) error {
return f . pacer . Call ( func ( ) ( bool , error ) {
2019-09-06 14:50:36 +02:00
err = f . svc . Buckets . Delete ( bucket ) . Context ( ctx ) . Do ( )
2021-03-11 15:44:01 +01:00
return shouldRetry ( ctx , err )
2019-08-15 17:26:16 +02:00
} )
2018-05-09 15:27:21 +02:00
} )
2014-07-13 18:54:03 +02:00
}
2015-09-22 19:47:16 +02:00
// Precision returns the precision
2015-11-07 12:14:46 +01:00
func ( f * Fs ) Precision ( ) time . Duration {
2014-07-13 18:54:03 +02:00
return time . Nanosecond
}
2020-10-13 23:43:40 +02:00
// Copy src to this remote using server-side copy operations.
2015-02-14 19:48:08 +01:00
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
2019-06-17 10:34:30 +02:00
func ( f * Fs ) Copy ( ctx context . Context , src fs . Object , remote string ) ( fs . Object , error ) {
2019-08-15 17:26:16 +02:00
dstBucket , dstPath := f . split ( remote )
2019-08-22 22:30:55 +02:00
err := f . makeBucket ( ctx , dstBucket )
2017-06-28 22:14:53 +02:00
if err != nil {
return nil , err
}
2015-11-07 12:14:46 +01:00
srcObj , ok := src . ( * Object )
2015-02-14 19:48:08 +01:00
if ! ok {
2017-02-09 12:01:20 +01:00
fs . Debugf ( src , "Can't copy - not same remote type" )
2015-02-14 19:48:08 +01:00
return nil , fs . ErrorCantCopy
}
2019-08-15 17:26:16 +02:00
srcBucket , srcPath := srcObj . split ( )
2015-02-14 19:48:08 +01:00
2015-11-07 12:14:46 +01:00
// Temporary Object under construction
dstObj := & Object {
fs : f ,
remote : remote ,
}
2015-02-14 19:48:08 +01:00
2020-11-30 17:18:41 +01:00
rewriteRequest := f . svc . Objects . Rewrite ( srcBucket , srcPath , dstBucket , dstPath , nil )
if ! f . opt . BucketPolicyOnly {
rewriteRequest . DestinationPredefinedAcl ( f . opt . ObjectACL )
}
var rewriteResponse * storage . RewriteResponse
for {
err = f . pacer . Call ( func ( ) ( bool , error ) {
rewriteResponse , err = rewriteRequest . Context ( ctx ) . Do ( )
2021-03-11 15:44:01 +01:00
return shouldRetry ( ctx , err )
2020-11-30 17:18:41 +01:00
} )
if err != nil {
return nil , err
2019-08-29 16:19:02 +02:00
}
2020-11-30 17:18:41 +01:00
if rewriteResponse . Done {
break
}
rewriteRequest . RewriteToken ( rewriteResponse . RewriteToken )
fs . Debugf ( dstObj , "Continuing rewrite %d bytes done" , rewriteResponse . TotalBytesRewritten )
2015-02-14 19:48:08 +01:00
}
// Set the metadata for the new object while we have it
2020-11-30 17:18:41 +01:00
dstObj . setMetaData ( rewriteResponse . Resource )
2015-02-14 19:48:08 +01:00
return dstObj , nil
}
2016-01-11 13:39:33 +01:00
// Hashes returns the supported hash sets.
2018-01-12 17:30:54 +01:00
func ( f * Fs ) Hashes ( ) hash . Set {
2018-01-18 21:27:52 +01:00
return hash . Set ( hash . MD5 )
2016-01-11 13:39:33 +01:00
}
2014-07-13 18:54:03 +02:00
// ------------------------------------------------------------
2015-09-22 19:47:16 +02:00
// Fs returns the parent Fs
2016-02-18 12:35:25 +01:00
func ( o * Object ) Fs ( ) fs . Info {
2015-11-07 12:14:46 +01:00
return o . fs
2014-07-13 18:54:03 +02:00
}
// Return a string version
2015-11-07 12:14:46 +01:00
func ( o * Object ) String ( ) string {
2014-07-13 18:54:03 +02:00
if o == nil {
return "<nil>"
}
return o . remote
}
2015-09-22 19:47:16 +02:00
// Remote returns the remote path
2015-11-07 12:14:46 +01:00
func ( o * Object ) Remote ( ) string {
2014-07-13 18:54:03 +02:00
return o . remote
}
2016-01-11 13:39:33 +01:00
// Hash returns the Md5sum of an object returning a lowercase hex string
2019-06-17 10:34:30 +02:00
func ( o * Object ) Hash ( ctx context . Context , t hash . Type ) ( string , error ) {
2018-01-18 21:27:52 +01:00
if t != hash . MD5 {
return "" , hash . ErrUnsupported
2016-01-11 13:39:33 +01:00
}
2014-07-13 18:54:03 +02:00
return o . md5sum , nil
}
// Size returns the size of an object in bytes
2015-11-07 12:14:46 +01:00
func ( o * Object ) Size ( ) int64 {
2014-07-13 18:54:03 +02:00
return o . bytes
}
// setMetaData sets the fs data from a storage.Object
2015-11-07 12:14:46 +01:00
func ( o * Object ) setMetaData ( info * storage . Object ) {
2014-07-13 18:54:03 +02:00
o . url = info . MediaLink
o . bytes = int64 ( info . Size )
2016-09-21 23:13:24 +02:00
o . mimeType = info . ContentType
2014-07-13 18:54:03 +02:00
// Read md5sum
md5sumData , err := base64 . StdEncoding . DecodeString ( info . Md5Hash )
if err != nil {
2017-02-09 12:01:20 +01:00
fs . Logf ( o , "Bad MD5 decode: %v" , err )
2014-07-13 18:54:03 +02:00
} else {
o . md5sum = hex . EncodeToString ( md5sumData )
}
// read mtime out of metadata if available
mtimeString , ok := info . Metadata [ metaMtime ]
if ok {
2021-05-21 17:01:32 +02:00
modTime , err := time . Parse ( timeFormat , mtimeString )
2014-07-13 18:54:03 +02:00
if err == nil {
o . modTime = modTime
return
}
2017-02-09 12:01:20 +01:00
fs . Debugf ( o , "Failed to read mtime from metadata: %s" , err )
2014-07-13 18:54:03 +02:00
}
2021-05-21 11:11:43 +02:00
// Fallback to GSUtil mtime
mtimeGsutilString , ok := info . Metadata [ metaMtimeGsutil ]
if ok {
unixTimeSec , err := strconv . ParseInt ( mtimeGsutilString , 10 , 64 )
if err == nil {
o . modTime = time . Unix ( unixTimeSec , 0 )
return
}
fs . Debugf ( o , "Failed to read GSUtil mtime from metadata: %s" , err )
}
2014-07-13 18:54:03 +02:00
// Fallback to the Updated time
2021-05-21 17:01:32 +02:00
modTime , err := time . Parse ( timeFormat , info . Updated )
2014-07-13 18:54:03 +02:00
if err != nil {
2017-02-09 12:01:20 +01:00
fs . Logf ( o , "Bad time decode: %v" , err )
2014-07-13 18:54:03 +02:00
} else {
o . modTime = modTime
}
}
2019-08-29 16:19:02 +02:00
// readObjectInfo reads the definition for an object
2019-09-06 14:50:36 +02:00
func ( o * Object ) readObjectInfo ( ctx context . Context ) ( object * storage . Object , err error ) {
2019-08-15 17:26:16 +02:00
bucket , bucketPath := o . split ( )
2018-05-09 15:27:21 +02:00
err = o . fs . pacer . Call ( func ( ) ( bool , error ) {
2019-09-06 14:50:36 +02:00
object , err = o . fs . svc . Objects . Get ( bucket , bucketPath ) . Context ( ctx ) . Do ( )
2021-03-11 15:44:01 +01:00
return shouldRetry ( ctx , err )
2018-05-09 15:27:21 +02:00
} )
2014-07-13 18:54:03 +02:00
if err != nil {
2016-06-25 22:23:20 +02:00
if gErr , ok := err . ( * googleapi . Error ) ; ok {
if gErr . Code == http . StatusNotFound {
2019-08-29 16:19:02 +02:00
return nil , fs . ErrorObjectNotFound
2016-06-25 22:23:20 +02:00
}
}
2019-08-29 16:19:02 +02:00
return nil , err
}
return object , nil
}
// readMetaData gets the metadata if it hasn't already been fetched
//
// it also sets the info
2019-09-06 14:50:36 +02:00
func ( o * Object ) readMetaData ( ctx context . Context ) ( err error ) {
2019-08-29 16:19:02 +02:00
if ! o . modTime . IsZero ( ) {
return nil
}
2019-09-06 14:50:36 +02:00
object , err := o . readObjectInfo ( ctx )
2019-08-29 16:19:02 +02:00
if err != nil {
2014-07-13 18:54:03 +02:00
return err
}
o . setMetaData ( object )
return nil
}
// ModTime returns the modification time of the object
//
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
2019-06-17 10:34:30 +02:00
func ( o * Object ) ModTime ( ctx context . Context ) time . Time {
2019-09-06 14:50:36 +02:00
err := o . readMetaData ( ctx )
2014-07-13 18:54:03 +02:00
if err != nil {
2017-02-09 12:01:20 +01:00
// fs.Logf(o, "Failed to read metadata: %v", err)
2014-07-13 18:54:03 +02:00
return time . Now ( )
}
return o . modTime
}
// Returns metadata for an object
func metadataFromModTime ( modTime time . Time ) map [ string ] string {
metadata := make ( map [ string ] string , 1 )
2021-05-21 17:01:32 +02:00
metadata [ metaMtime ] = modTime . Format ( timeFormat )
2021-05-21 11:11:43 +02:00
metadata [ metaMtimeGsutil ] = strconv . FormatInt ( modTime . Unix ( ) , 10 )
2014-07-13 18:54:03 +02:00
return metadata
}
2015-09-22 19:47:16 +02:00
// SetModTime sets the modification time of the local fs object
2019-06-17 10:34:30 +02:00
func ( o * Object ) SetModTime ( ctx context . Context , modTime time . Time ) ( err error ) {
2019-08-29 16:19:02 +02:00
// read the complete existing object first
2019-09-06 14:50:36 +02:00
object , err := o . readObjectInfo ( ctx )
2019-08-29 16:19:02 +02:00
if err != nil {
return err
2014-07-13 18:54:03 +02:00
}
2019-08-29 16:19:02 +02:00
// Add the mtime to the existing metadata
if object . Metadata == nil {
object . Metadata = make ( map [ string ] string , 1 )
}
2021-05-21 17:01:32 +02:00
object . Metadata [ metaMtime ] = modTime . Format ( timeFormat )
2021-05-21 11:11:43 +02:00
object . Metadata [ metaMtimeGsutil ] = strconv . FormatInt ( modTime . Unix ( ) , 10 )
2019-08-29 16:19:02 +02:00
// Copy the object to itself to update the metadata
// Using PATCH requires too many permissions
bucket , bucketPath := o . split ( )
2018-05-09 15:27:21 +02:00
var newObject * storage . Object
err = o . fs . pacer . Call ( func ( ) ( bool , error ) {
2019-08-29 16:19:02 +02:00
copyObject := o . fs . svc . Objects . Copy ( bucket , bucketPath , bucket , bucketPath , object )
if ! o . fs . opt . BucketPolicyOnly {
copyObject . DestinationPredefinedAcl ( o . fs . opt . ObjectACL )
}
2019-09-06 14:50:36 +02:00
newObject , err = copyObject . Context ( ctx ) . Do ( )
2021-03-11 15:44:01 +01:00
return shouldRetry ( ctx , err )
2018-05-09 15:27:21 +02:00
} )
2014-07-13 18:54:03 +02:00
if err != nil {
2016-03-22 16:07:10 +01:00
return err
2014-07-13 18:54:03 +02:00
}
2014-07-28 21:07:02 +02:00
o . setMetaData ( newObject )
2016-03-22 16:07:10 +01:00
return nil
2014-07-13 18:54:03 +02:00
}
2015-09-22 19:47:16 +02:00
// Storable returns a boolean as to whether this object is storable
2015-11-07 12:14:46 +01:00
func ( o * Object ) Storable ( ) bool {
2014-07-13 18:54:03 +02:00
return true
}
// Open an object for read
2019-06-17 10:34:30 +02:00
func ( o * Object ) Open ( ctx context . Context , options ... fs . OpenOption ) ( in io . ReadCloser , err error ) {
2021-02-03 18:41:27 +01:00
req , err := http . NewRequestWithContext ( ctx , "GET" , o . url , nil )
2014-07-15 12:18:43 +02:00
if err != nil {
return nil , err
}
2019-08-06 16:18:08 +02:00
fs . FixRangeOption ( options , o . bytes )
2016-09-10 12:29:57 +02:00
fs . OpenOptionAddHTTPHeaders ( req . Header , options )
2018-05-09 15:27:21 +02:00
var res * http . Response
err = o . fs . pacer . Call ( func ( ) ( bool , error ) {
res , err = o . fs . client . Do ( req )
if err == nil {
err = googleapi . CheckResponse ( res )
if err != nil {
_ = res . Body . Close ( ) // ignore error
}
}
2021-03-11 15:44:01 +01:00
return shouldRetry ( ctx , err )
2018-05-09 15:27:21 +02:00
} )
2014-07-13 18:54:03 +02:00
if err != nil {
return nil , err
}
2016-09-10 12:29:57 +02:00
_ , isRanging := req . Header [ "Range" ]
if ! ( res . StatusCode == http . StatusOK || ( isRanging && res . StatusCode == http . StatusPartialContent ) ) {
2014-07-25 19:19:49 +02:00
_ = res . Body . Close ( ) // ignore error
2016-06-12 16:06:02 +02:00
return nil , errors . Errorf ( "bad response: %d: %s" , res . StatusCode , res . Status )
2014-07-13 18:54:03 +02:00
}
return res . Body , nil
}
// Update the object with the contents of the io.Reader, modTime and size
//
// The new object may have been created if an error is returned
2019-06-17 10:34:30 +02:00
func ( o * Object ) Update ( ctx context . Context , in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) error {
2019-08-15 17:26:16 +02:00
bucket , bucketPath := o . split ( )
2019-08-22 22:30:55 +02:00
err := o . fs . makeBucket ( ctx , bucket )
2017-06-07 15:16:50 +02:00
if err != nil {
return err
}
2019-06-17 10:34:30 +02:00
modTime := src . ModTime ( ctx )
2016-02-18 12:35:25 +01:00
2014-07-13 18:54:03 +02:00
object := storage . Object {
2019-08-15 17:26:16 +02:00
Bucket : bucket ,
Name : bucketPath ,
2019-06-17 10:34:30 +02:00
ContentType : fs . MimeType ( ctx , src ) ,
2014-07-14 13:44:31 +02:00
Metadata : metadataFromModTime ( modTime ) ,
2014-07-13 18:54:03 +02:00
}
2020-05-02 10:15:28 +02:00
// Apply upload options
for _ , option := range options {
key , value := option . Header ( )
lowerKey := strings . ToLower ( key )
switch lowerKey {
case "" :
// ignore
case "cache-control" :
object . CacheControl = value
case "content-disposition" :
object . ContentDisposition = value
case "content-encoding" :
object . ContentEncoding = value
case "content-language" :
object . ContentLanguage = value
case "content-type" :
object . ContentType = value
2020-12-03 09:52:12 +01:00
case "x-goog-storage-class" :
object . StorageClass = value
2020-05-02 10:15:28 +02:00
default :
const googMetaPrefix = "x-goog-meta-"
if strings . HasPrefix ( lowerKey , googMetaPrefix ) {
metaKey := lowerKey [ len ( googMetaPrefix ) : ]
object . Metadata [ metaKey ] = value
} else {
fs . Errorf ( o , "Don't know how to set key %q on upload" , key )
}
}
}
2018-05-09 15:27:21 +02:00
var newObject * storage . Object
err = o . fs . pacer . CallNoRetry ( func ( ) ( bool , error ) {
2019-08-15 17:26:16 +02:00
insertObject := o . fs . svc . Objects . Insert ( bucket , & object ) . Media ( in , googleapi . ContentType ( "" ) ) . Name ( object . Name )
2019-03-04 15:52:54 +01:00
if ! o . fs . opt . BucketPolicyOnly {
insertObject . PredefinedAcl ( o . fs . opt . ObjectACL )
}
2019-09-06 14:50:36 +02:00
newObject , err = insertObject . Context ( ctx ) . Do ( )
2021-03-11 15:44:01 +01:00
return shouldRetry ( ctx , err )
2018-05-09 15:27:21 +02:00
} )
2014-07-21 22:25:46 +02:00
if err != nil {
return err
}
2014-07-16 13:12:36 +02:00
// Set the metadata for the new object while we have it
o . setMetaData ( newObject )
2014-07-21 22:25:46 +02:00
return nil
2014-07-13 18:54:03 +02:00
}
// Remove an object
2019-06-17 10:34:30 +02:00
func ( o * Object ) Remove ( ctx context . Context ) ( err error ) {
2019-08-15 17:26:16 +02:00
bucket , bucketPath := o . split ( )
2018-05-09 15:27:21 +02:00
err = o . fs . pacer . Call ( func ( ) ( bool , error ) {
2019-09-06 14:50:36 +02:00
err = o . fs . svc . Objects . Delete ( bucket , bucketPath ) . Context ( ctx ) . Do ( )
2021-03-11 15:44:01 +01:00
return shouldRetry ( ctx , err )
2018-05-09 15:27:21 +02:00
} )
return err
2014-07-13 18:54:03 +02:00
}
2016-09-21 23:13:24 +02:00
// MimeType of an Object if known, "" otherwise
2019-06-17 10:34:30 +02:00
func ( o * Object ) MimeType ( ctx context . Context ) string {
2016-09-21 23:13:24 +02:00
return o . mimeType
}
2014-07-13 18:54:03 +02:00
// Check the interfaces are satisfied
2015-11-07 12:14:46 +01:00
var (
2017-09-16 22:46:02 +02:00
_ fs . Fs = & Fs { }
_ fs . Copier = & Fs { }
_ fs . PutStreamer = & Fs { }
_ fs . ListRer = & Fs { }
_ fs . Object = & Object { }
_ fs . MimeTyper = & Object { }
2015-11-07 12:14:46 +01:00
)