diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 1674f0d42..e39a5a6c2 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -143,6 +143,41 @@ var ( _linkTemplates map[string]*template.Template // available link types ) +// rwChoices type for fs.Bits +type rwChoices struct{} + +func (rwChoices) Choices() []fs.BitsChoicesInfo { + return []fs.BitsChoicesInfo{ + {Bit: uint64(rwOff), Name: "off"}, + {Bit: uint64(rwRead), Name: "read"}, + {Bit: uint64(rwWrite), Name: "write"}, + } +} + +// rwChoice type alias +type rwChoice = fs.Bits[rwChoices] + +const ( + rwRead rwChoice = 1 << iota + rwWrite + rwOff rwChoice = 0 +) + +// Examples for the options +var rwExamples = fs.OptionExamples{{ + Value: rwOff.String(), + Help: "Do not read or write the value", +}, { + Value: rwRead.String(), + Help: "Read the value only", +}, { + Value: rwWrite.String(), + Help: "Write the value only", +}, { + Value: (rwRead | rwWrite).String(), + Help: "Read and Write the value.", +}} + // Parse the scopes option returning a slice of scopes func driveScopes(scopesString string) (scopes []string) { if scopesString == "" { @@ -250,6 +285,10 @@ func init() { } return nil, fmt.Errorf("unknown state %q", config.State) }, + MetadataInfo: &fs.MetadataInfo{ + System: systemMetadataInfo, + Help: `User metadata is stored in the properties field of the drive object.`, + }, Options: append(driveOAuthOptions(), []fs.Option{{ Name: "scope", Help: "Comma separated list of scopes that rclone should use when requesting access from drive.", @@ -639,6 +678,56 @@ having trouble with like many empty directories. `, Advanced: true, Default: true, + }, { + Name: "metadata_owner", + Help: `Control whether owner should be read or written in metadata. + +Owner is a standard part of the file metadata so is easy to read. But it +isn't always desirable to set the owner from the metadata. + +Note that you can't set the owner on Shared Drives, and that setting +ownership will generate an email to the new owner (this can't be +disabled), and you can't transfer ownership to someone outside your +organization. +`, + Advanced: true, + Default: rwRead, + Examples: rwExamples, + }, { + Name: "metadata_permissions", + Help: `Control whether permissions should be read or written in metadata. + +Reading permissions metadata from files can be done quickly, but it +isn't always desirable to set the permissions from the metadata. + +Note that rclone drops any inherited permissions on Shared Drives and +any owner permission on My Drives as these are duplicated in the owner +metadata. +`, + Advanced: true, + Default: rwOff, + Examples: rwExamples, + }, { + Name: "metadata_labels", + Help: `Control whether labels should be read or written in metadata. + +Reading labels metadata from files takes an extra API transaction and +will slow down listings. It isn't always desirable to set the labels +from the metadata. + +The format of labels is documented in the drive API documentation at +https://developers.google.com/drive/api/reference/rest/v3/Label - +rclone just provides a JSON dump of this format. + +When setting labels, the label and fields must already exist - rclone +will not create them. This means that if you are transferring labels +from two different accounts you will have to create the labels in +advance and use the metadata mapper to translate the IDs between the +two accounts. +`, + Advanced: true, + Default: rwOff, + Examples: rwExamples, }, { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, @@ -715,6 +804,9 @@ type Options struct { SkipDanglingShortcuts bool `config:"skip_dangling_shortcuts"` ResourceKey string `config:"resource_key"` FastListBugFix bool `config:"fast_list_bug_fix"` + MetadataOwner rwChoice `config:"metadata_owner"` + MetadataPermissions rwChoice `config:"metadata_permissions"` + MetadataLabels rwChoice `config:"metadata_labels"` Enc encoder.MultiEncoder `config:"encoding"` EnvAuth bool `config:"env_auth"` } @@ -738,21 +830,24 @@ type Fs struct { isTeamDrive bool // true if this is a team drive fileFields googleapi.Field // fields to fetch file info with m configmap.Mapper - grouping int32 // number of IDs to search at once in ListR - read with atomic - listRmu *sync.Mutex // protects listRempties - listRempties map[string]struct{} // IDs of supposedly empty directories which triggered grouping disable - dirResourceKeys *sync.Map // map directory ID to resource key + grouping int32 // number of IDs to search at once in ListR - read with atomic + listRmu *sync.Mutex // protects listRempties + listRempties map[string]struct{} // IDs of supposedly empty directories which triggered grouping disable + dirResourceKeys *sync.Map // map directory ID to resource key + permissionsMu *sync.Mutex // protect the below + permissions map[string]*drive.Permission // map permission IDs to Permissions } type baseObject struct { - fs *Fs // what this object is part of - remote string // The remote path - id string // Drive Id of this object - modifiedDate string // RFC3339 time it was last modified - mimeType string // The object MIME type - bytes int64 // size of the object - parents []string // IDs of the parent directories - resourceKey *string // resourceKey is needed for link shared objects + fs *Fs // what this object is part of + remote string // The remote path + id string // Drive Id of this object + modifiedDate string // RFC3339 time it was last modified + mimeType string // The object MIME type + bytes int64 // size of the object + parents []string // IDs of the parent directories + resourceKey *string // resourceKey is needed for link shared objects + metadata *fs.Metadata // metadata if known } type documentObject struct { baseObject @@ -1275,6 +1370,8 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err listRmu: new(sync.Mutex), listRempties: make(map[string]struct{}), dirResourceKeys: new(sync.Map), + permissionsMu: new(sync.Mutex), + permissions: make(map[string]*drive.Permission), } f.isTeamDrive = opt.TeamDriveID != "" f.fileFields = f.getFileFields() @@ -1285,6 +1382,9 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err CanHaveEmptyDirectories: true, ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs, FilterAware: true, + ReadMetadata: true, + WriteMetadata: true, + UserMetadata: true, }).Fill(ctx, f) // Create a new authorized Drive client. @@ -1389,7 +1489,7 @@ func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, e return f, nil } -func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject { +func (f *Fs) newBaseObject(ctx context.Context, remote string, info *drive.File) (o baseObject, err error) { modifiedDate := info.ModifiedTime if f.opt.UseCreatedDate { modifiedDate = info.CreatedTime @@ -1400,7 +1500,7 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject { if f.opt.SizeAsQuota { size = info.QuotaBytesUsed } - return baseObject{ + o = baseObject{ fs: f, remote: remote, id: info.Id, @@ -1409,6 +1509,11 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject { bytes: size, parents: info.Parents, } + err = nil + if f.ci.Metadata { + err = o.parseMetadata(ctx, info) + } + return o, err } // getFileFields gets the fields for a normal file Get or List @@ -1426,11 +1531,14 @@ func (f *Fs) getFileFields() (fields googleapi.Field) { if f.opt.SizeAsQuota { fields += ",quotaBytesUsed" } + if f.ci.Metadata { + fields += "," + metadataFields + } return fields } // newRegularObject creates an fs.Object for a normal drive.File -func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object { +func (f *Fs) newRegularObject(ctx context.Context, remote string, info *drive.File) (obj fs.Object, err error) { // wipe checksum if SkipChecksumGphotos and file is type Photo or Video if f.opt.SkipChecksumGphotos { for _, space := range info.Spaces { @@ -1443,27 +1551,33 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object { } } o := &Object{ - baseObject: f.newBaseObject(remote, info), url: fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)), md5sum: strings.ToLower(info.Md5Checksum), sha1sum: strings.ToLower(info.Sha1Checksum), sha256sum: strings.ToLower(info.Sha256Checksum), v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize), } + o.baseObject, err = f.newBaseObject(ctx, remote, info) + if err != nil { + return nil, err + } if info.ResourceKey != "" { o.resourceKey = &info.ResourceKey } - return o + return o, nil } // newDocumentObject creates an fs.Object for a google docs drive.File -func (f *Fs) newDocumentObject(remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) { +func (f *Fs) newDocumentObject(ctx context.Context, remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) { mediaType, _, err := mime.ParseMediaType(exportMimeType) if err != nil { return nil, err } url := info.ExportLinks[mediaType] - baseObject := f.newBaseObject(remote+extension, info) + baseObject, err := f.newBaseObject(ctx, remote+extension, info) + if err != nil { + return nil, err + } baseObject.bytes = -1 baseObject.mimeType = exportMimeType return &documentObject{ @@ -1475,7 +1589,7 @@ func (f *Fs) newDocumentObject(remote string, info *drive.File, extension, expor } // newLinkObject creates an fs.Object that represents a link a google docs drive.File -func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) { +func (f *Fs) newLinkObject(ctx context.Context, remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) { t := linkTemplate(exportMimeType) if t == nil { return nil, fmt.Errorf("unsupported link type %s", exportMimeType) @@ -1494,7 +1608,10 @@ func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMim return nil, fmt.Errorf("executing template failed: %w", err) } - baseObject := f.newBaseObject(remote+extension, info) + baseObject, err := f.newBaseObject(ctx, remote+extension, info) + if err != nil { + return nil, err + } baseObject.bytes = int64(buf.Len()) baseObject.mimeType = exportMimeType return &linkObject{ @@ -1510,7 +1627,7 @@ func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMim func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *drive.File) (fs.Object, error) { // If item has MD5 sum it is a file stored on drive if info.Md5Checksum != "" { - return f.newRegularObject(remote, info), nil + return f.newRegularObject(ctx, remote, info) } extension, exportName, exportMimeType, isDocument := f.findExportFormat(ctx, info) @@ -1541,15 +1658,15 @@ func (f *Fs) newObjectWithExportInfo( case info.MimeType == shortcutMimeTypeDangling: // Pretend a dangling shortcut is a regular object // It will error if used, but appear in listings so it can be deleted - return f.newRegularObject(remote, info), nil + return f.newRegularObject(ctx, remote, info) case info.Md5Checksum != "": // If item has MD5 sum it is a file stored on drive - return f.newRegularObject(remote, info), nil + return f.newRegularObject(ctx, remote, info) case f.opt.SkipGdocs: fs.Debugf(remote, "Skipping google document type %q", info.MimeType) return nil, fs.ErrorObjectNotFound case f.opt.ShowAllGdocs: - return f.newDocumentObject(remote, info, "", info.MimeType) + return f.newDocumentObject(ctx, remote, info, "", info.MimeType) default: // If item MimeType is in the ExportFormats then it is a google doc if !isDocument { @@ -1561,9 +1678,9 @@ func (f *Fs) newObjectWithExportInfo( return nil, fs.ErrorObjectNotFound } if isLinkMimeType(exportMimeType) { - return f.newLinkObject(remote, info, extension, exportMimeType) + return f.newLinkObject(ctx, remote, info, extension, exportMimeType) } - return f.newDocumentObject(remote, info, extension, exportMimeType) + return f.newDocumentObject(ctx, remote, info, extension, exportMimeType) } } @@ -2323,6 +2440,10 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, } else { createInfo.MimeType = fs.MimeTypeFromName(remote) } + updateMetadata, err := f.fetchAndUpdateMetadata(ctx, src, options, createInfo) + if err != nil { + return nil, err + } var info *drive.File if size >= 0 && size < int64(f.opt.UploadCutoff) { @@ -2347,6 +2468,10 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, return nil, err } } + err = updateMetadata(ctx, info) + if err != nil { + return nil, err + } return f.newObjectWithInfo(ctx, remote, info) } @@ -3944,10 +4069,20 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op MimeType: srcMimeType, ModifiedTime: src.ModTime(ctx).Format(timeFormatOut), } + + updateMetadata, err := o.fs.fetchAndUpdateMetadata(ctx, src, options, updateInfo) + if err != nil { + return err + } + info, err := o.baseObject.update(ctx, updateInfo, srcMimeType, in, src) if err != nil { return err } + err = updateMetadata(ctx, info) + if err != nil { + return err + } newO, err := o.fs.newObjectWithInfo(ctx, o.remote, info) if err != nil { return err @@ -4033,6 +4168,26 @@ func (o *baseObject) ParentID() string { return "" } +// Metadata returns metadata for an object +// +// It should return nil if there is no Metadata +func (o *baseObject) Metadata(ctx context.Context) (metadata fs.Metadata, err error) { + if o.metadata != nil { + return *o.metadata, nil + } + fs.Debugf(o, "Fetching metadata") + id := actualID(o.id) + info, err := o.fs.getFile(ctx, id, o.fs.fileFields) + if err != nil { + return nil, err + } + err = o.parseMetadata(ctx, info) + if err != nil { + return nil, err + } + return *o.metadata, nil +} + func (o *documentObject) ext() string { return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:] } @@ -4094,6 +4249,7 @@ var ( _ fs.MimeTyper = (*Object)(nil) _ fs.IDer = (*Object)(nil) _ fs.ParentIDer = (*Object)(nil) + _ fs.Metadataer = (*Object)(nil) _ fs.Object = (*documentObject)(nil) _ fs.MimeTyper = (*documentObject)(nil) _ fs.IDer = (*documentObject)(nil) diff --git a/backend/drive/metadata.go b/backend/drive/metadata.go new file mode 100644 index 000000000..8b3c87365 --- /dev/null +++ b/backend/drive/metadata.go @@ -0,0 +1,600 @@ +package drive + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "sync" + + "github.com/rclone/rclone/fs" + "golang.org/x/sync/errgroup" + drive "google.golang.org/api/drive/v3" + "google.golang.org/api/googleapi" +) + +// system metadata keys which this backend owns +var systemMetadataInfo = map[string]fs.MetadataHelp{ + "content-type": { + Help: "The MIME type of the file.", + Type: "string", + Example: "text/plain", + }, + "mtime": { + Help: "Time of last modification with mS accuracy.", + Type: "RFC 3339", + Example: "2006-01-02T15:04:05.999Z07:00", + }, + "btime": { + Help: "Time of file birth (creation) with mS accuracy.", + Type: "RFC 3339", + Example: "2006-01-02T15:04:05.999Z07:00", + }, + "copy-requires-writer-permission": { + Help: "Whether the options to copy, print, or download this file, should be disabled for readers and commenters.", + Type: "boolean", + Example: "true", + }, + "writers-can-share": { + Help: "Whether users with only writer permission can modify the file's permissions. Not populated for items in shared drives.", + Type: "boolean", + Example: "false", + }, + "viewed-by-me": { + Help: "Whether the file has been viewed by this user.", + Type: "boolean", + Example: "true", + ReadOnly: true, + }, + "owner": { + Help: "The owner of the file. Usually an email address. Enable with --drive-metadata-owner.", + Type: "string", + Example: "user@example.com", + }, + "permissions": { + Help: "Permissions in a JSON dump of Google drive format. On shared drives these will only be present if they aren't inherited. Enable with --drive-metadata-permissions.", + Type: "JSON", + Example: "{}", + }, + "folder-color-rgb": { + Help: "The color for a folder or a shortcut to a folder as an RGB hex string.", + Type: "string", + Example: "881133", + }, + "description": { + Help: "A short description of the file.", + Type: "string", + Example: "Contract for signing", + }, + "starred": { + Help: "Whether the user has starred the file.", + Type: "boolean", + Example: "false", + }, + "labels": { + Help: "Labels attached to this file in a JSON dump of Googled drive format. Enable with --drive-metadata-labels.", + Type: "JSON", + Example: "[]", + }, +} + +// Extra fields we need to fetch to implement the system metadata above +var metadataFields = googleapi.Field(strings.Join([]string{ + "copyRequiresWriterPermission", + "description", + "folderColorRgb", + "hasAugmentedPermissions", + "owners", + "permissionIds", + "permissions", + "properties", + "starred", + "viewedByMe", + "viewedByMeTime", + "writersCanShare", +}, ",")) + +// Fields we need to read from permissions +var permissionsFields = googleapi.Field(strings.Join([]string{ + "*", + "permissionDetails/*", +}, ",")) + +// getPermission returns permissions for the fileID and permissionID passed in +func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, useCache bool) (perm *drive.Permission, inherited bool, err error) { + f.permissionsMu.Lock() + defer f.permissionsMu.Unlock() + if useCache { + perm = f.permissions[permissionID] + if perm != nil { + return perm, false, nil + } + } + fs.Debugf(f, "Fetching permission %q", permissionID) + err = f.pacer.Call(func() (bool, error) { + perm, err = f.svc.Permissions.Get(fileID, permissionID). + Fields(permissionsFields). + SupportsAllDrives(true). + Context(ctx).Do() + return f.shouldRetry(ctx, err) + }) + if err != nil { + return nil, false, err + } + + inherited = len(perm.PermissionDetails) > 0 && perm.PermissionDetails[0].Inherited + + cleanPermission(perm) + + // cache the permission + f.permissions[permissionID] = perm + + return perm, inherited, err +} + +// Set the permissions on the info +func (f *Fs) setPermissions(ctx context.Context, info *drive.File, permissions []*drive.Permission) (err error) { + for _, perm := range permissions { + if perm.Role == "owner" { + // ignore owner permissions - these are set with owner + continue + } + cleanPermissionForWrite(perm) + err = f.pacer.Call(func() (bool, error) { + _, err = f.svc.Permissions.Create(info.Id, perm). + SupportsAllDrives(true). + Context(ctx).Do() + return f.shouldRetry(ctx, err) + }) + if err != nil { + return fmt.Errorf("failed to set permission: %w", err) + } + } + return nil +} + +// Clean attributes from permissions which we can't write +func cleanPermissionForWrite(perm *drive.Permission) { + perm.Deleted = false + perm.DisplayName = "" + perm.Id = "" + perm.Kind = "" + perm.PermissionDetails = nil + perm.TeamDrivePermissionDetails = nil +} + +// Clean and cache the permission if not already cached +func (f *Fs) cleanAndCachePermission(perm *drive.Permission) { + f.permissionsMu.Lock() + defer f.permissionsMu.Unlock() + cleanPermission(perm) + if _, found := f.permissions[perm.Id]; !found { + f.permissions[perm.Id] = perm + } +} + +// Clean fields we don't need to keep from the permission +func cleanPermission(perm *drive.Permission) { + // DisplayName: Output only. The "pretty" name of the value of the + // permission. The following is a list of examples for each type of + // permission: * `user` - User's full name, as defined for their Google + // account, such as "Joe Smith." * `group` - Name of the Google Group, + // such as "The Company Administrators." * `domain` - String domain + // name, such as "thecompany.com." * `anyone` - No `displayName` is + // present. + perm.DisplayName = "" + + // Kind: Output only. Identifies what kind of resource this is. Value: + // the fixed string "drive#permission". + perm.Kind = "" + + // PermissionDetails: Output only. Details of whether the permissions on + // this shared drive item are inherited or directly on this item. This + // is an output-only field which is present only for shared drive items. + perm.PermissionDetails = nil + + // PhotoLink: Output only. A link to the user's profile photo, if + // available. + perm.PhotoLink = "" + + // TeamDrivePermissionDetails: Output only. Deprecated: Output only. Use + // `permissionDetails` instead. + perm.TeamDrivePermissionDetails = nil +} + +// Fields we need to read from labels +var labelsFields = googleapi.Field(strings.Join([]string{ + "*", +}, ",")) + +// getLabels returns labels for the fileID passed in +func (f *Fs) getLabels(ctx context.Context, fileID string) (labels []*drive.Label, err error) { + fs.Debugf(f, "Fetching labels for %q", fileID) + listLabels := f.svc.Files.ListLabels(fileID). + Fields(labelsFields). + Context(ctx) + for { + var info *drive.LabelList + err = f.pacer.Call(func() (bool, error) { + info, err = listLabels.Do() + return f.shouldRetry(ctx, err) + }) + if err != nil { + return nil, err + } + labels = append(labels, info.Labels...) + if info.NextPageToken == "" { + break + } + listLabels.PageToken(info.NextPageToken) + } + for _, label := range labels { + cleanLabel(label) + } + return labels, nil +} + +// Set the labels on the info +func (f *Fs) setLabels(ctx context.Context, info *drive.File, labels []*drive.Label) (err error) { + if len(labels) == 0 { + return nil + } + req := drive.ModifyLabelsRequest{} + for _, label := range labels { + req.LabelModifications = append(req.LabelModifications, &drive.LabelModification{ + FieldModifications: labelFieldsToFieldModifications(label.Fields), + LabelId: label.Id, + }) + } + err = f.pacer.Call(func() (bool, error) { + _, err = f.svc.Files.ModifyLabels(info.Id, &req). + Context(ctx).Do() + return f.shouldRetry(ctx, err) + }) + if err != nil { + return fmt.Errorf("failed to set owner: %w", err) + } + return nil +} + +// Convert label fields into something which can set the fields +func labelFieldsToFieldModifications(fields map[string]drive.LabelField) (out []*drive.LabelFieldModification) { + for id, field := range fields { + var emails []string + for _, user := range field.User { + emails = append(emails, user.EmailAddress) + } + out = append(out, &drive.LabelFieldModification{ + // FieldId: The ID of the field to be modified. + FieldId: id, + + // SetDateValues: Replaces the value of a dateString Field with these + // new values. The string must be in the RFC 3339 full-date format: + // YYYY-MM-DD. + SetDateValues: field.DateString, + + // SetIntegerValues: Replaces the value of an `integer` field with these + // new values. + SetIntegerValues: field.Integer, + + // SetSelectionValues: Replaces a `selection` field with these new + // values. + SetSelectionValues: field.Selection, + + // SetTextValues: Sets the value of a `text` field. + SetTextValues: field.Text, + + // SetUserValues: Replaces a `user` field with these new values. The + // values must be valid email addresses. + SetUserValues: emails, + }) + } + return out +} + +// Clean fields we don't need to keep from the label +func cleanLabel(label *drive.Label) { + // Kind: This is always drive#label + label.Kind = "" + + for name, field := range label.Fields { + // Kind: This is always drive#labelField. + field.Kind = "" + + // Note the fields are copies so we need to write them + // back to the map + label.Fields[name] = field + } +} + +// Parse the metadata from drive item +// +// It should return nil if there is no Metadata +func (o *baseObject) parseMetadata(ctx context.Context, info *drive.File) (err error) { + metadata := make(fs.Metadata, 16) + + // Dump user metadata first as it overrides system metadata + for k, v := range info.Properties { + metadata[k] = v + } + + // System metadata + metadata["copy-requires-writer-permission"] = fmt.Sprint(info.CopyRequiresWriterPermission) + metadata["writers-can-share"] = fmt.Sprint(info.WritersCanShare) + metadata["viewed-by-me"] = fmt.Sprint(info.ViewedByMe) + metadata["content-type"] = info.MimeType + + // Owners: Output only. The owner of this file. Only certain legacy + // files may have more than one owner. This field isn't populated for + // items in shared drives. + if o.fs.opt.MetadataOwner.IsSet(rwRead) && len(info.Owners) > 0 { + user := info.Owners[0] + if len(info.Owners) > 1 { + fs.Logf(o, "Ignoring more than 1 owner") + } + if user != nil { + id := user.EmailAddress + if id == "" { + id = user.DisplayName + } + metadata["owner"] = id + } + } + + if o.fs.opt.MetadataPermissions.IsSet(rwRead) { + // We only write permissions out if they are not inherited. + // + // On My Drives permissions seem to be attached to every item + // so they will always be written out. + // + // On Shared Drives only non-inherited permissions will be + // written out. + + // To read the inherited permissions flag will mean we need to + // read the permissions for each object and the cache will be + // useless. However shared drives don't return permissions + // only permissionIds so will need to fetch them for each + // object. We use HasAugmentedPermissions to see if there are + // special permissions before fetching them to save transactions. + + // HasAugmentedPermissions: Output only. Whether there are permissions + // directly on this file. This field is only populated for items in + // shared drives. + if o.fs.isTeamDrive && !info.HasAugmentedPermissions { + // Don't process permissions if there aren't any specifically set + info.Permissions = nil + info.PermissionIds = nil + } + + // PermissionIds: Output only. List of permission IDs for users with + // access to this file. + // + // Only process these if we have no Permissions + if len(info.PermissionIds) > 0 && len(info.Permissions) == 0 { + info.Permissions = make([]*drive.Permission, 0, len(info.PermissionIds)) + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(o.fs.ci.Checkers) + var mu sync.Mutex // protect the info.Permissions from concurrent writes + for _, permissionID := range info.PermissionIds { + permissionID := permissionID + g.Go(func() error { + // must fetch the team drive ones individually to check the inherited flag + perm, inherited, err := o.fs.getPermission(gCtx, actualID(info.Id), permissionID, !o.fs.isTeamDrive) + if err != nil { + return fmt.Errorf("failed to read permission: %w", err) + } + // Don't write inherited permissions out + if inherited { + return nil + } + // Don't write owner role out - these are covered by the owner metadata + if perm.Role == "owner" { + return nil + } + mu.Lock() + info.Permissions = append(info.Permissions, perm) + mu.Unlock() + return nil + }) + } + err = g.Wait() + if err != nil { + return err + } + } else { + // Clean the fetched permissions + for _, perm := range info.Permissions { + o.fs.cleanAndCachePermission(perm) + } + } + + // Permissions: Output only. The full list of permissions for the file. + // This is only available if the requesting user can share the file. Not + // populated for items in shared drives. + if len(info.Permissions) > 0 { + buf, err := json.Marshal(info.Permissions) + if err != nil { + return fmt.Errorf("failed to marshal permissions: %w", err) + } + metadata["permissions"] = string(buf) + } + + // Permission propagation + // https://developers.google.com/drive/api/guides/manage-sharing#permission-propagation + // Leads me to believe that in non shared drives, permissions + // are added to each item when you set permissions for a + // folder whereas in shared drives they are inherited and + // placed on the item directly. + } + + if info.FolderColorRgb != "" { + metadata["folder-color-rgb"] = info.FolderColorRgb + } + if info.Description != "" { + metadata["description"] = info.Description + } + metadata["starred"] = fmt.Sprint(info.Starred) + metadata["btime"] = info.CreatedTime + metadata["mtime"] = info.ModifiedTime + + if o.fs.opt.MetadataLabels.IsSet(rwRead) { + // FIXME would be really nice if we knew if files had labels + // before listing but we need to know all possible label IDs + // to get it in the listing. + + labels, err := o.fs.getLabels(ctx, actualID(info.Id)) + if err != nil { + return fmt.Errorf("failed to fetch labels: %w", err) + } + buf, err := json.Marshal(labels) + if err != nil { + return fmt.Errorf("failed to marshal labels: %w", err) + } + metadata["labels"] = string(buf) + } + + o.metadata = &metadata + return nil +} + +// Set the owner on the info +func (f *Fs) setOwner(ctx context.Context, info *drive.File, owner string) (err error) { + perm := drive.Permission{ + Role: "owner", + EmailAddress: owner, + // Type: The type of the grantee. Valid values are: * `user` * `group` * + // `domain` * `anyone` When creating a permission, if `type` is `user` + // or `group`, you must provide an `emailAddress` for the user or group. + // When `type` is `domain`, you must provide a `domain`. There isn't + // extra information required for an `anyone` type. + Type: "user", + } + err = f.pacer.Call(func() (bool, error) { + _, err = f.svc.Permissions.Create(info.Id, &perm). + SupportsAllDrives(true). + TransferOwnership(true). + // SendNotificationEmail(false). - required apparently! + Context(ctx).Do() + return f.shouldRetry(ctx, err) + }) + if err != nil { + return fmt.Errorf("failed to set owner: %w", err) + } + return nil +} + +// Call back to set metadata that can't be set on the upload/update +// +// The *drive.File passed in holds the current state of the drive.File +// and this should update it with any modifications. +type updateMetadataFn func(context.Context, *drive.File) error + +// read the metadata from meta and write it into updateInfo +// +// It returns a callback which should be called to finish the updates +// after the data is uploaded. +func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs.Metadata) (callback updateMetadataFn, err error) { + callbackFns := []updateMetadataFn{} + callback = func(ctx context.Context, info *drive.File) error { + for _, fn := range callbackFns { + err := fn(ctx, info) + if err != nil { + return err + } + } + return nil + } + // merge metadata into request and user metadata + for k, v := range meta { + k, v := k, v + // parse a boolean from v and write into out + parseBool := func(out *bool) error { + b, err := strconv.ParseBool(v) + if err != nil { + return fmt.Errorf("can't parse metadata %q = %q: %w", k, v, err) + } + *out = b + return nil + } + switch k { + case "copy-requires-writer-permission": + if err := parseBool(&updateInfo.CopyRequiresWriterPermission); err != nil { + return nil, err + } + case "writers-can-share": + if err := parseBool(&updateInfo.WritersCanShare); err != nil { + return nil, err + } + case "viewed-by-me": + // Can't write this + case "content-type": + updateInfo.MimeType = v + case "owner": + if !f.opt.MetadataOwner.IsSet(rwWrite) { + continue + } + // Can't set Owner on upload so need to set afterwards + callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error { + return f.setOwner(ctx, info, v) + }) + case "permissions": + if !f.opt.MetadataPermissions.IsSet(rwWrite) { + continue + } + var perms []*drive.Permission + err := json.Unmarshal([]byte(v), &perms) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal permissions: %w", err) + } + // Can't set Permissions on upload so need to set afterwards + callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error { + return f.setPermissions(ctx, info, perms) + }) + case "labels": + if !f.opt.MetadataLabels.IsSet(rwWrite) { + continue + } + var labels []*drive.Label + err := json.Unmarshal([]byte(v), &labels) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal labels: %w", err) + } + // Can't set Labels on upload so need to set afterwards + callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error { + return f.setLabels(ctx, info, labels) + }) + case "folder-color-rgb": + updateInfo.FolderColorRgb = v + case "description": + updateInfo.Description = v + case "starred": + if err := parseBool(&updateInfo.Starred); err != nil { + return nil, err + } + case "btime": + updateInfo.CreatedTime = v + case "mtime": + updateInfo.ModifiedTime = v + default: + if updateInfo.Properties == nil { + updateInfo.Properties = make(map[string]string, 1) + } + updateInfo.Properties[k] = v + } + } + return callback, nil +} + +// Fetch metadata and update updateInfo if --metadata is in use +func (f *Fs) fetchAndUpdateMetadata(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption, updateInfo *drive.File) (callback updateMetadataFn, err error) { + meta, err := fs.GetMetadataOptions(ctx, src, options) + if err != nil { + return nil, fmt.Errorf("failed to read metadata from source object: %w", err) + } + callback, err = f.updateMetadata(ctx, updateInfo, meta) + if err != nil { + return nil, fmt.Errorf("failed to update metadata from source object: %w", err) + } + return callback, nil +}