mirror of
https://github.com/rclone/rclone.git
synced 2024-12-23 07:29:35 +01:00
drive: add read/write metadata support
- fetch metadata with listings and fetch permissions in parallel - only write permissions out if they are not inherited. - make setting labels, owner and permissions work controlled by flags - `--drive-metadata-labels`, `--drive-metadata-owner`, `--drive-metadata-permissions`
This commit is contained in:
parent
10774d297a
commit
9fdf3d548a
@ -143,6 +143,41 @@ var (
|
||||
_linkTemplates map[string]*template.Template // available link types
|
||||
)
|
||||
|
||||
// rwChoices type for fs.Bits
|
||||
type rwChoices struct{}
|
||||
|
||||
func (rwChoices) Choices() []fs.BitsChoicesInfo {
|
||||
return []fs.BitsChoicesInfo{
|
||||
{Bit: uint64(rwOff), Name: "off"},
|
||||
{Bit: uint64(rwRead), Name: "read"},
|
||||
{Bit: uint64(rwWrite), Name: "write"},
|
||||
}
|
||||
}
|
||||
|
||||
// rwChoice type alias
|
||||
type rwChoice = fs.Bits[rwChoices]
|
||||
|
||||
const (
|
||||
rwRead rwChoice = 1 << iota
|
||||
rwWrite
|
||||
rwOff rwChoice = 0
|
||||
)
|
||||
|
||||
// Examples for the options
|
||||
var rwExamples = fs.OptionExamples{{
|
||||
Value: rwOff.String(),
|
||||
Help: "Do not read or write the value",
|
||||
}, {
|
||||
Value: rwRead.String(),
|
||||
Help: "Read the value only",
|
||||
}, {
|
||||
Value: rwWrite.String(),
|
||||
Help: "Write the value only",
|
||||
}, {
|
||||
Value: (rwRead | rwWrite).String(),
|
||||
Help: "Read and Write the value.",
|
||||
}}
|
||||
|
||||
// Parse the scopes option returning a slice of scopes
|
||||
func driveScopes(scopesString string) (scopes []string) {
|
||||
if scopesString == "" {
|
||||
@ -250,6 +285,10 @@ func init() {
|
||||
}
|
||||
return nil, fmt.Errorf("unknown state %q", config.State)
|
||||
},
|
||||
MetadataInfo: &fs.MetadataInfo{
|
||||
System: systemMetadataInfo,
|
||||
Help: `User metadata is stored in the properties field of the drive object.`,
|
||||
},
|
||||
Options: append(driveOAuthOptions(), []fs.Option{{
|
||||
Name: "scope",
|
||||
Help: "Comma separated list of scopes that rclone should use when requesting access from drive.",
|
||||
@ -639,6 +678,56 @@ having trouble with like many empty directories.
|
||||
`,
|
||||
Advanced: true,
|
||||
Default: true,
|
||||
}, {
|
||||
Name: "metadata_owner",
|
||||
Help: `Control whether owner should be read or written in metadata.
|
||||
|
||||
Owner is a standard part of the file metadata so is easy to read. But it
|
||||
isn't always desirable to set the owner from the metadata.
|
||||
|
||||
Note that you can't set the owner on Shared Drives, and that setting
|
||||
ownership will generate an email to the new owner (this can't be
|
||||
disabled), and you can't transfer ownership to someone outside your
|
||||
organization.
|
||||
`,
|
||||
Advanced: true,
|
||||
Default: rwRead,
|
||||
Examples: rwExamples,
|
||||
}, {
|
||||
Name: "metadata_permissions",
|
||||
Help: `Control whether permissions should be read or written in metadata.
|
||||
|
||||
Reading permissions metadata from files can be done quickly, but it
|
||||
isn't always desirable to set the permissions from the metadata.
|
||||
|
||||
Note that rclone drops any inherited permissions on Shared Drives and
|
||||
any owner permission on My Drives as these are duplicated in the owner
|
||||
metadata.
|
||||
`,
|
||||
Advanced: true,
|
||||
Default: rwOff,
|
||||
Examples: rwExamples,
|
||||
}, {
|
||||
Name: "metadata_labels",
|
||||
Help: `Control whether labels should be read or written in metadata.
|
||||
|
||||
Reading labels metadata from files takes an extra API transaction and
|
||||
will slow down listings. It isn't always desirable to set the labels
|
||||
from the metadata.
|
||||
|
||||
The format of labels is documented in the drive API documentation at
|
||||
https://developers.google.com/drive/api/reference/rest/v3/Label -
|
||||
rclone just provides a JSON dump of this format.
|
||||
|
||||
When setting labels, the label and fields must already exist - rclone
|
||||
will not create them. This means that if you are transferring labels
|
||||
from two different accounts you will have to create the labels in
|
||||
advance and use the metadata mapper to translate the IDs between the
|
||||
two accounts.
|
||||
`,
|
||||
Advanced: true,
|
||||
Default: rwOff,
|
||||
Examples: rwExamples,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@ -715,6 +804,9 @@ type Options struct {
|
||||
SkipDanglingShortcuts bool `config:"skip_dangling_shortcuts"`
|
||||
ResourceKey string `config:"resource_key"`
|
||||
FastListBugFix bool `config:"fast_list_bug_fix"`
|
||||
MetadataOwner rwChoice `config:"metadata_owner"`
|
||||
MetadataPermissions rwChoice `config:"metadata_permissions"`
|
||||
MetadataLabels rwChoice `config:"metadata_labels"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
EnvAuth bool `config:"env_auth"`
|
||||
}
|
||||
@ -742,6 +834,8 @@ type Fs struct {
|
||||
listRmu *sync.Mutex // protects listRempties
|
||||
listRempties map[string]struct{} // IDs of supposedly empty directories which triggered grouping disable
|
||||
dirResourceKeys *sync.Map // map directory ID to resource key
|
||||
permissionsMu *sync.Mutex // protect the below
|
||||
permissions map[string]*drive.Permission // map permission IDs to Permissions
|
||||
}
|
||||
|
||||
type baseObject struct {
|
||||
@ -753,6 +847,7 @@ type baseObject struct {
|
||||
bytes int64 // size of the object
|
||||
parents []string // IDs of the parent directories
|
||||
resourceKey *string // resourceKey is needed for link shared objects
|
||||
metadata *fs.Metadata // metadata if known
|
||||
}
|
||||
type documentObject struct {
|
||||
baseObject
|
||||
@ -1275,6 +1370,8 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
|
||||
listRmu: new(sync.Mutex),
|
||||
listRempties: make(map[string]struct{}),
|
||||
dirResourceKeys: new(sync.Map),
|
||||
permissionsMu: new(sync.Mutex),
|
||||
permissions: make(map[string]*drive.Permission),
|
||||
}
|
||||
f.isTeamDrive = opt.TeamDriveID != ""
|
||||
f.fileFields = f.getFileFields()
|
||||
@ -1285,6 +1382,9 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
|
||||
CanHaveEmptyDirectories: true,
|
||||
ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
|
||||
FilterAware: true,
|
||||
ReadMetadata: true,
|
||||
WriteMetadata: true,
|
||||
UserMetadata: true,
|
||||
}).Fill(ctx, f)
|
||||
|
||||
// Create a new authorized Drive client.
|
||||
@ -1389,7 +1489,7 @@ func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, e
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
|
||||
func (f *Fs) newBaseObject(ctx context.Context, remote string, info *drive.File) (o baseObject, err error) {
|
||||
modifiedDate := info.ModifiedTime
|
||||
if f.opt.UseCreatedDate {
|
||||
modifiedDate = info.CreatedTime
|
||||
@ -1400,7 +1500,7 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
|
||||
if f.opt.SizeAsQuota {
|
||||
size = info.QuotaBytesUsed
|
||||
}
|
||||
return baseObject{
|
||||
o = baseObject{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
id: info.Id,
|
||||
@ -1409,6 +1509,11 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
|
||||
bytes: size,
|
||||
parents: info.Parents,
|
||||
}
|
||||
err = nil
|
||||
if f.ci.Metadata {
|
||||
err = o.parseMetadata(ctx, info)
|
||||
}
|
||||
return o, err
|
||||
}
|
||||
|
||||
// getFileFields gets the fields for a normal file Get or List
|
||||
@ -1426,11 +1531,14 @@ func (f *Fs) getFileFields() (fields googleapi.Field) {
|
||||
if f.opt.SizeAsQuota {
|
||||
fields += ",quotaBytesUsed"
|
||||
}
|
||||
if f.ci.Metadata {
|
||||
fields += "," + metadataFields
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
// newRegularObject creates an fs.Object for a normal drive.File
|
||||
func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
|
||||
func (f *Fs) newRegularObject(ctx context.Context, remote string, info *drive.File) (obj fs.Object, err error) {
|
||||
// wipe checksum if SkipChecksumGphotos and file is type Photo or Video
|
||||
if f.opt.SkipChecksumGphotos {
|
||||
for _, space := range info.Spaces {
|
||||
@ -1443,27 +1551,33 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
|
||||
}
|
||||
}
|
||||
o := &Object{
|
||||
baseObject: f.newBaseObject(remote, info),
|
||||
url: fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)),
|
||||
md5sum: strings.ToLower(info.Md5Checksum),
|
||||
sha1sum: strings.ToLower(info.Sha1Checksum),
|
||||
sha256sum: strings.ToLower(info.Sha256Checksum),
|
||||
v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize),
|
||||
}
|
||||
o.baseObject, err = f.newBaseObject(ctx, remote, info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.ResourceKey != "" {
|
||||
o.resourceKey = &info.ResourceKey
|
||||
}
|
||||
return o
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// newDocumentObject creates an fs.Object for a google docs drive.File
|
||||
func (f *Fs) newDocumentObject(remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
|
||||
func (f *Fs) newDocumentObject(ctx context.Context, remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
|
||||
mediaType, _, err := mime.ParseMediaType(exportMimeType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
url := info.ExportLinks[mediaType]
|
||||
baseObject := f.newBaseObject(remote+extension, info)
|
||||
baseObject, err := f.newBaseObject(ctx, remote+extension, info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
baseObject.bytes = -1
|
||||
baseObject.mimeType = exportMimeType
|
||||
return &documentObject{
|
||||
@ -1475,7 +1589,7 @@ func (f *Fs) newDocumentObject(remote string, info *drive.File, extension, expor
|
||||
}
|
||||
|
||||
// newLinkObject creates an fs.Object that represents a link a google docs drive.File
|
||||
func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
|
||||
func (f *Fs) newLinkObject(ctx context.Context, remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
|
||||
t := linkTemplate(exportMimeType)
|
||||
if t == nil {
|
||||
return nil, fmt.Errorf("unsupported link type %s", exportMimeType)
|
||||
@ -1494,7 +1608,10 @@ func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMim
|
||||
return nil, fmt.Errorf("executing template failed: %w", err)
|
||||
}
|
||||
|
||||
baseObject := f.newBaseObject(remote+extension, info)
|
||||
baseObject, err := f.newBaseObject(ctx, remote+extension, info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
baseObject.bytes = int64(buf.Len())
|
||||
baseObject.mimeType = exportMimeType
|
||||
return &linkObject{
|
||||
@ -1510,7 +1627,7 @@ func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMim
|
||||
func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *drive.File) (fs.Object, error) {
|
||||
// If item has MD5 sum it is a file stored on drive
|
||||
if info.Md5Checksum != "" {
|
||||
return f.newRegularObject(remote, info), nil
|
||||
return f.newRegularObject(ctx, remote, info)
|
||||
}
|
||||
|
||||
extension, exportName, exportMimeType, isDocument := f.findExportFormat(ctx, info)
|
||||
@ -1541,15 +1658,15 @@ func (f *Fs) newObjectWithExportInfo(
|
||||
case info.MimeType == shortcutMimeTypeDangling:
|
||||
// Pretend a dangling shortcut is a regular object
|
||||
// It will error if used, but appear in listings so it can be deleted
|
||||
return f.newRegularObject(remote, info), nil
|
||||
return f.newRegularObject(ctx, remote, info)
|
||||
case info.Md5Checksum != "":
|
||||
// If item has MD5 sum it is a file stored on drive
|
||||
return f.newRegularObject(remote, info), nil
|
||||
return f.newRegularObject(ctx, remote, info)
|
||||
case f.opt.SkipGdocs:
|
||||
fs.Debugf(remote, "Skipping google document type %q", info.MimeType)
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
case f.opt.ShowAllGdocs:
|
||||
return f.newDocumentObject(remote, info, "", info.MimeType)
|
||||
return f.newDocumentObject(ctx, remote, info, "", info.MimeType)
|
||||
default:
|
||||
// If item MimeType is in the ExportFormats then it is a google doc
|
||||
if !isDocument {
|
||||
@ -1561,9 +1678,9 @@ func (f *Fs) newObjectWithExportInfo(
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
if isLinkMimeType(exportMimeType) {
|
||||
return f.newLinkObject(remote, info, extension, exportMimeType)
|
||||
return f.newLinkObject(ctx, remote, info, extension, exportMimeType)
|
||||
}
|
||||
return f.newDocumentObject(remote, info, extension, exportMimeType)
|
||||
return f.newDocumentObject(ctx, remote, info, extension, exportMimeType)
|
||||
}
|
||||
}
|
||||
|
||||
@ -2323,6 +2440,10 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
} else {
|
||||
createInfo.MimeType = fs.MimeTypeFromName(remote)
|
||||
}
|
||||
updateMetadata, err := f.fetchAndUpdateMetadata(ctx, src, options, createInfo)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var info *drive.File
|
||||
if size >= 0 && size < int64(f.opt.UploadCutoff) {
|
||||
@ -2347,6 +2468,10 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
err = updateMetadata(ctx, info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.newObjectWithInfo(ctx, remote, info)
|
||||
}
|
||||
|
||||
@ -3944,10 +4069,20 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
MimeType: srcMimeType,
|
||||
ModifiedTime: src.ModTime(ctx).Format(timeFormatOut),
|
||||
}
|
||||
|
||||
updateMetadata, err := o.fs.fetchAndUpdateMetadata(ctx, src, options, updateInfo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
info, err := o.baseObject.update(ctx, updateInfo, srcMimeType, in, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = updateMetadata(ctx, info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newO, err := o.fs.newObjectWithInfo(ctx, o.remote, info)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -4033,6 +4168,26 @@ func (o *baseObject) ParentID() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Metadata returns metadata for an object
|
||||
//
|
||||
// It should return nil if there is no Metadata
|
||||
func (o *baseObject) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
|
||||
if o.metadata != nil {
|
||||
return *o.metadata, nil
|
||||
}
|
||||
fs.Debugf(o, "Fetching metadata")
|
||||
id := actualID(o.id)
|
||||
info, err := o.fs.getFile(ctx, id, o.fs.fileFields)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = o.parseMetadata(ctx, info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return *o.metadata, nil
|
||||
}
|
||||
|
||||
func (o *documentObject) ext() string {
|
||||
return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
|
||||
}
|
||||
@ -4094,6 +4249,7 @@ var (
|
||||
_ fs.MimeTyper = (*Object)(nil)
|
||||
_ fs.IDer = (*Object)(nil)
|
||||
_ fs.ParentIDer = (*Object)(nil)
|
||||
_ fs.Metadataer = (*Object)(nil)
|
||||
_ fs.Object = (*documentObject)(nil)
|
||||
_ fs.MimeTyper = (*documentObject)(nil)
|
||||
_ fs.IDer = (*documentObject)(nil)
|
||||
|
600
backend/drive/metadata.go
Normal file
600
backend/drive/metadata.go
Normal file
@ -0,0 +1,600 @@
|
||||
package drive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"golang.org/x/sync/errgroup"
|
||||
drive "google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/googleapi"
|
||||
)
|
||||
|
||||
// system metadata keys which this backend owns
|
||||
var systemMetadataInfo = map[string]fs.MetadataHelp{
|
||||
"content-type": {
|
||||
Help: "The MIME type of the file.",
|
||||
Type: "string",
|
||||
Example: "text/plain",
|
||||
},
|
||||
"mtime": {
|
||||
Help: "Time of last modification with mS accuracy.",
|
||||
Type: "RFC 3339",
|
||||
Example: "2006-01-02T15:04:05.999Z07:00",
|
||||
},
|
||||
"btime": {
|
||||
Help: "Time of file birth (creation) with mS accuracy.",
|
||||
Type: "RFC 3339",
|
||||
Example: "2006-01-02T15:04:05.999Z07:00",
|
||||
},
|
||||
"copy-requires-writer-permission": {
|
||||
Help: "Whether the options to copy, print, or download this file, should be disabled for readers and commenters.",
|
||||
Type: "boolean",
|
||||
Example: "true",
|
||||
},
|
||||
"writers-can-share": {
|
||||
Help: "Whether users with only writer permission can modify the file's permissions. Not populated for items in shared drives.",
|
||||
Type: "boolean",
|
||||
Example: "false",
|
||||
},
|
||||
"viewed-by-me": {
|
||||
Help: "Whether the file has been viewed by this user.",
|
||||
Type: "boolean",
|
||||
Example: "true",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"owner": {
|
||||
Help: "The owner of the file. Usually an email address. Enable with --drive-metadata-owner.",
|
||||
Type: "string",
|
||||
Example: "user@example.com",
|
||||
},
|
||||
"permissions": {
|
||||
Help: "Permissions in a JSON dump of Google drive format. On shared drives these will only be present if they aren't inherited. Enable with --drive-metadata-permissions.",
|
||||
Type: "JSON",
|
||||
Example: "{}",
|
||||
},
|
||||
"folder-color-rgb": {
|
||||
Help: "The color for a folder or a shortcut to a folder as an RGB hex string.",
|
||||
Type: "string",
|
||||
Example: "881133",
|
||||
},
|
||||
"description": {
|
||||
Help: "A short description of the file.",
|
||||
Type: "string",
|
||||
Example: "Contract for signing",
|
||||
},
|
||||
"starred": {
|
||||
Help: "Whether the user has starred the file.",
|
||||
Type: "boolean",
|
||||
Example: "false",
|
||||
},
|
||||
"labels": {
|
||||
Help: "Labels attached to this file in a JSON dump of Googled drive format. Enable with --drive-metadata-labels.",
|
||||
Type: "JSON",
|
||||
Example: "[]",
|
||||
},
|
||||
}
|
||||
|
||||
// Extra fields we need to fetch to implement the system metadata above
|
||||
var metadataFields = googleapi.Field(strings.Join([]string{
|
||||
"copyRequiresWriterPermission",
|
||||
"description",
|
||||
"folderColorRgb",
|
||||
"hasAugmentedPermissions",
|
||||
"owners",
|
||||
"permissionIds",
|
||||
"permissions",
|
||||
"properties",
|
||||
"starred",
|
||||
"viewedByMe",
|
||||
"viewedByMeTime",
|
||||
"writersCanShare",
|
||||
}, ","))
|
||||
|
||||
// Fields we need to read from permissions
|
||||
var permissionsFields = googleapi.Field(strings.Join([]string{
|
||||
"*",
|
||||
"permissionDetails/*",
|
||||
}, ","))
|
||||
|
||||
// getPermission returns permissions for the fileID and permissionID passed in
|
||||
func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, useCache bool) (perm *drive.Permission, inherited bool, err error) {
|
||||
f.permissionsMu.Lock()
|
||||
defer f.permissionsMu.Unlock()
|
||||
if useCache {
|
||||
perm = f.permissions[permissionID]
|
||||
if perm != nil {
|
||||
return perm, false, nil
|
||||
}
|
||||
}
|
||||
fs.Debugf(f, "Fetching permission %q", permissionID)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
perm, err = f.svc.Permissions.Get(fileID, permissionID).
|
||||
Fields(permissionsFields).
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
inherited = len(perm.PermissionDetails) > 0 && perm.PermissionDetails[0].Inherited
|
||||
|
||||
cleanPermission(perm)
|
||||
|
||||
// cache the permission
|
||||
f.permissions[permissionID] = perm
|
||||
|
||||
return perm, inherited, err
|
||||
}
|
||||
|
||||
// Set the permissions on the info
|
||||
func (f *Fs) setPermissions(ctx context.Context, info *drive.File, permissions []*drive.Permission) (err error) {
|
||||
for _, perm := range permissions {
|
||||
if perm.Role == "owner" {
|
||||
// ignore owner permissions - these are set with owner
|
||||
continue
|
||||
}
|
||||
cleanPermissionForWrite(perm)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.svc.Permissions.Create(info.Id, perm).
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to set permission: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Clean attributes from permissions which we can't write
|
||||
func cleanPermissionForWrite(perm *drive.Permission) {
|
||||
perm.Deleted = false
|
||||
perm.DisplayName = ""
|
||||
perm.Id = ""
|
||||
perm.Kind = ""
|
||||
perm.PermissionDetails = nil
|
||||
perm.TeamDrivePermissionDetails = nil
|
||||
}
|
||||
|
||||
// Clean and cache the permission if not already cached
|
||||
func (f *Fs) cleanAndCachePermission(perm *drive.Permission) {
|
||||
f.permissionsMu.Lock()
|
||||
defer f.permissionsMu.Unlock()
|
||||
cleanPermission(perm)
|
||||
if _, found := f.permissions[perm.Id]; !found {
|
||||
f.permissions[perm.Id] = perm
|
||||
}
|
||||
}
|
||||
|
||||
// Clean fields we don't need to keep from the permission
|
||||
func cleanPermission(perm *drive.Permission) {
|
||||
// DisplayName: Output only. The "pretty" name of the value of the
|
||||
// permission. The following is a list of examples for each type of
|
||||
// permission: * `user` - User's full name, as defined for their Google
|
||||
// account, such as "Joe Smith." * `group` - Name of the Google Group,
|
||||
// such as "The Company Administrators." * `domain` - String domain
|
||||
// name, such as "thecompany.com." * `anyone` - No `displayName` is
|
||||
// present.
|
||||
perm.DisplayName = ""
|
||||
|
||||
// Kind: Output only. Identifies what kind of resource this is. Value:
|
||||
// the fixed string "drive#permission".
|
||||
perm.Kind = ""
|
||||
|
||||
// PermissionDetails: Output only. Details of whether the permissions on
|
||||
// this shared drive item are inherited or directly on this item. This
|
||||
// is an output-only field which is present only for shared drive items.
|
||||
perm.PermissionDetails = nil
|
||||
|
||||
// PhotoLink: Output only. A link to the user's profile photo, if
|
||||
// available.
|
||||
perm.PhotoLink = ""
|
||||
|
||||
// TeamDrivePermissionDetails: Output only. Deprecated: Output only. Use
|
||||
// `permissionDetails` instead.
|
||||
perm.TeamDrivePermissionDetails = nil
|
||||
}
|
||||
|
||||
// Fields we need to read from labels
|
||||
var labelsFields = googleapi.Field(strings.Join([]string{
|
||||
"*",
|
||||
}, ","))
|
||||
|
||||
// getLabels returns labels for the fileID passed in
|
||||
func (f *Fs) getLabels(ctx context.Context, fileID string) (labels []*drive.Label, err error) {
|
||||
fs.Debugf(f, "Fetching labels for %q", fileID)
|
||||
listLabels := f.svc.Files.ListLabels(fileID).
|
||||
Fields(labelsFields).
|
||||
Context(ctx)
|
||||
for {
|
||||
var info *drive.LabelList
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
info, err = listLabels.Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
labels = append(labels, info.Labels...)
|
||||
if info.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
listLabels.PageToken(info.NextPageToken)
|
||||
}
|
||||
for _, label := range labels {
|
||||
cleanLabel(label)
|
||||
}
|
||||
return labels, nil
|
||||
}
|
||||
|
||||
// Set the labels on the info
|
||||
func (f *Fs) setLabels(ctx context.Context, info *drive.File, labels []*drive.Label) (err error) {
|
||||
if len(labels) == 0 {
|
||||
return nil
|
||||
}
|
||||
req := drive.ModifyLabelsRequest{}
|
||||
for _, label := range labels {
|
||||
req.LabelModifications = append(req.LabelModifications, &drive.LabelModification{
|
||||
FieldModifications: labelFieldsToFieldModifications(label.Fields),
|
||||
LabelId: label.Id,
|
||||
})
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.svc.Files.ModifyLabels(info.Id, &req).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to set owner: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Convert label fields into something which can set the fields
|
||||
func labelFieldsToFieldModifications(fields map[string]drive.LabelField) (out []*drive.LabelFieldModification) {
|
||||
for id, field := range fields {
|
||||
var emails []string
|
||||
for _, user := range field.User {
|
||||
emails = append(emails, user.EmailAddress)
|
||||
}
|
||||
out = append(out, &drive.LabelFieldModification{
|
||||
// FieldId: The ID of the field to be modified.
|
||||
FieldId: id,
|
||||
|
||||
// SetDateValues: Replaces the value of a dateString Field with these
|
||||
// new values. The string must be in the RFC 3339 full-date format:
|
||||
// YYYY-MM-DD.
|
||||
SetDateValues: field.DateString,
|
||||
|
||||
// SetIntegerValues: Replaces the value of an `integer` field with these
|
||||
// new values.
|
||||
SetIntegerValues: field.Integer,
|
||||
|
||||
// SetSelectionValues: Replaces a `selection` field with these new
|
||||
// values.
|
||||
SetSelectionValues: field.Selection,
|
||||
|
||||
// SetTextValues: Sets the value of a `text` field.
|
||||
SetTextValues: field.Text,
|
||||
|
||||
// SetUserValues: Replaces a `user` field with these new values. The
|
||||
// values must be valid email addresses.
|
||||
SetUserValues: emails,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Clean fields we don't need to keep from the label
|
||||
func cleanLabel(label *drive.Label) {
|
||||
// Kind: This is always drive#label
|
||||
label.Kind = ""
|
||||
|
||||
for name, field := range label.Fields {
|
||||
// Kind: This is always drive#labelField.
|
||||
field.Kind = ""
|
||||
|
||||
// Note the fields are copies so we need to write them
|
||||
// back to the map
|
||||
label.Fields[name] = field
|
||||
}
|
||||
}
|
||||
|
||||
// Parse the metadata from drive item
|
||||
//
|
||||
// It should return nil if there is no Metadata
|
||||
func (o *baseObject) parseMetadata(ctx context.Context, info *drive.File) (err error) {
|
||||
metadata := make(fs.Metadata, 16)
|
||||
|
||||
// Dump user metadata first as it overrides system metadata
|
||||
for k, v := range info.Properties {
|
||||
metadata[k] = v
|
||||
}
|
||||
|
||||
// System metadata
|
||||
metadata["copy-requires-writer-permission"] = fmt.Sprint(info.CopyRequiresWriterPermission)
|
||||
metadata["writers-can-share"] = fmt.Sprint(info.WritersCanShare)
|
||||
metadata["viewed-by-me"] = fmt.Sprint(info.ViewedByMe)
|
||||
metadata["content-type"] = info.MimeType
|
||||
|
||||
// Owners: Output only. The owner of this file. Only certain legacy
|
||||
// files may have more than one owner. This field isn't populated for
|
||||
// items in shared drives.
|
||||
if o.fs.opt.MetadataOwner.IsSet(rwRead) && len(info.Owners) > 0 {
|
||||
user := info.Owners[0]
|
||||
if len(info.Owners) > 1 {
|
||||
fs.Logf(o, "Ignoring more than 1 owner")
|
||||
}
|
||||
if user != nil {
|
||||
id := user.EmailAddress
|
||||
if id == "" {
|
||||
id = user.DisplayName
|
||||
}
|
||||
metadata["owner"] = id
|
||||
}
|
||||
}
|
||||
|
||||
if o.fs.opt.MetadataPermissions.IsSet(rwRead) {
|
||||
// We only write permissions out if they are not inherited.
|
||||
//
|
||||
// On My Drives permissions seem to be attached to every item
|
||||
// so they will always be written out.
|
||||
//
|
||||
// On Shared Drives only non-inherited permissions will be
|
||||
// written out.
|
||||
|
||||
// To read the inherited permissions flag will mean we need to
|
||||
// read the permissions for each object and the cache will be
|
||||
// useless. However shared drives don't return permissions
|
||||
// only permissionIds so will need to fetch them for each
|
||||
// object. We use HasAugmentedPermissions to see if there are
|
||||
// special permissions before fetching them to save transactions.
|
||||
|
||||
// HasAugmentedPermissions: Output only. Whether there are permissions
|
||||
// directly on this file. This field is only populated for items in
|
||||
// shared drives.
|
||||
if o.fs.isTeamDrive && !info.HasAugmentedPermissions {
|
||||
// Don't process permissions if there aren't any specifically set
|
||||
info.Permissions = nil
|
||||
info.PermissionIds = nil
|
||||
}
|
||||
|
||||
// PermissionIds: Output only. List of permission IDs for users with
|
||||
// access to this file.
|
||||
//
|
||||
// Only process these if we have no Permissions
|
||||
if len(info.PermissionIds) > 0 && len(info.Permissions) == 0 {
|
||||
info.Permissions = make([]*drive.Permission, 0, len(info.PermissionIds))
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(o.fs.ci.Checkers)
|
||||
var mu sync.Mutex // protect the info.Permissions from concurrent writes
|
||||
for _, permissionID := range info.PermissionIds {
|
||||
permissionID := permissionID
|
||||
g.Go(func() error {
|
||||
// must fetch the team drive ones individually to check the inherited flag
|
||||
perm, inherited, err := o.fs.getPermission(gCtx, actualID(info.Id), permissionID, !o.fs.isTeamDrive)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read permission: %w", err)
|
||||
}
|
||||
// Don't write inherited permissions out
|
||||
if inherited {
|
||||
return nil
|
||||
}
|
||||
// Don't write owner role out - these are covered by the owner metadata
|
||||
if perm.Role == "owner" {
|
||||
return nil
|
||||
}
|
||||
mu.Lock()
|
||||
info.Permissions = append(info.Permissions, perm)
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
err = g.Wait()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// Clean the fetched permissions
|
||||
for _, perm := range info.Permissions {
|
||||
o.fs.cleanAndCachePermission(perm)
|
||||
}
|
||||
}
|
||||
|
||||
// Permissions: Output only. The full list of permissions for the file.
|
||||
// This is only available if the requesting user can share the file. Not
|
||||
// populated for items in shared drives.
|
||||
if len(info.Permissions) > 0 {
|
||||
buf, err := json.Marshal(info.Permissions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal permissions: %w", err)
|
||||
}
|
||||
metadata["permissions"] = string(buf)
|
||||
}
|
||||
|
||||
// Permission propagation
|
||||
// https://developers.google.com/drive/api/guides/manage-sharing#permission-propagation
|
||||
// Leads me to believe that in non shared drives, permissions
|
||||
// are added to each item when you set permissions for a
|
||||
// folder whereas in shared drives they are inherited and
|
||||
// placed on the item directly.
|
||||
}
|
||||
|
||||
if info.FolderColorRgb != "" {
|
||||
metadata["folder-color-rgb"] = info.FolderColorRgb
|
||||
}
|
||||
if info.Description != "" {
|
||||
metadata["description"] = info.Description
|
||||
}
|
||||
metadata["starred"] = fmt.Sprint(info.Starred)
|
||||
metadata["btime"] = info.CreatedTime
|
||||
metadata["mtime"] = info.ModifiedTime
|
||||
|
||||
if o.fs.opt.MetadataLabels.IsSet(rwRead) {
|
||||
// FIXME would be really nice if we knew if files had labels
|
||||
// before listing but we need to know all possible label IDs
|
||||
// to get it in the listing.
|
||||
|
||||
labels, err := o.fs.getLabels(ctx, actualID(info.Id))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to fetch labels: %w", err)
|
||||
}
|
||||
buf, err := json.Marshal(labels)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal labels: %w", err)
|
||||
}
|
||||
metadata["labels"] = string(buf)
|
||||
}
|
||||
|
||||
o.metadata = &metadata
|
||||
return nil
|
||||
}
|
||||
|
||||
// Set the owner on the info
|
||||
func (f *Fs) setOwner(ctx context.Context, info *drive.File, owner string) (err error) {
|
||||
perm := drive.Permission{
|
||||
Role: "owner",
|
||||
EmailAddress: owner,
|
||||
// Type: The type of the grantee. Valid values are: * `user` * `group` *
|
||||
// `domain` * `anyone` When creating a permission, if `type` is `user`
|
||||
// or `group`, you must provide an `emailAddress` for the user or group.
|
||||
// When `type` is `domain`, you must provide a `domain`. There isn't
|
||||
// extra information required for an `anyone` type.
|
||||
Type: "user",
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.svc.Permissions.Create(info.Id, &perm).
|
||||
SupportsAllDrives(true).
|
||||
TransferOwnership(true).
|
||||
// SendNotificationEmail(false). - required apparently!
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to set owner: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Call back to set metadata that can't be set on the upload/update
|
||||
//
|
||||
// The *drive.File passed in holds the current state of the drive.File
|
||||
// and this should update it with any modifications.
|
||||
type updateMetadataFn func(context.Context, *drive.File) error
|
||||
|
||||
// read the metadata from meta and write it into updateInfo
|
||||
//
|
||||
// It returns a callback which should be called to finish the updates
|
||||
// after the data is uploaded.
|
||||
func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs.Metadata) (callback updateMetadataFn, err error) {
|
||||
callbackFns := []updateMetadataFn{}
|
||||
callback = func(ctx context.Context, info *drive.File) error {
|
||||
for _, fn := range callbackFns {
|
||||
err := fn(ctx, info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// merge metadata into request and user metadata
|
||||
for k, v := range meta {
|
||||
k, v := k, v
|
||||
// parse a boolean from v and write into out
|
||||
parseBool := func(out *bool) error {
|
||||
b, err := strconv.ParseBool(v)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't parse metadata %q = %q: %w", k, v, err)
|
||||
}
|
||||
*out = b
|
||||
return nil
|
||||
}
|
||||
switch k {
|
||||
case "copy-requires-writer-permission":
|
||||
if err := parseBool(&updateInfo.CopyRequiresWriterPermission); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case "writers-can-share":
|
||||
if err := parseBool(&updateInfo.WritersCanShare); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case "viewed-by-me":
|
||||
// Can't write this
|
||||
case "content-type":
|
||||
updateInfo.MimeType = v
|
||||
case "owner":
|
||||
if !f.opt.MetadataOwner.IsSet(rwWrite) {
|
||||
continue
|
||||
}
|
||||
// Can't set Owner on upload so need to set afterwards
|
||||
callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
|
||||
return f.setOwner(ctx, info, v)
|
||||
})
|
||||
case "permissions":
|
||||
if !f.opt.MetadataPermissions.IsSet(rwWrite) {
|
||||
continue
|
||||
}
|
||||
var perms []*drive.Permission
|
||||
err := json.Unmarshal([]byte(v), &perms)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal permissions: %w", err)
|
||||
}
|
||||
// Can't set Permissions on upload so need to set afterwards
|
||||
callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
|
||||
return f.setPermissions(ctx, info, perms)
|
||||
})
|
||||
case "labels":
|
||||
if !f.opt.MetadataLabels.IsSet(rwWrite) {
|
||||
continue
|
||||
}
|
||||
var labels []*drive.Label
|
||||
err := json.Unmarshal([]byte(v), &labels)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal labels: %w", err)
|
||||
}
|
||||
// Can't set Labels on upload so need to set afterwards
|
||||
callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
|
||||
return f.setLabels(ctx, info, labels)
|
||||
})
|
||||
case "folder-color-rgb":
|
||||
updateInfo.FolderColorRgb = v
|
||||
case "description":
|
||||
updateInfo.Description = v
|
||||
case "starred":
|
||||
if err := parseBool(&updateInfo.Starred); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case "btime":
|
||||
updateInfo.CreatedTime = v
|
||||
case "mtime":
|
||||
updateInfo.ModifiedTime = v
|
||||
default:
|
||||
if updateInfo.Properties == nil {
|
||||
updateInfo.Properties = make(map[string]string, 1)
|
||||
}
|
||||
updateInfo.Properties[k] = v
|
||||
}
|
||||
}
|
||||
return callback, nil
|
||||
}
|
||||
|
||||
// Fetch metadata and update updateInfo if --metadata is in use
|
||||
func (f *Fs) fetchAndUpdateMetadata(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption, updateInfo *drive.File) (callback updateMetadataFn, err error) {
|
||||
meta, err := fs.GetMetadataOptions(ctx, src, options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read metadata from source object: %w", err)
|
||||
}
|
||||
callback, err = f.updateMetadata(ctx, updateInfo, meta)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to update metadata from source object: %w", err)
|
||||
}
|
||||
return callback, nil
|
||||
}
|
Loading…
Reference in New Issue
Block a user