diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index d9b0cd845..9b86da75c 100755 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -142,6 +142,31 @@ memory. It can be set smaller if you are tight on memory.`, maxChunkSize), Help: "Impersonate this user when using a business account.", Default: "", Advanced: true, + }, { + Name: "shared_files", + Help: `Instructs rclone to work on individual shared files. + +In this mode rclone's features are extremely limited - only list (ls, lsl, etc.) +operations and read operations (e.g. downloading) are supported in this mode. +All other operations will be disabled.`, + Default: false, + Advanced: true, + }, { + Name: "shared_folders", + Help: `Instructs rclone to work on shared folders. + +When this flag is used with no path only the List operation is supported and +all available shared folders will be listed. If you specify a path the first part +will be interpreted as the name of shared folder. Rclone will then try to mount this +shared to the root namespace. On success shared folder rclone proceeds normally. +The shared folder is now pretty much a normal folder and all normal operations +are supported. + +Note that we don't unmount the shared folder afterwards so the +--dropbox-shared-folders can be omitted after the first use of a particular +shared folder.`, + Default: false, + Advanced: true, }, { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, @@ -161,9 +186,11 @@ memory. It can be set smaller if you are tight on memory.`, maxChunkSize), // Options defines the configuration for this backend type Options struct { - ChunkSize fs.SizeSuffix `config:"chunk_size"` - Impersonate string `config:"impersonate"` - Enc encoder.MultiEncoder `config:"encoding"` + ChunkSize fs.SizeSuffix `config:"chunk_size"` + Impersonate string `config:"impersonate"` + SharedFiles bool `config:"shared_files"` + SharedFolders bool `config:"shared_folders"` + Enc encoder.MultiEncoder `config:"encoding"` } // Fs represents a remote dropbox server @@ -186,7 +213,9 @@ type Fs struct { // // Dropbox Objects always have full metadata type Object struct { - fs *Fs // what this object is part of + fs *Fs // what this object is part of + id string + url string remote string // The remote path bytes int64 // size of the object modTime time.Time // time it was last modified @@ -332,8 +361,60 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { CaseInsensitive: true, ReadMimeType: true, CanHaveEmptyDirectories: true, - }).Fill(f) - f.setRoot(root) + }) + + // do not fill features yet + if f.opt.SharedFiles { + f.setRoot(root) + if f.root == "" { + return f, nil + } + _, err := f.findSharedFile(f.root) + f.root = "" + if err == nil { + return f, fs.ErrorIsFile + } + return f, nil + } + + if f.opt.SharedFolders { + f.setRoot(root) + if f.root == "" { + return f, nil // our root it empty so we probably want to list shared folders + } + + dir := path.Dir(f.root) + if dir == "." { + dir = f.root + } + + // root is not empty so we have find the right shared folder if it exists + id, err := f.findSharedFolder(dir) + if err != nil { + // if we didn't find the specified shared folder we have to bail out here + return nil, err + } + // we found the specified shared folder so let's mount it + // this will add it to the users normal root namespace and allows us + // to actually perform operations on it using the normal api endpoints. + err = f.mountSharedFolder(id) + if err != nil { + switch e := err.(type) { + case sharing.MountFolderAPIError: + if e.EndpointError == nil || (e.EndpointError != nil && e.EndpointError.Tag != sharing.MountFolderErrorAlreadyMounted) { + return nil, err + } + default: + return nil, err + } + // if the moint failed we have to abort here + } + // if the mount succeeded it's now a normal folder in the users root namespace + // we disable shared folder mode and proceed normally + f.opt.SharedFolders = false + } + + f.features.Fill(f) // If root starts with / then use the actual root if strings.HasPrefix(root, "/") { @@ -355,6 +436,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { } fs.Debugf(f, "Using root namespace %q", f.ns) } + f.setRoot(root) // See if the root is actually an object _, err = f.getFileMetadata(f.slashRoot) @@ -465,9 +547,158 @@ func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Obje // NewObject finds the Object at remote. If it can't be found // it returns the error fs.ErrorObjectNotFound. func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + if f.opt.SharedFiles { + return f.findSharedFile(remote) + } return f.newObjectWithInfo(remote, nil) } +// listSharedFoldersApi lists all available shared folders mounted and not mounted +// we'll need the id later so we have to return them in original format +func (f *Fs) listSharedFoldersAPI() (result []*sharing.SharedFolderMetadata, err error) { + started := false + var res *sharing.ListFoldersResult + for { + if !started { + arg := sharing.ListFoldersArgs{ + Limit: 100, + } + err := f.pacer.Call(func() (bool, error) { + res, err = f.sharing.ListFolders(&arg) + return shouldRetry(err) + }) + if err != nil { + return nil, err + } + started = true + } else { + arg := sharing.ListFoldersContinueArg{ + Cursor: res.Cursor, + } + err := f.pacer.Call(func() (bool, error) { + res, err = f.sharing.ListFoldersContinue(&arg) + return shouldRetry(err) + }) + if err != nil { + return nil, errors.Wrap(err, "list continue") + } + } + result = append(result, res.Entries...) + if res.Cursor == "" { + break + } + } + + return result, nil +} + +// listSharedFolders lists shared folders as normal dir entries +func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) { + result, err := f.listSharedFoldersAPI() + if err != nil { + return nil, err + } + for _, entry := range result { + entryPath := entry.Name + leaf := f.opt.Enc.ToStandardName(entryPath) + d := fs.NewDir(leaf, time.Now()) + entries = append(entries, d) + } + return entries, nil +} + +// findSharedFolder find the id for a given shared folder name +// somewhat annoyingly there is no endpoint to query a shared folder by it's name +// so our only option is to iterate over all shared folders +func (f *Fs) findSharedFolder(name string) (id string, err error) { + entries, err := f.listSharedFoldersAPI() + if err != nil { + return "", err + } + for _, entry := range entries { + if entry.Name == name { + return entry.SharedFolderId, nil + } + } + return "", fs.ErrorDirNotFound +} + +// mountSharedFolders mount a shared folder to the root namespace +func (f *Fs) mountSharedFolder(id string) error { + arg := sharing.MountFolderArg{ + SharedFolderId: id, + } + err := f.pacer.Call(func() (bool, error) { + _, err := f.sharing.MountFolder(&arg) + return shouldRetry(err) + }) + return err +} + +// listSharedFolders lists shared the user as access to (note this means individual +// files not files contained in shared folders) +func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) { + started := false + var res *sharing.ListFilesResult + for { + if !started { + arg := sharing.ListFilesArg{ + Limit: 100, + } + err := f.pacer.Call(func() (bool, error) { + res, err = f.sharing.ListReceivedFiles(&arg) + return shouldRetry(err) + }) + if err != nil { + return nil, err + } + started = true + } else { + arg := sharing.ListFilesContinueArg{ + Cursor: res.Cursor, + } + err := f.pacer.Call(func() (bool, error) { + res, err = f.sharing.ListReceivedFilesContinue(&arg) + return shouldRetry(err) + }) + if err != nil { + return nil, errors.Wrap(err, "list continue") + } + } + for _, entry := range res.Entries { + fmt.Printf("%+v\n", entry) + entryPath := entry.Name + o := &Object{ + fs: f, + url: entry.PreviewUrl, + remote: entryPath, + modTime: entry.TimeInvited, + } + if err != nil { + return nil, err + } + entries = append(entries, o) + } + if res.Cursor == "" { + break + } + } + return entries, nil +} + +func (f *Fs) findSharedFile(name string) (o *Object, err error) { + files, err := f.listReceivedFiles() + if err != nil { + return nil, err + } + for _, entry := range files { + if entry.(*Object).remote == name { + return entry.(*Object), nil + } + } + return nil, fs.ErrorObjectNotFound +} + // List the objects and directories in dir into entries. The // entries can be returned in any order but should be for a // complete directory. @@ -478,6 +709,13 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { // This should return ErrDirNotFound if the directory isn't // found. func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + if f.opt.SharedFiles { + return f.listReceivedFiles() + } + if f.opt.SharedFolders { + return f.listSharedFolders() + } + root := f.slashRoot if dir != "" { root += "/" + dir @@ -564,6 +802,9 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + if f.opt.SharedFiles || f.opt.SharedFolders { + return nil, fserrors.NoRetryError(errors.New("not support in shared files mode")) + } // Temporary Object under construction o := &Object{ fs: f, @@ -579,6 +820,9 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt // Mkdir creates the container if it doesn't exist func (f *Fs) Mkdir(ctx context.Context, dir string) error { + if f.opt.SharedFiles || f.opt.SharedFolders { + return fserrors.NoRetryError(errors.New("not support in shared files mode")) + } root := path.Join(f.slashRoot, dir) // can't create or run metadata on root @@ -656,6 +900,9 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error) // // Returns an error if it isn't empty func (f *Fs) Rmdir(ctx context.Context, dir string) error { + if f.opt.SharedFiles || f.opt.SharedFolders { + return fserrors.NoRetryError(errors.New("not support in shared files mode")) + } return f.purgeCheck(ctx, dir, true) } @@ -929,6 +1176,9 @@ func (o *Object) Remote() string { // Hash returns the dropbox special hash func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { + if o.fs.opt.SharedFiles || o.fs.opt.SharedFolders { + return "", fserrors.NoRetryError(errors.New("not support in shared files mode")) + } if t != DbHashType { return "", hash.ErrUnsupported } @@ -948,6 +1198,7 @@ func (o *Object) Size() int64 { // // This isn't a complete set of metadata and has an inacurate date func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error { + o.id = info.Id o.bytes = int64(info.Size) o.modTime = info.ClientModified o.hash = info.ContentHash @@ -1016,10 +1267,27 @@ func (o *Object) Storable() bool { // Open an object for read func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { + if o.fs.opt.SharedFiles { + if len(options) != 0 { + return nil, errors.New("OpenOptions not supported for shared files") + } + arg := sharing.GetSharedLinkMetadataArg{ + Url: o.url, + } + err = o.fs.pacer.Call(func() (bool, error) { + _, in, err = o.fs.sharing.GetSharedLinkFile(&arg) + return shouldRetry(err) + }) + if err != nil { + return nil, err + } + return + } + fs.FixRangeOption(options, o.bytes) headers := fs.OpenOptionHeaders(options) arg := files.DownloadArg{ - Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()), + Path: o.id, ExtraHeaders: headers, } err = o.fs.pacer.Call(func() (bool, error) { @@ -1153,6 +1421,9 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + if o.fs.opt.SharedFiles || o.fs.opt.SharedFolders { + return fserrors.NoRetryError(errors.New("not support in shared files mode")) + } remote := o.remotePath() if ignoredFiles.MatchString(remote) { return fserrors.NoRetryError(errors.Errorf("file name %q is disallowed - not uploading", path.Base(remote))) @@ -1181,6 +1452,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // Remove an object func (o *Object) Remove(ctx context.Context) (err error) { + if o.fs.opt.SharedFiles || o.fs.opt.SharedFolders { + return fserrors.NoRetryError(errors.New("not support in shared files mode")) + } err = o.fs.pacer.Call(func() (bool, error) { _, err = o.fs.srv.DeleteV2(&files.DeleteArg{ Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),