mirror of
https://github.com/rclone/rclone.git
synced 2024-11-08 09:35:26 +01:00
20da3e6352
* Add options to Put, PutUnchecked and Update for all Fses * Use these to create HashOption * Implement this in local * Pass the option in fs.Copy This has the effect that we only calculate hashes we need to in the local Fs which speeds up transfers significantly.
878 lines
22 KiB
Go
878 lines
22 KiB
Go
// Package dropbox provides an interface to Dropbox object storage
|
|
package dropbox
|
|
|
|
// FIXME put low level retries in
|
|
// FIXME dropbox for business would be quite easy to add
|
|
|
|
/*
|
|
FIXME is case folding of PathDisplay going to cause a problem?
|
|
|
|
From the docs
|
|
|
|
path_display String. The cased path to be used for display purposes
|
|
only. In rare instances the casing will not correctly match the user's
|
|
filesystem, but this behavior will match the path provided in the Core
|
|
API v1, and at least the last path component will have the correct
|
|
casing. Changes to only the casing of paths won't be returned by
|
|
list_folder/continue. This field will be null if the file or folder is
|
|
not mounted. This field is optional.
|
|
*/
|
|
|
|
import (
|
|
"crypto/md5"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"path"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"golang.org/x/oauth2"
|
|
|
|
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox"
|
|
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/files"
|
|
"github.com/ncw/rclone/fs"
|
|
"github.com/ncw/rclone/oauthutil"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
// Constants
|
|
const (
|
|
rcloneClientID = "5jcck7diasz0rqy"
|
|
rcloneEncryptedClientSecret = "fRS5vVLr2v6FbyXYnIgjwBuUAt0osq_QZTXAEcmZ7g"
|
|
)
|
|
|
|
var (
|
|
// Description of how to auth for this app
|
|
dropboxConfig = &oauth2.Config{
|
|
Scopes: []string{},
|
|
Endpoint: oauth2.Endpoint{
|
|
AuthURL: "https://www.dropbox.com/1/oauth2/authorize",
|
|
TokenURL: "https://api.dropboxapi.com/1/oauth2/token",
|
|
}, // FIXME replace with this once vendored dropbox.OAuthEndpoint(""),
|
|
ClientID: rcloneClientID,
|
|
ClientSecret: fs.MustReveal(rcloneEncryptedClientSecret),
|
|
RedirectURL: oauthutil.RedirectLocalhostURL,
|
|
}
|
|
// A regexp matching path names for files Dropbox ignores
|
|
// See https://www.dropbox.com/en/help/145 - Ignored files
|
|
ignoredFiles = regexp.MustCompile(`(?i)(^|/)(desktop\.ini|thumbs\.db|\.ds_store|icon\r|\.dropbox|\.dropbox.attr)$`)
|
|
// Upload chunk size - setting too small makes uploads slow.
|
|
// Chunks aren't buffered into memory though so can set large.
|
|
uploadChunkSize = fs.SizeSuffix(128 * 1024 * 1024)
|
|
maxUploadChunkSize = fs.SizeSuffix(150 * 1024 * 1024)
|
|
)
|
|
|
|
// Register with Fs
|
|
func init() {
|
|
fs.Register(&fs.RegInfo{
|
|
Name: "dropbox",
|
|
Description: "Dropbox",
|
|
NewFs: NewFs,
|
|
Config: func(name string) {
|
|
err := oauthutil.Config("dropbox", name, dropboxConfig)
|
|
if err != nil {
|
|
log.Fatalf("Failed to configure token: %v", err)
|
|
}
|
|
},
|
|
Options: []fs.Option{{
|
|
Name: "app_key",
|
|
Help: "Dropbox App Key - leave blank normally.",
|
|
}, {
|
|
Name: "app_secret",
|
|
Help: "Dropbox App Secret - leave blank normally.",
|
|
}},
|
|
})
|
|
fs.VarP(&uploadChunkSize, "dropbox-chunk-size", "", fmt.Sprintf("Upload chunk size. Max %v.", maxUploadChunkSize))
|
|
}
|
|
|
|
// Fs represents a remote dropbox server
|
|
type Fs struct {
|
|
name string // name of this remote
|
|
root string // the path we are working on
|
|
features *fs.Features // optional features
|
|
srv files.Client // the connection to the dropbox server
|
|
slashRoot string // root with "/" prefix, lowercase
|
|
slashRootSlash string // root with "/" prefix and postfix, lowercase
|
|
}
|
|
|
|
// Object describes a dropbox object
|
|
//
|
|
// Dropbox Objects always have full metadata
|
|
type Object struct {
|
|
fs *Fs // what this object is part of
|
|
remote string // The remote path
|
|
bytes int64 // size of the object
|
|
modTime time.Time // time it was last modified
|
|
hash string // content_hash of the object
|
|
}
|
|
|
|
// ------------------------------------------------------------
|
|
|
|
// Name of the remote (as passed into NewFs)
|
|
func (f *Fs) Name() string {
|
|
return f.name
|
|
}
|
|
|
|
// Root of the remote (as passed into NewFs)
|
|
func (f *Fs) Root() string {
|
|
return f.root
|
|
}
|
|
|
|
// String converts this Fs to a string
|
|
func (f *Fs) String() string {
|
|
return fmt.Sprintf("Dropbox root '%s'", f.root)
|
|
}
|
|
|
|
// Features returns the optional features of this Fs
|
|
func (f *Fs) Features() *fs.Features {
|
|
return f.features
|
|
}
|
|
|
|
// NewFs contstructs an Fs from the path, container:path
|
|
func NewFs(name, root string) (fs.Fs, error) {
|
|
if uploadChunkSize > maxUploadChunkSize {
|
|
return nil, errors.Errorf("chunk size too big, must be < %v", maxUploadChunkSize)
|
|
}
|
|
|
|
// Convert the old token if it exists. The old token was just
|
|
// just a string, the new one is a JSON blob
|
|
oldToken := strings.TrimSpace(fs.ConfigFileGet(name, fs.ConfigToken))
|
|
if oldToken != "" && oldToken[0] != '{' {
|
|
fs.Infof(name, "Converting token to new format")
|
|
newToken := fmt.Sprintf(`{"access_token":"%s","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}`, oldToken)
|
|
err := fs.ConfigSetValueAndSave(name, fs.ConfigToken, newToken)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "NewFS convert token")
|
|
}
|
|
}
|
|
|
|
oAuthClient, _, err := oauthutil.NewClient(name, dropboxConfig)
|
|
if err != nil {
|
|
log.Fatalf("Failed to configure dropbox: %v", err)
|
|
}
|
|
|
|
config := dropbox.Config{
|
|
Verbose: false, // enables verbose logging in the SDK
|
|
Client: oAuthClient, // maybe???
|
|
}
|
|
srv := files.New(config)
|
|
|
|
f := &Fs{
|
|
name: name,
|
|
srv: srv,
|
|
}
|
|
f.features = (&fs.Features{CaseInsensitive: true, ReadMimeType: true}).Fill(f)
|
|
f.setRoot(root)
|
|
|
|
// See if the root is actually an object
|
|
_, err = f.getFileMetadata(f.slashRoot)
|
|
if err == nil {
|
|
newRoot := path.Dir(f.root)
|
|
if newRoot == "." {
|
|
newRoot = ""
|
|
}
|
|
f.setRoot(newRoot)
|
|
// return an error with an fs which points to the parent
|
|
return f, fs.ErrorIsFile
|
|
}
|
|
return f, nil
|
|
}
|
|
|
|
// Sets root in f
|
|
func (f *Fs) setRoot(root string) {
|
|
f.root = strings.Trim(root, "/")
|
|
lowerCaseRoot := strings.ToLower(f.root)
|
|
|
|
f.slashRoot = "/" + lowerCaseRoot
|
|
f.slashRootSlash = f.slashRoot
|
|
if lowerCaseRoot != "" {
|
|
f.slashRootSlash += "/"
|
|
}
|
|
}
|
|
|
|
// getMetadata gets the metadata for a file or directory
|
|
func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool, err error) {
|
|
entry, err = f.srv.GetMetadata(&files.GetMetadataArg{Path: objPath})
|
|
if err != nil {
|
|
switch e := err.(type) {
|
|
case files.GetMetadataAPIError:
|
|
switch e.EndpointError.Path.Tag {
|
|
case files.LookupErrorNotFound:
|
|
notFound = true
|
|
err = nil
|
|
}
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
// getFileMetadata gets the metadata for a file
|
|
func (f *Fs) getFileMetadata(filePath string) (fileInfo *files.FileMetadata, err error) {
|
|
entry, notFound, err := f.getMetadata(filePath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if notFound {
|
|
return nil, fs.ErrorObjectNotFound
|
|
}
|
|
fileInfo, ok := entry.(*files.FileMetadata)
|
|
if !ok {
|
|
return nil, fs.ErrorNotAFile
|
|
}
|
|
return fileInfo, nil
|
|
}
|
|
|
|
// getDirMetadata gets the metadata for a directory
|
|
func (f *Fs) getDirMetadata(dirPath string) (dirInfo *files.FolderMetadata, err error) {
|
|
entry, notFound, err := f.getMetadata(dirPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if notFound {
|
|
return nil, fs.ErrorDirNotFound
|
|
}
|
|
dirInfo, ok := entry.(*files.FolderMetadata)
|
|
if !ok {
|
|
return nil, fs.ErrorIsFile
|
|
}
|
|
return dirInfo, nil
|
|
}
|
|
|
|
// Return an Object from a path
|
|
//
|
|
// If it can't be found it returns the error fs.ErrorObjectNotFound.
|
|
func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Object, error) {
|
|
o := &Object{
|
|
fs: f,
|
|
remote: remote,
|
|
}
|
|
var err error
|
|
if info != nil {
|
|
err = o.setMetadataFromEntry(info)
|
|
} else {
|
|
err = o.readEntryAndSetMetadata()
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return o, nil
|
|
}
|
|
|
|
// NewObject finds the Object at remote. If it can't be found
|
|
// it returns the error fs.ErrorObjectNotFound.
|
|
func (f *Fs) NewObject(remote string) (fs.Object, error) {
|
|
return f.newObjectWithInfo(remote, nil)
|
|
}
|
|
|
|
// Strips the root off path and returns it
|
|
func strip(path, root string) (string, error) {
|
|
if len(root) > 0 {
|
|
if root[0] != '/' {
|
|
root = "/" + root
|
|
}
|
|
if root[len(root)-1] != '/' {
|
|
root += "/"
|
|
}
|
|
} else if len(root) == 0 {
|
|
root = "/"
|
|
}
|
|
if !strings.HasPrefix(strings.ToLower(path), strings.ToLower(root)) {
|
|
return "", errors.Errorf("path %q is not under root %q", path, root)
|
|
}
|
|
return path[len(root):], nil
|
|
}
|
|
|
|
// Strips the root off path and returns it
|
|
func (f *Fs) stripRoot(path string) (string, error) {
|
|
return strip(path, f.slashRootSlash)
|
|
}
|
|
|
|
// Walk the root returning a channel of Objects
|
|
func (f *Fs) list(out fs.ListOpts, dir string, recursive bool) {
|
|
root := f.slashRoot
|
|
if dir != "" {
|
|
root += "/" + dir
|
|
}
|
|
|
|
started := false
|
|
var res *files.ListFolderResult
|
|
var err error
|
|
for {
|
|
if !started {
|
|
arg := files.ListFolderArg{
|
|
Path: root,
|
|
Recursive: recursive,
|
|
}
|
|
if root == "/" {
|
|
arg.Path = "" // Specify root folder as empty string
|
|
}
|
|
res, err = f.srv.ListFolder(&arg)
|
|
if err != nil {
|
|
switch e := err.(type) {
|
|
case files.ListFolderAPIError:
|
|
switch e.EndpointError.Path.Tag {
|
|
case files.LookupErrorNotFound:
|
|
err = fs.ErrorDirNotFound
|
|
}
|
|
}
|
|
out.SetError(err)
|
|
return
|
|
}
|
|
started = false
|
|
} else {
|
|
arg := files.ListFolderContinueArg{
|
|
Cursor: res.Cursor,
|
|
}
|
|
res, err = f.srv.ListFolderContinue(&arg)
|
|
if err != nil {
|
|
out.SetError(errors.Wrap(err, "list continue"))
|
|
return
|
|
}
|
|
}
|
|
for _, entry := range res.Entries {
|
|
var fileInfo *files.FileMetadata
|
|
var folderInfo *files.FolderMetadata
|
|
var metadata *files.Metadata
|
|
switch info := entry.(type) {
|
|
case *files.FolderMetadata:
|
|
folderInfo = info
|
|
metadata = &info.Metadata
|
|
case *files.FileMetadata:
|
|
fileInfo = info
|
|
metadata = &info.Metadata
|
|
default:
|
|
fs.Errorf(f, "Unknown type %T", entry)
|
|
continue
|
|
}
|
|
|
|
entryPath := metadata.PathDisplay // FIXME PathLower
|
|
|
|
if folderInfo != nil {
|
|
name, err := f.stripRoot(entryPath + "/")
|
|
if err != nil {
|
|
out.SetError(err)
|
|
return
|
|
}
|
|
name = strings.Trim(name, "/")
|
|
if name != "" && name != dir {
|
|
dir := &fs.Dir{
|
|
Name: name,
|
|
When: time.Now(),
|
|
//When: folderInfo.ClientMtime,
|
|
//Bytes: folderInfo.Bytes,
|
|
//Count: -1,
|
|
}
|
|
if out.AddDir(dir) {
|
|
return
|
|
}
|
|
}
|
|
} else if fileInfo != nil {
|
|
path, err := f.stripRoot(entryPath)
|
|
if err != nil {
|
|
out.SetError(err)
|
|
return
|
|
}
|
|
o, err := f.newObjectWithInfo(path, fileInfo)
|
|
if err != nil {
|
|
out.SetError(err)
|
|
return
|
|
}
|
|
if out.Add(o) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
if !res.HasMore {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
// List walks the path returning a channel of Objects
|
|
func (f *Fs) List(out fs.ListOpts, dir string) {
|
|
defer out.Finished()
|
|
level := out.Level()
|
|
switch level {
|
|
case 1:
|
|
f.list(out, dir, false)
|
|
case fs.MaxLevel:
|
|
f.list(out, dir, true)
|
|
default:
|
|
out.SetError(fs.ErrorLevelNotSupported)
|
|
}
|
|
}
|
|
|
|
// A read closer which doesn't close the input
|
|
type readCloser struct {
|
|
in io.Reader
|
|
}
|
|
|
|
// Read bytes from the object - see io.Reader
|
|
func (rc *readCloser) Read(p []byte) (n int, err error) {
|
|
return rc.in.Read(p)
|
|
}
|
|
|
|
// Dummy close function
|
|
func (rc *readCloser) Close() error {
|
|
return nil
|
|
}
|
|
|
|
// Put the object
|
|
//
|
|
// Copy the reader in to the new object which is returned
|
|
//
|
|
// The new object may have been created if an error is returned
|
|
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
|
// Temporary Object under construction
|
|
o := &Object{
|
|
fs: f,
|
|
remote: src.Remote(),
|
|
}
|
|
return o, o.Update(in, src, options...)
|
|
}
|
|
|
|
// Mkdir creates the container if it doesn't exist
|
|
func (f *Fs) Mkdir(dir string) error {
|
|
root := path.Join(f.slashRoot, dir)
|
|
|
|
// can't create or run metadata on root
|
|
if root == "/" {
|
|
return nil
|
|
}
|
|
|
|
// check directory doesn't exist
|
|
_, err := f.getDirMetadata(root)
|
|
if err == nil {
|
|
return nil // directory exists already
|
|
} else if err != fs.ErrorDirNotFound {
|
|
return err // some other error
|
|
}
|
|
|
|
// create it
|
|
arg2 := files.CreateFolderArg{
|
|
Path: root,
|
|
}
|
|
_, err = f.srv.CreateFolder(&arg2)
|
|
return err
|
|
}
|
|
|
|
// Rmdir deletes the container
|
|
//
|
|
// Returns an error if it isn't empty
|
|
func (f *Fs) Rmdir(dir string) error {
|
|
root := path.Join(f.slashRoot, dir)
|
|
|
|
// can't remove root
|
|
if root == "/" {
|
|
return errors.New("can't remove root directory")
|
|
}
|
|
|
|
// check directory exists
|
|
_, err := f.getDirMetadata(root)
|
|
if err != nil {
|
|
return errors.Wrap(err, "Rmdir")
|
|
}
|
|
|
|
// check directory empty
|
|
arg := files.ListFolderArg{
|
|
Path: root,
|
|
Recursive: false,
|
|
}
|
|
if root == "/" {
|
|
arg.Path = "" // Specify root folder as empty string
|
|
}
|
|
res, err := f.srv.ListFolder(&arg)
|
|
if err != nil {
|
|
return errors.Wrap(err, "Rmdir")
|
|
}
|
|
if len(res.Entries) != 0 {
|
|
return errors.New("directory not empty")
|
|
}
|
|
|
|
// remove it
|
|
_, err = f.srv.Delete(&files.DeleteArg{Path: root})
|
|
return err
|
|
}
|
|
|
|
// Precision returns the precision
|
|
func (f *Fs) Precision() time.Duration {
|
|
return time.Second
|
|
}
|
|
|
|
// Copy src to this remote using server side copy operations.
|
|
//
|
|
// This is stored with the remote path given
|
|
//
|
|
// It returns the destination Object and a possible error
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantCopy
|
|
func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
|
|
srcObj, ok := src.(*Object)
|
|
if !ok {
|
|
fs.Debugf(src, "Can't copy - not same remote type")
|
|
return nil, fs.ErrorCantCopy
|
|
}
|
|
|
|
// Temporary Object under construction
|
|
dstObj := &Object{
|
|
fs: f,
|
|
remote: remote,
|
|
}
|
|
|
|
// Copy
|
|
arg := files.RelocationArg{}
|
|
arg.FromPath = srcObj.remotePath()
|
|
arg.ToPath = dstObj.remotePath()
|
|
entry, err := f.srv.Copy(&arg)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "copy failed")
|
|
}
|
|
|
|
// Set the metadata
|
|
fileInfo, ok := entry.(*files.FileMetadata)
|
|
if !ok {
|
|
return nil, fs.ErrorNotAFile
|
|
}
|
|
err = dstObj.setMetadataFromEntry(fileInfo)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "copy failed")
|
|
}
|
|
|
|
return dstObj, nil
|
|
}
|
|
|
|
// Purge deletes all the files and the container
|
|
//
|
|
// Optional interface: Only implement this if you have a way of
|
|
// deleting all the files quicker than just running Remove() on the
|
|
// result of List()
|
|
func (f *Fs) Purge() error {
|
|
// Let dropbox delete the filesystem tree
|
|
_, err := f.srv.Delete(&files.DeleteArg{Path: f.slashRoot})
|
|
return err
|
|
}
|
|
|
|
// Move src to this remote using server side move operations.
|
|
//
|
|
// This is stored with the remote path given
|
|
//
|
|
// It returns the destination Object and a possible error
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantMove
|
|
func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
|
|
srcObj, ok := src.(*Object)
|
|
if !ok {
|
|
fs.Debugf(src, "Can't move - not same remote type")
|
|
return nil, fs.ErrorCantMove
|
|
}
|
|
|
|
// Temporary Object under construction
|
|
dstObj := &Object{
|
|
fs: f,
|
|
remote: remote,
|
|
}
|
|
|
|
// Do the move
|
|
arg := files.RelocationArg{}
|
|
arg.FromPath = srcObj.remotePath()
|
|
arg.ToPath = dstObj.remotePath()
|
|
entry, err := f.srv.Move(&arg)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "move failed")
|
|
}
|
|
|
|
// Set the metadata
|
|
fileInfo, ok := entry.(*files.FileMetadata)
|
|
if !ok {
|
|
return nil, fs.ErrorNotAFile
|
|
}
|
|
err = dstObj.setMetadataFromEntry(fileInfo)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "move failed")
|
|
}
|
|
return dstObj, nil
|
|
}
|
|
|
|
// DirMove moves src, srcRemote to this remote at dstRemote
|
|
// using server side move operations.
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantDirMove
|
|
//
|
|
// If destination exists then return fs.ErrorDirExists
|
|
func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
|
|
srcFs, ok := src.(*Fs)
|
|
if !ok {
|
|
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
|
return fs.ErrorCantDirMove
|
|
}
|
|
srcPath := path.Join(srcFs.slashRoot, srcRemote)
|
|
dstPath := path.Join(f.slashRoot, dstRemote)
|
|
|
|
// Check if destination exists
|
|
_, err := f.getDirMetadata(f.slashRoot)
|
|
if err == nil {
|
|
return fs.ErrorDirExists
|
|
} else if err != fs.ErrorDirNotFound {
|
|
return err
|
|
}
|
|
|
|
// Make sure the parent directory exists
|
|
// ...apparently not necessary
|
|
|
|
// Do the move
|
|
arg := files.RelocationArg{}
|
|
arg.FromPath = srcPath
|
|
arg.ToPath = dstPath
|
|
_, err = f.srv.Move(&arg)
|
|
if err != nil {
|
|
return errors.Wrap(err, "MoveDir failed")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Hashes returns the supported hash sets.
|
|
func (f *Fs) Hashes() fs.HashSet {
|
|
return fs.HashSet(fs.HashDropbox)
|
|
}
|
|
|
|
// ------------------------------------------------------------
|
|
|
|
// Fs returns the parent Fs
|
|
func (o *Object) Fs() fs.Info {
|
|
return o.fs
|
|
}
|
|
|
|
// Return a string version
|
|
func (o *Object) String() string {
|
|
if o == nil {
|
|
return "<nil>"
|
|
}
|
|
return o.remote
|
|
}
|
|
|
|
// Remote returns the remote path
|
|
func (o *Object) Remote() string {
|
|
return o.remote
|
|
}
|
|
|
|
// Hash returns the dropbox special hash
|
|
func (o *Object) Hash(t fs.HashType) (string, error) {
|
|
if t != fs.HashDropbox {
|
|
return "", fs.ErrHashUnsupported
|
|
}
|
|
err := o.readMetaData()
|
|
if err != nil {
|
|
return "", errors.Wrap(err, "failed to read hash from metadata")
|
|
}
|
|
return o.hash, nil
|
|
}
|
|
|
|
// Size returns the size of an object in bytes
|
|
func (o *Object) Size() int64 {
|
|
return o.bytes
|
|
}
|
|
|
|
// setMetadataFromEntry sets the fs data from a files.FileMetadata
|
|
//
|
|
// This isn't a complete set of metadata and has an inacurate date
|
|
func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
|
|
o.bytes = int64(info.Size)
|
|
o.modTime = info.ClientModified
|
|
o.hash = info.ContentHash
|
|
return nil
|
|
}
|
|
|
|
// Reads the entry for a file from dropbox
|
|
func (o *Object) readEntry() (*files.FileMetadata, error) {
|
|
return o.fs.getFileMetadata(o.remotePath())
|
|
}
|
|
|
|
// Read entry if not set and set metadata from it
|
|
func (o *Object) readEntryAndSetMetadata() error {
|
|
// Last resort set time from client
|
|
if !o.modTime.IsZero() {
|
|
return nil
|
|
}
|
|
entry, err := o.readEntry()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return o.setMetadataFromEntry(entry)
|
|
}
|
|
|
|
// Returns the remote path for the object
|
|
func (o *Object) remotePath() string {
|
|
return o.fs.slashRootSlash + o.remote
|
|
}
|
|
|
|
// Returns the key for the metadata database for a given path
|
|
func metadataKey(path string) string {
|
|
// NB File system is case insensitive
|
|
path = strings.ToLower(path)
|
|
hash := md5.New()
|
|
_, _ = hash.Write([]byte(path))
|
|
return fmt.Sprintf("%x", hash.Sum(nil))
|
|
}
|
|
|
|
// Returns the key for the metadata database
|
|
func (o *Object) metadataKey() string {
|
|
return metadataKey(o.remotePath())
|
|
}
|
|
|
|
// readMetaData gets the info if it hasn't already been fetched
|
|
func (o *Object) readMetaData() (err error) {
|
|
if !o.modTime.IsZero() {
|
|
return nil
|
|
}
|
|
// Last resort
|
|
return o.readEntryAndSetMetadata()
|
|
}
|
|
|
|
// ModTime returns the modification time of the object
|
|
//
|
|
// It attempts to read the objects mtime and if that isn't present the
|
|
// LastModified returned in the http headers
|
|
func (o *Object) ModTime() time.Time {
|
|
err := o.readMetaData()
|
|
if err != nil {
|
|
fs.Debugf(o, "Failed to read metadata: %v", err)
|
|
return time.Now()
|
|
}
|
|
return o.modTime
|
|
}
|
|
|
|
// SetModTime sets the modification time of the local fs object
|
|
//
|
|
// Commits the datastore
|
|
func (o *Object) SetModTime(modTime time.Time) error {
|
|
// Dropbox doesn't have a way of doing this so returning this
|
|
// error will cause the file to be re-uploaded to set the
|
|
// time.
|
|
return fs.ErrorCantSetModTime
|
|
}
|
|
|
|
// Storable returns whether this object is storable
|
|
func (o *Object) Storable() bool {
|
|
return true
|
|
}
|
|
|
|
// Open an object for read
|
|
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
|
headers := fs.OpenOptionHeaders(options)
|
|
arg := files.DownloadArg{Path: o.remotePath(), ExtraHeaders: headers}
|
|
_, in, err = o.fs.srv.Download(&arg)
|
|
|
|
switch e := err.(type) {
|
|
case files.DownloadAPIError:
|
|
// Don't attempt to retry copyright violation errors
|
|
if e.EndpointError.Path.Tag == files.LookupErrorRestrictedContent {
|
|
return nil, fs.NoRetryError(err)
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// uploadChunked uploads the object in parts
|
|
//
|
|
// Call only if size is >= uploadChunkSize
|
|
//
|
|
// FIXME rework for retries
|
|
func (o *Object) uploadChunked(in io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) {
|
|
chunkSize := int64(uploadChunkSize)
|
|
chunks := int(size/chunkSize) + 1
|
|
|
|
// write the first whole chunk
|
|
fs.Debugf(o, "Uploading chunk 1/%d", chunks)
|
|
res, err := o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, &io.LimitedReader{R: in, N: chunkSize})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
cursor := files.UploadSessionCursor{
|
|
SessionId: res.SessionId,
|
|
Offset: uint64(chunkSize),
|
|
}
|
|
appendArg := files.UploadSessionAppendArg{
|
|
Cursor: &cursor,
|
|
Close: false,
|
|
}
|
|
|
|
// write more whole chunks (if any)
|
|
for i := 2; i < chunks; i++ {
|
|
fs.Debugf(o, "Uploading chunk %d/%d", i, chunks)
|
|
err = o.fs.srv.UploadSessionAppendV2(&appendArg, &io.LimitedReader{R: in, N: chunkSize})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
cursor.Offset += uint64(chunkSize)
|
|
}
|
|
|
|
// write the remains
|
|
args := &files.UploadSessionFinishArg{
|
|
Cursor: &cursor,
|
|
Commit: commitInfo,
|
|
}
|
|
fs.Debugf(o, "Uploading chunk %d/%d", chunks, chunks)
|
|
entry, err = o.fs.srv.UploadSessionFinish(args, in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return entry, nil
|
|
}
|
|
|
|
// Update the already existing object
|
|
//
|
|
// Copy the reader into the object updating modTime and size
|
|
//
|
|
// The new object may have been created if an error is returned
|
|
func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
|
remote := o.remotePath()
|
|
if ignoredFiles.MatchString(remote) {
|
|
fs.Logf(o, "File name disallowed - not uploading")
|
|
return nil
|
|
}
|
|
commitInfo := files.NewCommitInfo(o.remotePath())
|
|
commitInfo.Mode.Tag = "overwrite"
|
|
// The Dropbox API only accepts timestamps in UTC with second precision.
|
|
commitInfo.ClientModified = src.ModTime().UTC().Round(time.Second)
|
|
|
|
size := src.Size()
|
|
var err error
|
|
var entry *files.FileMetadata
|
|
if size > int64(uploadChunkSize) {
|
|
entry, err = o.uploadChunked(in, commitInfo, size)
|
|
} else {
|
|
entry, err = o.fs.srv.Upload(commitInfo, in)
|
|
}
|
|
if err != nil {
|
|
return errors.Wrap(err, "upload failed")
|
|
}
|
|
return o.setMetadataFromEntry(entry)
|
|
}
|
|
|
|
// Remove an object
|
|
func (o *Object) Remove() error {
|
|
_, err := o.fs.srv.Delete(&files.DeleteArg{Path: o.remotePath()})
|
|
return err
|
|
}
|
|
|
|
// Check the interfaces are satisfied
|
|
var (
|
|
_ fs.Fs = (*Fs)(nil)
|
|
_ fs.Copier = (*Fs)(nil)
|
|
_ fs.Purger = (*Fs)(nil)
|
|
_ fs.Mover = (*Fs)(nil)
|
|
_ fs.DirMover = (*Fs)(nil)
|
|
_ fs.Object = (*Object)(nil)
|
|
)
|