Move all backends into backend directory

backend/dropbox/dropbox.go

@@ -0,0 +1,976 @@
+// Package dropbox provides an interface to Dropbox object storage
+package dropbox
+
+// FIXME dropbox for business would be quite easy to add
+
+/*
+The Case folding of PathDisplay problem
+
+From the docs:
+
+path_display String. The cased path to be used for display purposes
+only. In rare instances the casing will not correctly match the user's
+filesystem, but this behavior will match the path provided in the Core
+API v1, and at least the last path component will have the correct
+casing. Changes to only the casing of paths won't be returned by
+list_folder/continue. This field will be null if the file or folder is
+not mounted. This field is optional.
+
+We solve this by not implementing the ListR interface.  The dropbox
+remote will recurse directory by directory only using the last element
+of path_display and all will be well.
+*/
+
+import (
+	"crypto/md5"
+	"fmt"
+	"io"
+	"log"
+	"path"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox"
+	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/files"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/oauthutil"
+	"github.com/ncw/rclone/pacer"
+	"github.com/pkg/errors"
+	"golang.org/x/oauth2"
+)
+
+// Constants
+const (
+	rcloneClientID              = "5jcck7diasz0rqy"
+	rcloneEncryptedClientSecret = "fRS5vVLr2v6FbyXYnIgjwBuUAt0osq_QZTXAEcmZ7g"
+	minSleep                    = 10 * time.Millisecond
+	maxSleep                    = 2 * time.Second
+	decayConstant               = 2 // bigger for slower decay, exponential
+)
+
+var (
+	// Description of how to auth for this app
+	dropboxConfig = &oauth2.Config{
+		Scopes: []string{},
+		// Endpoint: oauth2.Endpoint{
+		// 	AuthURL:  "https://www.dropbox.com/1/oauth2/authorize",
+		// 	TokenURL: "https://api.dropboxapi.com/1/oauth2/token",
+		// },
+		Endpoint:     dropbox.OAuthEndpoint(""),
+		ClientID:     rcloneClientID,
+		ClientSecret: fs.MustReveal(rcloneEncryptedClientSecret),
+		RedirectURL:  oauthutil.RedirectLocalhostURL,
+	}
+	// A regexp matching path names for files Dropbox ignores
+	// See https://www.dropbox.com/en/help/145 - Ignored files
+	ignoredFiles = regexp.MustCompile(`(?i)(^|/)(desktop\.ini|thumbs\.db|\.ds_store|icon\r|\.dropbox|\.dropbox.attr)$`)
+	// Upload chunk size - setting too small makes uploads slow.
+	// Chunks are buffered into memory for retries.
+	//
+	// Speed vs chunk size uploading a 1 GB file on 2017-11-22
+	//
+	// Chunk Size MB, Speed Mbyte/s, % of max
+	// 1	1.364	11%
+	// 2	2.443	19%
+	// 4	4.288	33%
+	// 8	6.79	52%
+	// 16	8.916	69%
+	// 24	10.195	79%
+	// 32	10.427	81%
+	// 40	10.96	85%
+	// 48	11.828	91%
+	// 56	11.763	91%
+	// 64	12.047	93%
+	// 96	12.302	95%
+	// 128	12.945	100%
+	//
+	// Choose 48MB which is 91% of Maximum speed.  rclone by
+	// default does 4 transfers so this should use 4*48MB = 192MB
+	// by default.
+	uploadChunkSize    = fs.SizeSuffix(48 * 1024 * 1024)
+	maxUploadChunkSize = fs.SizeSuffix(150 * 1024 * 1024)
+)
+
+// Register with Fs
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "dropbox",
+		Description: "Dropbox",
+		NewFs:       NewFs,
+		Config: func(name string) {
+			err := oauthutil.ConfigNoOffline("dropbox", name, dropboxConfig)
+			if err != nil {
+				log.Fatalf("Failed to configure token: %v", err)
+			}
+		},
+		Options: []fs.Option{{
+			Name: "app_key",
+			Help: "Dropbox App Key - leave blank normally.",
+		}, {
+			Name: "app_secret",
+			Help: "Dropbox App Secret - leave blank normally.",
+		}},
+	})
+	fs.VarP(&uploadChunkSize, "dropbox-chunk-size", "", fmt.Sprintf("Upload chunk size. Max %v.", maxUploadChunkSize))
+}
+
+// Fs represents a remote dropbox server
+type Fs struct {
+	name           string       // name of this remote
+	root           string       // the path we are working on
+	features       *fs.Features // optional features
+	srv            files.Client // the connection to the dropbox server
+	slashRoot      string       // root with "/" prefix, lowercase
+	slashRootSlash string       // root with "/" prefix and postfix, lowercase
+	pacer          *pacer.Pacer // To pace the API calls
+}
+
+// Object describes a dropbox object
+//
+// Dropbox Objects always have full metadata
+type Object struct {
+	fs      *Fs       // what this object is part of
+	remote  string    // The remote path
+	bytes   int64     // size of the object
+	modTime time.Time // time it was last modified
+	hash    string    // content_hash of the object
+}
+
+// ------------------------------------------------------------
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+	return fmt.Sprintf("Dropbox root '%s'", f.root)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// shouldRetry returns a boolean as to whether this err deserves to be
+// retried.  It returns the err as a convenience
+func shouldRetry(err error) (bool, error) {
+	if err == nil {
+		return false, err
+	}
+	baseErrString := errors.Cause(err).Error()
+	// FIXME there is probably a better way of doing this!
+	if strings.Contains(baseErrString, "too_many_write_operations") || strings.Contains(baseErrString, "too_many_requests") {
+		return true, err
+	}
+	return fs.ShouldRetry(err), err
+}
+
+// NewFs contstructs an Fs from the path, container:path
+func NewFs(name, root string) (fs.Fs, error) {
+	if uploadChunkSize > maxUploadChunkSize {
+		return nil, errors.Errorf("chunk size too big, must be < %v", maxUploadChunkSize)
+	}
+
+	// Convert the old token if it exists.  The old token was just
+	// just a string, the new one is a JSON blob
+	oldToken := strings.TrimSpace(fs.ConfigFileGet(name, fs.ConfigToken))
+	if oldToken != "" && oldToken[0] != '{' {
+		fs.Infof(name, "Converting token to new format")
+		newToken := fmt.Sprintf(`{"access_token":"%s","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}`, oldToken)
+		err := fs.ConfigSetValueAndSave(name, fs.ConfigToken, newToken)
+		if err != nil {
+			return nil, errors.Wrap(err, "NewFS convert token")
+		}
+	}
+
+	oAuthClient, _, err := oauthutil.NewClient(name, dropboxConfig)
+	if err != nil {
+		log.Fatalf("Failed to configure dropbox: %v", err)
+	}
+
+	config := dropbox.Config{
+		LogLevel: dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo
+		Client:   oAuthClient,    // maybe???
+	}
+	srv := files.New(config)
+
+	f := &Fs{
+		name:  name,
+		srv:   srv,
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+	}
+	f.features = (&fs.Features{
+		CaseInsensitive:         true,
+		ReadMimeType:            true,
+		CanHaveEmptyDirectories: true,
+	}).Fill(f)
+	f.setRoot(root)
+
+	// See if the root is actually an object
+	_, err = f.getFileMetadata(f.slashRoot)
+	if err == nil {
+		newRoot := path.Dir(f.root)
+		if newRoot == "." {
+			newRoot = ""
+		}
+		f.setRoot(newRoot)
+		// return an error with an fs which points to the parent
+		return f, fs.ErrorIsFile
+	}
+	return f, nil
+}
+
+// Sets root in f
+func (f *Fs) setRoot(root string) {
+	f.root = strings.Trim(root, "/")
+	lowerCaseRoot := strings.ToLower(f.root)
+
+	f.slashRoot = "/" + lowerCaseRoot
+	f.slashRootSlash = f.slashRoot
+	if lowerCaseRoot != "" {
+		f.slashRootSlash += "/"
+	}
+}
+
+// getMetadata gets the metadata for a file or directory
+func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool, err error) {
+	err = f.pacer.Call(func() (bool, error) {
+		entry, err = f.srv.GetMetadata(&files.GetMetadataArg{Path: objPath})
+		return shouldRetry(err)
+	})
+	if err != nil {
+		switch e := err.(type) {
+		case files.GetMetadataAPIError:
+			switch e.EndpointError.Path.Tag {
+			case files.LookupErrorNotFound:
+				notFound = true
+				err = nil
+			}
+		}
+	}
+	return
+}
+
+// getFileMetadata gets the metadata for a file
+func (f *Fs) getFileMetadata(filePath string) (fileInfo *files.FileMetadata, err error) {
+	entry, notFound, err := f.getMetadata(filePath)
+	if err != nil {
+		return nil, err
+	}
+	if notFound {
+		return nil, fs.ErrorObjectNotFound
+	}
+	fileInfo, ok := entry.(*files.FileMetadata)
+	if !ok {
+		return nil, fs.ErrorNotAFile
+	}
+	return fileInfo, nil
+}
+
+// getDirMetadata gets the metadata for a directory
+func (f *Fs) getDirMetadata(dirPath string) (dirInfo *files.FolderMetadata, err error) {
+	entry, notFound, err := f.getMetadata(dirPath)
+	if err != nil {
+		return nil, err
+	}
+	if notFound {
+		return nil, fs.ErrorDirNotFound
+	}
+	dirInfo, ok := entry.(*files.FolderMetadata)
+	if !ok {
+		return nil, fs.ErrorIsFile
+	}
+	return dirInfo, nil
+}
+
+// Return an Object from a path
+//
+// If it can't be found it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Object, error) {
+	o := &Object{
+		fs:     f,
+		remote: remote,
+	}
+	var err error
+	if info != nil {
+		err = o.setMetadataFromEntry(info)
+	} else {
+		err = o.readEntryAndSetMetadata()
+	}
+	if err != nil {
+		return nil, err
+	}
+	return o, nil
+}
+
+// NewObject finds the Object at remote.  If it can't be found
+// it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) NewObject(remote string) (fs.Object, error) {
+	return f.newObjectWithInfo(remote, nil)
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
+	root := f.slashRoot
+	if dir != "" {
+		root += "/" + dir
+	}
+
+	started := false
+	var res *files.ListFolderResult
+	for {
+		if !started {
+			arg := files.ListFolderArg{
+				Path:      root,
+				Recursive: false,
+			}
+			if root == "/" {
+				arg.Path = "" // Specify root folder as empty string
+			}
+			err = f.pacer.Call(func() (bool, error) {
+				res, err = f.srv.ListFolder(&arg)
+				return shouldRetry(err)
+			})
+			if err != nil {
+				switch e := err.(type) {
+				case files.ListFolderAPIError:
+					switch e.EndpointError.Path.Tag {
+					case files.LookupErrorNotFound:
+						err = fs.ErrorDirNotFound
+					}
+				}
+				return nil, err
+			}
+			started = true
+		} else {
+			arg := files.ListFolderContinueArg{
+				Cursor: res.Cursor,
+			}
+			err = f.pacer.Call(func() (bool, error) {
+				res, err = f.srv.ListFolderContinue(&arg)
+				return shouldRetry(err)
+			})
+			if err != nil {
+				return nil, errors.Wrap(err, "list continue")
+			}
+		}
+		for _, entry := range res.Entries {
+			var fileInfo *files.FileMetadata
+			var folderInfo *files.FolderMetadata
+			var metadata *files.Metadata
+			switch info := entry.(type) {
+			case *files.FolderMetadata:
+				folderInfo = info
+				metadata = &info.Metadata
+			case *files.FileMetadata:
+				fileInfo = info
+				metadata = &info.Metadata
+			default:
+				fs.Errorf(f, "Unknown type %T", entry)
+				continue
+			}
+
+			// Only the last element is reliably cased in PathDisplay
+			entryPath := metadata.PathDisplay
+			leaf := path.Base(entryPath)
+			remote := path.Join(dir, leaf)
+			if folderInfo != nil {
+				d := fs.NewDir(remote, time.Now())
+				entries = append(entries, d)
+			} else if fileInfo != nil {
+				o, err := f.newObjectWithInfo(remote, fileInfo)
+				if err != nil {
+					return nil, err
+				}
+				entries = append(entries, o)
+			}
+		}
+		if !res.HasMore {
+			break
+		}
+	}
+	return entries, nil
+}
+
+// A read closer which doesn't close the input
+type readCloser struct {
+	in io.Reader
+}
+
+// Read bytes from the object - see io.Reader
+func (rc *readCloser) Read(p []byte) (n int, err error) {
+	return rc.in.Read(p)
+}
+
+// Dummy close function
+func (rc *readCloser) Close() error {
+	return nil
+}
+
+// Put the object
+//
+// Copy the reader in to the new object which is returned
+//
+// The new object may have been created if an error is returned
+func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	// Temporary Object under construction
+	o := &Object{
+		fs:     f,
+		remote: src.Remote(),
+	}
+	return o, o.Update(in, src, options...)
+}
+
+// PutStream uploads to the remote path with the modTime given of indeterminate size
+func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.Put(in, src, options...)
+}
+
+// Mkdir creates the container if it doesn't exist
+func (f *Fs) Mkdir(dir string) error {
+	root := path.Join(f.slashRoot, dir)
+
+	// can't create or run metadata on root
+	if root == "/" {
+		return nil
+	}
+
+	// check directory doesn't exist
+	_, err := f.getDirMetadata(root)
+	if err == nil {
+		return nil // directory exists already
+	} else if err != fs.ErrorDirNotFound {
+		return err // some other error
+	}
+
+	// create it
+	arg2 := files.CreateFolderArg{
+		Path: root,
+	}
+	err = f.pacer.Call(func() (bool, error) {
+		_, err = f.srv.CreateFolderV2(&arg2)
+		return shouldRetry(err)
+	})
+	return err
+}
+
+// Rmdir deletes the container
+//
+// Returns an error if it isn't empty
+func (f *Fs) Rmdir(dir string) error {
+	root := path.Join(f.slashRoot, dir)
+
+	// can't remove root
+	if root == "/" {
+		return errors.New("can't remove root directory")
+	}
+
+	// check directory exists
+	_, err := f.getDirMetadata(root)
+	if err != nil {
+		return errors.Wrap(err, "Rmdir")
+	}
+
+	// check directory empty
+	arg := files.ListFolderArg{
+		Path:      root,
+		Recursive: false,
+	}
+	if root == "/" {
+		arg.Path = "" // Specify root folder as empty string
+	}
+	var res *files.ListFolderResult
+	err = f.pacer.Call(func() (bool, error) {
+		res, err = f.srv.ListFolder(&arg)
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return errors.Wrap(err, "Rmdir")
+	}
+	if len(res.Entries) != 0 {
+		return errors.New("directory not empty")
+	}
+
+	// remove it
+	err = f.pacer.Call(func() (bool, error) {
+		_, err = f.srv.DeleteV2(&files.DeleteArg{Path: root})
+		return shouldRetry(err)
+	})
+	return err
+}
+
+// Precision returns the precision
+func (f *Fs) Precision() time.Duration {
+	return time.Second
+}
+
+// Copy src to this remote using server side copy operations.
+//
+// This is stored with the remote path given
+//
+// It returns the destination Object and a possible error
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantCopy
+func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't copy - not same remote type")
+		return nil, fs.ErrorCantCopy
+	}
+
+	// Temporary Object under construction
+	dstObj := &Object{
+		fs:     f,
+		remote: remote,
+	}
+
+	// Copy
+	arg := files.RelocationArg{}
+	arg.FromPath = srcObj.remotePath()
+	arg.ToPath = dstObj.remotePath()
+	var err error
+	var result *files.RelocationResult
+	err = f.pacer.Call(func() (bool, error) {
+		result, err = f.srv.CopyV2(&arg)
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "copy failed")
+	}
+
+	// Set the metadata
+	fileInfo, ok := result.Metadata.(*files.FileMetadata)
+	if !ok {
+		return nil, fs.ErrorNotAFile
+	}
+	err = dstObj.setMetadataFromEntry(fileInfo)
+	if err != nil {
+		return nil, errors.Wrap(err, "copy failed")
+	}
+
+	return dstObj, nil
+}
+
+// Purge deletes all the files and the container
+//
+// Optional interface: Only implement this if you have a way of
+// deleting all the files quicker than just running Remove() on the
+// result of List()
+func (f *Fs) Purge() (err error) {
+	// Let dropbox delete the filesystem tree
+	err = f.pacer.Call(func() (bool, error) {
+		_, err = f.srv.DeleteV2(&files.DeleteArg{Path: f.slashRoot})
+		return shouldRetry(err)
+	})
+	return err
+}
+
+// Move src to this remote using server side move operations.
+//
+// This is stored with the remote path given
+//
+// It returns the destination Object and a possible error
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantMove
+func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't move - not same remote type")
+		return nil, fs.ErrorCantMove
+	}
+
+	// Temporary Object under construction
+	dstObj := &Object{
+		fs:     f,
+		remote: remote,
+	}
+
+	// Do the move
+	arg := files.RelocationArg{}
+	arg.FromPath = srcObj.remotePath()
+	arg.ToPath = dstObj.remotePath()
+	var err error
+	var result *files.RelocationResult
+	err = f.pacer.Call(func() (bool, error) {
+		result, err = f.srv.MoveV2(&arg)
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "move failed")
+	}
+
+	// Set the metadata
+	fileInfo, ok := result.Metadata.(*files.FileMetadata)
+	if !ok {
+		return nil, fs.ErrorNotAFile
+	}
+	err = dstObj.setMetadataFromEntry(fileInfo)
+	if err != nil {
+		return nil, errors.Wrap(err, "move failed")
+	}
+	return dstObj, nil
+}
+
+// DirMove moves src, srcRemote to this remote at dstRemote
+// using server side move operations.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantDirMove
+//
+// If destination exists then return fs.ErrorDirExists
+func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
+	srcFs, ok := src.(*Fs)
+	if !ok {
+		fs.Debugf(srcFs, "Can't move directory - not same remote type")
+		return fs.ErrorCantDirMove
+	}
+	srcPath := path.Join(srcFs.slashRoot, srcRemote)
+	dstPath := path.Join(f.slashRoot, dstRemote)
+
+	// Check if destination exists
+	_, err := f.getDirMetadata(dstPath)
+	if err == nil {
+		return fs.ErrorDirExists
+	} else if err != fs.ErrorDirNotFound {
+		return err
+	}
+
+	// Make sure the parent directory exists
+	// ...apparently not necessary
+
+	// Do the move
+	arg := files.RelocationArg{}
+	arg.FromPath = srcPath
+	arg.ToPath = dstPath
+	err = f.pacer.Call(func() (bool, error) {
+		_, err = f.srv.MoveV2(&arg)
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return errors.Wrap(err, "MoveDir failed")
+	}
+
+	return nil
+}
+
+// Hashes returns the supported hash sets.
+func (f *Fs) Hashes() fs.HashSet {
+	return fs.HashSet(fs.HashDropbox)
+}
+
+// ------------------------------------------------------------
+
+// Fs returns the parent Fs
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Return a string version
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.remote
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// Hash returns the dropbox special hash
+func (o *Object) Hash(t fs.HashType) (string, error) {
+	if t != fs.HashDropbox {
+		return "", fs.ErrHashUnsupported
+	}
+	err := o.readMetaData()
+	if err != nil {
+		return "", errors.Wrap(err, "failed to read hash from metadata")
+	}
+	return o.hash, nil
+}
+
+// Size returns the size of an object in bytes
+func (o *Object) Size() int64 {
+	return o.bytes
+}
+
+// setMetadataFromEntry sets the fs data from a files.FileMetadata
+//
+// This isn't a complete set of metadata and has an inacurate date
+func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
+	o.bytes = int64(info.Size)
+	o.modTime = info.ClientModified
+	o.hash = info.ContentHash
+	return nil
+}
+
+// Reads the entry for a file from dropbox
+func (o *Object) readEntry() (*files.FileMetadata, error) {
+	return o.fs.getFileMetadata(o.remotePath())
+}
+
+// Read entry if not set and set metadata from it
+func (o *Object) readEntryAndSetMetadata() error {
+	// Last resort set time from client
+	if !o.modTime.IsZero() {
+		return nil
+	}
+	entry, err := o.readEntry()
+	if err != nil {
+		return err
+	}
+	return o.setMetadataFromEntry(entry)
+}
+
+// Returns the remote path for the object
+func (o *Object) remotePath() string {
+	return o.fs.slashRootSlash + o.remote
+}
+
+// Returns the key for the metadata database for a given path
+func metadataKey(path string) string {
+	// NB File system is case insensitive
+	path = strings.ToLower(path)
+	hash := md5.New()
+	_, _ = hash.Write([]byte(path))
+	return fmt.Sprintf("%x", hash.Sum(nil))
+}
+
+// Returns the key for the metadata database
+func (o *Object) metadataKey() string {
+	return metadataKey(o.remotePath())
+}
+
+// readMetaData gets the info if it hasn't already been fetched
+func (o *Object) readMetaData() (err error) {
+	if !o.modTime.IsZero() {
+		return nil
+	}
+	// Last resort
+	return o.readEntryAndSetMetadata()
+}
+
+// ModTime returns the modification time of the object
+//
+// It attempts to read the objects mtime and if that isn't present the
+// LastModified returned in the http headers
+func (o *Object) ModTime() time.Time {
+	err := o.readMetaData()
+	if err != nil {
+		fs.Debugf(o, "Failed to read metadata: %v", err)
+		return time.Now()
+	}
+	return o.modTime
+}
+
+// SetModTime sets the modification time of the local fs object
+//
+// Commits the datastore
+func (o *Object) SetModTime(modTime time.Time) error {
+	// Dropbox doesn't have a way of doing this so returning this
+	// error will cause the file to be deleted first then
+	// re-uploaded to set the time.
+	return fs.ErrorCantSetModTimeWithoutDelete
+}
+
+// Storable returns whether this object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// Open an object for read
+func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	headers := fs.OpenOptionHeaders(options)
+	arg := files.DownloadArg{Path: o.remotePath(), ExtraHeaders: headers}
+	err = o.fs.pacer.Call(func() (bool, error) {
+		_, in, err = o.fs.srv.Download(&arg)
+		return shouldRetry(err)
+	})
+
+	switch e := err.(type) {
+	case files.DownloadAPIError:
+		// Don't attempt to retry copyright violation errors
+		if e.EndpointError.Path.Tag == files.LookupErrorRestrictedContent {
+			return nil, fs.NoRetryError(err)
+		}
+	}
+
+	return
+}
+
+// uploadChunked uploads the object in parts
+//
+// Will work optimally if size is >= uploadChunkSize. If the size is either
+// unknown (i.e. -1) or smaller than uploadChunkSize, the method incurs an
+// avoidable request to the Dropbox API that does not carry payload.
+func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) {
+	chunkSize := int64(uploadChunkSize)
+	chunks := 0
+	if size != -1 {
+		chunks = int(size/chunkSize) + 1
+	}
+	in := fs.NewCountingReader(in0)
+	buf := make([]byte, int(chunkSize))
+
+	fmtChunk := func(cur int, last bool) {
+		if chunks == 0 && last {
+			fs.Debugf(o, "Streaming chunk %d/%d", cur, cur)
+		} else if chunks == 0 {
+			fs.Debugf(o, "Streaming chunk %d/unknown", cur)
+		} else {
+			fs.Debugf(o, "Uploading chunk %d/%d", cur, chunks)
+		}
+	}
+
+	// write the first chunk
+	fmtChunk(1, false)
+	var res *files.UploadSessionStartResult
+	chunk := fs.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
+	err = o.fs.pacer.Call(func() (bool, error) {
+		// seek to the start in case this is a retry
+		if _, err = chunk.Seek(0, 0); err != nil {
+			return false, nil
+		}
+		res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, chunk)
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	cursor := files.UploadSessionCursor{
+		SessionId: res.SessionId,
+		Offset:    0,
+	}
+	appendArg := files.UploadSessionAppendArg{
+		Cursor: &cursor,
+		Close:  false,
+	}
+
+	// write more whole chunks (if any)
+	currentChunk := 2
+	for {
+		if chunks > 0 && currentChunk >= chunks {
+			// if the size is known, only upload full chunks. Remaining bytes are uploaded with
+			// the UploadSessionFinish request.
+			break
+		} else if chunks == 0 && in.BytesRead()-cursor.Offset < uint64(chunkSize) {
+			// if the size is unknown, upload as long as we can read full chunks from the reader.
+			// The UploadSessionFinish request will not contain any payload.
+			break
+		}
+		cursor.Offset = in.BytesRead()
+		fmtChunk(currentChunk, false)
+		chunk = fs.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
+		err = o.fs.pacer.Call(func() (bool, error) {
+			// seek to the start in case this is a retry
+			if _, err = chunk.Seek(0, 0); err != nil {
+				return false, nil
+			}
+			err = o.fs.srv.UploadSessionAppendV2(&appendArg, chunk)
+			// after the first chunk is uploaded, we retry everything
+			return err != nil, err
+		})
+		if err != nil {
+			return nil, err
+		}
+		currentChunk++
+	}
+
+	// write the remains
+	cursor.Offset = in.BytesRead()
+	args := &files.UploadSessionFinishArg{
+		Cursor: &cursor,
+		Commit: commitInfo,
+	}
+	fmtChunk(currentChunk, true)
+	chunk = fs.NewRepeatableReaderBuffer(in, buf)
+	err = o.fs.pacer.Call(func() (bool, error) {
+		// seek to the start in case this is a retry
+		if _, err = chunk.Seek(0, 0); err != nil {
+			return false, nil
+		}
+		entry, err = o.fs.srv.UploadSessionFinish(args, chunk)
+		// after the first chunk is uploaded, we retry everything
+		return err != nil, err
+	})
+	if err != nil {
+		return nil, err
+	}
+	return entry, nil
+}
+
+// Update the already existing object
+//
+// Copy the reader into the object updating modTime and size
+//
+// The new object may have been created if an error is returned
+func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	remote := o.remotePath()
+	if ignoredFiles.MatchString(remote) {
+		fs.Logf(o, "File name disallowed - not uploading")
+		return nil
+	}
+	commitInfo := files.NewCommitInfo(o.remotePath())
+	commitInfo.Mode.Tag = "overwrite"
+	// The Dropbox API only accepts timestamps in UTC with second precision.
+	commitInfo.ClientModified = src.ModTime().UTC().Round(time.Second)
+
+	size := src.Size()
+	var err error
+	var entry *files.FileMetadata
+	if size > int64(uploadChunkSize) || size == -1 {
+		entry, err = o.uploadChunked(in, commitInfo, size)
+	} else {
+		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+			entry, err = o.fs.srv.Upload(commitInfo, in)
+			return shouldRetry(err)
+		})
+	}
+	if err != nil {
+		return errors.Wrap(err, "upload failed")
+	}
+	return o.setMetadataFromEntry(entry)
+}
+
+// Remove an object
+func (o *Object) Remove() (err error) {
+	err = o.fs.pacer.Call(func() (bool, error) {
+		_, err = o.fs.srv.DeleteV2(&files.DeleteArg{Path: o.remotePath()})
+		return shouldRetry(err)
+	})
+	return err
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs          = (*Fs)(nil)
+	_ fs.Copier      = (*Fs)(nil)
+	_ fs.Purger      = (*Fs)(nil)
+	_ fs.PutStreamer = (*Fs)(nil)
+	_ fs.Mover       = (*Fs)(nil)
+	_ fs.DirMover    = (*Fs)(nil)
+	_ fs.Object      = (*Object)(nil)
+)