putio: add new backend

README.md

@@ -53,7 +53,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
   * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
   * premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/)
-  * put.io [:page_facing_up:](https://rclone.org/webdav/#put-io)
+  * put.io [:page_facing_up:](https://rclone.org/putio/)
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)

backend/all/all.go

@@ -25,6 +25,7 @@ import (
 	_ "github.com/rclone/rclone/backend/opendrive"
 	_ "github.com/rclone/rclone/backend/pcloud"
 	_ "github.com/rclone/rclone/backend/premiumizeme"
+	_ "github.com/rclone/rclone/backend/putio"
 	_ "github.com/rclone/rclone/backend/qingstor"
 	_ "github.com/rclone/rclone/backend/s3"
 	_ "github.com/rclone/rclone/backend/sftp"

backend/putio/fs.go

@@ -0,0 +1,691 @@
+package putio
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/putdotio/go-putio/putio"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/oauthutil"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/readers"
+)
+
+// Fs represents a remote Putio server
+type Fs struct {
+	name        string             // name of this remote
+	root        string             // the path we are working on
+	features    *fs.Features       // optional features
+	client      *putio.Client      // client for making API calls to Put.io
+	pacer       *fs.Pacer          // To pace the API calls
+	dirCache    *dircache.DirCache // Map of directory path to directory id
+	oAuthClient *http.Client
+}
+
+// ------------------------------------------------------------
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+	return fmt.Sprintf("Putio root '%s'", f.root)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// shouldRetry returns a boolean as to whether this err deserves to be
+// retried.  It returns the err as a convenience
+func shouldRetry(err error) (bool, error) {
+	if err == nil {
+		return false, nil
+	}
+	if fserrors.ShouldRetry(err) {
+		return true, err
+	}
+	if perr, ok := err.(*putio.ErrorResponse); ok {
+		if perr.Response.StatusCode == 429 || perr.Response.StatusCode >= 500 {
+			return true, err
+		}
+	}
+	return false, err
+}
+
+// NewFs constructs an Fs from the path, container:path
+func NewFs(name, root string, m configmap.Mapper) (f fs.Fs, err error) {
+	// defer log.Trace(name, "root=%v", root)("f=%+v, err=%v", &f, &err)
+	oAuthClient, _, err := oauthutil.NewClient(name, m, putioConfig)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to configure putio")
+	}
+	p := &Fs{
+		name:        name,
+		root:        root,
+		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		client:      putio.NewClient(oAuthClient),
+		oAuthClient: oAuthClient,
+	}
+	p.features = (&fs.Features{
+		DuplicateFiles:          true,
+		ReadMimeType:            true,
+		CanHaveEmptyDirectories: true,
+	}).Fill(p)
+	p.dirCache = dircache.New(root, "0", p)
+	ctx := context.Background()
+	// Find the current root
+	err = p.dirCache.FindRoot(ctx, false)
+	if err != nil {
+		// Assume it is a file
+		newRoot, remote := dircache.SplitPath(root)
+		tempF := *p
+		tempF.dirCache = dircache.New(newRoot, "0", &tempF)
+		tempF.root = newRoot
+		// Make new Fs which is the parent
+		err = tempF.dirCache.FindRoot(ctx, false)
+		if err != nil {
+			// No root so return old f
+			return p, nil
+		}
+		_, err := tempF.NewObject(ctx, remote)
+		if err != nil {
+			// unable to list folder so return old f
+			return p, nil
+		}
+		// XXX: update the old f here instead of returning tempF, since
+		// `features` were already filled with functions having *f as a receiver.
+		// See https://github.com/rclone/rclone/issues/2182
+		p.dirCache = tempF.dirCache
+		p.root = tempF.root
+		return p, fs.ErrorIsFile
+	}
+	// fs.Debugf(p, "Root id: %s", p.dirCache.RootID())
+	return p, nil
+}
+
+func itoa(i int64) string {
+	return strconv.FormatInt(i, 10)
+}
+
+func atoi(a string) int64 {
+	i, err := strconv.ParseInt(a, 10, 64)
+	if err != nil {
+		panic(err)
+	}
+	return i
+}
+
+// CreateDir makes a directory with pathID as parent and name leaf
+func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
+	// defer log.Trace(f, "pathID=%v, leaf=%v", pathID, leaf)("newID=%v, err=%v", newID, &err)
+	parentID := atoi(pathID)
+	var entry putio.File
+	err = f.pacer.Call(func() (bool, error) {
+		// fs.Debugf(f, "creating folder. part: %s, parentID: %d", leaf, parentID)
+		entry, err = f.client.Files.CreateFolder(ctx, leaf, parentID)
+		return shouldRetry(err)
+	})
+	return itoa(entry.ID), err
+}
+
+// FindLeaf finds a directory of name leaf in the folder with ID pathID
+func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
+	// defer log.Trace(f, "pathID=%v, leaf=%v", pathID, leaf)("pathIDOut=%v, found=%v, err=%v", pathIDOut, found, &err)
+	if pathID == "0" && leaf == "" {
+		// that's the root directory
+		return pathID, true, nil
+	}
+	fileID := atoi(pathID)
+	var children []putio.File
+	err = f.pacer.Call(func() (bool, error) {
+		// fs.Debugf(f, "listing file: %d", fileID)
+		children, _, err = f.client.Files.List(ctx, fileID)
+		return shouldRetry(err)
+	})
+	if err != nil {
+		if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode == 404 {
+			err = nil
+		}
+		return
+	}
+	for _, child := range children {
+		if child.Name == leaf {
+			found = true
+			pathIDOut = itoa(child.ID)
+			if !child.IsDir() {
+				err = fs.ErrorNotAFile
+			}
+			return
+		}
+	}
+	return
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	// defer log.Trace(f, "dir=%v", dir)("err=%v", &err)
+	err = f.dirCache.FindRoot(ctx, false)
+	if err != nil {
+		return nil, err
+	}
+	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
+	if err != nil {
+		return nil, err
+	}
+	parentID := atoi(directoryID)
+	var children []putio.File
+	err = f.pacer.Call(func() (bool, error) {
+		// fs.Debugf(f, "listing files inside List: %d", parentID)
+		children, _, err = f.client.Files.List(ctx, parentID)
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return
+	}
+	for _, child := range children {
+		remote := path.Join(dir, child.Name)
+		// fs.Debugf(f, "child: %s", remote)
+		if child.IsDir() {
+			f.dirCache.Put(remote, itoa(child.ID))
+			d := fs.NewDir(remote, child.UpdatedAt.Time)
+			entries = append(entries, d)
+		} else {
+			o, err := f.newObjectWithInfo(ctx, remote, child)
+			if err != nil {
+				return nil, err
+			}
+			entries = append(entries, o)
+		}
+	}
+	return
+}
+
+// Put the object
+//
+// Copy the reader in to the new object which is returned
+//
+// The new object may have been created if an error is returned
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (o fs.Object, err error) {
+	// defer log.Trace(f, "src=%+v", src)("o=%+v, err=%v", &o, &err)
+	exisitingObj, err := f.NewObject(ctx, src.Remote())
+	switch err {
+	case nil:
+		return exisitingObj, exisitingObj.Update(ctx, in, src, options...)
+	case fs.ErrorObjectNotFound:
+		// Not found so create it
+		return f.PutUnchecked(ctx, in, src, options...)
+	default:
+		return nil, err
+	}
+}
+
+// PutUnchecked uploads the object
+//
+// This will create a duplicate if we upload a new file without
+// checking to see if there is one already - use Put() for that.
+func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (o fs.Object, err error) {
+	// defer log.Trace(f, "src=%+v", src)("o=%+v, err=%v", &o, &err)
+	size := src.Size()
+	remote := src.Remote()
+	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	if err != nil {
+		return nil, err
+	}
+	loc, err := f.createUpload(ctx, leaf, size, directoryID, src.ModTime(ctx))
+	if err != nil {
+		return nil, err
+	}
+	fileID, err := f.sendUpload(ctx, loc, size, in)
+	if err != nil {
+		return nil, err
+	}
+	var entry putio.File
+	err = f.pacer.Call(func() (bool, error) {
+		// fs.Debugf(f, "getting file: %d", fileID)
+		entry, err = f.client.Files.Get(ctx, fileID)
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return f.newObjectWithInfo(ctx, remote, entry)
+}
+
+func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID string, modTime time.Time) (location string, err error) {
+	// defer log.Trace(f, "name=%v, size=%v, parentID=%v, modTime=%v", name, size, parentID, modTime.String())("location=%v, err=%v", location, &err)
+	err = f.pacer.Call(func() (bool, error) {
+		req, err := http.NewRequest("POST", "https://upload.put.io/files/", nil)
+		if err != nil {
+			return false, err
+		}
+		req.Header.Set("tus-resumable", "1.0.0")
+		req.Header.Set("upload-length", strconv.FormatInt(size, 10))
+		b64name := base64.StdEncoding.EncodeToString([]byte(name))
+		b64true := base64.StdEncoding.EncodeToString([]byte("true"))
+		b64parentID := base64.StdEncoding.EncodeToString([]byte(parentID))
+		b64modifiedAt := base64.StdEncoding.EncodeToString([]byte(modTime.Format(time.RFC3339)))
+		req.Header.Set("upload-metadata", fmt.Sprintf("name %s,no-torrent %s,parent_id %s,updated-at %s", b64name, b64true, b64parentID, b64modifiedAt))
+		resp, err := f.oAuthClient.Do(req)
+		retry, err := shouldRetry(err)
+		if retry {
+			return true, err
+		}
+		if err != nil {
+			return false, err
+		}
+		if resp.StatusCode != 201 {
+			return false, fmt.Errorf("unexpected status code from upload create: %d", resp.StatusCode)
+		}
+		location = resp.Header.Get("location")
+		if location == "" {
+			return false, errors.New("empty location header from upload create")
+		}
+		return false, nil
+	})
+	return
+}
+
+func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.Reader) (fileID int64, err error) {
+	// defer log.Trace(f, "location=%v, size=%v", location, size)("fileID=%v, err=%v", fileID, &err)
+	if size == 0 {
+		err = f.pacer.Call(func() (bool, error) {
+			fs.Debugf(f, "Sending zero length chunk")
+			fileID, err = f.transferChunk(ctx, location, 0, bytes.NewReader([]byte{}), 0)
+			return shouldRetry(err)
+		})
+		return
+	}
+	var start int64
+	buf := make([]byte, defaultChunkSize)
+	for start < size {
+		reqSize := size - start
+		if reqSize >= int64(defaultChunkSize) {
+			reqSize = int64(defaultChunkSize)
+		}
+		chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, reqSize)
+
+		// Transfer the chunk
+		err = f.pacer.Call(func() (bool, error) {
+			fs.Debugf(f, "Sending chunk. start: %d length: %d", start, reqSize)
+			// TODO get file offset and seek to the position
+			fileID, err = f.transferChunk(ctx, location, start, chunk, reqSize)
+			return shouldRetry(err)
+		})
+		if err != nil {
+			return
+		}
+
+		start += reqSize
+	}
+	return
+}
+
+func (f *Fs) transferChunk(ctx context.Context, location string, start int64, chunk io.ReadSeeker, chunkSize int64) (fileID int64, err error) {
+	// defer log.Trace(f, "location=%v, start=%v, chunkSize=%v", location, start, chunkSize)("fileID=%v, err=%v", fileID, &err)
+	_, _ = chunk.Seek(0, io.SeekStart)
+	req, err := f.makeUploadPatchRequest(location, chunk, start, chunkSize)
+	if err != nil {
+		return 0, err
+	}
+	req = req.WithContext(ctx)
+	res, err := f.oAuthClient.Do(req)
+	if err != nil {
+		return 0, err
+	}
+	defer func() {
+		_ = res.Body.Close()
+	}()
+	if res.StatusCode != 204 {
+		return 0, fmt.Errorf("unexpected status code while transferring chunk: %d", res.StatusCode)
+	}
+	sfid := res.Header.Get("putio-file-id")
+	if sfid != "" {
+		fileID, err = strconv.ParseInt(sfid, 10, 64)
+		if err != nil {
+			return 0, err
+		}
+	}
+	return fileID, nil
+}
+
+func (f *Fs) makeUploadPatchRequest(location string, in io.Reader, offset, length int64) (*http.Request, error) {
+	req, err := http.NewRequest("PATCH", location, in)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("tus-resumable", "1.0.0")
+	req.Header.Set("upload-offset", strconv.FormatInt(offset, 10))
+	req.Header.Set("content-length", strconv.FormatInt(length, 10))
+	req.Header.Set("content-type", "application/offset+octet-stream")
+	return req, nil
+}
+
+// Mkdir creates the container if it doesn't exist
+func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
+	// defer log.Trace(f, "dir=%v", dir)("err=%v", &err)
+	err = f.dirCache.FindRoot(ctx, true)
+	if err != nil {
+		return err
+	}
+	if dir != "" {
+		_, err = f.dirCache.FindDir(ctx, dir, true)
+	}
+	return err
+}
+
+// Rmdir deletes the container
+//
+// Returns an error if it isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
+	// defer log.Trace(f, "dir=%v", dir)("err=%v", &err)
+
+	root := strings.Trim(path.Join(f.root, dir), "/")
+
+	// can't remove root
+	if root == "" {
+		return errors.New("can't remove root directory")
+	}
+
+	// check directory exists
+	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
+	if err != nil {
+		return errors.Wrap(err, "Rmdir")
+	}
+	dirID := atoi(directoryID)
+
+	// check directory empty
+	var children []putio.File
+	err = f.pacer.Call(func() (bool, error) {
+		// fs.Debugf(f, "listing files: %d", dirID)
+		children, _, err = f.client.Files.List(ctx, dirID)
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return errors.Wrap(err, "Rmdir")
+	}
+	if len(children) != 0 {
+		return errors.New("directory not empty")
+	}
+
+	// remove it
+	err = f.pacer.Call(func() (bool, error) {
+		// fs.Debugf(f, "deleting file: %d", dirID)
+		err = f.client.Files.Delete(ctx, dirID)
+		return shouldRetry(err)
+	})
+	f.dirCache.FlushDir(dir)
+	return err
+}
+
+// Precision returns the precision
+func (f *Fs) Precision() time.Duration {
+	return time.Second
+}
+
+// Purge deletes all the files and the container
+//
+// Optional interface: Only implement this if you have a way of
+// deleting all the files quicker than just running Remove() on the
+// result of List()
+func (f *Fs) Purge(ctx context.Context) (err error) {
+	// defer log.Trace(f, "")("err=%v", &err)
+
+	if f.root == "" {
+		return errors.New("can't purge root directory")
+	}
+	err = f.dirCache.FindRoot(ctx, false)
+	if err != nil {
+		return err
+	}
+
+	rootID := atoi(f.dirCache.RootID())
+	// Let putio delete the filesystem tree
+	err = f.pacer.Call(func() (bool, error) {
+		// fs.Debugf(f, "deleting file: %d", rootID)
+		err = f.client.Files.Delete(ctx, rootID)
+		return shouldRetry(err)
+	})
+	f.dirCache.ResetRoot()
+	return err
+}
+
+// Copy src to this remote using server side copy operations.
+//
+// This is stored with the remote path given
+//
+// It returns the destination Object and a possible error
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantCopy
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (o fs.Object, err error) {
+	// defer log.Trace(f, "src=%+v, remote=%v", src, remote)("o=%+v, err=%v", &o, &err)
+	srcObj, ok := src.(*Object)
+	if !ok {
+		return nil, fs.ErrorCantCopy
+	}
+	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	if err != nil {
+		return nil, err
+	}
+	err = f.pacer.Call(func() (bool, error) {
+		params := url.Values{}
+		params.Set("file_id", strconv.FormatInt(srcObj.file.ID, 10))
+		params.Set("parent_id", directoryID)
+		params.Set("name", leaf)
+		req, err := f.client.NewRequest(ctx, "POST", "/v2/files/copy", strings.NewReader(params.Encode()))
+		if err != nil {
+			return false, err
+		}
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+		// fs.Debugf(f, "copying file (%d) to parent_id: %s", srcObj.file.ID, directoryID)
+		_, err = f.client.Do(req, nil)
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return f.NewObject(ctx, remote)
+}
+
+// Move src to this remote using server side move operations.
+//
+// This is stored with the remote path given
+//
+// It returns the destination Object and a possible error
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantMove
+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (o fs.Object, err error) {
+	// defer log.Trace(f, "src=%+v, remote=%v", src, remote)("o=%+v, err=%v", &o, &err)
+	srcObj, ok := src.(*Object)
+	if !ok {
+		return nil, fs.ErrorCantMove
+	}
+	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	if err != nil {
+		return nil, err
+	}
+	err = f.pacer.Call(func() (bool, error) {
+		params := url.Values{}
+		params.Set("file_id", strconv.FormatInt(srcObj.file.ID, 10))
+		params.Set("parent_id", directoryID)
+		params.Set("name", leaf)
+		req, err := f.client.NewRequest(ctx, "POST", "/v2/files/move", strings.NewReader(params.Encode()))
+		if err != nil {
+			return false, err
+		}
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+		// fs.Debugf(f, "moving file (%d) to parent_id: %s", srcObj.file.ID, directoryID)
+		_, err = f.client.Do(req, nil)
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return f.NewObject(ctx, remote)
+}
+
+// DirMove moves src, srcRemote to this remote at dstRemote
+// using server side move operations.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantDirMove
+//
+// If destination exists then return fs.ErrorDirExists
+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) {
+	// defer log.Trace(f, "src=%+v, srcRemote=%v, dstRemote", src, srcRemote, dstRemote)("err=%v", &err)
+	srcFs, ok := src.(*Fs)
+	if !ok {
+		return fs.ErrorCantDirMove
+	}
+	srcPath := path.Join(srcFs.root, srcRemote)
+	dstPath := path.Join(f.root, dstRemote)
+
+	// Refuse to move to or from the root
+	if srcPath == "" || dstPath == "" {
+		return errors.New("can't move root directory")
+	}
+
+	// find the root src directory
+	err = srcFs.dirCache.FindRoot(ctx, false)
+	if err != nil {
+		return err
+	}
+
+	// find the root dst directory
+	if dstRemote != "" {
+		err = f.dirCache.FindRoot(ctx, true)
+		if err != nil {
+			return err
+		}
+	} else {
+		if f.dirCache.FoundRoot() {
+			return fs.ErrorDirExists
+		}
+	}
+
+	// Find ID of dst parent, creating subdirs if necessary
+	var leaf, dstDirectoryID string
+	findPath := dstRemote
+	if dstRemote == "" {
+		findPath = f.root
+	}
+	leaf, dstDirectoryID, err = f.dirCache.FindPath(ctx, findPath, true)
+	if err != nil {
+		return err
+	}
+
+	// Check destination does not exist
+	if dstRemote != "" {
+		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
+		if err == fs.ErrorDirNotFound {
+			// OK
+		} else if err != nil {
+			return err
+		} else {
+			return fs.ErrorDirExists
+		}
+	}
+
+	// Find ID of src
+	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
+	if err != nil {
+		return err
+	}
+
+	err = f.pacer.Call(func() (bool, error) {
+		params := url.Values{}
+		params.Set("file_id", srcID)
+		params.Set("parent_id", dstDirectoryID)
+		params.Set("name", leaf)
+		req, err := f.client.NewRequest(ctx, "POST", "/v2/files/move", strings.NewReader(params.Encode()))
+		if err != nil {
+			return false, err
+		}
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+		// fs.Debugf(f, "moving file (%s) to parent_id: %s", srcID, dstDirectoryID)
+		_, err = f.client.Do(req, nil)
+		return shouldRetry(err)
+	})
+	srcFs.dirCache.FlushDir(srcRemote)
+	return err
+}
+
+// About gets quota information
+func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
+	// defer log.Trace(f, "")("usage=%+v, err=%v", usage, &err)
+	var ai putio.AccountInfo
+	err = f.pacer.Call(func() (bool, error) {
+		// fs.Debugf(f, "getting account info")
+		ai, err = f.client.Account.Info(ctx)
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "about failed")
+	}
+	return &fs.Usage{
+		Total: fs.NewUsageValue(ai.Disk.Size),  // quota of bytes that can be used
+		Used:  fs.NewUsageValue(ai.Disk.Used),  // bytes in use
+		Free:  fs.NewUsageValue(ai.Disk.Avail), // bytes which can be uploaded before reaching the quota
+	}, nil
+}
+
+// Hashes returns the supported hash sets.
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hash.CRC32)
+}
+
+// DirCacheFlush resets the directory cache - used in testing as an
+// optional interface
+func (f *Fs) DirCacheFlush() {
+	// defer log.Trace(f, "")("")
+	f.dirCache.ResetRoot()
+}
+
+// CleanUp the trash in the Fs
+func (f *Fs) CleanUp(ctx context.Context) (err error) {
+	// defer log.Trace(f, "")("err=%v", &err)
+	return f.pacer.Call(func() (bool, error) {
+		req, err := f.client.NewRequest(ctx, "POST", "/v2/trash/empty", nil)
+		if err != nil {
+			return false, err
+		}
+		// fs.Debugf(f, "emptying trash")
+		_, err = f.client.Do(req, nil)
+		return shouldRetry(err)
+	})
+}

backend/putio/object.go

@@ -0,0 +1,276 @@
+package putio
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"strconv"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/putdotio/go-putio/putio"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/fs/hash"
+)
+
+// Object describes a Putio object
+//
+// Putio Objects always have full metadata
+type Object struct {
+	fs      *Fs // what this object is part of
+	file    *putio.File
+	remote  string // The remote path
+	modtime time.Time
+}
+
+// NewObject finds the Object at remote.  If it can't be found
+// it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
+	// defer log.Trace(f, "remote=%v", remote)("o=%+v, err=%v", &o, &err)
+	obj := &Object{
+		fs:     f,
+		remote: remote,
+	}
+	err = obj.readEntryAndSetMetadata(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return obj, err
+}
+
+// Return an Object from a path
+//
+// If it can't be found it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info putio.File) (o fs.Object, err error) {
+	// defer log.Trace(f, "remote=%v, info=+v", remote, &info)("o=%+v, err=%v", &o, &err)
+	obj := &Object{
+		fs:     f,
+		remote: remote,
+	}
+	err = obj.setMetadataFromEntry(info)
+	if err != nil {
+		return nil, err
+	}
+	return obj, err
+}
+
+// Fs returns the parent Fs
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Return a string version
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.remote
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// Hash returns the dropbox special hash
+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
+	if t != hash.CRC32 {
+		return "", hash.ErrUnsupported
+	}
+	err := o.readEntryAndSetMetadata(ctx)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to read hash from metadata")
+	}
+	return o.file.CRC32, nil
+}
+
+// Size returns the size of an object in bytes
+func (o *Object) Size() int64 {
+	if o.file == nil {
+		return 0
+	}
+	return o.file.Size
+}
+
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	if o.file == nil {
+		return ""
+	}
+	return itoa(o.file.ID)
+}
+
+// MimeType returns the content type of the Object if
+// known, or "" if not
+func (o *Object) MimeType(ctx context.Context) string {
+	err := o.readEntryAndSetMetadata(ctx)
+	if err != nil {
+		return ""
+	}
+	return o.file.ContentType
+}
+
+// setMetadataFromEntry sets the fs data from a putio.File
+//
+// This isn't a complete set of metadata and has an inacurate date
+func (o *Object) setMetadataFromEntry(info putio.File) error {
+	o.file = &info
+	o.modtime = info.UpdatedAt.Time
+	return nil
+}
+
+// Reads the entry for a file from putio
+func (o *Object) readEntry(ctx context.Context) (f *putio.File, err error) {
+	// defer log.Trace(o, "")("f=%+v, err=%v", f, &err)
+	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, o.remote, false)
+	if err != nil {
+		if err == fs.ErrorDirNotFound {
+			return nil, fs.ErrorObjectNotFound
+		}
+		return nil, err
+	}
+	var resp struct {
+		File putio.File `json:"file"`
+	}
+	err = o.fs.pacer.Call(func() (bool, error) {
+		// fs.Debugf(o, "requesting child. directoryID: %s, name: %s", directoryID, leaf)
+		req, err := o.fs.client.NewRequest(ctx, "GET", "/v2/files/"+directoryID+"/child?name="+url.PathEscape(leaf), nil)
+		if err != nil {
+			return false, err
+		}
+		_, err = o.fs.client.Do(req, &resp)
+		if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode == 404 {
+			return false, fs.ErrorObjectNotFound
+		}
+		return shouldRetry(err)
+	})
+	return &resp.File, err
+}
+
+// Read entry if not set and set metadata from it
+func (o *Object) readEntryAndSetMetadata(ctx context.Context) error {
+	if o.file != nil {
+		return nil
+	}
+	entry, err := o.readEntry(ctx)
+	if err != nil {
+		return err
+	}
+	return o.setMetadataFromEntry(*entry)
+}
+
+// Returns the remote path for the object
+func (o *Object) remotePath() string {
+	return path.Join(o.fs.root, o.remote)
+}
+
+// ModTime returns the modification time of the object
+//
+// It attempts to read the objects mtime and if that isn't present the
+// LastModified returned in the http headers
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	if o.modtime.IsZero() {
+		err := o.readEntryAndSetMetadata(ctx)
+		if err != nil {
+			fs.Debugf(o, "Failed to read metadata: %v", err)
+			return time.Now()
+		}
+	}
+	return o.modtime
+}
+
+// SetModTime sets the modification time of the local fs object
+//
+// Commits the datastore
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error) {
+	// defer log.Trace(o, "modTime=%v", modTime.String())("err=%v", &err)
+	req, err := o.fs.client.NewRequest(ctx, "POST", "/v2/files/touch?file_id="+strconv.FormatInt(o.file.ID, 10)+"&updated_at="+url.QueryEscape(modTime.Format(time.RFC3339)), nil)
+	if err != nil {
+		return err
+	}
+	// fs.Debugf(o, "setting modtime: %s", modTime.String())
+	_, err = o.fs.client.Do(req, nil)
+	if err != nil {
+		return err
+	}
+	o.modtime = modTime
+	if o.file != nil {
+		o.file.UpdatedAt.Time = modTime
+	}
+	return nil
+}
+
+// Storable returns whether this object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// Open an object for read
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	// defer log.Trace(o, "")("err=%v", &err)
+	var storageURL string
+	err = o.fs.pacer.Call(func() (bool, error) {
+		storageURL, err = o.fs.client.Files.URL(ctx, o.file.ID, true)
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return
+	}
+
+	var resp *http.Response
+	headers := fs.OpenOptionHeaders(options)
+	err = o.fs.pacer.Call(func() (bool, error) {
+		req, _ := http.NewRequest(http.MethodGet, storageURL, nil)
+		req.Header.Set("User-Agent", o.fs.client.UserAgent)
+
+		// merge headers with extra headers
+		for header, value := range headers {
+			req.Header.Set(header, value)
+		}
+		// fs.Debugf(o, "opening file: id=%d", o.file.ID)
+		resp, err = http.DefaultClient.Do(req)
+		return shouldRetry(err)
+	})
+	if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode >= 400 && perr.Response.StatusCode <= 499 {
+		_ = resp.Body.Close()
+		return nil, fserrors.NoRetryError(err)
+	}
+	return resp.Body, err
+}
+
+// Update the already existing object
+//
+// Copy the reader into the object updating modTime and size
+//
+// The new object may have been created if an error is returned
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
+	// defer log.Trace(o, "src=%+v", src)("err=%v", &err)
+	remote := o.remotePath()
+	if ignoredFiles.MatchString(remote) {
+		fs.Logf(o, "File name disallowed - not uploading")
+		return nil
+	}
+	err = o.Remove(ctx)
+	if err != nil {
+		return err
+	}
+	newObj, err := o.fs.PutUnchecked(ctx, in, src, options...)
+	if err != nil {
+		return err
+	}
+	*o = *(newObj.(*Object))
+	return err
+}
+
+// Remove an object
+func (o *Object) Remove(ctx context.Context) (err error) {
+	// defer log.Trace(o, "")("err=%v", &err)
+	return o.fs.pacer.Call(func() (bool, error) {
+		// fs.Debugf(o, "removing file: id=%d", o.file.ID)
+		err = o.fs.client.Files.Delete(ctx, o.file.ID)
+		return shouldRetry(err)
+	})
+}

backend/putio/putio.go

@@ -0,0 +1,72 @@
+package putio
+
+import (
+	"log"
+	"regexp"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/oauthutil"
+	"golang.org/x/oauth2"
+)
+
+// Constants
+const (
+	rcloneClientID             = "4131"
+	rcloneObscuredClientSecret = "cMwrjWVmrHZp3gf1ZpCrlyGAmPpB-YY5BbVnO1fj-G9evcd8"
+	minSleep                   = 10 * time.Millisecond
+	maxSleep                   = 2 * time.Second
+	decayConstant              = 2 // bigger for slower decay, exponential
+	defaultChunkSize           = 48 * fs.MebiByte
+)
+
+var (
+	// Description of how to auth for this app
+	putioConfig = &oauth2.Config{
+		Scopes: []string{},
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  "https://api.put.io/v2/oauth2/authenticate",
+			TokenURL: "https://api.put.io/v2/oauth2/access_token",
+		},
+		ClientID:     rcloneClientID,
+		ClientSecret: obscure.MustReveal(rcloneObscuredClientSecret),
+		RedirectURL:  oauthutil.RedirectLocalhostURL,
+	}
+	// A regexp matching path names for ignoring unnecessary files
+	ignoredFiles = regexp.MustCompile(`(?i)(^|/)(desktop\.ini|thumbs\.db|\.ds_store|icon\r)$`)
+)
+
+// Register with Fs
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "putio",
+		Description: "Put.io",
+		NewFs:       NewFs,
+		Config: func(name string, m configmap.Mapper) {
+			err := oauthutil.ConfigNoOffline("putio", name, m, putioConfig)
+			if err != nil {
+				log.Fatalf("Failed to configure token: %v", err)
+			}
+		},
+	})
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs              = (*Fs)(nil)
+	_ fs.Purger          = (*Fs)(nil)
+	_ fs.PutUncheckeder  = (*Fs)(nil)
+	_ fs.Abouter         = (*Fs)(nil)
+	_ fs.Mover           = (*Fs)(nil)
+	_ fs.DirMover        = (*Fs)(nil)
+	_ dircache.DirCacher = (*Fs)(nil)
+	_ fs.DirCacheFlusher = (*Fs)(nil)
+	_ fs.CleanUpper      = (*Fs)(nil)
+	_ fs.Copier          = (*Fs)(nil)
+	_ fs.Object          = (*Object)(nil)
+	_ fs.MimeTyper       = (*Object)(nil)
+	_ fs.IDer            = (*Object)(nil)
+)

backend/putio/putio_test.go

@@ -0,0 +1,16 @@
+// Test Put.io filesystem interface
+package putio
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestPutio:",
+		NilObject:  (*Object)(nil),
+	})
+}

bin/make_manual.py

@@ -48,6 +48,7 @@ docs = [
     "swift.md",
     "pcloud.md",
     "premiumize.md",
+    "putio.md",
     "sftp.md",
     "union.md",
     "webdav.md",

docs/content/about.md

@@ -46,7 +46,7 @@ Rclone is a command line program to sync files and directories to and from:
 * {{< provider name="ownCloud" home="https://owncloud.org/" config="/webdav/#owncloud" >}}
 * {{< provider name="pCloud" home="https://www.pcloud.com/" config="/pcloud/" >}}
 * {{< provider name="premiumize.me" home="https://premiumize.me/" config="/premiumizeme/" >}}
-* {{< provider name="put.io" home="https://put.io/" config="/webdav/#put-io" >}}
+* {{< provider name="put.io" home="https://put.io/" config="/putio/" >}}
 * {{< provider name="QingStor" home="https://www.qingcloud.com/products/storage" config="/qingstor/" >}}
 * {{< provider name="Rackspace Cloud Files" home="https://www.rackspace.com/cloud/files" config="/swift/" >}}
 * {{< provider name="rsync.net" home="https://rsync.net/products/rclone.html" config="/sftp/#rsync-net" >}}

docs/content/docs.md

@@ -44,6 +44,7 @@ See the following for detailed instructions for
   * [OpenDrive](/opendrive/)
   * [Pcloud](/pcloud/)
   * [premiumize.me](/premiumizeme/)
+  * [put.io](/putio/)
   * [QingStor](/qingstor/)
   * [SFTP](/sftp/)
   * [Union](/union/)

docs/content/overview.md

@@ -38,6 +38,7 @@ Here is an overview of the major features of each cloud storage system.
 | Openstack Swift              | MD5         | Yes     | No               | No              | R/W       |
 | pCloud                       | MD5, SHA1   | Yes     | No               | No              | W         |
 | premiumize.me                | -           | No      | Yes              | No              | R         |
+| put.io                       | CRC-32      | Yes     | No               | Yes             | R         |
 | QingStor                     | MD5         | No      | No               | No              | R/W       |
 | SFTP                         | MD5, SHA1 ‡ | Yes     | Depends          | No              | -         |
 | WebDAV                       | MD5, SHA1 ††| Yes ††† | Depends          | No              | -         |
@@ -154,6 +155,7 @@ operations more efficient.
 | Openstack Swift              | Yes † | Yes  | No   | No      | No      | Yes   | Yes          | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes | No |
 | pCloud                       | Yes   | Yes  | Yes  | Yes     | Yes     | No    | No           | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes | Yes |
 | premiumize.me                | Yes   | No   | Yes  | Yes     | No      | No    | No           | Yes         | Yes | Yes |
+| put.io                       | Yes   | No   | Yes  | Yes     | Yes     | No    | Yes          | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes | Yes |
 | QingStor                     | No    | Yes  | No   | No      | No      | Yes   | No           | No [#2178](https://github.com/rclone/rclone/issues/2178) | No  | No |
 | SFTP                         | No    | No   | Yes  | Yes     | No      | No    | Yes          | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes  | Yes |
 | WebDAV                       | Yes   | Yes  | Yes  | Yes     | No      | No    | Yes ‡        | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes  | Yes |

docs/content/putio.md

@@ -0,0 +1,97 @@
+---
+title: "put.io"
+description: "Rclone docs for put.io"
+date: "2019-08-08"
+---
+
+<i class="fas fa-parking"></i> put.io
+---------------------------------
+
+Paths are specified as `remote:path`
+
+put.io paths may be as deep as required, eg
+`remote:directory/subdirectory`.
+
+The initial setup for put.io involves getting a token from put.io
+which you need to do in your browser.  `rclone config` walks you
+through it.
+
+Here is an example of how to make a remote called `remote`.  First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+```
+No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> putio
+Type of storage to configure.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+[snip]
+25 / Put.io
+   \ "putio"
+[snip]
+Storage> putio
+** See help for putio backend at: https://rclone.org/putio/ **
+
+Remote config
+Use auto config?
+ * Say Y if not sure
+ * Say N if you are working on a remote or headless machine
+y) Yes
+n) No
+y/n> y
+If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+--------------------
+[putio]
+type = putio
+token = {"access_token":"XXXXXXXX","expiry":"0001-01-01T00:00:00Z"}
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+Current remotes:
+
+Name                 Type
+====                 ====
+putio                putio
+
+e) Edit existing remote
+n) New remote
+d) Delete remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+e/n/d/r/c/s/q> q
+```
+
+Note that rclone runs a webserver on your local machine to collect the
+token as returned from Google if you use auto config mode. This only
+runs from the moment it opens your browser to the moment you get back
+the verification code.  This is on `http://127.0.0.1:53682/` and this
+it may require you to unblock it temporarily if you are running a host
+firewall, or use manual mode.
+
+You can then use it like this,
+
+List directories in top level of your put.io
+
+    rclone lsd remote:
+
+List all the files in your put.io
+
+    rclone ls remote:
+
+To copy a local directory to a put.io directory called backup
+
+    rclone copy /home/source remote:backup

docs/content/webdav.md

@@ -189,31 +189,6 @@ Owncloud does. This [may be
 fixed](https://github.com/nextcloud/nextcloud-snap/issues/365) in the
 future.
 
-### Put.io ###
-
-put.io can be accessed in a read only way using webdav.
-
-Configure the `url` as `https://webdav.put.io` and use your normal
-account username and password for `user` and `pass`.  Set the `vendor`
-to `other`.
-
-Your config file should end up looking like this:
-
-```
-[putio]
-type = webdav
-url = https://webdav.put.io
-vendor = other
-user = YourUserName
-pass = encryptedpassword
-```
-
-If you are using `put.io` with `rclone mount` then use the
-`--read-only` flag to signal to the OS that it can't write to the
-mount.
-
-For more help see [the put.io webdav docs](http://help.put.io/apps-and-integrations/ftp-and-webdav).
-
 ### Sharepoint ###
 
 Rclone can be used with Sharepoint provided by OneDrive for Business

docs/layouts/chrome/navbar.html

@@ -80,6 +80,7 @@
                     <li><a href="/swift/"><i class="fa fa-space-shuttle"></i> Openstack Swift</a></li>
                     <li><a href="/pcloud/"><i class="fa fa-cloud"></i> pCloud</a></li>
                     <li><a href="/premiumizeme/"><i class="fa fa-user"></i> premiumize.me</a></li>
+                    <li><a href="/putio/"><i class="fa fas-parking"></i> put.io</a></li>
                     <li><a href="/sftp/"><i class="fa fa-server"></i> SFTP</a></li>
                     <li><a href="/union/"><i class="fa fa-link"></i> Union (merge backends)</a></li>
                     <li><a href="/webdav/"><i class="fa fa-server"></i> WebDAV</a></li>

fstest/test_all/config.yaml

@@ -125,3 +125,7 @@ backends:
  - backend:  "premiumizeme"
    remote:   "TestPremiumizeMe:"
    fastlist: false
+ - backend:  "putio"
+   remote:   "TestPutio:"
+   subdir:   false
+   fastlist: false