diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index df4cc964e..91cb3af78 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -354,7 +354,7 @@ Integration tests * Add your backend to `fstest/test_all/config.yaml` * Once you've done that then you can use the integration test framework from the project root: * go install ./... - * test_all -backend remote + * test_all -backends remote Or if you want to run the integration tests manually: diff --git a/README.md b/README.md index b40ab2537..9a3f6ef3c 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/) * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway) * SFTP [:page_facing_up:](https://rclone.org/sftp/) + * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/) * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi) * WebDAV [:page_facing_up:](https://rclone.org/webdav/) * Yandex Disk [:page_facing_up:](https://rclone.org/yandex/) diff --git a/backend/all/all.go b/backend/all/all.go index 9c81bc1af..3fc9239b9 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -32,6 +32,7 @@ import ( _ "github.com/rclone/rclone/backend/s3" _ "github.com/rclone/rclone/backend/sftp" _ "github.com/rclone/rclone/backend/sharefile" + _ "github.com/rclone/rclone/backend/sugarsync" _ "github.com/rclone/rclone/backend/swift" _ "github.com/rclone/rclone/backend/union" _ "github.com/rclone/rclone/backend/webdav" diff --git a/backend/sugarsync/api/types.go b/backend/sugarsync/api/types.go new file mode 100644 index 000000000..5edda687b --- /dev/null +++ b/backend/sugarsync/api/types.go @@ -0,0 +1,160 @@ +// Package api has type definitions for sugarsync +// +// Converted from the API docs with help from https://www.onlinetool.io/xmltogo/ +package api + +import ( + "encoding/xml" + "time" +) + +// AppAuthorization is used to request a refresh token +// +// The token is returned in the Location: field +type AppAuthorization struct { + XMLName xml.Name `xml:"appAuthorization"` + Username string `xml:"username"` + Password string `xml:"password"` + Application string `xml:"application"` + AccessKeyID string `xml:"accessKeyId"` + PrivateAccessKey string `xml:"privateAccessKey"` +} + +// TokenAuthRequest is the request to get Authorization +type TokenAuthRequest struct { + XMLName xml.Name `xml:"tokenAuthRequest"` + AccessKeyID string `xml:"accessKeyId"` + PrivateAccessKey string `xml:"privateAccessKey"` + RefreshToken string `xml:"refreshToken"` +} + +// Authorization is returned from the TokenAuthRequest +type Authorization struct { + XMLName xml.Name `xml:"authorization"` + Expiration time.Time `xml:"expiration"` + User string `xml:"user"` +} + +// File represents a single file +type File struct { + Name string `xml:"displayName"` + Ref string `xml:"ref"` + DsID string `xml:"dsid"` + TimeCreated time.Time `xml:"timeCreated"` + Parent string `xml:"parent"` + Size int64 `xml:"size"` + LastModified time.Time `xml:"lastModified"` + MediaType string `xml:"mediaType"` + PresentOnServer bool `xml:"presentOnServer"` + FileData string `xml:"fileData"` + Versions string `xml:"versions"` + PublicLink PublicLink +} + +// Collection represents +// - Workspace Collection +// - Sync Folders collection +// - Folder +type Collection struct { + Type string `xml:"type,attr"` + Name string `xml:"displayName"` + Ref string `xml:"ref"` // only for Folder + DsID string `xml:"dsid"` + TimeCreated time.Time `xml:"timeCreated"` + Parent string `xml:"parent"` + Collections string `xml:"collections"` + Files string `xml:"files"` + Contents string `xml:"contents"` + // Sharing bool `xml:"sharing>enabled,attr"` +} + +// CollectionContents is the result of a list call +type CollectionContents struct { + //XMLName xml.Name `xml:"collectionContents"` + Start int `xml:"start,attr"` + HasMore bool `xml:"hasMore,attr"` + End int `xml:"end,attr"` + Collections []Collection `xml:"collection"` + Files []File `xml:"file"` +} + +// User is returned from the /user call +type User struct { + XMLName xml.Name `xml:"user"` + Username string `xml:"username"` + Nickname string `xml:"nickname"` + Quota struct { + Limit int64 `xml:"limit"` + Usage int64 `xml:"usage"` + } `xml:"quota"` + Workspaces string `xml:"workspaces"` + SyncFolders string `xml:"syncfolders"` + Deleted string `xml:"deleted"` + MagicBriefcase string `xml:"magicBriefcase"` + WebArchive string `xml:"webArchive"` + MobilePhotos string `xml:"mobilePhotos"` + Albums string `xml:"albums"` + RecentActivities string `xml:"recentActivities"` + ReceivedShares string `xml:"receivedShares"` + PublicLinks string `xml:"publicLinks"` + MaximumPublicLinkSize int `xml:"maximumPublicLinkSize"` +} + +// CreateFolder is posted to a folder URL to create a folder +type CreateFolder struct { + XMLName xml.Name `xml:"folder"` + Name string `xml:"displayName"` +} + +// MoveFolder is posted to a folder URL to move a folder +type MoveFolder struct { + XMLName xml.Name `xml:"folder"` + Name string `xml:"displayName"` + Parent string `xml:"parent"` +} + +// CreateSyncFolder is posted to the root folder URL to create a sync folder +type CreateSyncFolder struct { + XMLName xml.Name `xml:"syncFolder"` + Name string `xml:"displayName"` +} + +// CreateFile is posted to a folder URL to create a file +type CreateFile struct { + XMLName xml.Name `xml:"file"` + Name string `xml:"displayName"` + MediaType string `xml:"mediaType"` +} + +// MoveFile is posted to a file URL to create a file +type MoveFile struct { + XMLName xml.Name `xml:"file"` + Name string `xml:"displayName"` + Parent string `xml:"parent"` +} + +// CopyFile copies a file from source +type CopyFile struct { + XMLName xml.Name `xml:"fileCopy"` + Source string `xml:"source,attr"` + Name string `xml:"displayName"` +} + +// PublicLink is the URL and enabled flag for a public link +type PublicLink struct { + XMLName xml.Name `xml:"publicLink"` + URL string `xml:",chardata"` + Enabled bool `xml:"enabled,attr"` +} + +// SetPublicLink can be used to enable the file for sharing +type SetPublicLink struct { + XMLName xml.Name `xml:"file"` + PublicLink PublicLink +} + +// SetLastModified sets the modified time for a file +type SetLastModified struct { + XMLName xml.Name `xml:"file"` + LastModified time.Time `xml:"lastModified"` +} diff --git a/backend/sugarsync/sugarsync.go b/backend/sugarsync/sugarsync.go new file mode 100644 index 000000000..2df0b0087 --- /dev/null +++ b/backend/sugarsync/sugarsync.go @@ -0,0 +1,1362 @@ +// Package sugarsync provides an interface to the Sugarsync +// object storage system. +package sugarsync + +/* FIXME + +DirMove tests fails with: Can not move sync folder. + +go test -v -short -run TestIntegration/FsMkdir/FsPutFiles/FsDirMove -verbose -dump-bodies + +To work around this we use the remote "TestSugarSync:Test" to test with. + +*/ + +import ( + "context" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/pkg/errors" + "github.com/rclone/rclone/backend/sugarsync/api" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/config/obscure" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/lib/dircache" + "github.com/rclone/rclone/lib/encoder" + "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/rest" +) + +/* +maxFileLength = 16383 +canWriteUnnormalized = true +canReadUnnormalized = true +canReadRenormalized = false +canStream = true +*/ + +const ( + appID = "/sc/9068489/215_1736969337" + accessKeyID = "OTA2ODQ4OTE1NzEzNDAwNTI4Njc" + encryptedPrivateAccessKey = "JONdXuRLNSRI5ue2Cr-vn-5m_YxyMNq9yHRKUQevqo8uaZjH502Z-x1axhyqOa8cDyldGq08RfFxozo" + minSleep = 10 * time.Millisecond + maxSleep = 2 * time.Second + decayConstant = 2 // bigger for slower decay, exponential + rootURL = "https://api.sugarsync.com" + listChunks = 500 // chunk size to read directory listings + expiryLeeway = 5 * time.Minute // time before the token expires to renew +) + +// withDefault returns value but if value is "" then it returns defaultValue +func withDefault(key, defaultValue string) (value string) { + if value == "" { + value = defaultValue + } + return value +} + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "sugarsync", + Description: "Sugarsync", + NewFs: NewFs, + Config: func(name string, m configmap.Mapper) { + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + log.Fatalf("Failed to read options: %v", err) + } + + if opt.RefreshToken != "" { + fmt.Printf("Already have a token - refresh?\n") + if !config.ConfirmWithConfig(m, "config_refresh_token", true) { + return + } + } + fmt.Printf("Username (email address)> ") + username := config.ReadLine() + password := config.GetPassword("Your Sugarsync password is only required during setup and will not be stored.") + + authRequest := api.AppAuthorization{ + Username: username, + Password: password, + Application: withDefault(opt.AppID, appID), + AccessKeyID: withDefault(opt.AccessKeyID, accessKeyID), + PrivateAccessKey: withDefault(opt.PrivateAccessKey, obscure.MustReveal(encryptedPrivateAccessKey)), + } + + var resp *http.Response + opts := rest.Opts{ + Method: "POST", + Path: "/app-authorization", + } + srv := rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(rootURL) // FIXME + + // FIXME + //err = f.pacer.Call(func() (bool, error) { + resp, err = srv.CallXML(context.Background(), &opts, &authRequest, nil) + // return shouldRetry(resp, err) + //}) + if err != nil { + log.Fatalf("Failed to get token: %v", err) + } + opt.RefreshToken = resp.Header.Get("Location") + m.Set("refresh_token", opt.RefreshToken) + }, + Options: []fs.Option{{ + Name: "app_id", + Help: "Sugarsync App ID.\n\nLeave blank to use rclone's.", + }, { + Name: "access_key_id", + Help: "Sugarsync Access Key ID.\n\nLeave blank to use rclone's.", + }, { + Name: "private_access_key", + Help: "Sugarsync Private Access Key\n\nLeave blank to use rclone's.", + }, { + Name: "hard_delete", + Help: "Permanently delete files if true\notherwise put them in the deleted files.", + Default: false, + }, { + Name: "refresh_token", + Help: "Sugarsync refresh token\n\nLeave blank normally, will be auto configured by rclone.", + Advanced: true, + }, { + Name: "authorization", + Help: "Sugarsync authorization\n\nLeave blank normally, will be auto configured by rclone.", + Advanced: true, + }, { + Name: "authorization_expiry", + Help: "Sugarsync authorization expiry\n\nLeave blank normally, will be auto configured by rclone.", + Advanced: true, + }, { + Name: "user", + Help: "Sugarsync user\n\nLeave blank normally, will be auto configured by rclone.", + Advanced: true, + }, { + Name: "root_id", + Help: "Sugarsync root id\n\nLeave blank normally, will be auto configured by rclone.", + Advanced: true, + }, { + Name: "deleted_id", + Help: "Sugarsync deleted folder id\n\nLeave blank normally, will be auto configured by rclone.", + Advanced: true, + }, { + Name: config.ConfigEncoding, + Help: config.ConfigEncodingHelp, + Advanced: true, + Default: (encoder.Base | + encoder.EncodeCtl | + encoder.EncodeInvalidUtf8), + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + AppID string `config:"app_id"` + AccessKeyID string `config:"access_key_id"` + PrivateAccessKey string `config:"private_access_key"` + HardDelete bool `config:"hard_delete"` + RefreshToken string `config:"refresh_token"` + Authorization string `config:"authorization"` + AuthorizationExpiry string `config:"authorization_expiry"` + User string `config:"user"` + RootID string `config:"root_id"` + DeletedID string `config:"deleted_id"` + Enc encoder.MultiEncoder `config:"encoding"` +} + +// Fs represents a remote sugarsync +type Fs struct { + name string // name of this remote + root string // the path we are working on + opt Options // parsed options + features *fs.Features // optional features + srv *rest.Client // the connection to the one drive server + dirCache *dircache.DirCache // Map of directory path to directory id + pacer *fs.Pacer // pacer for API calls + m configmap.Mapper // config file access + authMu sync.Mutex // used when doing authorization + authExpiry time.Time // time the authorization expires +} + +// Object describes a sugarsync object +// +// Will definitely have info but maybe not meta +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + hasMetaData bool // whether info below has been set + size int64 // size of the object + modTime time.Time // modification time of the object + id string // ID of the object +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("sugarsync root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// parsePath parses an sugarsync 'url' +func parsePath(path string) (root string) { + root = strings.Trim(path, "/") + return +} + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 429, // Too Many Requests. + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout + 509, // Bandwidth Limit Exceeded +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func shouldRetry(resp *http.Response, err error) (bool, error) { + return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +// readMetaDataForPath reads the metadata from the path +func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.File, err error) { + // defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err) + leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false) + if err != nil { + if err == fs.ErrorDirNotFound { + return nil, fs.ErrorObjectNotFound + } + return nil, err + } + + found, err := f.listAll(ctx, directoryID, func(item *api.File) bool { + if item.Name == leaf { + info = item + return true + } + return false + }, nil) + if err != nil { + return nil, err + } + if !found { + return nil, fs.ErrorObjectNotFound + } + return info, nil +} + +// readMetaDataForID reads the metadata for a file from the ID +func (f *Fs) readMetaDataForID(ctx context.Context, ID string) (info *api.File, err error) { + var resp *http.Response + opts := rest.Opts{ + Method: "GET", + RootURL: ID, + } + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(ctx, &opts, nil, &info) + return shouldRetry(resp, err) + }) + if err != nil { + if resp != nil && resp.StatusCode == http.StatusNotFound { + return nil, fs.ErrorObjectNotFound + } + return nil, errors.Wrap(err, "failed to get authorization") + } + return info, nil +} + +// getAuthToken gets an Auth token from the refresh token +func (f *Fs) getAuthToken(ctx context.Context) error { + fs.Debugf(f, "Renewing token") + + var authRequest = api.TokenAuthRequest{ + AccessKeyID: withDefault(f.opt.AccessKeyID, accessKeyID), + PrivateAccessKey: withDefault(f.opt.PrivateAccessKey, obscure.MustReveal(encryptedPrivateAccessKey)), + RefreshToken: f.opt.RefreshToken, + } + + if authRequest.RefreshToken == "" { + return errors.New("no refresh token found - run `rclone config reconnect`") + } + + var authResponse api.Authorization + var err error + var resp *http.Response + opts := rest.Opts{ + Method: "POST", + Path: "/authorization", + ExtraHeaders: map[string]string{ + "Authorization": "", // unset Authorization + }, + } + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(ctx, &opts, &authRequest, &authResponse) + return shouldRetry(resp, err) + }) + if err != nil { + return errors.Wrap(err, "failed to get authorization") + } + f.opt.Authorization = resp.Header.Get("Location") + f.authExpiry = authResponse.Expiration + f.opt.User = authResponse.User + + // Cache the results + f.m.Set("authorization", f.opt.Authorization) + f.m.Set("authorization_expiry", f.authExpiry.Format(time.RFC3339)) + f.m.Set("user", f.opt.User) + return nil +} + +// Read the auth from the config file and refresh it if it is expired, setting it in srv +func (f *Fs) getAuth(req *http.Request) (err error) { + f.authMu.Lock() + defer f.authMu.Unlock() + ctx := req.Context() + + // if have auth, check it is in date + if f.opt.Authorization == "" || f.opt.User == "" || f.authExpiry.IsZero() || time.Until(f.authExpiry) < expiryLeeway { + // Get the auth token + f.srv.SetSigner(nil) // temporariliy remove the signer so we don't infinitely recurse + err = f.getAuthToken(ctx) + f.srv.SetSigner(f.getAuth) // replace signer + if err != nil { + return err + } + } + + // Set Authorization header + req.Header.Set("Authorization", f.opt.Authorization) + + return nil +} + +// Read the user info into f +func (f *Fs) getUser(ctx context.Context) (user *api.User, err error) { + var resp *http.Response + opts := rest.Opts{ + Method: "GET", + Path: "/user", + } + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(ctx, &opts, nil, &user) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, errors.Wrap(err, "failed to get user") + } + return user, nil +} + +// Read the expiry time from a string +func parseExpiry(expiryString string) time.Time { + if expiryString == "" { + return time.Time{} + } + expiry, err := time.Parse(time.RFC3339, expiryString) + if err != nil { + fs.Debugf("sugarsync", "Invalid expiry time %q read from config", expiryString) + return time.Time{} + } + return expiry +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + ctx := context.Background() + + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + root = parsePath(root) + client := fshttp.NewClient(fs.Config) + f := &Fs{ + name: name, + root: root, + opt: *opt, + srv: rest.NewClient(client).SetRoot(rootURL), + pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), + m: m, + authExpiry: parseExpiry(opt.AuthorizationExpiry), + } + f.features = (&fs.Features{ + CaseInsensitive: true, + CanHaveEmptyDirectories: true, + }).Fill(f) + f.srv.SetSigner(f.getAuth) // use signing hook to get the auth + f.srv.SetErrorHandler(errorHandler) + + // Get rootID + if f.opt.RootID == "" { + user, err := f.getUser(ctx) + if err != nil { + return nil, err + } + f.opt.RootID = user.SyncFolders + if strings.HasSuffix(f.opt.RootID, "/contents") { + f.opt.RootID = f.opt.RootID[:len(f.opt.RootID)-9] + } else { + return nil, errors.Errorf("unexpected rootID %q", f.opt.RootID) + } + // Cache the results + f.m.Set("root_id", f.opt.RootID) + f.opt.DeletedID = user.Deleted + f.m.Set("deleted_id", f.opt.DeletedID) + } + f.dirCache = dircache.New(root, f.opt.RootID, f) + + // Find the current root + err = f.dirCache.FindRoot(ctx, false) + if err != nil { + // Assume it is a file + newRoot, remote := dircache.SplitPath(root) + oldDirCache := f.dirCache + f.dirCache = dircache.New(newRoot, f.opt.RootID, f) + f.root = newRoot + resetF := func() { + f.dirCache = oldDirCache + f.root = root + } + // Make new Fs which is the parent + err = f.dirCache.FindRoot(ctx, false) + if err != nil { + // No root so return old f + resetF() + return f, nil + } + _, err := f.newObjectWithInfo(ctx, remote, nil) + if err != nil { + if err == fs.ErrorObjectNotFound { + // File doesn't exist so return old f + resetF() + return f, nil + } + return nil, err + } + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + return f, nil +} + +var findError = regexp.MustCompile(`

(.*?)

`) + +// errorHandler parses errors from the body +// +// Errors seem to be HTML with

containing the error text +//

Can not move sync folder.

+func errorHandler(resp *http.Response) (err error) { + body, err := rest.ReadBody(resp) + if err != nil { + return errors.Wrap(err, "error reading error out of body") + } + match := findError.FindSubmatch(body) + if match == nil || len(match) < 2 || len(match[1]) == 0 { + return errors.Errorf("HTTP error %v (%v) returned body: %q", resp.StatusCode, resp.Status, body) + } + return errors.Errorf("HTTP error %v (%v): %s", resp.StatusCode, resp.Status, match[1]) +} + +// rootSlash returns root with a slash on if it is empty, otherwise empty string +func (f *Fs) rootSlash() string { + if f.root == "" { + return f.root + } + return f.root + "/" +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.File) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + var err error + if info != nil { + // Set info + err = o.setMetaData(info) + } else { + err = o.readMetaData(ctx) // reads info and meta, returning an error + } + if err != nil { + return nil, err + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + return f.newObjectWithInfo(ctx, remote, nil) +} + +// FindLeaf finds a directory of name leaf in the folder with ID pathID +func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { + //fs.Debugf(f, "FindLeaf(%q, %q)", pathID, leaf) + // Find the leaf in pathID + found, err = f.listAll(ctx, pathID, nil, func(item *api.Collection) bool { + if item.Name == leaf { + pathIDOut = item.Ref + return true + } + return false + }) + // fs.Debugf(f, ">FindLeaf %q, %v, %v", pathIDOut, found, err) + return pathIDOut, found, err +} + +// CreateDir makes a directory with pathID as parent and name leaf +func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) { + // fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf) + var resp *http.Response + opts := rest.Opts{ + Method: "POST", + RootURL: pathID, + NoResponse: true, + } + var mkdir interface{} + if pathID == f.opt.RootID { + // folders at the root are syncFolders + mkdir = &api.CreateSyncFolder{ + Name: f.opt.Enc.FromStandardName(leaf), + } + opts.ExtraHeaders = map[string]string{ + "*X-SugarSync-API-Version": "1.5", // non canonical header + } + + } else { + mkdir = &api.CreateFolder{ + Name: f.opt.Enc.FromStandardName(leaf), + } + } + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(ctx, &opts, mkdir, nil) + return shouldRetry(resp, err) + }) + if err != nil { + return "", err + } + newID = resp.Header.Get("Location") + if newID == "" { + // look up ID if not returned (eg for syncFolder) + var found bool + newID, found, err = f.FindLeaf(ctx, pathID, leaf) + if err != nil { + return "", err + } + if !found { + return "", errors.Errorf("couldn't find ID for newly created directory %q", leaf) + } + + } + return newID, nil +} + +// list the objects into the function supplied +// +// Should return true to finish processing +type listAllFileFn func(*api.File) bool + +// list the folders into the function supplied +// +// Should return true to finish processing +type listAllFolderFn func(*api.Collection) bool + +// Lists the directory required calling the user function on each item found +// +// If the user fn ever returns true then it early exits with found = true +func (f *Fs) listAll(ctx context.Context, dirID string, fileFn listAllFileFn, folderFn listAllFolderFn) (found bool, err error) { + opts := rest.Opts{ + Method: "GET", + RootURL: dirID, + Path: "/contents", + Parameters: url.Values{}, + } + opts.Parameters.Set("max", strconv.Itoa(listChunks)) + start := 0 +OUTER: + for { + opts.Parameters.Set("start", strconv.Itoa(start)) + + var result api.CollectionContents + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(ctx, &opts, nil, &result) + return shouldRetry(resp, err) + }) + if err != nil { + return found, errors.Wrap(err, "couldn't list files") + } + if fileFn != nil { + for i := range result.Files { + item := &result.Files[i] + item.Name = f.opt.Enc.ToStandardName(item.Name) + if fileFn(item) { + found = true + break OUTER + } + } + } + if folderFn != nil { + for i := range result.Collections { + item := &result.Collections[i] + item.Name = f.opt.Enc.ToStandardName(item.Name) + if folderFn(item) { + found = true + break OUTER + } + } + } + if !result.HasMore { + break + } + start = result.End + 1 + } + return +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + err = f.dirCache.FindRoot(ctx, false) + if err != nil { + return nil, err + } + directoryID, err := f.dirCache.FindDir(ctx, dir, false) + if err != nil { + return nil, err + } + var iErr error + _, err = f.listAll(ctx, directoryID, + func(info *api.File) bool { + remote := path.Join(dir, info.Name) + o, err := f.newObjectWithInfo(ctx, remote, info) + if err != nil { + iErr = err + return true + } + entries = append(entries, o) + return false + }, + func(info *api.Collection) bool { + remote := path.Join(dir, info.Name) + id := info.Ref + // cache the directory ID for later lookups + f.dirCache.Put(remote, id) + d := fs.NewDir(remote, info.TimeCreated).SetID(id) + entries = append(entries, d) + return false + }) + if err != nil { + return nil, err + } + if iErr != nil { + return nil, iErr + } + return entries, nil +} + +// Creates from the parameters passed in a half finished Object which +// must have setMetaData called on it +// +// Returns the object, leaf, directoryID and error +// +// Used to create new objects +func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { + // Create the directory for the object if it doesn't exist + leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true) + if err != nil { + return + } + // Temporary Object under construction + o = &Object{ + fs: f, + remote: remote, + } + return o, leaf, directoryID, nil +} + +// Put the object +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + existingObj, err := f.newObjectWithInfo(ctx, src.Remote(), nil) + switch err { + case nil: + return existingObj, existingObj.Update(ctx, in, src, options...) + case fs.ErrorObjectNotFound: + // Not found so create it + return f.PutUnchecked(ctx, in, src) + default: + return nil, err + } +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.Put(ctx, in, src, options...) +} + +// PutUnchecked the object into the container +// +// This will produce an error if the object already exists +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + remote := src.Remote() + size := src.Size() + modTime := src.ModTime(ctx) + + o, _, _, err := f.createObject(ctx, remote, modTime, size) + if err != nil { + return nil, err + } + return o, o.Update(ctx, in, src, options...) +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir(ctx context.Context, dir string) error { + err := f.dirCache.FindRoot(ctx, true) + if err != nil { + return err + } + if dir != "" { + _, err = f.dirCache.FindDir(ctx, dir, true) + } + return err +} + +// delete removes an object or directory by ID either putting it +// in the Deleted files or deleting it permanently +func (f *Fs) delete(ctx context.Context, isFile bool, id string, remote string, hardDelete bool) (err error) { + if hardDelete { + opts := rest.Opts{ + Method: "DELETE", + RootURL: id, + NoResponse: true, + } + return f.pacer.Call(func() (bool, error) { + resp, err := f.srv.Call(ctx, &opts) + return shouldRetry(resp, err) + }) + } + // Move file/dir to deleted files if not hard delete + leaf := path.Base(remote) + if isFile { + _, err = f.moveFile(ctx, id, leaf, f.opt.DeletedID) + } else { + err = f.moveDir(ctx, id, leaf, f.opt.DeletedID) + } + return err +} + +// purgeCheck removes the root directory, if check is set then it +// refuses to do so if it has anything in +func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { + root := path.Join(f.root, dir) + if root == "" { + return errors.New("can't purge root directory") + } + dc := f.dirCache + err := dc.FindRoot(ctx, false) + if err != nil { + return err + } + directoryID, err := dc.FindDir(ctx, dir, false) + if err != nil { + return err + } + + if check { + found, err := f.listAll(ctx, directoryID, func(item *api.File) bool { + return true + }, func(item *api.Collection) bool { + return true + }) + if err != nil { + return err + } + if found { + return fs.ErrorDirectoryNotEmpty + } + } + + err = f.delete(ctx, false, directoryID, root, f.opt.HardDelete || check) + if err != nil { + return err + } + + f.dirCache.FlushDir(dir) + return nil +} + +// Rmdir deletes the root folder +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(ctx context.Context, dir string) error { + return f.purgeCheck(ctx, dir, true) +} + +// Precision return the precision of this Fs +func (f *Fs) Precision() time.Duration { + return fs.ModTimeNotSupported +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + err := srcObj.readMetaData(ctx) + if err != nil { + return nil, err + } + + srcPath := srcObj.fs.rootSlash() + srcObj.remote + dstPath := f.rootSlash() + remote + if strings.ToLower(srcPath) == strings.ToLower(dstPath) { + return nil, errors.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath) + } + + // Create temporary object + dstObj, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size) + if err != nil { + return nil, err + } + + // Copy the object + opts := rest.Opts{ + Method: "POST", + RootURL: directoryID, + NoResponse: true, + } + copyFile := api.CopyFile{ + Name: f.opt.Enc.FromStandardName(leaf), + Source: srcObj.id, + } + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(ctx, &opts, ©File, nil) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + dstObj.id = resp.Header.Get("Location") + err = dstObj.readMetaData(ctx) + if err != nil { + return nil, err + } + return dstObj, nil +} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge(ctx context.Context) error { + // Caution: Deleting a folder may orphan objects. It's important + // to remove the contents of the folder before you delete the + // folder. That's because removing a folder using DELETE does not + // remove the objects contained within the folder. If you delete + // a folder without first deleting its contents, the contents may + // be rendered inaccessible. + // + // An alternative to permanently deleting a folder is moving it to the + // Deleted Files folder. A folder (and all its contents) in the + // Deleted Files folder can be recovered. Your app can retrieve the + // link to the user's Deleted Files folder from the element + // in the user resource representation. Your application can then move + // a folder to the Deleted Files folder by issuing an HTTP PUT request + // to the URL that represents the file resource and provide as input, + // XML that specifies in the element the link to the Deleted + // Files folder. + if f.opt.HardDelete { + return fs.ErrorCantPurge + } + return f.purgeCheck(ctx, "", false) +} + +// moveFile moves a file server side +func (f *Fs) moveFile(ctx context.Context, id, leaf, directoryID string) (info *api.File, err error) { + opts := rest.Opts{ + Method: "PUT", + RootURL: id, + } + move := api.MoveFile{ + Name: f.opt.Enc.FromStandardName(leaf), + Parent: directoryID, + } + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(ctx, &opts, &move, &info) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + // The docs say that there is nothing returned but apparently + // there is... however it doesn't have Ref + // + // If ref not set, assume it hasn't changed + if info.Ref == "" { + info.Ref = id + } + return info, nil +} + +// moveDir moves a folder server side +func (f *Fs) moveDir(ctx context.Context, id, leaf, directoryID string) (err error) { + // Move the object + opts := rest.Opts{ + Method: "PUT", + RootURL: id, + NoResponse: true, + } + move := api.MoveFolder{ + Name: f.opt.Enc.FromStandardName(leaf), + Parent: directoryID, + } + var resp *http.Response + return f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(ctx, &opts, &move, nil) + return shouldRetry(resp, err) + }) +} + +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + // Create temporary object + dstObj, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size) + if err != nil { + return nil, err + } + + // Do the move + info, err := f.moveFile(ctx, srcObj.id, leaf, directoryID) + if err != nil { + return nil, err + } + + err = dstObj.setMetaData(info) + if err != nil { + return nil, err + } + return dstObj, nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + srcPath := path.Join(srcFs.root, srcRemote) + dstPath := path.Join(f.root, dstRemote) + + // Refuse to move to or from the root + if srcPath == "" || dstPath == "" { + fs.Debugf(src, "DirMove error: Can't move root") + return errors.New("can't move root directory") + } + + // find the root src directory + err := srcFs.dirCache.FindRoot(ctx, false) + if err != nil { + return err + } + + // find the root dst directory + if dstRemote != "" { + err = f.dirCache.FindRoot(ctx, true) + if err != nil { + return err + } + } else { + if f.dirCache.FoundRoot() { + return fs.ErrorDirExists + } + } + + // Find ID of dst parent, creating subdirs if necessary + var leaf, directoryID string + findPath := dstRemote + if dstRemote == "" { + findPath = f.root + } + leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true) + if err != nil { + return err + } + + // Check destination does not exist + if dstRemote != "" { + _, err = f.dirCache.FindDir(ctx, dstRemote, false) + if err == fs.ErrorDirNotFound { + // OK + } else if err != nil { + return err + } else { + return fs.ErrorDirExists + } + } + + // Find ID of src + srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false) + if err != nil { + return err + } + + // Do the move + err = f.moveDir(ctx, srcID, leaf, directoryID) + if err != nil { + return err + } + srcFs.dirCache.FlushDir(srcRemote) + return nil +} + +// PublicLink adds a "readable by anyone with link" permission on the given file or folder. +func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) { + obj, err := f.NewObject(ctx, remote) + if err != nil { + return "", err + } + o, ok := obj.(*Object) + if !ok { + return "", errors.New("internal error: not an Object") + } + opts := rest.Opts{ + Method: "PUT", + RootURL: o.id, + } + linkFile := api.SetPublicLink{ + PublicLink: api.PublicLink{Enabled: true}, + } + var resp *http.Response + var info *api.File + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(ctx, &opts, &linkFile, &info) + return shouldRetry(resp, err) + }) + if err != nil { + return "", err + } + return info.PublicLink.URL, err +} + +// DirCacheFlush resets the directory cache - used in testing as an +// optional interface +func (f *Fs) DirCacheFlush() { + f.dirCache.ResetRoot() +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.None) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the SHA-1 of an object returning a lowercase hex string +func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { + return "", hash.ErrUnsupported +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + err := o.readMetaData(context.TODO()) + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return 0 + } + return o.size +} + +// setMetaData sets the metadata from info +func (o *Object) setMetaData(info *api.File) (err error) { + o.hasMetaData = true + o.size = info.Size + o.modTime = info.LastModified + if info.Ref != "" { + o.id = info.Ref + } else if o.id == "" { + return errors.New("no ID found in response") + } + return nil +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +func (o *Object) readMetaData(ctx context.Context) (err error) { + if o.hasMetaData { + return nil + } + var info *api.File + if o.id != "" { + info, err = o.fs.readMetaDataForID(ctx, o.id) + } else { + info, err = o.fs.readMetaDataForPath(ctx, o.remote) + } + if err != nil { + return err + } + return o.setMetaData(info) +} + +// ModTime returns the modification time of the object +// +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime(ctx context.Context) time.Time { + err := o.readMetaData(ctx) + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return time.Now() + } + return o.modTime +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { + // Sugarsync doesn't support setting the mod time. + // + // In theory (but not in the docs) you could patch the object, + // however it doesn't work. + return fs.ErrorCantSetModTime +} + +// Storable returns a boolean showing whether this object storable +func (o *Object) Storable() bool { + return true +} + +// Open an object for read +func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { + if o.id == "" { + return nil, errors.New("can't download - no id") + } + fs.FixRangeOption(options, o.size) + var resp *http.Response + opts := rest.Opts{ + Method: "GET", + RootURL: o.id, + Path: "/data", + Options: options, + } + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(ctx, &opts) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + return resp.Body, err +} + +// createFile makes an (empty) file with pathID as parent and name leaf and returns the ID +func (f *Fs) createFile(ctx context.Context, pathID, leaf, mimeType string) (newID string, err error) { + var resp *http.Response + opts := rest.Opts{ + Method: "POST", + RootURL: pathID, + NoResponse: true, + } + mkdir := api.CreateFile{ + Name: f.opt.Enc.FromStandardName(leaf), + MediaType: mimeType, + } + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallXML(ctx, &opts, &mkdir, nil) + return shouldRetry(resp, err) + }) + if err != nil { + return "", err + } + return resp.Header.Get("Location"), nil +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// If existing is set then it updates the object rather than creating a new one +// +// The new object may have been created if an error is returned +func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + size := src.Size() + // modTime := src.ModTime(ctx) + remote := o.Remote() + + // Create the directory for the object if it doesn't exist + leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true) + if err != nil { + return err + } + + // if file doesn't exist, create it + if o.id == "" { + o.id, err = o.fs.createFile(ctx, directoryID, leaf, fs.MimeType(ctx, src)) + if err != nil { + return errors.Wrap(err, "failed to create file") + } + if o.id == "" { + return errors.New("failed to create file: no ID") + } + // if created the file and returning an error then delete the file + defer func() { + if err != nil { + delErr := o.fs.delete(ctx, true, o.id, remote, o.fs.opt.HardDelete) + if delErr != nil { + fs.Errorf(o, "failed to remove failed upload: %v", delErr) + } + } + }() + } + + var resp *http.Response + opts := rest.Opts{ + Method: "PUT", + RootURL: o.id, + Path: "/data", + NoResponse: true, + Body: in, + } + if size >= 0 { + opts.ContentLength = &size + } + err = o.fs.pacer.CallNoRetry(func() (bool, error) { + resp, err = o.fs.srv.Call(ctx, &opts) + return shouldRetry(resp, err) + }) + if err != nil { + return errors.Wrap(err, "failed to upload file") + } + + o.hasMetaData = false + return o.readMetaData(ctx) +} + +// Remove an object +func (o *Object) Remove(ctx context.Context) error { + return o.fs.delete(ctx, true, o.id, o.remote, o.fs.opt.HardDelete) +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + return o.id +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.PutStreamer = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.DirCacheFlusher = (*Fs)(nil) + _ fs.PublicLinker = (*Fs)(nil) + _ fs.Object = (*Object)(nil) + _ fs.IDer = (*Object)(nil) +) diff --git a/backend/sugarsync/sugarsync_internal_test.go b/backend/sugarsync/sugarsync_internal_test.go new file mode 100644 index 000000000..020970e04 --- /dev/null +++ b/backend/sugarsync/sugarsync_internal_test.go @@ -0,0 +1,59 @@ +package sugarsync + +import ( + "bytes" + "io/ioutil" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestErrorHandler(t *testing.T) { + for _, test := range []struct { + name string + body string + code int + status string + want string + }{ + { + name: "empty", + body: "", + code: 500, + status: "internal error", + want: `HTTP error 500 (internal error) returned body: ""`, + }, + { + name: "unknown", + body: "

unknown

", + code: 500, + status: "internal error", + want: `HTTP error 500 (internal error) returned body: "

unknown

"`, + }, + { + name: "blank", + body: "Nothing here

", + code: 500, + status: "internal error", + want: `HTTP error 500 (internal error) returned body: "Nothing here

"`, + }, + { + name: "real", + body: "

an error

\n

Can not move sync folder.

\n

more stuff

", + code: 500, + status: "internal error", + want: `HTTP error 500 (internal error): Can not move sync folder.`, + }, + } { + t.Run(test.name, func(t *testing.T) { + resp := http.Response{ + Body: ioutil.NopCloser(bytes.NewBufferString(test.body)), + StatusCode: test.code, + Status: test.status, + } + got := errorHandler(&resp) + assert.Equal(t, test.want, got.Error()) + }) + } +} diff --git a/backend/sugarsync/sugarsync_test.go b/backend/sugarsync/sugarsync_test.go new file mode 100644 index 000000000..619d9f883 --- /dev/null +++ b/backend/sugarsync/sugarsync_test.go @@ -0,0 +1,17 @@ +// Test Sugarsync filesystem interface +package sugarsync_test + +import ( + "testing" + + "github.com/rclone/rclone/backend/sugarsync" + "github.com/rclone/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestSugarSync:Test", + NilObject: (*sugarsync.Object)(nil), + }) +} diff --git a/bin/make_manual.py b/bin/make_manual.py index a44c3fd05..6c681f3ad 100755 --- a/bin/make_manual.py +++ b/bin/make_manual.py @@ -54,6 +54,7 @@ docs = [ "premiumizeme.md", "putio.md", "sftp.md", + "sugarsync.md", "union.md", "webdav.md", "yandex.md", diff --git a/docs/content/about.md b/docs/content/about.md index 7e2bc3507..75174b4d3 100644 --- a/docs/content/about.md +++ b/docs/content/about.md @@ -51,6 +51,7 @@ Rclone is a command line program to sync files and directories to and from: * {{< provider name="rsync.net" home="https://rsync.net/products/rclone.html" config="/sftp/#rsync-net" >}} * {{< provider name="Scaleway" home="https://www.scaleway.com/object-storage/" config="/s3/#scaleway" >}} * {{< provider name="SFTP" home="https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol" config="/sftp/" >}} +* {{< provider name="SugarSync" home="https://sugarsync.com/" config="/sugarsync/" >}} * {{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" >}} * {{< provider name="WebDAV" home="https://en.wikipedia.org/wiki/WebDAV" config="/webdav/" >}} * {{< provider name="Yandex Disk" home="https://disk.yandex.com/" config="/yandex/" >}} diff --git a/docs/content/docs.md b/docs/content/docs.md index 488fd413a..f2cf1d72e 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -50,6 +50,7 @@ See the following for detailed instructions for * [put.io](/putio/) * [QingStor](/qingstor/) * [SFTP](/sftp/) + * [SugarSync](/sugarsync/) * [Union](/union/) * [WebDAV](/webdav/) * [Yandex Disk](/yandex/) diff --git a/docs/content/overview.md b/docs/content/overview.md index 0b6305152..2691b676b 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -43,6 +43,7 @@ Here is an overview of the major features of each cloud storage system. | put.io | CRC-32 | Yes | No | Yes | R | | QingStor | MD5 | No | No | No | R/W | | SFTP | MD5, SHA1 ‡ | Yes | Depends | No | - | +| SugarSync | - | No | No | No | - | | WebDAV | MD5, SHA1 ††| Yes ††† | Depends | No | - | | Yandex Disk | MD5 | Yes | No | No | R/W | | The local filesystem | All | Yes | Depends | No | - | @@ -340,6 +341,7 @@ operations more efficient. | put.io | Yes | No | Yes | Yes | Yes | No | Yes | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes | Yes | | QingStor | No | Yes | No | No | No | Yes | No | No [#2178](https://github.com/rclone/rclone/issues/2178) | No | No | | SFTP | No | No | Yes | Yes | No | No | Yes | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes | Yes | +| SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | Yes | No | Yes | | WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ‡ | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes | Yes | | Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | Yes | | The local filesystem | Yes | No | Yes | Yes | No | No | Yes | No | Yes | Yes | diff --git a/docs/content/sugarsync.md b/docs/content/sugarsync.md new file mode 100644 index 000000000..360c2487c --- /dev/null +++ b/docs/content/sugarsync.md @@ -0,0 +1,252 @@ +--- +title: "SugarSync" +description: "Rclone docs for SugarSync" +date: "2020-01-17" +--- + + SugarSync +----------------------------------------- + +[SugarSync](https://sugarsync.com) is a cloud service that enables +active synchronization of files across computers and other devices for +file backup, access, syncing, and sharing. + +The initial setup for SugarSync involves getting a token from SugarSync which you +can do with rclone. `rclone config` walks you through it. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value +[snip] +XX / Sugarsync + \ "sugarsync" +[snip] +Storage> sugarsync +** See help for sugarsync backend at: https://rclone.org/sugarsync/ ** + +Sugarsync App ID. +Leave blank to use rclone's. +Enter a string value. Press Enter for the default (""). +app_id> +Sugarsync Access Key ID. +Leave blank to use rclone's. +Enter a string value. Press Enter for the default (""). +access_key_id> +Sugarsync Private Access Key +Leave blank to use rclone's. +Enter a string value. Press Enter for the default (""). +private_access_key> +Permanently delete files if true +otherwise put them in the deleted files. +Enter a boolean value (true or false). Press Enter for the default ("false"). +hard_delete> +Edit advanced config? (y/n) +y) Yes +n) No (default) +y/n> n +Remote config +Username (email address)> nick@craig-wood.com +Your Sugarsync password is only required during setup and will not be stored. +password: +-------------------- +[remote] +type = sugarsync +refresh_token = https://api.sugarsync.com/app-authorization/XXXXXXXXXXXXXXXXXX +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +Note that the config asks for your email and password but doesn't +store them, it only uses them to get the initial token. + +Once configured you can then use `rclone` like this, + +List directories (sync folders) in top level of your SugarSync + + rclone lsd remote: + +List all the files in your SugarSync folder "Test" + + rclone ls remote:Test + +To copy a local directory to an SugarSync folder called backup + + rclone copy /home/source remote:backup + +Paths are specified as `remote:path` + +Paths may be as deep as required, eg `remote:directory/subdirectory`. + +**NB** you can't create files in the top level folder you have to +create a folder, which rclone will create as a "Sync Folder" with +SugarSync. + +### Modified time and hashes ### + +SugarSync does not support modification times or hashes, therefore +syncing will default to `--size-only` checking. Note that using +`--update` will work as rclone can read the time files were uploaded. + +#### Restricted filename characters + +SugarSync replaces the [default restricted characters set](/overview/#restricted-characters) +except for DEL. + +Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8), +as they can't be used in XML strings. + +### Deleting files ### + +Deleted files will be moved to the "Deleted items" folder by default. + +However you can supply the flag `--sugarsync-hard-delete` or set the +config parameter `hard_delete = true` if you would like files to be +deleted straight away. + + + +### Standard Options + +Here are the standard options specific to sugarsync (Sugarsync). + +#### --sugarsync-app-id + +Sugarsync App ID. + +Leave blank to use rclone's. + +- Config: app_id +- Env Var: RCLONE_SUGARSYNC_APP_ID +- Type: string +- Default: "" + +#### --sugarsync-access-key-id + +Sugarsync Access Key ID. + +Leave blank to use rclone's. + +- Config: access_key_id +- Env Var: RCLONE_SUGARSYNC_ACCESS_KEY_ID +- Type: string +- Default: "" + +#### --sugarsync-private-access-key + +Sugarsync Private Access Key + +Leave blank to use rclone's. + +- Config: private_access_key +- Env Var: RCLONE_SUGARSYNC_PRIVATE_ACCESS_KEY +- Type: string +- Default: "" + +#### --sugarsync-hard-delete + +Permanently delete files if true +otherwise put them in the deleted files. + +- Config: hard_delete +- Env Var: RCLONE_SUGARSYNC_HARD_DELETE +- Type: bool +- Default: false + +### Advanced Options + +Here are the advanced options specific to sugarsync (Sugarsync). + +#### --sugarsync-refresh-token + +Sugarsync refresh token + +Leave blank normally, will be auto configured by rclone. + +- Config: refresh_token +- Env Var: RCLONE_SUGARSYNC_REFRESH_TOKEN +- Type: string +- Default: "" + +#### --sugarsync-authorization + +Sugarsync authorization + +Leave blank normally, will be auto configured by rclone. + +- Config: authorization +- Env Var: RCLONE_SUGARSYNC_AUTHORIZATION +- Type: string +- Default: "" + +#### --sugarsync-authorization-expiry + +Sugarsync authorization expiry + +Leave blank normally, will be auto configured by rclone. + +- Config: authorization_expiry +- Env Var: RCLONE_SUGARSYNC_AUTHORIZATION_EXPIRY +- Type: string +- Default: "" + +#### --sugarsync-user + +Sugarsync user + +Leave blank normally, will be auto configured by rclone. + +- Config: user +- Env Var: RCLONE_SUGARSYNC_USER +- Type: string +- Default: "" + +#### --sugarsync-root-id + +Sugarsync root id + +Leave blank normally, will be auto configured by rclone. + +- Config: root_id +- Env Var: RCLONE_SUGARSYNC_ROOT_ID +- Type: string +- Default: "" + +#### --sugarsync-deleted-id + +Sugarsync deleted folder id + +Leave blank normally, will be auto configured by rclone. + +- Config: deleted_id +- Env Var: RCLONE_SUGARSYNC_DELETED_ID +- Type: string +- Default: "" + +#### --sugarsync-encoding + +This sets the encoding for the backend. + +See: the [encoding section in the overview](/overview/#encoding) for more info. + +- Config: encoding +- Env Var: RCLONE_SUGARSYNC_ENCODING +- Type: MultiEncoder +- Default: Slash,Ctl,InvalidUtf8,Dot + + diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index 4000bfe8d..f1796bb8e 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -86,6 +86,7 @@
  • premiumize.me
  • put.io
  • SFTP
    • +
  • SugarSync
  • Union (merge backends)
  • WebDAV
  • Yandex Disk
    • diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index 32f98c374..5cc2bd0a4 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -143,6 +143,11 @@ backends: - backend: "sftp" remote: "TestSftp:" fastlist: false + - backend: "sugarsync" + remote: "TestSugarSync:Test" + fastlist: false + ignore: + - TestIntegration/FsMkdir/FsPutFiles/PublicLink - backend: "swift" remote: "TestSwift:" fastlist: true