rclone/backend/jottacloud/jottacloud.go

package jottacloud

import (
	"bytes"
	"context"
	"crypto/md5"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/rclone/rclone/backend/jottacloud/api"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/accounting"
	"github.com/rclone/rclone/fs/config"
	"github.com/rclone/rclone/fs/config/configmap"
	"github.com/rclone/rclone/fs/config/configstruct"
	"github.com/rclone/rclone/fs/config/obscure"
	"github.com/rclone/rclone/fs/fserrors"
	"github.com/rclone/rclone/fs/fshttp"
	"github.com/rclone/rclone/fs/hash"
	"github.com/rclone/rclone/fs/walk"
	"github.com/rclone/rclone/lib/encoder"
	"github.com/rclone/rclone/lib/oauthutil"
	"github.com/rclone/rclone/lib/pacer"
	"github.com/rclone/rclone/lib/rest"
	"golang.org/x/oauth2"
)

// Globals
const (
	minSleep           = 10 * time.Millisecond
	maxSleep           = 2 * time.Second
	decayConstant      = 2 // bigger for slower decay, exponential
	defaultDevice      = "Jotta"
	defaultMountpoint  = "Archive"
	jfsURL             = "https://jfs.jottacloud.com/jfs/"
	apiURL             = "https://api.jottacloud.com/"
	wwwURL             = "https://www.jottacloud.com/"
	cachePrefix        = "rclone-jcmd5-"
	configDevice       = "device"
	configMountpoint   = "mountpoint"
	configTokenURL     = "tokenURL"
	configClientID     = "client_id"
	configClientSecret = "client_secret"
	configUsername     = "username"
	configVersion      = 1

	defaultTokenURL = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
	defaultClientID = "jottacli"

	legacyTokenURL              = "https://api.jottacloud.com/auth/v1/token"
	legacyRegisterURL           = "https://api.jottacloud.com/auth/v1/register"
	legacyClientID              = "nibfk8biu12ju7hpqomr8b1e40"
	legacyEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
	legacyConfigVersion         = 0

	teliaCloudTokenURL = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/token"
	teliaCloudAuthURL  = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/auth"
	teliaCloudClientID = "desktop"

	tele2CloudTokenURL = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/token"
	tele2CloudAuthURL  = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/auth"
	tele2CloudClientID = "desktop"
)

// Register with Fs
func init() {
	// needs to be done early so we can use oauth during config
	fs.Register(&fs.RegInfo{
		Name:        "jottacloud",
		Description: "Jottacloud",
		NewFs:       NewFs,
		Config:      Config,
		Options: []fs.Option{{
			Name:     "md5_memory_limit",
			Help:     "Files bigger than this will be cached on disk to calculate the MD5 if required.",
			Default:  fs.SizeSuffix(10 * 1024 * 1024),
			Advanced: true,
		}, {
			Name:     "trashed_only",
			Help:     "Only show files that are in the trash.\n\nThis will show trashed files in their original directory structure.",
			Default:  false,
			Advanced: true,
		}, {
			Name:     "hard_delete",
			Help:     "Delete files permanently rather than putting them into the trash.",
			Default:  false,
			Advanced: true,
		}, {
			Name:     "upload_resume_limit",
			Help:     "Files bigger than this can be resumed if the upload fail's.",
			Default:  fs.SizeSuffix(10 * 1024 * 1024),
			Advanced: true,
		}, {
			Name:     "no_versions",
			Help:     "Avoid server side versioning by deleting files and recreating files instead of overwriting them.",
			Default:  false,
			Advanced: true,
		}, {
			Name:     config.ConfigEncoding,
			Help:     config.ConfigEncodingHelp,
			Advanced: true,
			// Encode invalid UTF-8 bytes as xml doesn't handle them properly.
			//
			// Also: '*', '/', ':', '<', '>', '?', '\"', '\x00', '|'
			Default: (encoder.Display |
				encoder.EncodeWin | // :?"*<>|
				encoder.EncodeInvalidUtf8),
		}},
	})
}

// Config runs the backend configuration protocol
func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
	switch config.State {
	case "":
		return fs.ConfigChooseExclusiveFixed("auth_type_done", "config_type", `Select authentication type.`, []fs.OptionExample{{
			Value: "standard",
			Help:  "Standard authentication.\nUse this if you're a normal Jottacloud user.",
		}, {
			Value: "legacy",
			Help:  "Legacy authentication.\nThis is only required for certain whitelabel versions of Jottacloud and not recommended for normal users.",
		}, {
			Value: "telia",
			Help:  "Telia Cloud authentication.\nUse this if you are using Telia Cloud.",
		}, {
			Value: "tele2",
			Help:  "Tele2 Cloud authentication.\nUse this if you are using Tele2 Cloud.",
		}})
	case "auth_type_done":
		// Jump to next state according to config chosen
		return fs.ConfigGoto(config.Result)
	case "standard": // configure a jottacloud backend using the modern JottaCli token based authentication
		m.Set("configVersion", fmt.Sprint(configVersion))
		return fs.ConfigInput("standard_token", "config_login_token", "Personal login token.\nGenerate here: https://www.jottacloud.com/web/secure")
	case "standard_token":
		loginToken := config.Result
		m.Set(configClientID, defaultClientID)
		m.Set(configClientSecret, "")

		srv := rest.NewClient(fshttp.NewClient(ctx))
		token, tokenEndpoint, username, err := doTokenAuth(ctx, srv, loginToken)
		if err != nil {
			return nil, fmt.Errorf("failed to get oauth token: %w", err)
		}
		m.Set(configTokenURL, tokenEndpoint)
		err = oauthutil.PutToken(name, m, &token, true)
		if err != nil {
			return nil, fmt.Errorf("error while saving token: %w", err)
		}
		m.Set(configUsername, username)
		return fs.ConfigGoto("choose_device")
	case "legacy": // configure a jottacloud backend using legacy authentication
		m.Set("configVersion", fmt.Sprint(legacyConfigVersion))
		return fs.ConfigConfirm("legacy_api", false, "config_machine_specific", `Do you want to create a machine specific API key?

Rclone has it's own Jottacloud API KEY which works fine as long as one
only uses rclone on a single machine. When you want to use rclone with
this account on more than one machine it's recommended to create a
machine specific API key. These keys can NOT be shared between
machines.`)
	case "legacy_api":
		srv := rest.NewClient(fshttp.NewClient(ctx))
		if config.Result == "true" {
			deviceRegistration, err := registerDevice(ctx, srv)
			if err != nil {
				return nil, fmt.Errorf("failed to register device: %w", err)
			}
			m.Set(configClientID, deviceRegistration.ClientID)
			m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
			fs.Debugf(nil, "Got clientID %q and clientSecret %q", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
		}
		return fs.ConfigInput("legacy_username", "config_username", "Username (e-mail address)")
	case "legacy_username":
		m.Set(configUsername, config.Result)
		return fs.ConfigPassword("legacy_password", "config_password", "Password (only used in setup, will not be stored)")
	case "legacy_password":
		m.Set("password", config.Result)
		m.Set("auth_code", "")
		return fs.ConfigGoto("legacy_do_auth")
	case "legacy_auth_code":
		authCode := strings.ReplaceAll(config.Result, "-", "") // remove any "-" contained in the code so we have a 6 digit number
		m.Set("auth_code", authCode)
		return fs.ConfigGoto("legacy_do_auth")
	case "legacy_do_auth":
		username, _ := m.Get(configUsername)
		password, _ := m.Get("password")
		password = obscure.MustReveal(password)
		authCode, _ := m.Get("auth_code")

		srv := rest.NewClient(fshttp.NewClient(ctx))
		clientID, ok := m.Get(configClientID)
		if !ok {
			clientID = legacyClientID
		}
		clientSecret, ok := m.Get(configClientSecret)
		if !ok {
			clientSecret = legacyEncryptedClientSecret
		}

		oauthConfig := &oauth2.Config{
			Endpoint: oauth2.Endpoint{
				AuthURL: legacyTokenURL,
			},
			ClientID:     clientID,
			ClientSecret: obscure.MustReveal(clientSecret),
		}
		token, err := doLegacyAuth(ctx, srv, oauthConfig, username, password, authCode)
		if err == errAuthCodeRequired {
			return fs.ConfigInput("legacy_auth_code", "config_auth_code", "Verification Code\nThis account uses 2 factor authentication you will receive a verification code via SMS.")
		}
		m.Set("password", "")
		m.Set("auth_code", "")
		if err != nil {
			return nil, fmt.Errorf("failed to get oauth token: %w", err)
		}
		err = oauthutil.PutToken(name, m, &token, true)
		if err != nil {
			return nil, fmt.Errorf("error while saving token: %w", err)
		}
		return fs.ConfigGoto("choose_device")
	case "telia": // telia cloud config
		m.Set("configVersion", fmt.Sprint(configVersion))
		m.Set(configClientID, teliaCloudClientID)
		m.Set(configTokenURL, teliaCloudTokenURL)
		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
			OAuth2Config: &oauth2.Config{
				Endpoint: oauth2.Endpoint{
					AuthURL:  teliaCloudAuthURL,
					TokenURL: teliaCloudTokenURL,
				},
				ClientID:    teliaCloudClientID,
				Scopes:      []string{"openid", "jotta-default", "offline_access"},
				RedirectURL: oauthutil.RedirectLocalhostURL,
			},
		})
	case "tele2": // tele2 cloud config
		m.Set("configVersion", fmt.Sprint(configVersion))
		m.Set(configClientID, tele2CloudClientID)
		m.Set(configTokenURL, tele2CloudTokenURL)
		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
			OAuth2Config: &oauth2.Config{
				Endpoint: oauth2.Endpoint{
					AuthURL:  tele2CloudAuthURL,
					TokenURL: tele2CloudTokenURL,
				},
				ClientID:    tele2CloudClientID,
				Scopes:      []string{"openid", "jotta-default", "offline_access"},
				RedirectURL: oauthutil.RedirectLocalhostURL,
			},
		})
	case "choose_device":
		return fs.ConfigConfirm("choose_device_query", false, "config_non_standard", `Use a non-standard device/mountpoint?
Choosing no, the default, will let you access the storage used for the archive
section of the official Jottacloud client. If you instead want to access the
sync or the backup section, for example, you must choose yes.`)

	case "choose_device_query":
		if config.Result != "true" {
			m.Set(configDevice, "")
			m.Set(configMountpoint, "")
		}
		username, userOk := m.Get(configUsername)
		if userOk && config.Result != "true" {
			return fs.ConfigGoto("end")
		}
		oAuthClient, _, err := getOAuthClient(ctx, name, m)
		if err != nil {
			return nil, err
		}
		if !userOk {
			apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
			cust, err := getCustomerInfo(ctx, apiSrv)
			if err != nil {
				return nil, err
			}
			username = cust.Username
			m.Set(configUsername, username)
			if config.Result != "true" {
				return fs.ConfigGoto("end")
			}
		}

		jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL)
		acc, err := getDriveInfo(ctx, jfsSrv, username)
		if err != nil {
			return nil, err
		}

		deviceNames := make([]string, len(acc.Devices))
		for i, dev := range acc.Devices {
			if i > 0 && dev.Name == defaultDevice {
				// Insert the special Jotta device as first entry, making it the default choice.
				copy(deviceNames[1:i+1], deviceNames[0:i])
				deviceNames[0] = dev.Name
			} else {
				deviceNames[i] = dev.Name
			}
		}

		help := fmt.Sprintf(`The device to use. In standard setup the built-in %s device is used,
which contains predefined mountpoints for archive, sync etc. All other devices
are treated as backup devices by the official Jottacloud client. You may create
a new by entering a unique name.`, defaultDevice)
		return fs.ConfigChoose("choose_device_result", "config_device", help, len(deviceNames), func(i int) (string, string) {
			return deviceNames[i], ""
		})
	case "choose_device_result":
		device := config.Result

		oAuthClient, _, err := getOAuthClient(ctx, name, m)
		if err != nil {
			return nil, err
		}
		jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL)

		username, _ := m.Get(configUsername)

		acc, err := getDriveInfo(ctx, jfsSrv, username)
		if err != nil {
			return nil, err
		}
		isNew := true
		for _, dev := range acc.Devices {
			if strings.EqualFold(dev.Name, device) { // If device name exists with different casing we prefer the existing (not sure if and how the api handles the opposite)
				device = dev.Name // Prefer same casing as existing, e.g. if user entered "jotta" we use the standard casing "Jotta" instead
				isNew = false
				break
			}
		}
		var dev *api.JottaDevice
		if isNew {
			fs.Debugf(nil, "Creating new device: %s", device)
			dev, err = createDevice(ctx, jfsSrv, path.Join(username, device))
			if err != nil {
				return nil, err
			}
		}
		m.Set(configDevice, device)

		if !isNew {
			dev, err = getDeviceInfo(ctx, jfsSrv, path.Join(username, device))
			if err != nil {
				return nil, err
			}
		}

		var help string
		if device == defaultDevice {
			// With built-in Jotta device the mountpoint choice is exclusive,
			// we do not want to risk any problems by creating new mountpoints on it.
			help = fmt.Sprintf(`The mountpoint to use on the built-in device %s.
The standard setup is to use the %s mountpoint. Most other mountpoints
have very limited support in rclone and should generally be avoided.`, defaultDevice, defaultMountpoint)
			return fs.ConfigChooseExclusive("choose_device_mountpoint", "config_mountpoint", help, len(dev.MountPoints), func(i int) (string, string) {
				return dev.MountPoints[i].Name, ""
			})
		}
		help = fmt.Sprintf(`The mountpoint to use on the non-standard device %s.
You may create a new by entering a unique name.`, device)
		return fs.ConfigChoose("choose_device_mountpoint", "config_mountpoint", help, len(dev.MountPoints), func(i int) (string, string) {
			return dev.MountPoints[i].Name, ""
		})
	case "choose_device_mountpoint":
		mountpoint := config.Result

		oAuthClient, _, err := getOAuthClient(ctx, name, m)
		if err != nil {
			return nil, err
		}
		jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL)

		username, _ := m.Get(configUsername)
		device, _ := m.Get(configDevice)

		dev, err := getDeviceInfo(ctx, jfsSrv, path.Join(username, device))
		if err != nil {
			return nil, err
		}
		isNew := true
		for _, mnt := range dev.MountPoints {
			if strings.EqualFold(mnt.Name, mountpoint) {
				mountpoint = mnt.Name
				isNew = false
				break
			}
		}

		if isNew {
			if device == defaultDevice {
				return nil, fmt.Errorf("custom mountpoints not supported on built-in %s device: %w", defaultDevice, err)
			}
			fs.Debugf(nil, "Creating new mountpoint: %s", mountpoint)
			_, err := createMountPoint(ctx, jfsSrv, path.Join(username, device, mountpoint))
			if err != nil {
				return nil, err
			}
		}
		m.Set(configMountpoint, mountpoint)

		return fs.ConfigGoto("end")
	case "end":
		// All the config flows end up here in case we need to carry on with something
		return nil, nil
	}
	return nil, fmt.Errorf("unknown state %q", config.State)
}

// Options defines the configuration for this backend
type Options struct {
	Device             string               `config:"device"`
	Mountpoint         string               `config:"mountpoint"`
	MD5MemoryThreshold fs.SizeSuffix        `config:"md5_memory_limit"`
	TrashedOnly        bool                 `config:"trashed_only"`
	HardDelete         bool                 `config:"hard_delete"`
	NoVersions         bool                 `config:"no_versions"`
	UploadThreshold    fs.SizeSuffix        `config:"upload_resume_limit"`
	Enc                encoder.MultiEncoder `config:"encoding"`
}

// Fs represents a remote jottacloud
type Fs struct {
	name             string
	root             string
	user             string
	opt              Options
	features         *fs.Features
	fileEndpoint     string
	allocateEndpoint string
	jfsSrv           *rest.Client
	apiSrv           *rest.Client
	pacer            *fs.Pacer
	tokenRenewer     *oauthutil.Renew // renew the token on expiry
}

// Object describes a jottacloud object
//
// Will definitely have info but maybe not meta
type Object struct {
	fs          *Fs
	remote      string
	hasMetaData bool
	size        int64
	modTime     time.Time
	md5         string
	mimeType    string
}

// ------------------------------------------------------------

// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
	return f.name
}

// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
	return f.root
}

// String converts this Fs to a string
func (f *Fs) String() string {
	return fmt.Sprintf("jottacloud root '%s'", f.root)
}

// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
	return f.features
}

// joinPath joins two path/url elements
//
// Does not perform clean on the result like path.Join does,
// which breaks urls by changing prefix "https://" into "https:/".
func joinPath(base string, rel string) string {
	if rel == "" {
		return base
	}
	if strings.HasSuffix(base, "/") {
		return base + strings.TrimPrefix(rel, "/")
	}
	if strings.HasPrefix(rel, "/") {
		return strings.TrimSuffix(base, "/") + rel
	}
	return base + "/" + rel
}

// retryErrorCodes is a slice of error codes that we will retry
var retryErrorCodes = []int{
	429, // Too Many Requests.
	500, // Internal Server Error
	502, // Bad Gateway
	503, // Service Unavailable
	504, // Gateway Timeout
	509, // Bandwidth Limit Exceeded
}

// shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried.  It returns the err as a convenience
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
	if fserrors.ContextError(ctx, &err) {
		return false, err
	}
	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
}

// registerDevice register a new device for use with the jottacloud API
func registerDevice(ctx context.Context, srv *rest.Client) (reg *api.DeviceRegistrationResponse, err error) {
	// random generator to generate random device names
	seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
	randonDeviceNamePartLength := 21
	randomDeviceNamePart := make([]byte, randonDeviceNamePartLength)
	charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
	for i := range randomDeviceNamePart {
		randomDeviceNamePart[i] = charset[seededRand.Intn(len(charset))]
	}
	randomDeviceName := "rclone-" + string(randomDeviceNamePart)
	fs.Debugf(nil, "Trying to register device '%s'", randomDeviceName)

	values := url.Values{}
	values.Set("device_id", randomDeviceName)

	opts := rest.Opts{
		Method:       "POST",
		RootURL:      legacyRegisterURL,
		ContentType:  "application/x-www-form-urlencoded",
		ExtraHeaders: map[string]string{"Authorization": "Bearer c2xrZmpoYWRsZmFramhkc2xma2phaHNkbGZramhhc2xkZmtqaGFzZGxrZmpobGtq"},
		Parameters:   values,
	}

	var deviceRegistration *api.DeviceRegistrationResponse
	_, err = srv.CallJSON(ctx, &opts, nil, &deviceRegistration)
	return deviceRegistration, err
}

var errAuthCodeRequired = errors.New("auth code required")

// doLegacyAuth runs the actual token request for V1 authentication
//
// Call this first with blank authCode. If errAuthCodeRequired is
// returned then call it again with an authCode
func doLegacyAuth(ctx context.Context, srv *rest.Client, oauthConfig *oauth2.Config, username, password, authCode string) (token oauth2.Token, err error) {
	// prepare out token request with username and password
	values := url.Values{}
	values.Set("grant_type", "PASSWORD")
	values.Set("password", password)
	values.Set("username", username)
	values.Set("client_id", oauthConfig.ClientID)
	values.Set("client_secret", oauthConfig.ClientSecret)
	opts := rest.Opts{
		Method:      "POST",
		RootURL:     oauthConfig.Endpoint.AuthURL,
		ContentType: "application/x-www-form-urlencoded",
		Parameters:  values,
	}
	if authCode != "" {
		opts.ExtraHeaders = make(map[string]string)
		opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
	}

	// do the first request
	var jsonToken api.TokenJSON
	resp, err := srv.CallJSON(ctx, &opts, nil, &jsonToken)
	if err != nil && authCode == "" {
		// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
		if resp != nil {
			if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
				return token, errAuthCodeRequired
			}
		}
	}

	token.AccessToken = jsonToken.AccessToken
	token.RefreshToken = jsonToken.RefreshToken
	token.TokenType = jsonToken.TokenType
	token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second)
	return token, err
}

// doTokenAuth runs the actual token request for V2 authentication
func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 string) (token oauth2.Token, tokenEndpoint string, username string, err error) {
	loginTokenBytes, err := base64.RawURLEncoding.DecodeString(loginTokenBase64)
	if err != nil {
		return token, "", "", err
	}

	// decode login token
	var loginToken api.LoginToken
	decoder := json.NewDecoder(bytes.NewReader(loginTokenBytes))
	err = decoder.Decode(&loginToken)
	if err != nil {
		return token, "", "", err
	}

	// retrieve endpoint urls
	opts := rest.Opts{
		Method:  "GET",
		RootURL: loginToken.WellKnownLink,
	}
	var wellKnown api.WellKnown
	_, err = apiSrv.CallJSON(ctx, &opts, nil, &wellKnown)
	if err != nil {
		return token, "", "", err
	}

	// prepare out token request with username and password
	values := url.Values{}
	values.Set("client_id", defaultClientID)
	values.Set("grant_type", "password")
	values.Set("password", loginToken.AuthToken)
	values.Set("scope", "openid offline_access")
	values.Set("username", loginToken.Username)
	values.Encode()
	opts = rest.Opts{
		Method:      "POST",
		RootURL:     wellKnown.TokenEndpoint,
		ContentType: "application/x-www-form-urlencoded",
		Body:        strings.NewReader(values.Encode()),
	}

	// do the first request
	var jsonToken api.TokenJSON
	_, err = apiSrv.CallJSON(ctx, &opts, nil, &jsonToken)
	if err != nil {
		return token, "", "", err
	}

	token.AccessToken = jsonToken.AccessToken
	token.RefreshToken = jsonToken.RefreshToken
	token.TokenType = jsonToken.TokenType
	token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second)
	return token, wellKnown.TokenEndpoint, loginToken.Username, err
}

// getCustomerInfo queries general information about the account
func getCustomerInfo(ctx context.Context, apiSrv *rest.Client) (info *api.CustomerInfo, err error) {
	opts := rest.Opts{
		Method: "GET",
		Path:   "account/v1/customer",
	}

	_, err = apiSrv.CallJSON(ctx, &opts, nil, &info)
	if err != nil {
		return nil, fmt.Errorf("couldn't get customer info: %w", err)
	}

	return info, nil
}

// getDriveInfo queries general information about the account and the available devices and mountpoints.
func getDriveInfo(ctx context.Context, srv *rest.Client, username string) (info *api.DriveInfo, err error) {
	opts := rest.Opts{
		Method: "GET",
		Path:   username,
	}

	_, err = srv.CallXML(ctx, &opts, nil, &info)
	if err != nil {
		return nil, fmt.Errorf("couldn't get drive info: %w", err)
	}

	return info, nil
}

// getDeviceInfo queries Information about a jottacloud device
func getDeviceInfo(ctx context.Context, srv *rest.Client, path string) (info *api.JottaDevice, err error) {
	opts := rest.Opts{
		Method: "GET",
		Path:   urlPathEscape(path),
	}

	_, err = srv.CallXML(ctx, &opts, nil, &info)
	if err != nil {
		return nil, fmt.Errorf("couldn't get device info: %w", err)
	}

	return info, nil
}

// createDevice makes a device
func createDevice(ctx context.Context, srv *rest.Client, path string) (info *api.JottaDevice, err error) {
	opts := rest.Opts{
		Method:     "POST",
		Path:       urlPathEscape(path),
		Parameters: url.Values{},
	}

	opts.Parameters.Set("type", "WORKSTATION")

	_, err = srv.CallXML(ctx, &opts, nil, &info)
	if err != nil {
		return nil, fmt.Errorf("couldn't create device: %w", err)
	}
	return info, nil
}

// createMountPoint makes a mount point
func createMountPoint(ctx context.Context, srv *rest.Client, path string) (info *api.JottaMountPoint, err error) {
	opts := rest.Opts{
		Method: "POST",
		Path:   urlPathEscape(path),
	}

	_, err = srv.CallXML(ctx, &opts, nil, &info)
	if err != nil {
		return nil, fmt.Errorf("couldn't create mountpoint: %w", err)
	}
	return info, nil
}

// setEndpoints generates the API endpoints
func (f *Fs) setEndpoints() {
	if f.opt.Device == "" {
		f.opt.Device = defaultDevice
	}
	if f.opt.Mountpoint == "" {
		f.opt.Mountpoint = defaultMountpoint
	}
	f.fileEndpoint = path.Join(f.user, f.opt.Device, f.opt.Mountpoint)
	f.allocateEndpoint = path.Join("/jfs", f.opt.Device, f.opt.Mountpoint)
}

// readMetaDataForPath reads the metadata from the path
func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.JottaFile, err error) {
	opts := rest.Opts{
		Method: "GET",
		Path:   f.filePath(path),
	}
	var result api.JottaFile
	var resp *http.Response
	err = f.pacer.Call(func() (bool, error) {
		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &result)
		return shouldRetry(ctx, resp, err)
	})

	if apiErr, ok := err.(*api.Error); ok {
		// does not exist
		if apiErr.StatusCode == http.StatusNotFound {
			return nil, fs.ErrorObjectNotFound
		}
	}

	if err != nil {
		return nil, fmt.Errorf("read metadata failed: %w", err)
	}
	if result.XMLName.Local == "folder" {
		return nil, fs.ErrorIsDir
	} else if result.XMLName.Local != "file" {
		return nil, fs.ErrorNotAFile
	}
	return &result, nil
}

// errorHandler parses a non 2xx error response into an error
func errorHandler(resp *http.Response) error {
	// Decode error response
	errResponse := new(api.Error)
	err := rest.DecodeXML(resp, &errResponse)
	if err != nil {
		fs.Debugf(nil, "Couldn't decode error response: %v", err)
	}
	if errResponse.Message == "" {
		errResponse.Message = resp.Status
	}
	if errResponse.StatusCode == 0 {
		errResponse.StatusCode = resp.StatusCode
	}
	return errResponse
}

// Jottacloud wants '+' to be URL encoded even though the RFC states it's not reserved
func urlPathEscape(in string) string {
	return strings.ReplaceAll(rest.URLPathEscape(in), "+", "%2B")
}

// filePathRaw returns an unescaped file path (f.root, file)
// Optionally made absolute by prefixing with "/", typically required when used
// as request parameter instead of the path (which is relative to some root url).
func (f *Fs) filePathRaw(file string, absolute bool) string {
	prefix := ""
	if absolute {
		prefix = "/"
	}
	return path.Join(prefix, f.fileEndpoint, f.opt.Enc.FromStandardPath(path.Join(f.root, file)))
}

// filePath returns an escaped file path (f.root, file)
func (f *Fs) filePath(file string) string {
	return urlPathEscape(f.filePathRaw(file, false))
}

// allocatePathRaw returns an unescaped allocate file path (f.root, file)
// Optionally made absolute by prefixing with "/", typically required when used
// as request parameter instead of the path (which is relative to some root url).
func (f *Fs) allocatePathRaw(file string, absolute bool) string {
	prefix := ""
	if absolute {
		prefix = "/"
	}
	return path.Join(prefix, f.allocateEndpoint, f.opt.Enc.FromStandardPath(path.Join(f.root, file)))
}

// Jottacloud requires the grant_type 'refresh_token' string
// to be uppercase and throws a 400 Bad Request if we use the
// lower case used by the oauth2 module
//
// This filter catches all refresh requests, reads the body,
// changes the case and then sends it on
func grantTypeFilter(req *http.Request) {
	if legacyTokenURL == req.URL.String() {
		// read the entire body
		refreshBody, err := ioutil.ReadAll(req.Body)
		if err != nil {
			return
		}
		_ = req.Body.Close()

		// make the refresh token upper case
		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))

		// set the new ReadCloser (with a dummy Close())
		req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
	}
}

func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuthClient *http.Client, ts *oauthutil.TokenSource, err error) {
	// Check config version
	var ver int
	version, ok := m.Get("configVersion")
	if ok {
		ver, err = strconv.Atoi(version)
		if err != nil {
			return nil, nil, errors.New("failed to parse config version")
		}
		ok = (ver == configVersion) || (ver == legacyConfigVersion)
	}
	if !ok {
		return nil, nil, errors.New("outdated config - please reconfigure this backend")
	}

	baseClient := fshttp.NewClient(ctx)
	oauthConfig := &oauth2.Config{
		Endpoint: oauth2.Endpoint{
			AuthURL:  defaultTokenURL,
			TokenURL: defaultTokenURL,
		},
	}
	if ver == configVersion {
		oauthConfig.ClientID = defaultClientID
		// if custom endpoints are set use them else stick with defaults
		if tokenURL, ok := m.Get(configTokenURL); ok {
			oauthConfig.Endpoint.TokenURL = tokenURL
			// jottacloud is weird. we need to use the tokenURL as authURL
			oauthConfig.Endpoint.AuthURL = tokenURL
		}
	} else if ver == legacyConfigVersion {
		clientID, ok := m.Get(configClientID)
		if !ok {
			clientID = legacyClientID
		}
		clientSecret, ok := m.Get(configClientSecret)
		if !ok {
			clientSecret = legacyEncryptedClientSecret
		}
		oauthConfig.ClientID = clientID
		oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)

		oauthConfig.Endpoint.TokenURL = legacyTokenURL
		oauthConfig.Endpoint.AuthURL = legacyTokenURL

		// add the request filter to fix token refresh
		if do, ok := baseClient.Transport.(interface {
			SetRequestFilter(f func(req *http.Request))
		}); ok {
			do.SetRequestFilter(grantTypeFilter)
		} else {
			fs.Debugf(name+":", "Couldn't add request filter - uploads will fail")
		}
	}

	// Create OAuth Client
	oAuthClient, ts, err = oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to configure Jottacloud oauth client: %w", err)
	}
	return oAuthClient, ts, nil
}

// NewFs constructs an Fs from the path, container:path
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
	// Parse config into Options struct
	opt := new(Options)
	err := configstruct.Set(m, opt)
	if err != nil {
		return nil, err
	}

	oAuthClient, ts, err := getOAuthClient(ctx, name, m)
	if err != nil {
		return nil, err
	}

	rootIsDir := strings.HasSuffix(root, "/")
	root = strings.Trim(root, "/")

	f := &Fs{
		name:   name,
		root:   root,
		opt:    *opt,
		jfsSrv: rest.NewClient(oAuthClient).SetRoot(jfsURL),
		apiSrv: rest.NewClient(oAuthClient).SetRoot(apiURL),
		pacer:  fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
	}
	f.features = (&fs.Features{
		CaseInsensitive:         true,
		CanHaveEmptyDirectories: true,
		ReadMimeType:            true,
		WriteMimeType:           false,
	}).Fill(ctx, f)
	f.jfsSrv.SetErrorHandler(errorHandler)
	if opt.TrashedOnly { // we cannot support showing Trashed Files when using ListR right now
		f.features.ListR = nil
	}

	// Renew the token in the background
	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
		_, err := f.readMetaDataForPath(ctx, "")
		if err == fs.ErrorNotAFile || err == fs.ErrorIsDir {
			err = nil
		}
		return err
	})

	user, userOk := m.Get(configUsername)
	if userOk {
		f.user = user
	} else {
		fs.Infof(nil, "Username not found in config and must be looked up, reconfigure to avoid the extra request")
		cust, err := getCustomerInfo(ctx, f.apiSrv)
		if err != nil {
			return nil, err
		}
		f.user = cust.Username
	}
	f.setEndpoints()

	if root != "" && !rootIsDir {
		// Check to see if the root actually an existing file
		remote := path.Base(root)
		f.root = path.Dir(root)
		if f.root == "." {
			f.root = ""
		}
		_, err := f.NewObject(context.TODO(), remote)
		if err != nil {
			if errors.Is(err, fs.ErrorObjectNotFound) || errors.Is(err, fs.ErrorNotAFile) || errors.Is(err, fs.ErrorIsDir) {
				// File doesn't exist so return old f
				f.root = root
				return f, nil
			}
			return nil, err
		}
		// return an error with an fs which points to the parent
		return f, fs.ErrorIsFile
	}
	return f, nil
}

// Return an Object from a path
//
// If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.JottaFile) (fs.Object, error) {
	o := &Object{
		fs:     f,
		remote: remote,
	}
	var err error
	if info != nil {
		if !f.validFile(info) {
			return nil, fs.ErrorObjectNotFound
		}
		err = o.setMetaData(info) // sets the info
	} else {
		err = o.readMetaData(ctx, false) // reads info and meta, returning an error
	}
	if err != nil {
		return nil, err
	}
	return o, nil
}

// NewObject finds the Object at remote.  If it can't be found
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
	return f.newObjectWithInfo(ctx, remote, nil)
}

// CreateDir makes a directory
func (f *Fs) CreateDir(ctx context.Context, path string) (jf *api.JottaFolder, err error) {
	// fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf)
	var resp *http.Response
	opts := rest.Opts{
		Method:     "POST",
		Path:       f.filePath(path),
		Parameters: url.Values{},
	}

	opts.Parameters.Set("mkDir", "true")

	err = f.pacer.Call(func() (bool, error) {
		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &jf)
		return shouldRetry(ctx, resp, err)
	})
	if err != nil {
		//fmt.Printf("...Error %v\n", err)
		return nil, err
	}
	// fmt.Printf("...Id %q\n", *info.Id)
	return jf, nil
}

// List the objects and directories in dir into entries.  The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
	opts := rest.Opts{
		Method: "GET",
		Path:   f.filePath(dir),
	}

	var resp *http.Response
	var result api.JottaFolder
	err = f.pacer.Call(func() (bool, error) {
		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &result)
		return shouldRetry(ctx, resp, err)
	})

	if err != nil {
		if apiErr, ok := err.(*api.Error); ok {
			// does not exist
			if apiErr.StatusCode == http.StatusNotFound {
				return nil, fs.ErrorDirNotFound
			}
		}
		return nil, fmt.Errorf("couldn't list files: %w", err)
	}

	if !f.validFolder(&result) {
		return nil, fs.ErrorDirNotFound
	}

	for i := range result.Folders {
		item := &result.Folders[i]
		if f.validFolder(item) {
			remote := path.Join(dir, f.opt.Enc.ToStandardName(item.Name))
			d := fs.NewDir(remote, time.Time(item.ModifiedAt))
			entries = append(entries, d)
		}
	}

	for i := range result.Files {
		item := &result.Files[i]
		if f.validFile(item) {
			remote := path.Join(dir, f.opt.Enc.ToStandardName(item.Name))
			if o, err := f.newObjectWithInfo(ctx, remote, item); err == nil {
				entries = append(entries, o)
			}
		}
	}
	return entries, nil
}

func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback func(fs.DirEntry) error) error {

	type stats struct {
		Folders int `xml:"folders"`
		Files   int `xml:"files"`
	}
	var expected, actual stats

	type xmlFile struct {
		Path     string          `xml:"path"`
		Name     string          `xml:"filename"`
		Checksum string          `xml:"md5"`
		Size     int64           `xml:"size"`
		Modified api.Rfc3339Time `xml:"modified"` // Note: Liststream response includes 3 decimal milliseconds, but we ignore them since there is second precision everywhere else
		Created  api.Rfc3339Time `xml:"created"`
	}

	type xmlFolder struct {
		Path string `xml:"path"`
	}

	addFolder := func(path string) error {
		return callback(fs.NewDir(filesystem.opt.Enc.ToStandardPath(path), time.Time{}))
	}

	addFile := func(f *xmlFile) error {
		return callback(&Object{
			hasMetaData: true,
			fs:          filesystem,
			remote:      filesystem.opt.Enc.ToStandardPath(path.Join(f.Path, f.Name)),
			size:        f.Size,
			md5:         f.Checksum,
			modTime:     time.Time(f.Modified),
		})
	}

	// liststream paths are /mountpoint/root/path
	// so the returned paths should have /mountpoint/root/ trimmed
	// as the caller is expecting path.
	pathPrefix := filesystem.opt.Enc.FromStandardPath(path.Join("/", filesystem.opt.Mountpoint, filesystem.root))
	trimPathPrefix := func(p string) string {
		p = strings.TrimPrefix(p, pathPrefix)
		p = strings.TrimPrefix(p, "/")
		return p
	}

	uniqueFolders := map[string]bool{}
	decoder := xml.NewDecoder(r)

	for {
		t, err := decoder.Token()
		if err != nil {
			if err != io.EOF {
				return err
			}
			break
		}
		switch se := t.(type) {
		case xml.StartElement:
			switch se.Name.Local {
			case "file":
				var f xmlFile
				if err := decoder.DecodeElement(&f, &se); err != nil {
					return err
				}
				f.Path = trimPathPrefix(f.Path)
				actual.Files++
				if !uniqueFolders[f.Path] {
					uniqueFolders[f.Path] = true
					actual.Folders++
					if err := addFolder(f.Path); err != nil {
						return err
					}
				}
				if err := addFile(&f); err != nil {
					return err
				}
			case "folder":
				var f xmlFolder
				if err := decoder.DecodeElement(&f, &se); err != nil {
					return err
				}
				f.Path = trimPathPrefix(f.Path)
				uniqueFolders[f.Path] = true
				actual.Folders++
				if err := addFolder(f.Path); err != nil {
					return err
				}
			case "stats":
				if err := decoder.DecodeElement(&expected, &se); err != nil {
					return err
				}
			}
		}
	}

	if expected.Folders != actual.Folders ||
		expected.Files != actual.Files {
		return fmt.Errorf("invalid result from listStream: expected[%#v] != actual[%#v]", expected, actual)
	}
	return nil
}

// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
	opts := rest.Opts{
		Method:     "GET",
		Path:       f.filePath(dir),
		Parameters: url.Values{},
	}
	opts.Parameters.Set("mode", "liststream")
	list := walk.NewListRHelper(callback)

	var resp *http.Response
	err = f.pacer.Call(func() (bool, error) {
		resp, err = f.jfsSrv.Call(ctx, &opts)
		if err != nil {
			return shouldRetry(ctx, resp, err)
		}

		err = parseListRStream(ctx, resp.Body, f, func(d fs.DirEntry) error {
			if d.Remote() == dir {
				return nil
			}
			return list.Add(d)
		})
		_ = resp.Body.Close()
		return shouldRetry(ctx, resp, err)
	})
	if err != nil {
		if apiErr, ok := err.(*api.Error); ok {
			// does not exist
			if apiErr.StatusCode == http.StatusNotFound {
				return fs.ErrorDirNotFound
			}
		}
		return fmt.Errorf("couldn't list files: %w", err)
	}
	if err != nil {
		return err
	}
	return list.Flush()
}

// Creates from the parameters passed in a half finished Object which
// must have setMetaData called on it
//
// Used to create new objects
func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object) {
	// Temporary Object under construction
	o = &Object{
		fs:      f,
		remote:  remote,
		size:    size,
		modTime: modTime,
	}
	return o
}

// Put the object
//
// Copy the reader in to the new object which is returned
//
// The new object may have been created if an error is returned
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
	o := f.createObject(src.Remote(), src.ModTime(ctx), src.Size())
	return o, o.Update(ctx, in, src, options...)
}

// mkParentDir makes the parent of the native path dirPath if
// necessary and any directories above that
func (f *Fs) mkParentDir(ctx context.Context, dirPath string) error {
	// defer log.Trace(dirPath, "")("")
	// chop off trailing / if it exists
	parent := path.Dir(strings.TrimSuffix(dirPath, "/"))
	if parent == "." {
		parent = ""
	}
	return f.Mkdir(ctx, parent)
}

// Mkdir creates the container if it doesn't exist
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
	_, err := f.CreateDir(ctx, dir)
	return err
}

// purgeCheck removes the root directory, if check is set then it
// refuses to do so if it has anything in
func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error) {
	root := path.Join(f.root, dir)
	if root == "" {
		return errors.New("can't purge root directory")
	}

	// check that the directory exists
	entries, err := f.List(ctx, dir)
	if err != nil {
		return err
	}

	if check {
		if len(entries) != 0 {
			return fs.ErrorDirectoryNotEmpty
		}
	}

	opts := rest.Opts{
		Method:     "POST",
		Path:       f.filePath(dir),
		Parameters: url.Values{},
		NoResponse: true,
	}

	if f.opt.HardDelete {
		opts.Parameters.Set("rmDir", "true")
	} else {
		opts.Parameters.Set("dlDir", "true")
	}

	var resp *http.Response
	err = f.pacer.Call(func() (bool, error) {
		resp, err = f.jfsSrv.Call(ctx, &opts)
		return shouldRetry(ctx, resp, err)
	})
	if err != nil {
		return fmt.Errorf("couldn't purge directory: %w", err)
	}

	return nil
}

// Rmdir deletes the root folder
//
// Returns an error if it isn't empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
	return f.purgeCheck(ctx, dir, true)
}

// Precision return the precision of this Fs
func (f *Fs) Precision() time.Duration {
	return time.Second
}

// Purge deletes all the files and the container
func (f *Fs) Purge(ctx context.Context, dir string) error {
	return f.purgeCheck(ctx, dir, false)
}

// createOrUpdate tries to make remote file match without uploading.
// If the remote file exists, and has matching size and md5, only
// timestamps are updated. If the file does not exist or does does
// not match size and md5, but matching content can be constructed
// from deduplication, the file will be updated/created. If the file
// is currently in trash, but can be made to match, it will be
// restored. Returns ErrorObjectNotFound if upload will be necessary
// to get a matching remote file.
func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
	opts := rest.Opts{
		Method:       "POST",
		Path:         f.filePath(file),
		Parameters:   url.Values{},
		ExtraHeaders: make(map[string]string),
	}

	opts.Parameters.Set("cphash", "true")

	fileDate := api.JottaTime(modTime).String()
	opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
	opts.ExtraHeaders["JMd5"] = md5
	opts.ExtraHeaders["JCreated"] = fileDate
	opts.ExtraHeaders["JModified"] = fileDate

	var resp *http.Response
	err = f.pacer.Call(func() (bool, error) {
		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &info)
		return shouldRetry(ctx, resp, err)
	})

	if apiErr, ok := err.(*api.Error); ok {
		// does not exist, i.e. not matching size and md5, and not possible to make it by deduplication
		if apiErr.StatusCode == http.StatusNotFound {
			return nil, fs.ErrorObjectNotFound
		}
	}
	return info, nil
}

// copyOrMoves copies or moves directories or files depending on the method parameter
func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *api.JottaFile, err error) {
	opts := rest.Opts{
		Method:     "POST",
		Path:       src,
		Parameters: url.Values{},
	}

	opts.Parameters.Set(method, f.filePathRaw(dest, true))

	var resp *http.Response
	err = f.pacer.Call(func() (bool, error) {
		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &info)
		return shouldRetry(ctx, resp, err)
	})
	if err != nil {
		return nil, err
	}
	return info, nil
}

// Copy src to this remote using server-side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
	srcObj, ok := src.(*Object)
	if !ok {
		fs.Debugf(src, "Can't copy - not same remote type")
		return nil, fs.ErrorCantMove
	}

	err := f.mkParentDir(ctx, remote)
	if err != nil {
		return nil, err
	}
	info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)

	// if destination was a trashed file then after a successfull copy the copied file is still in trash (bug in api?)
	if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
		fs.Debugf(src, "Server-side copied to trashed destination, restoring")
		info, err = f.createOrUpdate(ctx, remote, srcObj.modTime, srcObj.size, srcObj.md5)
	}

	if err != nil {
		return nil, fmt.Errorf("couldn't copy file: %w", err)
	}

	return f.newObjectWithInfo(ctx, remote, info)
	//return f.newObjectWithInfo(remote, &result)
}

// Move src to this remote using server-side move operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantMove
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
	srcObj, ok := src.(*Object)
	if !ok {
		fs.Debugf(src, "Can't move - not same remote type")
		return nil, fs.ErrorCantMove
	}

	err := f.mkParentDir(ctx, remote)
	if err != nil {
		return nil, err
	}
	info, err := f.copyOrMove(ctx, "mv", srcObj.filePath(), remote)

	if err != nil {
		return nil, fmt.Errorf("couldn't move file: %w", err)
	}

	return f.newObjectWithInfo(ctx, remote, info)
	//return f.newObjectWithInfo(remote, result)
}

// DirMove moves src, srcRemote to this remote at dstRemote
// using server-side move operations.
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantDirMove
//
// If destination exists then return fs.ErrorDirExists
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
	srcFs, ok := src.(*Fs)
	if !ok {
		fs.Debugf(srcFs, "Can't move directory - not same remote type")
		return fs.ErrorCantDirMove
	}
	srcPath := path.Join(srcFs.root, srcRemote)
	dstPath := path.Join(f.root, dstRemote)

	// Refuse to move to or from the root
	if srcPath == "" || dstPath == "" {
		fs.Debugf(src, "DirMove error: Can't move root")
		return errors.New("can't move root directory")
	}
	//fmt.Printf("Move src: %s (FullPath %s), dst: %s (FullPath: %s)\n", srcRemote, srcPath, dstRemote, dstPath)

	var err error
	_, err = f.List(ctx, dstRemote)
	if err == fs.ErrorDirNotFound {
		// OK
	} else if err != nil {
		return err
	} else {
		return fs.ErrorDirExists
	}

	_, err = f.copyOrMove(ctx, "mvDir", path.Join(f.fileEndpoint, f.opt.Enc.FromStandardPath(srcPath))+"/", dstRemote)

	if err != nil {
		return fmt.Errorf("couldn't move directory: %w", err)
	}
	return nil
}

// PublicLink generates a public link to the remote path (usually readable by anyone)
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
	opts := rest.Opts{
		Method:     "GET",
		Path:       f.filePath(remote),
		Parameters: url.Values{},
	}

	if unlink {
		opts.Parameters.Set("mode", "disableShare")
	} else {
		opts.Parameters.Set("mode", "enableShare")
	}

	var resp *http.Response
	var result api.JottaFile
	err = f.pacer.Call(func() (bool, error) {
		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &result)
		return shouldRetry(ctx, resp, err)
	})

	if apiErr, ok := err.(*api.Error); ok {
		// does not exist
		if apiErr.StatusCode == http.StatusNotFound {
			return "", fs.ErrorObjectNotFound
		}
	}
	if err != nil {
		if unlink {
			return "", fmt.Errorf("couldn't remove public link: %w", err)
		}
		return "", fmt.Errorf("couldn't create public link: %w", err)
	}
	if unlink {
		if result.PublicURI != "" {
			return "", fmt.Errorf("couldn't remove public link - %q", result.PublicURI)
		}
		return "", nil
	}
	if result.PublicURI == "" {
		return "", errors.New("couldn't create public link - no uri received")
	}
	if result.PublicSharePath != "" {
		webLink := joinPath(wwwURL, result.PublicSharePath)
		fs.Debugf(nil, "Web link: %s", webLink)
	} else {
		fs.Debugf(nil, "No web link received")
	}
	directLink := joinPath(wwwURL, fmt.Sprintf("opin/io/downloadPublic/%s/%s", f.user, result.PublicURI))
	fs.Debugf(nil, "Direct link: %s", directLink)
	return directLink, nil
}

// About gets quota information
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
	info, err := getDriveInfo(ctx, f.jfsSrv, f.user)
	if err != nil {
		return nil, err
	}

	usage := &fs.Usage{
		Used: fs.NewUsageValue(info.Usage),
	}
	if info.Capacity > 0 {
		usage.Total = fs.NewUsageValue(info.Capacity)
		usage.Free = fs.NewUsageValue(info.Capacity - info.Usage)
	}
	return usage, nil
}

// UserInfo fetches info about the current user
func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err error) {
	cust, err := getCustomerInfo(ctx, f.apiSrv)
	if err != nil {
		return nil, err
	}
	return map[string]string{
		"Username":         cust.Username,
		"Email":            cust.Email,
		"Name":             cust.Name,
		"AccountType":      cust.AccountType,
		"SubscriptionType": cust.SubscriptionType,
	}, nil
}

// CleanUp empties the trash
func (f *Fs) CleanUp(ctx context.Context) error {
	opts := rest.Opts{
		Method: "POST",
		Path:   "files/v1/purge_trash",
	}

	var info api.TrashResponse
	_, err := f.apiSrv.CallJSON(ctx, &opts, nil, &info)
	if err != nil {
		return fmt.Errorf("couldn't empty trash: %w", err)
	}

	return nil
}

// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
	return hash.Set(hash.MD5)
}

// ---------------------------------------------

// Fs returns the parent Fs
func (o *Object) Fs() fs.Info {
	return o.fs
}

// Return a string version
func (o *Object) String() string {
	if o == nil {
		return "<nil>"
	}
	return o.remote
}

// Remote returns the remote path
func (o *Object) Remote() string {
	return o.remote
}

// filePath returns an escaped file path (f.root, remote)
func (o *Object) filePath() string {
	return o.fs.filePath(o.remote)
}

// Hash returns the MD5 of an object returning a lowercase hex string
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
	if t != hash.MD5 {
		return "", hash.ErrUnsupported
	}
	return o.md5, nil
}

// Size returns the size of an object in bytes
func (o *Object) Size() int64 {
	ctx := context.TODO()
	err := o.readMetaData(ctx, false)
	if err != nil {
		fs.Logf(o, "Failed to read metadata: %v", err)
		return 0
	}
	return o.size
}

// MimeType of an Object if known, "" otherwise
func (o *Object) MimeType(ctx context.Context) string {
	return o.mimeType
}

// validFile checks if info indicates file is valid
func (f *Fs) validFile(info *api.JottaFile) bool {
	if info.State != "COMPLETED" {
		return false // File is incomplete or corrupt
	}
	if !info.Deleted {
		return !f.opt.TrashedOnly // Regular file; return false if TrashedOnly, else true
	}
	return f.opt.TrashedOnly // Deleted file; return true if TrashedOnly, else false
}

// validFolder checks if info indicates folder is valid
func (f *Fs) validFolder(info *api.JottaFolder) bool {
	// Returns true if folder is not deleted.
	// If TrashedOnly option then always returns true, because a folder not
	// in trash must be traversed to get to files/subfolders that are.
	return !bool(info.Deleted) || f.opt.TrashedOnly
}

// setMetaData sets the metadata from info
func (o *Object) setMetaData(info *api.JottaFile) (err error) {
	o.hasMetaData = true
	o.size = info.Size
	o.md5 = info.MD5
	o.mimeType = info.MimeType
	o.modTime = time.Time(info.ModifiedAt)
	return nil
}

// readMetaData reads and updates the metadata for an object
func (o *Object) readMetaData(ctx context.Context, force bool) (err error) {
	if o.hasMetaData && !force {
		return nil
	}
	info, err := o.fs.readMetaDataForPath(ctx, o.remote)
	if err != nil {
		return err
	}
	if !o.fs.validFile(info) {
		return fs.ErrorObjectNotFound
	}
	return o.setMetaData(info)
}

// ModTime returns the modification time of the object
//
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
func (o *Object) ModTime(ctx context.Context) time.Time {
	err := o.readMetaData(ctx, false)
	if err != nil {
		fs.Logf(o, "Failed to read metadata: %v", err)
		return time.Now()
	}
	return o.modTime
}

// SetModTime sets the modification time of the local fs object
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
	// make sure metadata is available, we need its current size and md5
	err := o.readMetaData(ctx, false)
	if err != nil {
		fs.Logf(o, "Failed to read metadata: %v", err)
		return err
	}

	// request check/update with existing metadata and new modtime
	// (note that if size/md5 does not match, the file content will
	// also be modified if deduplication is possible, i.e. it is
	// important to use correct/latest values)
	_, err = o.fs.createOrUpdate(ctx, o.remote, modTime, o.size, o.md5)
	if err != nil {
		if err == fs.ErrorObjectNotFound {
			// file was modified (size/md5 changed) between readMetaData and createOrUpdate?
			return errors.New("metadata did not match")
		}
		return err
	}

	// update local metadata
	o.modTime = modTime
	return nil
}

// Storable returns a boolean showing whether this object storable
func (o *Object) Storable() bool {
	return true
}

// Open an object for read
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
	fs.FixRangeOption(options, o.size)
	var resp *http.Response
	opts := rest.Opts{
		Method:     "GET",
		Path:       o.filePath(),
		Parameters: url.Values{},
		Options:    options,
	}

	opts.Parameters.Set("mode", "bin")

	err = o.fs.pacer.Call(func() (bool, error) {
		resp, err = o.fs.jfsSrv.Call(ctx, &opts)
		return shouldRetry(ctx, resp, err)
	})
	if err != nil {
		return nil, err
	}
	return resp.Body, err
}

// Read the md5 of in returning a reader which will read the same contents
//
// The cleanup function should be called when out is finished with
// regardless of whether this function returned an error or not.
func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader, cleanup func(), err error) {
	// we need an MD5
	md5Hasher := md5.New()
	// use the teeReader to write to the local file AND calculate the MD5 while doing so
	teeReader := io.TeeReader(in, md5Hasher)

	// nothing to clean up by default
	cleanup = func() {}

	// don't cache small files on disk to reduce wear of the disk
	if size > threshold {
		var tempFile *os.File

		// create the cache file
		tempFile, err = ioutil.TempFile("", cachePrefix)
		if err != nil {
			return
		}

		_ = os.Remove(tempFile.Name()) // Delete the file - may not work on Windows

		// clean up the file after we are done downloading
		cleanup = func() {
			// the file should normally already be close, but just to make sure
			_ = tempFile.Close()
			_ = os.Remove(tempFile.Name()) // delete the cache file after we are done - may be deleted already
		}

		// copy the ENTIRE file to disc and calculate the MD5 in the process
		if _, err = io.Copy(tempFile, teeReader); err != nil {
			return
		}
		// jump to the start of the local file so we can pass it along
		if _, err = tempFile.Seek(0, 0); err != nil {
			return
		}

		// replace the already read source with a reader of our cached file
		out = tempFile
	} else {
		// that's a small file, just read it into memory
		var inData []byte
		inData, err = ioutil.ReadAll(teeReader)
		if err != nil {
			return
		}

		// set the reader to our read memory block
		out = bytes.NewReader(inData)
	}
	return hex.EncodeToString(md5Hasher.Sum(nil)), out, cleanup, nil
}

// Update the object with the contents of the io.Reader, modTime and size
//
// If existing is set then it updates the object rather than creating a new one
//
// The new object may have been created if an error is returned
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
	if o.fs.opt.NoVersions {
		err := o.readMetaData(ctx, false)
		if err == nil {
			// if the object exists delete it
			err = o.remove(ctx, true)
			if err != nil {
				return fmt.Errorf("failed to remove old object: %w", err)
			}
		}
		// if the object does not exist we can just continue but if the error is something different we should report that
		if err != fs.ErrorObjectNotFound {
			return err
		}
	}
	o.fs.tokenRenewer.Start()
	defer o.fs.tokenRenewer.Stop()
	size := src.Size()
	md5String, err := src.Hash(ctx, hash.MD5)
	if err != nil || md5String == "" {
		// unwrap the accounting from the input, we use wrap to put it
		// back on after the buffering
		var wrap accounting.WrapFn
		in, wrap = accounting.UnWrap(in)
		var cleanup func()
		md5String, in, cleanup, err = readMD5(in, size, int64(o.fs.opt.MD5MemoryThreshold))
		defer cleanup()
		if err != nil {
			return fmt.Errorf("failed to calculate MD5: %w", err)
		}
		// Wrap the accounting back onto the stream
		in = wrap(in)
	}

	// use the api to allocate the file first and get resume / deduplication info
	var resp *http.Response
	opts := rest.Opts{
		Method:       "POST",
		Path:         "files/v1/allocate",
		Options:      options,
		ExtraHeaders: make(map[string]string),
	}
	fileDate := api.Rfc3339Time(src.ModTime(ctx)).String()

	// the allocate request
	var request = api.AllocateFileRequest{
		Bytes:    size,
		Created:  fileDate,
		Modified: fileDate,
		Md5:      md5String,
		Path:     o.fs.allocatePathRaw(o.remote, true),
	}

	// send it
	var response api.AllocateFileResponse
	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
		resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response)
		return shouldRetry(ctx, resp, err)
	})
	if err != nil {
		return err
	}

	// If the file state is INCOMPLETE and CORRUPT, try to upload a then
	if response.State != "COMPLETED" {
		// how much do we still have to upload?
		remainingBytes := size - response.ResumePos
		opts = rest.Opts{
			Method:        "POST",
			RootURL:       response.UploadURL,
			ContentLength: &remainingBytes,
			ContentType:   "application/octet-stream",
			Body:          in,
			ExtraHeaders:  make(map[string]string),
		}
		if response.ResumePos != 0 {
			opts.ExtraHeaders["Range"] = "bytes=" + strconv.FormatInt(response.ResumePos, 10) + "-" + strconv.FormatInt(size-1, 10)
		}

		// copy the already uploaded bytes into the trash :)
		var result api.UploadResponse
		_, err = io.CopyN(ioutil.Discard, in, response.ResumePos)
		if err != nil {
			return err
		}

		// send the remaining bytes
		resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result)
		if err != nil {
			return err
		}

		// finally update the meta data
		o.hasMetaData = true
		o.size = result.Bytes
		o.md5 = result.Md5
		o.modTime = time.Unix(result.Modified/1000, 0)
	} else {
		// If the file state is COMPLETE we don't need to upload it because the file was already found but we still ned to update our metadata
		return o.readMetaData(ctx, true)
	}

	return nil
}

func (o *Object) remove(ctx context.Context, hard bool) error {
	opts := rest.Opts{
		Method:     "POST",
		Path:       o.filePath(),
		Parameters: url.Values{},
		NoResponse: true,
	}

	if hard {
		opts.Parameters.Set("rm", "true")
	} else {
		opts.Parameters.Set("dl", "true")
	}

	return o.fs.pacer.Call(func() (bool, error) {
		resp, err := o.fs.jfsSrv.CallXML(ctx, &opts, nil, nil)
		return shouldRetry(ctx, resp, err)
	})
}

// Remove an object
func (o *Object) Remove(ctx context.Context) error {
	return o.remove(ctx, o.fs.opt.HardDelete)
}

// Check the interfaces are satisfied
var (
	_ fs.Fs           = (*Fs)(nil)
	_ fs.Purger       = (*Fs)(nil)
	_ fs.Copier       = (*Fs)(nil)
	_ fs.Mover        = (*Fs)(nil)
	_ fs.DirMover     = (*Fs)(nil)
	_ fs.ListRer      = (*Fs)(nil)
	_ fs.PublicLinker = (*Fs)(nil)
	_ fs.Abouter      = (*Fs)(nil)
	_ fs.UserInfoer   = (*Fs)(nil)
	_ fs.CleanUpper   = (*Fs)(nil)
	_ fs.Object       = (*Object)(nil)
	_ fs.MimeTyper    = (*Object)(nil)
)