mirror of
https://github.com/rclone/rclone.git
synced 2024-11-22 16:34:30 +01:00
956 lines
27 KiB
Go
956 lines
27 KiB
Go
package operations
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"mime"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"path"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/config"
|
|
"github.com/rclone/rclone/fs/hash"
|
|
"github.com/rclone/rclone/fs/rc"
|
|
"github.com/rclone/rclone/lib/diskusage"
|
|
)
|
|
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "operations/list",
|
|
AuthRequired: true,
|
|
Fn: rcList,
|
|
Title: "List the given remote and path in JSON format",
|
|
Help: `This takes the following parameters:
|
|
|
|
- fs - a remote name string e.g. "drive:"
|
|
- remote - a path within that remote e.g. "dir"
|
|
- opt - a dictionary of options to control the listing (optional)
|
|
- recurse - If set recurse directories
|
|
- noModTime - If set return modification time
|
|
- showEncrypted - If set show decrypted names
|
|
- showOrigIDs - If set show the IDs for each item if known
|
|
- showHash - If set return a dictionary of hashes
|
|
- noMimeType - If set don't show mime types
|
|
- dirsOnly - If set only show directories
|
|
- filesOnly - If set only show files
|
|
- metadata - If set return metadata of objects also
|
|
- hashTypes - array of strings of hash types to show if showHash set
|
|
|
|
Returns:
|
|
|
|
- list
|
|
- This is an array of objects as described in the lsjson command
|
|
|
|
See the [lsjson](/commands/rclone_lsjson/) command for more information on the above and examples.
|
|
`,
|
|
})
|
|
}
|
|
|
|
// List the directory
|
|
func rcList(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
f, remote, err := rc.GetFsAndRemote(ctx, in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var opt ListJSONOpt
|
|
err = in.GetStruct("opt", &opt)
|
|
if rc.NotErrParamNotFound(err) {
|
|
return nil, err
|
|
}
|
|
var list = []*ListJSONItem{}
|
|
err = ListJSON(ctx, f, remote, &opt, func(item *ListJSONItem) error {
|
|
list = append(list, item)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = make(rc.Params)
|
|
out["list"] = list
|
|
return out, nil
|
|
}
|
|
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "operations/stat",
|
|
AuthRequired: true,
|
|
Fn: rcStat,
|
|
Title: "Give information about the supplied file or directory",
|
|
Help: `This takes the following parameters
|
|
|
|
- fs - a remote name string eg "drive:"
|
|
- remote - a path within that remote eg "dir"
|
|
- opt - a dictionary of options to control the listing (optional)
|
|
- see operations/list for the options
|
|
|
|
The result is
|
|
|
|
- item - an object as described in the lsjson command. Will be null if not found.
|
|
|
|
Note that if you are only interested in files then it is much more
|
|
efficient to set the filesOnly flag in the options.
|
|
|
|
See the [lsjson](/commands/rclone_lsjson/) command for more information on the above and examples.
|
|
`,
|
|
})
|
|
}
|
|
|
|
// List the directory
|
|
func rcStat(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
f, remote, err := rc.GetFsAndRemote(ctx, in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var opt ListJSONOpt
|
|
err = in.GetStruct("opt", &opt)
|
|
if rc.NotErrParamNotFound(err) {
|
|
return nil, err
|
|
}
|
|
item, err := StatJSON(ctx, f, remote, &opt)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = make(rc.Params)
|
|
out["item"] = item
|
|
return out, nil
|
|
}
|
|
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "operations/about",
|
|
AuthRequired: true,
|
|
Fn: rcAbout,
|
|
Title: "Return the space used on the remote",
|
|
Help: `This takes the following parameters:
|
|
|
|
- fs - a remote name string e.g. "drive:"
|
|
|
|
The result is as returned from rclone about --json
|
|
|
|
See the [about](/commands/rclone_about/) command for more information on the above.
|
|
`,
|
|
})
|
|
}
|
|
|
|
// About the remote
|
|
func rcAbout(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
f, err := rc.GetFs(ctx, in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
doAbout := f.Features().About
|
|
if doAbout == nil {
|
|
return nil, fmt.Errorf("%v doesn't support about", f)
|
|
}
|
|
u, err := doAbout(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("about call failed: %w", err)
|
|
}
|
|
err = rc.Reshape(&out, u)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("about Reshape failed: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func init() {
|
|
for _, copy := range []bool{false, true} {
|
|
copy := copy
|
|
name := "Move"
|
|
if copy {
|
|
name = "Copy"
|
|
}
|
|
rc.Add(rc.Call{
|
|
Path: "operations/" + strings.ToLower(name) + "file",
|
|
AuthRequired: true,
|
|
Fn: func(ctx context.Context, in rc.Params) (rc.Params, error) {
|
|
return rcMoveOrCopyFile(ctx, in, copy)
|
|
},
|
|
Title: name + " a file from source remote to destination remote",
|
|
Help: `This takes the following parameters:
|
|
|
|
- srcFs - a remote name string e.g. "drive:" for the source, "/" for local filesystem
|
|
- srcRemote - a path within that remote e.g. "file.txt" for the source
|
|
- dstFs - a remote name string e.g. "drive2:" for the destination, "/" for local filesystem
|
|
- dstRemote - a path within that remote e.g. "file2.txt" for the destination
|
|
`,
|
|
})
|
|
}
|
|
}
|
|
|
|
// Copy a file
|
|
func rcMoveOrCopyFile(ctx context.Context, in rc.Params, cp bool) (out rc.Params, err error) {
|
|
srcFs, srcRemote, err := rc.GetFsAndRemoteNamed(ctx, in, "srcFs", "srcRemote")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
dstFs, dstRemote, err := rc.GetFsAndRemoteNamed(ctx, in, "dstFs", "dstRemote")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return nil, moveOrCopyFile(ctx, dstFs, srcFs, dstRemote, srcRemote, cp)
|
|
}
|
|
|
|
func init() {
|
|
for _, op := range []struct {
|
|
name string
|
|
title string
|
|
help string
|
|
noRemote bool
|
|
needsRequest bool
|
|
}{
|
|
{name: "mkdir", title: "Make a destination directory or container"},
|
|
{name: "rmdir", title: "Remove an empty directory or container"},
|
|
{name: "purge", title: "Remove a directory or container and all of its contents"},
|
|
{name: "rmdirs", title: "Remove all the empty directories in the path", help: "- leaveRoot - boolean, set to true not to delete the root\n"},
|
|
{name: "delete", title: "Remove files in the path", noRemote: true},
|
|
{name: "deletefile", title: "Remove the single file pointed to"},
|
|
{name: "copyurl", title: "Copy the URL to the object", help: "- url - string, URL to read from\n - autoFilename - boolean, set to true to retrieve destination file name from url\n"},
|
|
{name: "uploadfile", title: "Upload file using multiform/form-data", help: "- each part in body represents a file to be uploaded\n", needsRequest: true},
|
|
{name: "cleanup", title: "Remove trashed files in the remote or path", noRemote: true},
|
|
{name: "settier", title: "Changes storage tier or class on all files in the path", noRemote: true},
|
|
{name: "settierfile", title: "Changes storage tier or class on the single file pointed to"},
|
|
} {
|
|
op := op
|
|
remote := "- remote - a path within that remote e.g. \"dir\"\n"
|
|
if op.noRemote {
|
|
remote = ""
|
|
}
|
|
rc.Add(rc.Call{
|
|
Path: "operations/" + op.name,
|
|
AuthRequired: true,
|
|
NeedsRequest: op.needsRequest,
|
|
Fn: func(ctx context.Context, in rc.Params) (rc.Params, error) {
|
|
return rcSingleCommand(ctx, in, op.name, op.noRemote)
|
|
},
|
|
Title: op.title,
|
|
Help: `This takes the following parameters:
|
|
|
|
- fs - a remote name string e.g. "drive:"
|
|
` + remote + op.help + `
|
|
See the [` + op.name + `](/commands/rclone_` + op.name + `/) command for more information on the above.
|
|
`,
|
|
})
|
|
}
|
|
}
|
|
|
|
// Run a single command, e.g. Mkdir
|
|
func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bool) (out rc.Params, err error) {
|
|
var (
|
|
f fs.Fs
|
|
remote string
|
|
)
|
|
if noRemote {
|
|
f, err = rc.GetFs(ctx, in)
|
|
} else {
|
|
f, remote, err = rc.GetFsAndRemote(ctx, in)
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
switch name {
|
|
case "mkdir":
|
|
return nil, Mkdir(ctx, f, remote)
|
|
case "rmdir":
|
|
return nil, Rmdir(ctx, f, remote)
|
|
case "purge":
|
|
return nil, Purge(ctx, f, remote)
|
|
case "rmdirs":
|
|
leaveRoot, err := in.GetBool("leaveRoot")
|
|
if rc.NotErrParamNotFound(err) {
|
|
return nil, err
|
|
}
|
|
return nil, Rmdirs(ctx, f, remote, leaveRoot)
|
|
case "delete":
|
|
return nil, Delete(ctx, f)
|
|
case "deletefile":
|
|
o, err := f.NewObject(ctx, remote)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return nil, DeleteFile(ctx, o)
|
|
case "copyurl":
|
|
url, err := in.GetString("url")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
autoFilename, _ := in.GetBool("autoFilename")
|
|
noClobber, _ := in.GetBool("noClobber")
|
|
headerFilename, _ := in.GetBool("headerFilename")
|
|
|
|
_, err = CopyURL(ctx, f, remote, url, autoFilename, headerFilename, noClobber)
|
|
return nil, err
|
|
case "uploadfile":
|
|
|
|
var request *http.Request
|
|
request, err := in.GetHTTPRequest()
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
contentType := request.Header.Get("Content-Type")
|
|
mediaType, params, err := mime.ParseMediaType(contentType)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if strings.HasPrefix(mediaType, "multipart/") {
|
|
mr := multipart.NewReader(request.Body, params["boundary"])
|
|
for {
|
|
p, err := mr.NextPart()
|
|
if err == io.EOF {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if p.FileName() != "" {
|
|
obj, err := Rcat(ctx, f, path.Join(remote, p.FileName()), p, time.Now(), nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
fs.Debugf(obj, "Upload Succeeded")
|
|
}
|
|
}
|
|
}
|
|
return nil, nil
|
|
case "cleanup":
|
|
return nil, CleanUp(ctx, f)
|
|
case "settier":
|
|
if !f.Features().SetTier {
|
|
return nil, fmt.Errorf("remote %s does not support settier", f.Name())
|
|
}
|
|
tier, err := in.GetString("tier")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return nil, SetTier(ctx, f, tier)
|
|
case "settierfile":
|
|
if !f.Features().SetTier {
|
|
return nil, fmt.Errorf("remote %s does not support settier", f.Name())
|
|
}
|
|
tier, err := in.GetString("tier")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
o, err := f.NewObject(ctx, remote)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return nil, SetTierFile(ctx, o, tier)
|
|
}
|
|
panic("unknown rcSingleCommand type")
|
|
}
|
|
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "operations/size",
|
|
AuthRequired: true,
|
|
Fn: rcSize,
|
|
Title: "Count the number of bytes and files in remote",
|
|
Help: `This takes the following parameters:
|
|
|
|
- fs - a remote name string e.g. "drive:path/to/dir"
|
|
|
|
Returns:
|
|
|
|
- count - number of files
|
|
- bytes - number of bytes in those files
|
|
|
|
See the [size](/commands/rclone_size/) command for more information on the above.
|
|
`,
|
|
})
|
|
}
|
|
|
|
// Size a directory
|
|
func rcSize(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
f, err := rc.GetFs(ctx, in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
count, bytes, sizeless, err := Count(ctx, f)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = make(rc.Params)
|
|
out["count"] = count
|
|
out["bytes"] = bytes
|
|
out["sizeless"] = sizeless
|
|
return out, nil
|
|
}
|
|
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "operations/publiclink",
|
|
AuthRequired: true,
|
|
Fn: rcPublicLink,
|
|
Title: "Create or retrieve a public link to the given file or folder.",
|
|
Help: `This takes the following parameters:
|
|
|
|
- fs - a remote name string e.g. "drive:"
|
|
- remote - a path within that remote e.g. "dir"
|
|
- unlink - boolean - if set removes the link rather than adding it (optional)
|
|
- expire - string - the expiry time of the link e.g. "1d" (optional)
|
|
|
|
Returns:
|
|
|
|
- url - URL of the resource
|
|
|
|
See the [link](/commands/rclone_link/) command for more information on the above.
|
|
`,
|
|
})
|
|
}
|
|
|
|
// Make a public link
|
|
func rcPublicLink(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
f, remote, err := rc.GetFsAndRemote(ctx, in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
unlink, _ := in.GetBool("unlink")
|
|
expire, err := in.GetDuration("expire")
|
|
if rc.IsErrParamNotFound(err) {
|
|
expire = time.Duration(fs.DurationOff)
|
|
} else if err != nil {
|
|
return nil, err
|
|
}
|
|
url, err := PublicLink(ctx, f, remote, fs.Duration(expire), unlink)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = make(rc.Params)
|
|
out["url"] = url
|
|
return out, nil
|
|
}
|
|
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "operations/fsinfo",
|
|
Fn: rcFsInfo,
|
|
Title: "Return information about the remote",
|
|
Help: `This takes the following parameters:
|
|
|
|
- fs - a remote name string e.g. "drive:"
|
|
|
|
This returns info about the remote passed in;
|
|
|
|
` + "```" + `
|
|
{
|
|
// optional features and whether they are available or not
|
|
"Features": {
|
|
"About": true,
|
|
"BucketBased": false,
|
|
"BucketBasedRootOK": false,
|
|
"CanHaveEmptyDirectories": true,
|
|
"CaseInsensitive": false,
|
|
"ChangeNotify": false,
|
|
"CleanUp": false,
|
|
"Command": true,
|
|
"Copy": false,
|
|
"DirCacheFlush": false,
|
|
"DirMove": true,
|
|
"Disconnect": false,
|
|
"DuplicateFiles": false,
|
|
"GetTier": false,
|
|
"IsLocal": true,
|
|
"ListR": false,
|
|
"MergeDirs": false,
|
|
"MetadataInfo": true,
|
|
"Move": true,
|
|
"OpenWriterAt": true,
|
|
"PublicLink": false,
|
|
"Purge": true,
|
|
"PutStream": true,
|
|
"PutUnchecked": false,
|
|
"ReadMetadata": true,
|
|
"ReadMimeType": false,
|
|
"ServerSideAcrossConfigs": false,
|
|
"SetTier": false,
|
|
"SetWrapper": false,
|
|
"Shutdown": false,
|
|
"SlowHash": true,
|
|
"SlowModTime": false,
|
|
"UnWrap": false,
|
|
"UserInfo": false,
|
|
"UserMetadata": true,
|
|
"WrapFs": false,
|
|
"WriteMetadata": true,
|
|
"WriteMimeType": false
|
|
},
|
|
// Names of hashes available
|
|
"Hashes": [
|
|
"md5",
|
|
"sha1",
|
|
"whirlpool",
|
|
"crc32",
|
|
"sha256",
|
|
"dropbox",
|
|
"mailru",
|
|
"quickxor"
|
|
],
|
|
"Name": "local", // Name as created
|
|
"Precision": 1, // Precision of timestamps in ns
|
|
"Root": "/", // Path as created
|
|
"String": "Local file system at /", // how the remote will appear in logs
|
|
// Information about the system metadata for this backend
|
|
"MetadataInfo": {
|
|
"System": {
|
|
"atime": {
|
|
"Help": "Time of last access",
|
|
"Type": "RFC 3339",
|
|
"Example": "2006-01-02T15:04:05.999999999Z07:00"
|
|
},
|
|
"btime": {
|
|
"Help": "Time of file birth (creation)",
|
|
"Type": "RFC 3339",
|
|
"Example": "2006-01-02T15:04:05.999999999Z07:00"
|
|
},
|
|
"gid": {
|
|
"Help": "Group ID of owner",
|
|
"Type": "decimal number",
|
|
"Example": "500"
|
|
},
|
|
"mode": {
|
|
"Help": "File type and mode",
|
|
"Type": "octal, unix style",
|
|
"Example": "0100664"
|
|
},
|
|
"mtime": {
|
|
"Help": "Time of last modification",
|
|
"Type": "RFC 3339",
|
|
"Example": "2006-01-02T15:04:05.999999999Z07:00"
|
|
},
|
|
"rdev": {
|
|
"Help": "Device ID (if special file)",
|
|
"Type": "hexadecimal",
|
|
"Example": "1abc"
|
|
},
|
|
"uid": {
|
|
"Help": "User ID of owner",
|
|
"Type": "decimal number",
|
|
"Example": "500"
|
|
}
|
|
},
|
|
"Help": "Textual help string\n"
|
|
}
|
|
}
|
|
` + "```" + `
|
|
|
|
This command does not have a command line equivalent so use this instead:
|
|
|
|
rclone rc --loopback operations/fsinfo fs=remote:
|
|
|
|
`,
|
|
})
|
|
}
|
|
|
|
// Fsinfo the remote
|
|
func rcFsInfo(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
f, err := rc.GetFs(ctx, in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
info := GetFsInfo(f)
|
|
err = rc.Reshape(&out, info)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("fsinfo Reshape failed: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "backend/command",
|
|
AuthRequired: true,
|
|
Fn: rcBackend,
|
|
Title: "Runs a backend command.",
|
|
Help: `This takes the following parameters:
|
|
|
|
- command - a string with the command name
|
|
- fs - a remote name string e.g. "drive:"
|
|
- arg - a list of arguments for the backend command
|
|
- opt - a map of string to string of options
|
|
|
|
Returns:
|
|
|
|
- result - result from the backend command
|
|
|
|
Example:
|
|
|
|
rclone rc backend/command command=noop fs=. -o echo=yes -o blue -a path1 -a path2
|
|
|
|
Returns
|
|
|
|
` + "```" + `
|
|
{
|
|
"result": {
|
|
"arg": [
|
|
"path1",
|
|
"path2"
|
|
],
|
|
"name": "noop",
|
|
"opt": {
|
|
"blue": "",
|
|
"echo": "yes"
|
|
}
|
|
}
|
|
}
|
|
` + "```" + `
|
|
|
|
Note that this is the direct equivalent of using this "backend"
|
|
command:
|
|
|
|
rclone backend noop . -o echo=yes -o blue path1 path2
|
|
|
|
Note that arguments must be preceded by the "-a" flag
|
|
|
|
See the [backend](/commands/rclone_backend/) command for more information.
|
|
`,
|
|
})
|
|
}
|
|
|
|
// Make a public link
|
|
func rcBackend(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
f, err := rc.GetFs(ctx, in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
doCommand := f.Features().Command
|
|
if doCommand == nil {
|
|
return nil, fmt.Errorf("%v: doesn't support backend commands", f)
|
|
}
|
|
command, err := in.GetString("command")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var opt = map[string]string{}
|
|
err = in.GetStructMissingOK("opt", &opt)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var arg = []string{}
|
|
err = in.GetStructMissingOK("arg", &arg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
result, err := doCommand(context.Background(), command, arg, opt)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("command %q failed: %w", command, err)
|
|
|
|
}
|
|
out = make(rc.Params)
|
|
out["result"] = result
|
|
return out, nil
|
|
}
|
|
|
|
// This should really be in fs/rc/internal.go but can't go there due
|
|
// to a circular dependency on config.
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "core/du",
|
|
Fn: rcDu,
|
|
Title: "Returns disk usage of a locally attached disk.",
|
|
Help: `
|
|
This returns the disk usage for the local directory passed in as dir.
|
|
|
|
If the directory is not passed in, it defaults to the directory
|
|
pointed to by --cache-dir.
|
|
|
|
- dir - string (optional)
|
|
|
|
Returns:
|
|
|
|
` + "```" + `
|
|
{
|
|
"dir": "/",
|
|
"info": {
|
|
"Available": 361769115648,
|
|
"Free": 361785892864,
|
|
"Total": 982141468672
|
|
}
|
|
}
|
|
` + "```" + `
|
|
`,
|
|
})
|
|
}
|
|
|
|
// Terminates app
|
|
func rcDu(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
dir, err := in.GetString("dir")
|
|
if rc.IsErrParamNotFound(err) {
|
|
dir = config.GetCacheDir()
|
|
|
|
} else if err != nil {
|
|
return nil, err
|
|
}
|
|
info, err := diskusage.New(dir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = rc.Params{
|
|
"dir": dir,
|
|
"info": info,
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "operations/check",
|
|
AuthRequired: true,
|
|
Fn: rcCheck,
|
|
Title: "check the source and destination are the same",
|
|
Help: `Checks the files in the source and destination match. It compares
|
|
sizes and hashes and logs a report of files that don't
|
|
match. It doesn't alter the source or destination.
|
|
|
|
This takes the following parameters:
|
|
|
|
- srcFs - a remote name string e.g. "drive:" for the source, "/" for local filesystem
|
|
- dstFs - a remote name string e.g. "drive2:" for the destination, "/" for local filesystem
|
|
- download - check by downloading rather than with hash
|
|
- checkFileHash - treat checkFileFs:checkFileRemote as a SUM file with hashes of given type
|
|
- checkFileFs - treat checkFileFs:checkFileRemote as a SUM file with hashes of given type
|
|
- checkFileRemote - treat checkFileFs:checkFileRemote as a SUM file with hashes of given type
|
|
- oneWay - check one way only, source files must exist on remote
|
|
- combined - make a combined report of changes (default false)
|
|
- missingOnSrc - report all files missing from the source (default true)
|
|
- missingOnDst - report all files missing from the destination (default true)
|
|
- match - report all matching files (default false)
|
|
- differ - report all non-matching files (default true)
|
|
- error - report all files with errors (hashing or reading) (default true)
|
|
|
|
If you supply the download flag, it will download the data from
|
|
both remotes and check them against each other on the fly. This can
|
|
be useful for remotes that don't support hashes or if you really want
|
|
to check all the data.
|
|
|
|
If you supply the size-only global flag, it will only compare the sizes not
|
|
the hashes as well. Use this for a quick check.
|
|
|
|
If you supply the checkFileHash option with a valid hash name, the
|
|
checkFileFs:checkFileRemote must point to a text file in the SUM
|
|
format. This treats the checksum file as the source and dstFs as the
|
|
destination. Note that srcFs is not used and should not be supplied in
|
|
this case.
|
|
|
|
Returns:
|
|
|
|
- success - true if no error, false otherwise
|
|
- status - textual summary of check, OK or text string
|
|
- hashType - hash used in check, may be missing
|
|
- combined - array of strings of combined report of changes
|
|
- missingOnSrc - array of strings of all files missing from the source
|
|
- missingOnDst - array of strings of all files missing from the destination
|
|
- match - array of strings of all matching files
|
|
- differ - array of strings of all non-matching files
|
|
- error - array of strings of all files with errors (hashing or reading)
|
|
|
|
`,
|
|
})
|
|
}
|
|
|
|
// Writer which writes into the slice provided
|
|
type stringWriter struct {
|
|
out *[]string
|
|
}
|
|
|
|
// Write writes len(p) bytes from p to the underlying data stream. It returns
|
|
// the number of bytes written from p (0 <= n <= len(p)) and any error
|
|
// encountered that caused the write to stop early. Write must return a non-nil
|
|
// error if it returns n < len(p). Write must not modify the slice data,
|
|
// even temporarily.
|
|
//
|
|
// Implementations must not retain p.
|
|
func (s stringWriter) Write(p []byte) (n int, err error) {
|
|
result := string(p)
|
|
result = strings.TrimSuffix(result, "\n")
|
|
*s.out = append(*s.out, result)
|
|
return len(p), nil
|
|
}
|
|
|
|
// Check two directories
|
|
func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
srcFs, err := rc.GetFsNamed(ctx, in, "srcFs")
|
|
if err != nil && !rc.IsErrParamNotFound(err) {
|
|
return nil, err
|
|
}
|
|
|
|
dstFs, err := rc.GetFsNamed(ctx, in, "dstFs")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
checkFileFs, checkFileRemote, err := rc.GetFsAndRemoteNamed(ctx, in, "checkFileFs", "checkFileRemote")
|
|
if err != nil && !rc.IsErrParamNotFound(err) {
|
|
return nil, err
|
|
}
|
|
|
|
checkFileHash, err := in.GetString("checkFileHash")
|
|
if err != nil && !rc.IsErrParamNotFound(err) {
|
|
return nil, err
|
|
}
|
|
|
|
checkFileSet := 0
|
|
if checkFileHash != "" {
|
|
checkFileSet++
|
|
}
|
|
if checkFileFs != nil {
|
|
checkFileSet++
|
|
}
|
|
if checkFileRemote != "" {
|
|
checkFileSet++
|
|
}
|
|
if checkFileSet > 0 && checkFileSet < 3 {
|
|
return nil, fmt.Errorf("need all of checkFileFs, checkFileRemote, checkFileHash to be set together")
|
|
}
|
|
|
|
var checkFileHashType hash.Type
|
|
if checkFileHash != "" {
|
|
if err := checkFileHashType.Set(checkFileHash); err != nil {
|
|
return nil, err
|
|
}
|
|
if srcFs != nil {
|
|
return nil, rc.NewErrParamInvalid(errors.New("only supply dstFs when using checkFileHash"))
|
|
}
|
|
} else {
|
|
if srcFs == nil {
|
|
return nil, rc.NewErrParamInvalid(errors.New("need srcFs parameter when not using checkFileHash"))
|
|
}
|
|
}
|
|
|
|
oneway, _ := in.GetBool("oneway")
|
|
download, _ := in.GetBool("download")
|
|
|
|
opt := &CheckOpt{
|
|
Fsrc: srcFs,
|
|
Fdst: dstFs,
|
|
OneWay: oneway,
|
|
}
|
|
|
|
out = rc.Params{}
|
|
|
|
getOutput := func(name string, Default bool) io.Writer {
|
|
active, err := in.GetBool(name)
|
|
if err != nil {
|
|
active = Default
|
|
}
|
|
if !active {
|
|
return nil
|
|
}
|
|
result := []string{}
|
|
out[name] = &result
|
|
return stringWriter{&result}
|
|
}
|
|
|
|
opt.Combined = getOutput("combined", false)
|
|
opt.MissingOnSrc = getOutput("missingOnSrc", true)
|
|
opt.MissingOnDst = getOutput("missingOnDst", true)
|
|
opt.Match = getOutput("match", false)
|
|
opt.Differ = getOutput("differ", true)
|
|
opt.Error = getOutput("error", true)
|
|
|
|
if checkFileHash != "" {
|
|
out["hashType"] = checkFileHashType.String()
|
|
err = CheckSum(context.Background(), dstFs, checkFileFs, checkFileRemote, checkFileHashType, opt, download)
|
|
} else {
|
|
if download {
|
|
err = CheckDownload(context.Background(), opt)
|
|
} else {
|
|
out["hashType"] = srcFs.Hashes().Overlap(dstFs.Hashes()).GetOne().String()
|
|
err = Check(context.Background(), opt)
|
|
}
|
|
}
|
|
if err != nil {
|
|
out["status"] = err.Error()
|
|
out["success"] = false
|
|
} else {
|
|
out["status"] = "OK"
|
|
out["success"] = true
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "operations/hashsum",
|
|
AuthRequired: true,
|
|
Fn: rcHashsum,
|
|
Title: "Produces a hashsum file for all the objects in the path.",
|
|
Help: `Produces a hash file for all the objects in the path using the hash
|
|
named. The output is in the same format as the standard
|
|
md5sum/sha1sum tool.
|
|
|
|
This takes the following parameters:
|
|
|
|
- fs - a remote name string e.g. "drive:" for the source, "/" for local filesystem
|
|
- this can point to a file and just that file will be returned in the listing.
|
|
- hashType - type of hash to be used
|
|
- download - check by downloading rather than with hash (boolean)
|
|
- base64 - output the hashes in base64 rather than hex (boolean)
|
|
|
|
If you supply the download flag, it will download the data from the
|
|
remote and create the hash on the fly. This can be useful for remotes
|
|
that don't support the given hash or if you really want to check all
|
|
the data.
|
|
|
|
Note that if you wish to supply a checkfile to check hashes against
|
|
the current files then you should use operations/check instead of
|
|
operations/hashsum.
|
|
|
|
Returns:
|
|
|
|
- hashsum - array of strings of the hashes
|
|
- hashType - type of hash used
|
|
|
|
Example:
|
|
|
|
$ rclone rc --loopback operations/hashsum fs=bin hashType=MD5 download=true base64=true
|
|
{
|
|
"hashType": "md5",
|
|
"hashsum": [
|
|
"WTSVLpuiXyJO_kGzJerRLg== backend-versions.sh",
|
|
"v1b_OlWCJO9LtNq3EIKkNQ== bisect-go-rclone.sh",
|
|
"VHbmHzHh4taXzgag8BAIKQ== bisect-rclone.sh",
|
|
]
|
|
}
|
|
|
|
See the [hashsum](/commands/rclone_hashsum/) command for more information on the above.
|
|
`,
|
|
})
|
|
}
|
|
|
|
// Hashsum a directory
|
|
func rcHashsum(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
ctx, f, err := rc.GetFsNamedFileOK(ctx, in, "fs")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
download, _ := in.GetBool("download")
|
|
base64, _ := in.GetBool("base64")
|
|
hashType, err := in.GetString("hashType")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s\n%w", hash.HelpString(0), err)
|
|
}
|
|
var ht hash.Type
|
|
err = ht.Set(hashType)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s\n%w", hash.HelpString(0), err)
|
|
}
|
|
|
|
hashes := []string{}
|
|
err = HashLister(ctx, ht, base64, download, f, stringWriter{&hashes})
|
|
out = rc.Params{
|
|
"hashType": ht.String(),
|
|
"hashsum": hashes,
|
|
}
|
|
return out, err
|
|
}
|