backend: add new backend command for backend specific commands

These commands are for implementing backend specific
functionality. They have documentation which is placed automatically
into the backend doc.

There is a simple test for the feature in the backend tests.

bin/make_backend_docs.py

@@ -4,6 +4,7 @@ Make backend documentation
 """
 
 import os
+import io
 import subprocess
 
 marker = "<!--- autogenerated options"
@@ -19,6 +20,11 @@ def output_docs(backend, out):
     out.flush()
     subprocess.check_call(["rclone", "help", "backend", backend], stdout=out)
 
+def output_backend_tool_docs(backend, out):
+    """Output documentation for backend tool to out"""
+    out.flush()
+    subprocess.call(["rclone", "backend", "help", backend], stdout=out, stderr=subprocess.DEVNULL)
+    
 def alter_doc(backend):
     """Alter the documentation for backend"""
     doc_file = "docs/content/"+backend+".md"
@@ -35,6 +41,7 @@ def alter_doc(backend):
                     start_full = start + " - DO NOT EDIT, instead edit fs.RegInfo in backend/%s/%s.go then run make backenddocs -->\n" % (backend, backend)
                     out_file.write(start_full)
                     output_docs(backend, out_file)
+                    output_backend_tool_docs(backend, out_file)
                     out_file.write(stop+" -->\n")
                     altered = True
             if not in_docs:

cmd/all/all.go

@@ -6,6 +6,7 @@ import (
 	_ "github.com/rclone/rclone/cmd"
 	_ "github.com/rclone/rclone/cmd/about"
 	_ "github.com/rclone/rclone/cmd/authorize"
+	_ "github.com/rclone/rclone/cmd/backend"
 	_ "github.com/rclone/rclone/cmd/cachestats"
 	_ "github.com/rclone/rclone/cmd/cat"
 	_ "github.com/rclone/rclone/cmd/check"

cmd/backend/backend.go

@@ -0,0 +1,169 @@
+package backend
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"sort"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/cmd/rc"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/spf13/cobra"
+)
+
+var (
+	options []string
+)
+
+func init() {
+	cmd.Root.AddCommand(commandDefinition)
+	cmdFlags := commandDefinition.Flags()
+	flags.StringArrayVarP(cmdFlags, &options, "option", "o", options, "Option in the form name=value or name.")
+}
+
+var commandDefinition = &cobra.Command{
+	Use:   "backend <command> remote:path [opts] <args>",
+	Short: `Run a backend specific command.`,
+	Long: `
+This runs a backend specific command. The commands themselves (except
+for "help" and "features") are defined by the backends and you should
+see the backend docs for definitions.
+
+You can discover what commands a backend implements by using
+
+    rclone backend help remote:
+    rclone backend help <backendname>
+
+You can also discover information about the backend using (see
+[operations/fsinfo](/rc/#operations/fsinfo) in the remote control docs
+for more info).
+
+    rclone backend features remote:
+
+Pass options to the backend command with -o. This should be key=value or key, eg:
+
+    rclone backend stats remote:path stats -o format=json -o long
+
+Pass arguments to the backend by placing them on the end of the line
+
+    rclone backend cleanup remote:path file1 file2 file3
+
+Note to run these commands on a running backend then see
+[backend/command](/rc/#backend/command) in the rc docs.
+`,
+	RunE: func(command *cobra.Command, args []string) error {
+		cmd.CheckArgs(2, 1E6, command, args)
+		name, remote := args[0], args[1]
+		cmd.Run(false, false, command, func() error {
+			// show help if remote is a backend name
+			if name == "help" {
+				fsInfo, err := fs.Find(remote)
+				if err == nil {
+					return showHelp(fsInfo)
+				}
+			}
+			// Create remote
+			fsInfo, configName, fsPath, config, err := fs.ConfigFs(remote)
+			if err != nil {
+				return err
+			}
+			f, err := fsInfo.NewFs(configName, fsPath, config)
+			if err != nil {
+				return err
+			}
+			// Run the command
+			var out interface{}
+			switch name {
+			case "help":
+				return showHelp(fsInfo)
+			case "features":
+				out = operations.GetFsInfo(f)
+			default:
+				doCommand := f.Features().Command
+				if doCommand == nil {
+					return errors.Errorf("%v: doesn't support backend commands", f)
+				}
+				arg := args[2:]
+				opt := rc.ParseOptions(options)
+				out, err = doCommand(context.Background(), name, arg, opt)
+			}
+			if err != nil {
+				return errors.Wrapf(err, "command %q failed", name)
+
+			}
+			// Output the result
+			switch x := out.(type) {
+			case nil:
+			case string:
+				fmt.Println(out)
+			case []string:
+				for line := range x {
+					fmt.Println(line)
+				}
+			default:
+				// Write indented JSON to the output
+				enc := json.NewEncoder(os.Stdout)
+				enc.SetIndent("", "\t")
+				err = enc.Encode(out)
+				if err != nil {
+					return errors.Wrap(err, "failed to write JSON")
+				}
+			}
+			return nil
+		})
+		return nil
+	},
+}
+
+// show help for a backend
+func showHelp(fsInfo *fs.RegInfo) error {
+	cmds := fsInfo.CommandHelp
+	name := fsInfo.Name
+	if len(cmds) == 0 {
+		return errors.Errorf("%s backend has no commands", name)
+	}
+	fmt.Printf("### Backend commands\n\n")
+	fmt.Printf(`Here are the commands specific to the %s backend.
+
+Run them with with
+
+    rclone backend COMMAND remote:
+
+The help below will explain what arguments each command takes.
+
+See [the "rclone backend" command](/commands/rclone_backend/) for more
+info on how to pass options and arguments.
+
+These can be run on a running backend using the rc command
+[backend/command](/rc/#backend/command).
+
+`, name)
+	for _, cmd := range cmds {
+		fmt.Printf("#### %s\n\n", cmd.Name)
+		fmt.Printf("%s\n\n", cmd.Short)
+		fmt.Printf("    rclone backend %s remote: [options] [<arguments>+]\n\n", cmd.Name)
+		if cmd.Long != "" {
+			fmt.Printf("%s\n\n", cmd.Long)
+		}
+		if len(cmd.Opts) != 0 {
+			fmt.Printf("Options:\n\n")
+
+			ks := []string{}
+			for k := range cmd.Opts {
+				ks = append(ks, k)
+			}
+			sort.Strings(ks)
+			for _, k := range ks {
+				v := cmd.Opts[k]
+				fmt.Printf("- %q: %s\n", k, v)
+			}
+			fmt.Printf("\n")
+		}
+	}
+	return nil
+}

fs/fs.go

@@ -70,6 +70,7 @@ var (
 	ErrorPermissionDenied            = errors.New("permission denied")
 	ErrorCantShareDirectories        = errors.New("this backend can't share directories with link")
 	ErrorNotImplemented              = errors.New("optional feature not implemented")
+	ErrorCommandNotFound             = errors.New("command not found")
 )
 
 // RegInfo provides information about a filesystem
@@ -88,6 +89,8 @@ type RegInfo struct {
 	Config func(name string, config configmap.Mapper) `json:"-"`
 	// Options for the Fs configuration
 	Options Options
+	// The command help, if any
+	CommandHelp []CommandHelp
 }
 
 // FileName returns the on disk file name for this backend
@@ -634,6 +637,17 @@ type Features struct {
 
 	// Disconnect the current user
 	Disconnect func(ctx context.Context) error
+
+	// Command the backend to run a named command
+	//
+	// The command run is name
+	// args may be used to read arguments from
+	// opts may be used to read optional arguments from
+	//
+	// The result should be capable of being JSON encoded
+	// If it is a string or a []string it will be shown to the user
+	// otherwise it will be JSON encoded and shown to the user like that
+	Command func(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error)
 }
 
 // Disable nil's out the named feature.  If it isn't found then it
@@ -755,6 +769,9 @@ func (ft *Features) Fill(f Fs) *Features {
 	if do, ok := f.(Disconnecter); ok {
 		ft.Disconnect = do.Disconnect
 	}
+	if do, ok := f.(Commander); ok {
+		ft.Command = do.Command
+	}
 	return ft.DisableList(Config.DisableFeatures)
 }
 
@@ -830,6 +847,7 @@ func (ft *Features) Mask(f Fs) *Features {
 	if mask.Disconnect == nil {
 		ft.Disconnect = nil
 	}
+	// Command is always local so we don't mask it
 	return ft.DisableList(Config.DisableFeatures)
 }
 
@@ -1051,6 +1069,30 @@ type Disconnecter interface {
 	Disconnect(ctx context.Context) error
 }
 
+// CommandHelp describes a single backend Command
+//
+// These are automatically inserted in the docs
+type CommandHelp struct {
+	Name  string            // Name of the command, eg "link"
+	Short string            // Single line description
+	Long  string            // Long multi-line description
+	Opts  map[string]string // maps option name to a single line help
+}
+
+// Commander is an iterface to wrap the Command function
+type Commander interface {
+	// Command the backend to run a named command
+	//
+	// The command run is name
+	// args may be used to read arguments from
+	// opts may be used to read optional arguments from
+	//
+	// The result should be capable of being JSON encoded
+	// If it is a string or a []string it will be shown to the user
+	// otherwise it will be JSON encoded and shown to the user like that
+	Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error)
+}
+
 // ObjectsChan is a channel of Objects
 type ObjectsChan chan Object
 

fstest/fstests/fstests.go

@@ -292,9 +292,10 @@ func Run(t *testing.T, opt *Opt) {
 			ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
 			Path:    `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`,
 		}
-		isLocalRemote bool
+		isLocalRemote        bool
-		purged        bool // whether the dir has been purged or not
+		purged               bool // whether the dir has been purged or not
-		ctx           = context.Background()
+		ctx                  = context.Background()
+		unwrappableFsMethods = []string{"Command"} // these Fs methods don't need to be wrapped ever
 	)
 
 	if strings.HasSuffix(os.Getenv("RCLONE_CONFIG"), "/notfound") && *fstest.RemoteName == "" {
@@ -398,6 +399,9 @@ func Run(t *testing.T, opt *Opt) {
 			if stringsContains(vName, opt.UnimplementableFsMethods) {
 				continue
 			}
+			if stringsContains(vName, unwrappableFsMethods) {
+				continue
+			}
 			field := v.Field(i)
 			// skip the bools
 			if field.Type().Kind() == reflect.Bool {
@@ -409,6 +413,22 @@ func Run(t *testing.T, opt *Opt) {
 		}
 	})
 
+	// Check to see if Fs advertises commands and they work and have docs
+	t.Run("FsCommand", func(t *testing.T) {
+		skipIfNotOk(t)
+		doCommand := remote.Features().Command
+		if doCommand == nil {
+			t.Skip("No commands in this remote")
+		}
+		// Check the correct error is generated
+		_, err := doCommand(context.Background(), "NOTFOUND", nil, nil)
+		assert.Equal(t, fs.ErrorCommandNotFound, err, "Incorrect error generated on command not found")
+		// Check there are some commands in the fsInfo
+		fsInfo, _, _, _, err := fs.ConfigFs(remoteName)
+		require.NoError(t, err)
+		assert.True(t, len(fsInfo.CommandHelp) > 0, "Command is declared, must return some help in CommandHelp")
+	})
+
 	// TestFsRmdirNotFound tests deleting a non existent directory
 	t.Run("FsRmdirNotFound", func(t *testing.T) {
 		skipIfNotOk(t)