cmd: make auto completion work for all shells and reduce the size

This updates the bash completion to work with GenBashCompletionV2
which cuts down the size of the completion file dramatically.

See: https://forum.rclone.org/t/request-make-remote-path-completion-work-for-fish-and-zsh/42982/
See: #7000

cmd/completion.go

@@ -0,0 +1,172 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/cache"
+	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/fspath"
+	"github.com/spf13/cobra"
+)
+
+// Make a debug message while doing the completion.
+//
+// These end up in the file specified by BASH_COMP_DEBUG_FILE
+func compLogf(format string, a ...any) {
+	cobra.CompDebugln(fmt.Sprintf(format, a...), true)
+}
+
+// Add remotes to the completions being built up
+func addRemotes(toComplete string, completions []string) []string {
+	remotes := config.FileSections()
+	for _, remote := range remotes {
+		remote += ":"
+		if strings.HasPrefix(remote, toComplete) {
+			completions = append(completions, remote)
+		}
+	}
+	return completions
+}
+
+// Add local files to the completions being built up
+func addLocalFiles(toComplete string, result cobra.ShellCompDirective, completions []string) (cobra.ShellCompDirective, []string) {
+	path := filepath.Clean(toComplete)
+	dir, file := filepath.Split(path)
+	if dir == "" {
+		dir = "."
+	}
+	if len(dir) > 0 && dir[0] != filepath.Separator && dir[0] != '/' {
+		dir = strings.TrimRight(dir, string(filepath.Separator))
+		dir = strings.TrimRight(dir, "/")
+	}
+	fi, err := os.Stat(toComplete)
+	if err == nil {
+		if fi.IsDir() {
+			dir = toComplete
+			file = ""
+		}
+	}
+	fis, err := os.ReadDir(dir)
+	if err != nil {
+		compLogf("Failed to read directory %q: %v", dir, err)
+		return result, completions
+	}
+	for _, fi := range fis {
+		name := fi.Name()
+		if strings.HasPrefix(name, file) {
+			path := filepath.Join(dir, name)
+			if fi.IsDir() {
+				path += string(filepath.Separator)
+				result |= cobra.ShellCompDirectiveNoSpace
+			}
+			completions = append(completions, path)
+		}
+	}
+	return result, completions
+}
+
+// Add remote files to the completions being built up
+func addRemoteFiles(toComplete string, result cobra.ShellCompDirective, completions []string) (cobra.ShellCompDirective, []string) {
+	ctx := context.Background()
+	parent, _, err := fspath.Split(toComplete)
+	if err != nil {
+		compLogf("Failed to split path %q: %v", toComplete, err)
+		return result, completions
+	}
+	f, err := cache.Get(ctx, parent)
+	if err == fs.ErrorIsFile {
+		completions = append(completions, toComplete)
+		return result, completions
+	} else if err != nil {
+		compLogf("Failed to make Fs %q: %v", parent, err)
+		return result, completions
+	}
+	fis, err := f.List(ctx, "")
+	if err != nil {
+		compLogf("Failed to list Fs %q: %v", parent, err)
+		return result, completions
+	}
+	for _, fi := range fis {
+		remote := fi.Remote()
+		path := parent + remote
+		if strings.HasPrefix(path, toComplete) {
+			if _, ok := fi.(fs.Directory); ok {
+				path += "/"
+				result |= cobra.ShellCompDirectiveNoSpace
+			}
+			completions = append(completions, path)
+		}
+	}
+	return result, completions
+}
+
+// Workaround doesn't seem to be needed for BashCompletionV2
+const useColonWorkaround = false
+
+// do command completion
+//
+// This is called by the command completion scripts using a hidden __complete or __completeNoDesc commands.
+func validArgs(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	compLogf("ValidArgsFunction called with args=%q toComplete=%q", args, toComplete)
+
+	fixBug := -1
+	if useColonWorkaround {
+		// Work around what I think is a bug in cobra's bash
+		// completion which seems to be splitting the arguments on :
+		// Or there is something I don't understand - ncw
+		args = append(args, toComplete)
+		colonArg := -1
+		for i, arg := range args {
+			if arg == ":" {
+				colonArg = i
+			}
+		}
+		if colonArg > 0 {
+			newToComplete := strings.Join(args[colonArg-1:], "")
+			fixBug = len(newToComplete) - len(toComplete)
+			toComplete = newToComplete
+		}
+		compLogf("...shuffled args=%q toComplete=%q", args, toComplete)
+	}
+
+	result := cobra.ShellCompDirectiveDefault
+	completions := []string{}
+
+	// See whether we have a valid remote yet
+	_, err := fspath.Parse(toComplete)
+	parseOK := err == nil
+	hasColon := strings.ContainsRune(toComplete, ':')
+	validRemote := parseOK && hasColon
+	compLogf("valid remote = %v", validRemote)
+
+	// Add remotes for completion
+	if !validRemote {
+		completions = addRemotes(toComplete, completions)
+	}
+
+	// Add local files for completion
+	if !validRemote {
+		result, completions = addLocalFiles(toComplete, result, completions)
+	}
+
+	// Add remote files for completion
+	if validRemote {
+		result, completions = addRemoteFiles(toComplete, result, completions)
+	}
+
+	// If using bug workaround, adjust completions to start with :
+	if useColonWorkaround && fixBug >= 0 {
+		for i := range completions {
+			if len(completions[i]) >= fixBug {
+				completions[i] = completions[i][fixBug:]
+			}
+		}
+	}
+
+	return completions, result
+}

cmd/genautocomplete/genautocomplete_bash.go

@@ -38,7 +38,7 @@ If output_file is "-", then the output will be written to stdout.
 		out := "/etc/bash_completion.d/rclone"
 		if len(args) > 0 {
 			if args[0] == "-" {
-				err := cmd.Root.GenBashCompletion(os.Stdout)
+				err := cmd.Root.GenBashCompletionV2(os.Stdout, false)
 				if err != nil {
 					log.Fatal(err)
 				}
@@ -46,7 +46,7 @@ If output_file is "-", then the output will be written to stdout.
 			}
 			out = args[0]
 		}
-		err := cmd.Root.GenBashCompletionFile(out)
+		err := cmd.Root.GenBashCompletionFileV2(out, false)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/help.go

@@ -38,58 +38,10 @@ documentation, changelog and configuration walkthroughs.
 		fs.Debugf("rclone", "Version %q finishing with parameters %q", fs.Version, os.Args)
 		atexit.Run()
 	},
-	BashCompletionFunction: bashCompletionFunc,
+	ValidArgsFunction: validArgs,
 	DisableAutoGenTag: true,
 }
 
-const (
-	bashCompletionFunc = `
-__rclone_custom_func() {
-    if [[ ${#COMPREPLY[@]} -eq 0 ]]; then
-        local cur cword prev words
-        if declare -F _init_completion > /dev/null; then
-            _init_completion -n : || return
-        else
-            __rclone_init_completion -n : || return
-        fi
-	local rclone=(command rclone --ask-password=false)
-        if [[ $cur != *:* ]]; then
-            local ifs=$IFS
-            IFS=$'\n'
-            local remotes=($("${rclone[@]}" listremotes 2> /dev/null))
-            IFS=$ifs
-            local remote
-            for remote in "${remotes[@]}"; do
-                [[ $remote != $cur* ]] || COMPREPLY+=("$remote")
-            done
-            if [[ ${COMPREPLY[@]} ]]; then
-                local paths=("$cur"*)
-                [[ ! -f ${paths[0]} ]] || COMPREPLY+=("${paths[@]}")
-            fi
-        else
-            local path=${cur#*:}
-            if [[ $path == */* ]]; then
-                local prefix=$(eval printf '%s' "${path%/*}")
-            else
-                local prefix=
-            fi
-            local ifs=$IFS
-            IFS=$'\n'
-            local lines=($("${rclone[@]}" lsf "${cur%%:*}:$prefix" 2> /dev/null))
-            IFS=$ifs
-            local line
-            for line in "${lines[@]}"; do
-                local reply=${prefix:+$prefix/}$line
-                [[ $reply != $path* ]] || COMPREPLY+=("$reply")
-            done
-	    [[ ! ${COMPREPLY[@]} || $(type -t compopt) != builtin ]] || compopt -o filenames
-        fi
-        [[ ! ${COMPREPLY[@]} || $(type -t compopt) != builtin ]] || compopt -o nospace
-    fi
-}
-`
-)
-
 // GeneratingDocs is set by rclone gendocs to alter the format of the
 // output suitable for the documentation.
 var GeneratingDocs = false
@@ -220,10 +172,25 @@ func setupRootCommand(rootCmd *cobra.Command) {
 	helpCommand.AddCommand(helpBackends)
 	helpCommand.AddCommand(helpBackend)
 
+	// Set command completion for all functions to be the same
+	traverseCommands(rootCmd, func(cmd *cobra.Command) {
+		cmd.ValidArgsFunction = validArgs
+	})
+
 	cobra.OnInitialize(initConfig)
 
 }
 
+// Traverse the tree of commands running fn on each
+//
+// I was surprised there wasn't a cobra command to do this
+func traverseCommands(cmd *cobra.Command, fn func(*cobra.Command)) {
+	fn(cmd)
+	for _, childCmd := range cmd.Commands() {
+		traverseCommands(childCmd, fn)
+	}
+}
+
 var usageTemplate = `Usage:{{if .Runnable}}
   {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}}
   {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}}