config create: add --non-interactive and --continue parameters #3455

This adds a mechanism to add external interfaces to rclone using the
state based configuration.
This commit is contained in:
Nick Craig-Wood 2021-05-05 11:45:30 +01:00
parent e57553930f
commit 095cf9e4be
6 changed files with 254 additions and 83 deletions

View File

@ -105,12 +105,9 @@ var configProvidersCommand = &cobra.Command{
},
}
var (
configObscure bool
configNoObscure bool
)
var updateRemoteOpt config.UpdateRemoteOpt
const configPasswordHelp = `
var configPasswordHelp = strings.ReplaceAll(`
If any of the parameters passed is a password field, then rclone will
automatically obscure them if they aren't already obscured before
putting them in the config file.
@ -119,12 +116,57 @@ putting them in the config file.
consists only of base64 characters then rclone can get confused about
whether the password is already obscured or not and put unobscured
passwords into the config file. If you want to be 100% certain that
the passwords get obscured then use the "--obscure" flag, or if you
the passwords get obscured then use the |--obscure| flag, or if you
are 100% certain you are already passing obscured passwords then use
"--no-obscure". You can also set obscured passwords using the
"rclone config password" command.
`
|--no-obscure|. You can also set obscured passwords using the
|rclone config password| command.
The flag |--non-interactive| is for use by applications that wish to
configure rclone themeselves, rather than using rclone's text based
configuration questions. If this flag is set, and rclone needs to ask
the user a question, a JSON blob will be returned with the question in
it.
This will look something like (some irrelevant detail removed):
{
"State": "*oauth-islocal,teamdrive,,",
"Option": {
"Name": "config_is_local",
"Help": "Use auto config?\n * Say Y if not sure\n * Say N if you are working on a remote or headless machine\n",
"Default": true,
"Examples": [
{
"Value": "true",
"Help": "Yes"
},
{
"Value": "false",
"Help": "No"
}
],
"Required": false,
"IsPassword": false,
"Type": "bool"
},
"Error": "",
}
The format of |Option| is the same as returned by |rclone config
providers|. The question should be asked to the user and returned to
rclone as a string parameter along with the state parameter.
If |Error| is set then it should be shown to the user at the same
time as the question.
rclone config update name --continue state "*oauth-islocal,teamdrive,," result "true"
Note that when using |--continue| all passwords should be passed in
the clear (not obscured).
At the end of the non interactive process, rclone will return a result
with |State| as empty string.
`, "|", "`")
var configCreateCommand = &cobra.Command{
Use: "create `name` `type` [`key` `value`]*",
Short: `Create a new remote with name, type and options.`,
@ -152,19 +194,39 @@ using remote authorization you would do this:
if err != nil {
return err
}
err = config.CreateRemote(context.Background(), args[0], args[1], in, configObscure, configNoObscure)
return doConfig(args[0], in, func(opts config.UpdateRemoteOpt) (*fs.ConfigOut, error) {
return config.CreateRemote(context.Background(), args[0], args[1], in, opts)
})
},
}
func doConfig(name string, in rc.Params, do func(config.UpdateRemoteOpt) (*fs.ConfigOut, error)) error {
out, err := do(updateRemoteOpt)
if err != nil {
return err
}
if !(updateRemoteOpt.NonInteractive || updateRemoteOpt.Continue) {
config.ShowRemote(name)
} else {
if out == nil {
out = &fs.ConfigOut{}
}
outBytes, err := json.MarshalIndent(out, "", "\t")
if err != nil {
return err
}
config.ShowRemote(args[0])
return nil
},
_, _ = os.Stdout.Write(outBytes)
_, _ = os.Stdout.WriteString("\n")
}
return nil
}
func init() {
for _, cmdFlags := range []*pflag.FlagSet{configCreateCommand.Flags(), configUpdateCommand.Flags()} {
flags.BoolVarP(cmdFlags, &configObscure, "obscure", "", false, "Force any passwords to be obscured.")
flags.BoolVarP(cmdFlags, &configNoObscure, "no-obscure", "", false, "Force any passwords not to be obscured.")
flags.BoolVarP(cmdFlags, &updateRemoteOpt.Obscure, "obscure", "", false, "Force any passwords to be obscured.")
flags.BoolVarP(cmdFlags, &updateRemoteOpt.NoObscure, "no-obscure", "", false, "Force any passwords not to be obscured.")
flags.BoolVarP(cmdFlags, &updateRemoteOpt.NonInteractive, "non-interactive", "", false, "Don't interact with user and return questions.")
flags.BoolVarP(cmdFlags, &updateRemoteOpt.Continue, "continue", "", false, "Continue the configuration process with an answer.")
}
}
@ -191,12 +253,9 @@ require this add an extra parameter thus:
if err != nil {
return err
}
err = config.UpdateRemote(context.Background(), args[0], in, configObscure, configNoObscure)
if err != nil {
return err
}
config.ShowRemote(args[0])
return nil
return doConfig(args[0], in, func(opts config.UpdateRemoteOpt) (*fs.ConfigOut, error) {
return config.UpdateRemote(context.Background(), args[0], in, opts)
})
},
}

View File

@ -258,10 +258,42 @@ func StatePop(state string) (newState string, value string) {
//
// It wraps any OAuth transactions as necessary so only straight forward config questions are emitted
func BackendConfig(ctx context.Context, name string, m configmap.Mapper, ri *RegInfo, in ConfigIn) (out *ConfigOut, err error) {
for {
out, err = backendConfigStep(ctx, name, m, ri, in)
if err != nil {
break
}
if out == nil || out.State == "" {
// finished
break
}
if out.Option != nil {
// question to ask user
break
}
if out.Error != "" {
// error to show user
break
}
// non terminal state, but no question to ask or error to show - loop here
in = ConfigIn{
State: out.State,
Result: out.Result,
}
}
return out, err
}
func backendConfigStep(ctx context.Context, name string, m configmap.Mapper, ri *RegInfo, in ConfigIn) (out *ConfigOut, err error) {
ci := GetConfig(ctx)
if ri.Config == nil {
return nil, nil
}
Debugf(name, "config in: state=%q, result=%q", in.State, in.Result)
defer func() {
Debugf(name, "config out: out=%+v, err=%v", out, err)
}()
switch {
case strings.HasPrefix(in.State, "*oauth"):
// Do internal oauth states
@ -299,7 +331,7 @@ func BackendConfig(ctx context.Context, name string, m configmap.Mapper, ri *Reg
// If AutoConfirm is set, choose the default value
if ci.AutoConfirm {
result := fmt.Sprint(out.Option.Default)
Debugf(nil, "Auto confirm is set, choosing default %q for state %q", result, out.State)
Debugf(nil, "Auto confirm is set, choosing default %q for state %q, override by setting config parameter %q", result, out.State, out.Option.Name)
return ConfigResult(out.State, result)
}
}

View File

@ -408,76 +408,117 @@ func getWithDefault(name, key, defaultValue string) string {
return value
}
// UpdateRemoteOpt configures the remote update
type UpdateRemoteOpt struct {
// Treat all passwords as plain that need obscuring
Obscure bool `json:"obscure"`
// Treat all passwords as obscured
NoObscure bool `json:"noObscure"`
// Don't interact with the user - return questions
NonInteractive bool `json:"nonInteractive"`
// If set then supply state and result parameters to continue the process
Continue bool `json:"continue"`
}
// UpdateRemote adds the keyValues passed in to the remote of name.
// keyValues should be key, value pairs.
func UpdateRemote(ctx context.Context, name string, keyValues rc.Params, doObscure, noObscure bool) error {
if doObscure && noObscure {
return errors.New("can't use --obscure and --no-obscure together")
func UpdateRemote(ctx context.Context, name string, keyValues rc.Params, opt UpdateRemoteOpt) (out *fs.ConfigOut, err error) {
if opt.Obscure && opt.NoObscure {
return nil, errors.New("can't use --obscure and --no-obscure together")
}
err := fspath.CheckConfigName(name)
err = fspath.CheckConfigName(name)
if err != nil {
return err
return nil, err
}
interactive := !(opt.NonInteractive || opt.Continue)
if interactive {
ctx = suppressConfirm(ctx)
}
ctx = suppressConfirm(ctx)
// Work out which options need to be obscured
needsObscure := map[string]struct{}{}
if !noObscure {
if fsType := FileGet(name, "type"); fsType != "" {
if ri, err := fs.Find(fsType); err != nil {
fs.Debugf(nil, "Couldn't find fs for type %q", fsType)
} else {
for _, opt := range ri.Options {
if opt.IsPassword {
needsObscure[opt.Name] = struct{}{}
fsType := FileGet(name, "type")
if fsType == "" {
return nil, errors.New("couldn't find type field in config")
}
ri, err := fs.Find(fsType)
if err != nil {
return nil, errors.Errorf("couldn't find backend for type %q", fsType)
}
if !opt.Continue {
// Work out which options need to be obscured
needsObscure := map[string]struct{}{}
if !opt.NoObscure {
for _, option := range ri.Options {
if option.IsPassword {
needsObscure[option.Name] = struct{}{}
}
}
}
// Set the config
for k, v := range keyValues {
vStr := fmt.Sprint(v)
// Obscure parameter if necessary
if _, ok := needsObscure[k]; ok {
_, err := obscure.Reveal(vStr)
if err != nil || opt.Obscure {
// If error => not already obscured, so obscure it
// or we are forced to obscure
vStr, err = obscure.Obscure(vStr)
if err != nil {
return nil, errors.Wrap(err, "UpdateRemote: obscure failed")
}
}
}
} else {
fs.Debugf(nil, "UpdateRemote: Couldn't find fs type")
LoadedData().SetValue(name, k, vStr)
}
}
// Set the config
for k, v := range keyValues {
vStr := fmt.Sprint(v)
// Obscure parameter if necessary
if _, ok := needsObscure[k]; ok {
_, err := obscure.Reveal(vStr)
if err != nil || doObscure {
// If error => not already obscured, so obscure it
// or we are forced to obscure
vStr, err = obscure.Obscure(vStr)
if err != nil {
return errors.Wrap(err, "UpdateRemote: obscure failed")
}
if interactive {
err = RemoteConfig(ctx, name)
} else {
// Start the config state machine
m := fs.ConfigMap(ri, name, nil)
in := fs.ConfigIn{}
if opt.Continue {
if state, ok := keyValues["state"]; ok {
in.State = fmt.Sprint(state)
} else {
return nil, errors.New("UpdateRemote: need state parameter with --continue")
}
if result, ok := keyValues["result"]; ok {
in.Result = fmt.Sprint(result)
} else {
return nil, errors.New("UpdateRemote: need result parameter with --continue")
}
}
LoadedData().SetValue(name, k, vStr)
out, err = fs.BackendConfig(ctx, name, m, ri, in)
}
err = RemoteConfig(ctx, name)
if err != nil {
return err
return nil, err
}
SaveConfig()
cache.ClearConfig(name) // remove any remotes based on this config from the cache
return nil
return out, nil
}
// CreateRemote creates a new remote with name, provider and a list of
// parameters which are key, value pairs. If update is set then it
// adds the new keys rather than replacing all of them.
func CreateRemote(ctx context.Context, name string, provider string, keyValues rc.Params, doObscure, noObscure bool) error {
err := fspath.CheckConfigName(name)
func CreateRemote(ctx context.Context, name string, provider string, keyValues rc.Params, opts UpdateRemoteOpt) (out *fs.ConfigOut, err error) {
err = fspath.CheckConfigName(name)
if err != nil {
return err
return nil, err
}
if !opts.Continue {
// Delete the old config if it exists
LoadedData().DeleteSection(name)
// Set the type
LoadedData().SetValue(name, "type", provider)
}
// Delete the old config if it exists
LoadedData().DeleteSection(name)
// Set the type
LoadedData().SetValue(name, "type", provider)
// Set the remaining values
return UpdateRemote(ctx, name, keyValues, doObscure, noObscure)
return UpdateRemote(ctx, name, keyValues, opts)
}
// PasswordRemote adds the keyValues passed in to the remote of name.
@ -491,7 +532,10 @@ func PasswordRemote(ctx context.Context, name string, keyValues rc.Params) error
for k, v := range keyValues {
keyValues[k] = obscure.MustObscure(fmt.Sprint(v))
}
return UpdateRemote(ctx, name, keyValues, false, true)
_, err = UpdateRemote(ctx, name, keyValues, UpdateRemoteOpt{
NoObscure: true,
})
return err
}
// JSONListProviders prints all the providers and options in JSON format

View File

@ -3,6 +3,7 @@ package config
import (
"context"
"github.com/pkg/errors"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/rc"
)
@ -112,8 +113,12 @@ func init() {
extraHelp = "- type - type of the new remote\n"
}
if name == "create" || name == "update" {
extraHelp += "- obscure - optional bool - forces obscuring of passwords\n"
extraHelp += "- noObscure - optional bool - forces passwords not to be obscured\n"
extraHelp += `- opt - a dictionary of options to control the configuration
- obscure - declare passwords are plain and need obscuring
- noObscure - declare passwords are already obscured and don't need obscuring
- nonInteractive - don't interact with a user, return questions
- continue - continue the config process with an answer
`
}
rc.Add(rc.Call{
Path: "config/" + name,
@ -144,21 +149,47 @@ func rcConfig(ctx context.Context, in rc.Params, what string) (out rc.Params, er
if err != nil {
return nil, err
}
doObscure, _ := in.GetBool("obscure")
noObscure, _ := in.GetBool("noObscure")
var opt UpdateRemoteOpt
err = in.GetStruct("opt", &opt)
if err != nil && !rc.IsErrParamNotFound(err) {
return nil, err
}
// Backwards compatibility
if value, err := in.GetBool("obscure"); err == nil {
opt.Obscure = value
}
if value, err := in.GetBool("noObscure"); err == nil {
opt.NoObscure = value
}
var configOut *fs.ConfigOut
switch what {
case "create":
remoteType, err := in.GetString("type")
if err != nil {
return nil, err
remoteType, typeErr := in.GetString("type")
if typeErr != nil {
return nil, typeErr
}
return nil, CreateRemote(ctx, name, remoteType, parameters, doObscure, noObscure)
configOut, err = CreateRemote(ctx, name, remoteType, parameters, opt)
case "update":
return nil, UpdateRemote(ctx, name, parameters, doObscure, noObscure)
configOut, err = UpdateRemote(ctx, name, parameters, opt)
case "password":
return nil, PasswordRemote(ctx, name, parameters)
err = PasswordRemote(ctx, name, parameters)
default:
err = errors.New("unknown rcConfig type")
}
panic("unknown rcConfig type")
if err != nil {
return nil, err
}
if !opt.NonInteractive {
return nil, nil
}
if configOut == nil {
configOut = &fs.ConfigOut{}
}
err = rc.Reshape(&out, configOut)
if err != nil {
return nil, err
}
return out, nil
}
func init() {

View File

@ -252,7 +252,6 @@ func PostConfig(ctx context.Context, name string, m configmap.Mapper, ri *fs.Reg
State: "",
}
for {
fs.Debugf(name, "config: state=%q, result=%q", in.State, in.Result)
out, err := fs.BackendConfig(ctx, name, m, ri, in)
if err != nil {
return err
@ -266,7 +265,7 @@ func PostConfig(ctx context.Context, name string, m configmap.Mapper, ri *fs.Reg
in.State = out.State
in.Result = out.Result
if out.Option != nil {
fs.Debugf(name, "config: reading config item named %q", out.Option.Name)
fs.Debugf(name, "config: reading config parameter %q", out.Option.Name)
if out.Option.Default == nil {
out.Option.Default = ""
}

View File

@ -197,10 +197,15 @@ func TestCreateUpdatePasswordRemote(t *testing.T) {
break
}
t.Run(fmt.Sprintf("doObscure=%v,noObscure=%v", doObscure, noObscure), func(t *testing.T) {
require.NoError(t, config.CreateRemote(ctx, "test2", "config_test_remote", rc.Params{
opt := config.UpdateRemoteOpt{
Obscure: doObscure,
NoObscure: noObscure,
}
_, err := config.CreateRemote(ctx, "test2", "config_test_remote", rc.Params{
"bool": true,
"pass": "potato",
}, doObscure, noObscure))
}, opt)
require.NoError(t, err)
assert.Equal(t, []string{"test2"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test2", "type"))
@ -212,11 +217,12 @@ func TestCreateUpdatePasswordRemote(t *testing.T) {
assert.Equal(t, "potato", gotPw)
wantPw := obscure.MustObscure("potato2")
require.NoError(t, config.UpdateRemote(ctx, "test2", rc.Params{
_, err = config.UpdateRemote(ctx, "test2", rc.Params{
"bool": false,
"pass": wantPw,
"spare": "spare",
}, doObscure, noObscure))
}, opt)
require.NoError(t, err)
assert.Equal(t, []string{"test2"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test2", "type"))