fs: convert main options to new config system

There are some flags which haven't been converted which could be
converted in the future.

cmd/cmd.go

@@ -14,7 +14,6 @@ import (
 	"os"
 	"os/exec"
 	"path"
-	"regexp"
 	"runtime"
 	"runtime/pprof"
 	"strconv"
@@ -49,7 +48,6 @@ var (
 	cpuProfile    = flags.StringP("cpuprofile", "", "", "Write cpu profile to file", "Debugging")
 	memProfile    = flags.StringP("memprofile", "", "", "Write memory profile to file", "Debugging")
 	statsInterval = flags.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable)", "Logging")
-	dataRateUnit  = flags.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes' per second", "Logging")
 	version       bool
 	// Errors
 	errorCommandNotFound    = errors.New("command not found")
@@ -472,13 +470,6 @@ func initConfig() {
 			}
 		})
 	}
-
-	if m, _ := regexp.MatchString("^(bits|bytes)$", *dataRateUnit); !m {
-		fs.Errorf(nil, "Invalid unit passed to --stats-unit. Defaulting to bytes.")
-		ci.DataRateUnit = "bytes"
-	} else {
-		ci.DataRateUnit = *dataRateUnit
-	}
 }
 
 func resolveExitCode(err error) {

cmdtest/environment_test.go

@@ -66,7 +66,7 @@ func TestEnvironmentVariables(t *testing.T) {
 		assert.NotContains(t, out, "fileAA1.txt") // depth 4
 	}
 
-	// Test of debug logging while initialising flags from environment (tests #5241 Enhance1)
+	// Test of debug logging while initialising flags from environment (tests #5341 Enhance1)
 	env = "RCLONE_STATS=173ms"
 	out, err = rcloneEnv(env, "version", "-vv")
 	if assert.NoError(t, err) {

fs/config.go

@@ -3,6 +3,7 @@ package fs
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net"
 	"os"
 	"strconv"
@@ -13,7 +14,7 @@ import (
 // Global
 var (
 	// globalConfig for rclone
-	globalConfig = NewConfig()
+	globalConfig = new(ConfigInfo)
 
 	// Read a value from the config file
 	//
@@ -49,184 +50,693 @@ var (
 	ConfigEdit = "config_fs_edit"
 )
 
+// ConfigOptionsInfo describes the Options in use
+var ConfigOptionsInfo = Options{{
+	Name:    "modify_window",
+	Default: time.Nanosecond,
+	Help:    "Max time diff to be considered the same",
+	Groups:  "Copy",
+}, {
+	Name:    "checkers",
+	Default: 8,
+	Help:    "Number of checkers to run in parallel",
+	Groups:  "Performance",
+}, {
+	Name:    "transfers",
+	Default: 4,
+	Help:    "Number of file transfers to run in parallel",
+	Groups:  "Performance",
+}, {
+	Name:     "checksum",
+	ShortOpt: "c",
+	Default:  false,
+	Help:     "Check for changes with size & checksum (if available, or fallback to size only).",
+	Groups:   "Copy",
+}, {
+	Name:    "size_only",
+	Default: false,
+	Help:    "Skip based on size only, not modtime or checksum",
+	Groups:  "Copy",
+}, {
+	Name:     "ignore_times",
+	ShortOpt: "I",
+	Default:  false,
+	Help:     "Don't skip items that match size and time - transfer all unconditionally",
+	Groups:   "Copy",
+}, {
+	Name:    "ignore_existing",
+	Default: false,
+	Help:    "Skip all files that exist on destination",
+	Groups:  "Copy",
+}, {
+	Name:    "ignore_errors",
+	Default: false,
+	Help:    "Delete even if there are I/O errors",
+	Groups:  "Sync",
+}, {
+	Name:     "dry_run",
+	ShortOpt: "n",
+	Default:  false,
+	Help:     "Do a trial run with no permanent changes",
+	Groups:   "Config,Important",
+}, {
+	Name:     "interactive",
+	ShortOpt: "i",
+	Default:  false,
+	Help:     "Enable interactive mode",
+	Groups:   "Config,Important",
+}, {
+	Name:    "contimeout",
+	Default: 60 * time.Second,
+	Help:    "Connect timeout",
+	Groups:  "Networking",
+}, {
+	Name:    "timeout",
+	Default: 5 * 60 * time.Second,
+	Help:    "IO idle timeout",
+	Groups:  "Networking",
+}, {
+	Name:    "expect_continue_timeout",
+	Default: 1 * time.Second,
+	Help:    "Timeout when using expect / 100-continue in HTTP",
+	Groups:  "Networking",
+}, {
+	Name:    "no_check_certificate",
+	Default: false,
+	Help:    "Do not verify the server SSL certificate (insecure)",
+	Groups:  "Networking",
+}, {
+	Name:    "ask_password",
+	Default: true,
+	Help:    "Allow prompt for password for encrypted configuration",
+	Groups:  "Config",
+}, {
+	Name:    "password_command",
+	Default: SpaceSepList{},
+	Help:    "Command for supplying password for encrypted configuration",
+	Groups:  "Config",
+}, {
+	Name:    "max_delete",
+	Default: int64(-1),
+	Help:    "When synchronizing, limit the number of deletes",
+	Groups:  "Sync",
+}, {
+	Name:    "max_delete_size",
+	Default: SizeSuffix(-1),
+	Help:    "When synchronizing, limit the total size of deletes",
+	Groups:  "Sync",
+}, {
+	Name:    "track_renames",
+	Default: false,
+	Help:    "When synchronizing, track file renames and do a server-side move if possible",
+	Groups:  "Sync",
+}, {
+	Name:    "track_renames_strategy",
+	Default: "hash",
+	Help:    "Strategies to use when synchronizing using track-renames hash|modtime|leaf",
+	Groups:  "Sync",
+}, {
+	Name:    "retries",
+	Default: 3,
+	Help:    "Retry operations this many times if they fail",
+	Groups:  "Config",
+}, {
+	Name:    "retries_sleep",
+	Default: time.Duration(0),
+	Help:    "Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)",
+	Groups:  "Config",
+}, {
+	Name:    "low_level_retries",
+	Default: 10,
+	Help:    "Number of low level retries to do",
+	Groups:  "Config",
+}, {
+	Name:     "update",
+	ShortOpt: "u",
+	Default:  false,
+	Help:     "Skip files that are newer on the destination",
+	Groups:   "Copy",
+}, {
+	Name:    "use_server_modtime",
+	Default: false,
+	Help:    "Use server modified time instead of object metadata",
+	Groups:  "Config",
+}, {
+	Name:    "no_gzip_encoding",
+	Default: false,
+	Help:    "Don't set Accept-Encoding: gzip",
+	Groups:  "Networking",
+}, {
+	Name:    "max_depth",
+	Default: -1,
+	Help:    "If set limits the recursion depth to this",
+	Groups:  "Filter",
+}, {
+	Name:    "ignore_size",
+	Default: false,
+	Help:    "Ignore size when skipping use modtime or checksum",
+	Groups:  "Copy",
+}, {
+	Name:    "ignore_checksum",
+	Default: false,
+	Help:    "Skip post copy check of checksums",
+	Groups:  "Copy",
+}, {
+	Name:    "ignore_case_sync",
+	Default: false,
+	Help:    "Ignore case when synchronizing",
+	Groups:  "Copy",
+}, {
+	Name:    "fix_case",
+	Default: false,
+	Help:    "Force rename of case insensitive dest to match source",
+	Groups:  "Sync",
+}, {
+	Name:    "no_traverse",
+	Default: false,
+	Help:    "Don't traverse destination file system on copy",
+	Groups:  "Copy",
+}, {
+	Name:    "check_first",
+	Default: false,
+	Help:    "Do all the checks before starting transfers",
+	Groups:  "Copy",
+}, {
+	Name:    "no_check_dest",
+	Default: false,
+	Help:    "Don't check the destination, copy regardless",
+	Groups:  "Copy",
+}, {
+	Name:    "no_unicode_normalization",
+	Default: false,
+	Help:    "Don't normalize unicode characters in filenames",
+	Groups:  "Config",
+}, {
+	Name:    "no_update_modtime",
+	Default: false,
+	Help:    "Don't update destination modtime if files identical",
+	Groups:  "Copy",
+}, {
+	Name:    "no_update_dir_modtime",
+	Default: false,
+	Help:    "Don't update directory modification times",
+	Groups:  "Copy",
+}, {
+	Name:    "compare_dest",
+	Default: []string{},
+	Help:    "Include additional server-side paths during comparison",
+	Groups:  "Copy",
+}, {
+	Name:    "copy_dest",
+	Default: []string{},
+	Help:    "Implies --compare-dest but also copies files from paths into destination",
+	Groups:  "Copy",
+}, {
+	Name:    "backup_dir",
+	Default: "",
+	Help:    "Make backups into hierarchy based in DIR",
+	Groups:  "Sync",
+}, {
+	Name:    "suffix",
+	Default: "",
+	Help:    "Suffix to add to changed files",
+	Groups:  "Sync",
+}, {
+	Name:    "suffix_keep_extension",
+	Default: false,
+	Help:    "Preserve the extension when using --suffix",
+	Groups:  "Sync",
+}, {
+	Name:    "fast_list",
+	Default: false,
+	Help:    "Use recursive list if available; uses more memory but fewer transactions",
+	Groups:  "Listing",
+}, {
+	Name:    "tpslimit",
+	Default: 0.0,
+	Help:    "Limit HTTP transactions per second to this",
+	Groups:  "Networking",
+}, {
+	Name:    "tpslimit_burst",
+	Default: 1,
+	Help:    "Max burst of transactions for --tpslimit",
+	Groups:  "Networking",
+}, {
+	Name:    "user_agent",
+	Default: "rclone/" + Version,
+	Help:    "Set the user-agent to a specified string",
+	Groups:  "Networking",
+}, {
+	Name:    "immutable",
+	Default: false,
+	Help:    "Do not modify files, fail if existing files have been modified",
+	Groups:  "Copy",
+}, {
+	Name:    "auto_confirm",
+	Default: false,
+	Help:    "If enabled, do not request console confirmation",
+	Groups:  "Config",
+}, {
+	Name:    "stats_unit",
+	Default: "bytes",
+	Help:    "Show data rate in stats as either 'bits' or 'bytes' per second",
+	Groups:  "Logging",
+}, {
+	Name:    "stats_file_name_length",
+	Default: 45,
+	Help:    "Max file name length in stats (0 for no limit)",
+	Groups:  "Logging",
+}, {
+	Name:    "log_level",
+	Default: LogLevelNotice,
+	Help:    "Log level DEBUG|INFO|NOTICE|ERROR",
+	Groups:  "Logging",
+}, {
+	Name:    "stats_log_level",
+	Default: LogLevelInfo,
+	Help:    "Log level to show --stats output DEBUG|INFO|NOTICE|ERROR",
+	Groups:  "Logging",
+}, {
+	Name:    "bwlimit",
+	Default: BwTimetable{},
+	Help:    "Bandwidth limit in KiB/s, or use suffix B|K|M|G|T|P or a full timetable",
+	Groups:  "Networking",
+}, {
+	Name:    "bwlimit_file",
+	Default: BwTimetable{},
+	Help:    "Bandwidth limit per file in KiB/s, or use suffix B|K|M|G|T|P or a full timetable",
+	Groups:  "Networking",
+}, {
+	Name:    "buffer_size",
+	Default: SizeSuffix(16 << 20),
+	Help:    "In memory buffer size when reading files for each --transfer",
+	Groups:  "Performance",
+}, {
+	Name:    "streaming_upload_cutoff",
+	Default: SizeSuffix(100 * 1024),
+	Help:    "Cutoff for switching to chunked upload if file size is unknown, upload starts after reaching cutoff or when file ends",
+	Groups:  "Copy",
+}, {
+	Name:    "dump",
+	Default: DumpFlags(0),
+	Help:    "List of items to dump from: " + DumpFlagsList,
+	Groups:  "Debugging",
+}, {
+	Name:    "max_transfer",
+	Default: SizeSuffix(-1),
+	Help:    "Maximum size of data to transfer",
+	Groups:  "Copy",
+}, {
+	Name:    "max_duration",
+	Default: time.Duration(0),
+	Help:    "Maximum duration rclone will transfer data for",
+	Groups:  "Copy",
+}, {
+	Name:    "cutoff_mode",
+	Default: CutoffMode(0),
+	Help:    "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS",
+	Groups:  "Copy",
+}, {
+	Name:    "max_backlog",
+	Default: 10000,
+	Help:    "Maximum number of objects in sync or check backlog",
+	Groups:  "Copy,Check",
+}, {
+	Name:    "max_stats_groups",
+	Default: 1000,
+	Help:    "Maximum number of stats groups to keep in memory, on max oldest is discarded",
+	Groups:  "Logging",
+}, {
+	Name:    "stats_one_line",
+	Default: false,
+	Help:    "Make the stats fit on one line",
+	Groups:  "Logging",
+}, {
+	Name:    "stats_one_line_date",
+	Default: false,
+	Help:    "Enable --stats-one-line and add current date/time prefix",
+	Groups:  "Logging",
+}, {
+	Name:    "stats_one_line_date_format",
+	Default: "",
+	Help:    "Enable --stats-one-line-date and use custom formatted date: Enclose date string in double quotes (\"), see https://golang.org/pkg/time/#Time.Format",
+	Groups:  "Logging",
+}, {
+	Name:    "error_on_no_transfer",
+	Default: false,
+	Help:    "Sets exit code 9 if no files are transferred, useful in scripts",
+	Groups:  "Config",
+}, {
+	Name:     "progress",
+	ShortOpt: "P",
+	Default:  false,
+	Help:     "Show progress during transfer",
+	Groups:   "Logging",
+}, {
+	Name:    "progress_terminal_title",
+	Default: false,
+	Help:    "Show progress on the terminal title (requires -P/--progress)",
+	Groups:  "Logging",
+}, {
+	Name:    "use_cookies",
+	Default: false,
+	Help:    "Enable session cookiejar",
+	Groups:  "Networking",
+}, {
+	Name:    "use_mmap",
+	Default: false,
+	Help:    "Use mmap allocator (see docs)",
+	Groups:  "Config",
+}, {
+	Name:    "ca_cert",
+	Default: []string{},
+	Help:    "CA certificate used to verify servers",
+	Groups:  "Networking",
+}, {
+	Name:    "client_cert",
+	Default: "",
+	Help:    "Client SSL certificate (PEM) for mutual TLS auth",
+	Groups:  "Networking",
+}, {
+	Name:    "client_key",
+	Default: "",
+	Help:    "Client SSL private key (PEM) for mutual TLS auth",
+	Groups:  "Networking",
+}, {
+	Name:    "multi_thread_cutoff",
+	Default: SizeSuffix(256 * 1024 * 1024),
+	Help:    "Use multi-thread downloads for files above this size",
+	Groups:  "Copy",
+}, {
+	Name:    "multi_thread_streams",
+	Default: 4,
+	Help:    "Number of streams to use for multi-thread downloads",
+	Groups:  "Copy",
+}, {
+	Name:    "multi_thread_write_buffer_size",
+	Default: SizeSuffix(128 * 1024),
+	Help:    "In memory buffer size for writing when in multi-thread mode",
+	Groups:  "Copy",
+}, {
+	Name:    "multi_thread_chunk_size",
+	Default: SizeSuffix(64 * 1024 * 1024),
+	Help:    "Chunk size for multi-thread downloads / uploads, if not set by filesystem",
+	Groups:  "Copy",
+}, {
+	Name:    "use_json_log",
+	Default: false,
+	Help:    "Use json log format",
+	Groups:  "Logging",
+}, {
+	Name:    "order_by",
+	Default: "",
+	Help:    "Instructions on how to order the transfers, e.g. 'size,descending'",
+	Groups:  "Copy",
+}, {
+	Name:    "refresh_times",
+	Default: false,
+	Help:    "Refresh the modtime of remote files",
+	Groups:  "Copy",
+}, {
+	Name:    "no_console",
+	Default: false,
+	Help:    "Hide console window (supported on Windows only)",
+	Groups:  "Config",
+}, {
+	Name:    "fs_cache_expire_duration",
+	Default: 300 * time.Second,
+	Help:    "Cache remotes for this long (0 to disable caching)",
+	Groups:  "Config",
+}, {
+	Name:    "fs_cache_expire_interval",
+	Default: 60 * time.Second,
+	Help:    "Interval to check for expired remotes",
+	Groups:  "Config",
+}, {
+	Name:    "disable_http2",
+	Default: false,
+	Help:    "Disable HTTP/2 in the global transport",
+	Groups:  "Networking",
+}, {
+	Name:    "human_readable",
+	Default: false,
+	Help:    "Print numbers in a human-readable format, sizes with suffix Ki|Mi|Gi|Ti|Pi",
+	Groups:  "Config",
+}, {
+	Name:    "kv_lock_time",
+	Default: 1 * time.Second,
+	Help:    "Maximum time to keep key-value database locked by process",
+	Groups:  "Config",
+}, {
+	Name:    "disable_http_keep_alives",
+	Default: false,
+	Help:    "Disable HTTP keep-alives and use each connection once.",
+	Groups:  "Networking",
+}, {
+	Name:     "metadata",
+	ShortOpt: "M",
+	Default:  false,
+	Help:     "If set, preserve metadata when copying objects",
+	Groups:   "Metadata,Copy",
+}, {
+	Name:    "server_side_across_configs",
+	Default: false,
+	Help:    "Allow server-side operations (e.g. copy) to work across different configs",
+	Groups:  "Copy",
+}, {
+	Name:    "color",
+	Default: TerminalColorMode(0),
+	Help:    "When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS",
+	Groups:  "Config",
+}, {
+	Name:    "default_time",
+	Default: Time(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)),
+	Help:    "Time to show if modtime is unknown for files and directories",
+	Groups:  "Config,Listing",
+}, {
+	Name:    "inplace",
+	Default: false,
+	Help:    "Download directly to destination file instead of atomic download to temp/rename",
+	Groups:  "Copy",
+}, {
+	Name:    "metadata_mapper",
+	Default: SpaceSepList{},
+	Help:    "Program to run to transforming metadata before upload",
+	Groups:  "Metadata",
+}, {
+	Name:    "partial_suffix",
+	Default: ".partial",
+	Help:    "Add partial-suffix to temporary file name when --inplace is not used",
+	Groups:  "Copy",
+}}
+
 // ConfigInfo is filesystem config options
 type ConfigInfo struct {
-	LogLevel                   LogLevel
+	LogLevel                   LogLevel          `config:"log_level"`
-	StatsLogLevel              LogLevel
+	StatsLogLevel              LogLevel          `config:"stats_log_level"`
-	UseJSONLog                 bool
+	UseJSONLog                 bool              `config:"use_json_log"`
-	DryRun                     bool
+	DryRun                     bool              `config:"dry_run"`
-	Interactive                bool
+	Interactive                bool              `config:"interactive"`
-	CheckSum                   bool
+	CheckSum                   bool              `config:"checksum"`
-	SizeOnly                   bool
+	SizeOnly                   bool              `config:"size_only"`
-	IgnoreTimes                bool
+	IgnoreTimes                bool              `config:"ignore_times"`
-	IgnoreExisting             bool
+	IgnoreExisting             bool              `config:"ignore_existing"`
-	IgnoreErrors               bool
+	IgnoreErrors               bool              `config:"ignore_errors"`
-	ModifyWindow               time.Duration
+	ModifyWindow               time.Duration     `config:"modify_window"`
-	Checkers                   int
+	Checkers                   int               `config:"checkers"`
-	Transfers                  int
+	Transfers                  int               `config:"transfers"`
-	ConnectTimeout             time.Duration // Connect timeout
+	ConnectTimeout             time.Duration     `config:"contimeout"` // Connect timeout
-	Timeout                    time.Duration // Data channel timeout
+	Timeout                    time.Duration     `config:"timeout"`    // Data channel timeout
-	ExpectContinueTimeout      time.Duration
+	ExpectContinueTimeout      time.Duration     `config:"expect_continue_timeout"`
-	Dump                       DumpFlags
+	Dump                       DumpFlags         `config:"dump"`
-	InsecureSkipVerify         bool // Skip server certificate verification
+	InsecureSkipVerify         bool              `config:"no_check_certificate"` // Skip server certificate verification
-	DeleteMode                 DeleteMode
+	DeleteMode                 DeleteMode        `config:"delete_mode"`
-	MaxDelete                  int64
+	MaxDelete                  int64             `config:"max_delete"`
-	MaxDeleteSize              SizeSuffix
+	MaxDeleteSize              SizeSuffix        `config:"max_delete_size"`
-	TrackRenames               bool          // Track file renames.
+	TrackRenames               bool              `config:"track_renames"`          // Track file renames.
-	TrackRenamesStrategy       string        // Comma separated list of strategies used to track renames
+	TrackRenamesStrategy       string            `config:"track_renames_strategy"` // Comma separated list of strategies used to track renames
-	Retries                    int           // High-level retries
+	Retries                    int               `config:"retries"`                // High-level retries
-	RetriesInterval            time.Duration // --retries-sleep
+	RetriesInterval            time.Duration     `config:"retries_sleep"`
-	LowLevelRetries            int
+	LowLevelRetries            int               `config:"low_level_retries"`
-	UpdateOlder                bool // Skip files that are newer on the destination
+	UpdateOlder                bool              `config:"update"`           // Skip files that are newer on the destination
-	NoGzip                     bool // Disable compression
+	NoGzip                     bool              `config:"no_gzip_encoding"` // Disable compression
-	MaxDepth                   int
+	MaxDepth                   int               `config:"max_depth"`
-	IgnoreSize                 bool
+	IgnoreSize                 bool              `config:"ignore_size"`
-	IgnoreChecksum             bool
+	IgnoreChecksum             bool              `config:"ignore_checksum"`
-	IgnoreCaseSync             bool
+	IgnoreCaseSync             bool              `config:"ignore_case_sync"`
-	FixCase                    bool
+	FixCase                    bool              `config:"fix_case"`
-	NoTraverse                 bool
+	NoTraverse                 bool              `config:"no_traverse"`
-	CheckFirst                 bool
+	CheckFirst                 bool              `config:"check_first"`
-	NoCheckDest                bool
+	NoCheckDest                bool              `config:"no_check_dest"`
-	NoUnicodeNormalization     bool
+	NoUnicodeNormalization     bool              `config:"no_unicode_normalization"`
-	NoUpdateModTime            bool
+	NoUpdateModTime            bool              `config:"no_update_modtime"`
-	NoUpdateDirModTime         bool
+	NoUpdateDirModTime         bool              `config:"no_update_dir_modtime"`
-	DataRateUnit               string
+	DataRateUnit               string            `config:"stats_unit"`
-	CompareDest                []string
+	CompareDest                []string          `config:"compare_dest"`
-	CopyDest                   []string
+	CopyDest                   []string          `config:"copy_dest"`
-	BackupDir                  string
+	BackupDir                  string            `config:"backup_dir"`
-	Suffix                     string
+	Suffix                     string            `config:"suffix"`
-	SuffixKeepExtension        bool
+	SuffixKeepExtension        bool              `config:"suffix_keep_extension"`
-	UseListR                   bool
+	UseListR                   bool              `config:"fast_list"`
-	BufferSize                 SizeSuffix
+	BufferSize                 SizeSuffix        `config:"buffer_size"`
-	BwLimit                    BwTimetable
+	BwLimit                    BwTimetable       `config:"bwlimit"`
-	BwLimitFile                BwTimetable
+	BwLimitFile                BwTimetable       `config:"bwlimit_file"`
-	TPSLimit                   float64
+	TPSLimit                   float64           `config:"tpslimit"`
-	TPSLimitBurst              int
+	TPSLimitBurst              int               `config:"tpslimit_burst"`
-	BindAddr                   net.IP
+	BindAddr                   net.IP            `config:"bind_addr"`
-	DisableFeatures            []string
+	DisableFeatures            []string          `config:"disable"`
-	UserAgent                  string
+	UserAgent                  string            `config:"user_agent"`
-	Immutable                  bool
+	Immutable                  bool              `config:"immutable"`
-	AutoConfirm                bool
+	AutoConfirm                bool              `config:"auto_confirm"`
-	StreamingUploadCutoff      SizeSuffix
+	StreamingUploadCutoff      SizeSuffix        `config:"streaming_upload_cutoff"`
-	StatsFileNameLength        int
+	StatsFileNameLength        int               `config:"stats_file_name_length"`
-	AskPassword                bool
+	AskPassword                bool              `config:"ask_password"`
-	PasswordCommand            SpaceSepList
+	PasswordCommand            SpaceSepList      `config:"password_command"`
-	UseServerModTime           bool
+	UseServerModTime           bool              `config:"use_server_modtime"`
-	MaxTransfer                SizeSuffix
+	MaxTransfer                SizeSuffix        `config:"max_transfer"`
-	MaxDuration                time.Duration
+	MaxDuration                time.Duration     `config:"max_duration"`
-	CutoffMode                 CutoffMode
+	CutoffMode                 CutoffMode        `config:"cutoff_mode"`
-	MaxBacklog                 int
+	MaxBacklog                 int               `config:"max_backlog"`
-	MaxStatsGroups             int
+	MaxStatsGroups             int               `config:"max_stats_groups"`
-	StatsOneLine               bool
+	StatsOneLine               bool              `config:"stats_one_line"`
-	StatsOneLineDate           bool   // If we want a date prefix at all
+	StatsOneLineDate           bool              `config:"stats_one_line_date"`        // If we want a date prefix at all
-	StatsOneLineDateFormat     string // If we want to customize the prefix
+	StatsOneLineDateFormat     string            `config:"stats_one_line_date_format"` // If we want to customize the prefix
-	ErrorOnNoTransfer          bool   // Set appropriate exit code if no files transferred
+	ErrorOnNoTransfer          bool              `config:"error_on_no_transfer"`       // Set appropriate exit code if no files transferred
-	Progress                   bool
+	Progress                   bool              `config:"progress"`
-	ProgressTerminalTitle      bool
+	ProgressTerminalTitle      bool              `config:"progress_terminal_title"`
-	Cookie                     bool
+	Cookie                     bool              `config:"use_cookies"`
-	UseMmap                    bool
+	UseMmap                    bool              `config:"use_mmap"`
-	CaCert                     []string // Client Side CA
+	CaCert                     []string          `config:"ca_cert"`     // Client Side CA
-	ClientCert                 string   // Client Side Cert
+	ClientCert                 string            `config:"client_cert"` // Client Side Cert
-	ClientKey                  string   // Client Side Key
+	ClientKey                  string            `config:"client_key"`  // Client Side Key
-	MultiThreadCutoff          SizeSuffix
+	MultiThreadCutoff          SizeSuffix        `config:"multi_thread_cutoff"`
-	MultiThreadStreams         int
+	MultiThreadStreams         int               `config:"multi_thread_streams"`
-	MultiThreadSet             bool       // whether MultiThreadStreams was set (set in fs/config/configflags)
+	MultiThreadSet             bool              `config:"multi_thread_set"`        // whether MultiThreadStreams was set (set in fs/config/configflags)
-	MultiThreadChunkSize       SizeSuffix // Chunk size for multi-thread downloads / uploads, if not set by filesystem
+	MultiThreadChunkSize       SizeSuffix        `config:"multi_thread_chunk_size"` // Chunk size for multi-thread downloads / uploads, if not set by filesystem
-	MultiThreadWriteBufferSize SizeSuffix
+	MultiThreadWriteBufferSize SizeSuffix        `config:"multi_thread_write_buffer_size"`
-	OrderBy                    string // instructions on how to order the transfer
+	OrderBy                    string            `config:"order_by"` // instructions on how to order the transfer
-	UploadHeaders              []*HTTPOption
+	UploadHeaders              []*HTTPOption     `config:"upload_headers"`
-	DownloadHeaders            []*HTTPOption
+	DownloadHeaders            []*HTTPOption     `config:"download_headers"`
-	Headers                    []*HTTPOption
+	Headers                    []*HTTPOption     `config:"headers"`
-	MetadataSet                Metadata // extra metadata to write when uploading
+	MetadataSet                Metadata          `config:"metadata_set"` // extra metadata to write when uploading
-	RefreshTimes               bool
+	RefreshTimes               bool              `config:"refresh_times"`
-	NoConsole                  bool
+	NoConsole                  bool              `config:"no_console"`
-	TrafficClass               uint8
+	TrafficClass               uint8             `config:"traffic_class"`
-	FsCacheExpireDuration      time.Duration
+	FsCacheExpireDuration      time.Duration     `config:"fs_cache_expire_duration"`
-	FsCacheExpireInterval      time.Duration
+	FsCacheExpireInterval      time.Duration     `config:"fs_cache_expire_interval"`
-	DisableHTTP2               bool
+	DisableHTTP2               bool              `config:"disable_http2"`
-	HumanReadable              bool
+	HumanReadable              bool              `config:"human_readable"`
-	KvLockTime                 time.Duration // maximum time to keep key-value database locked by process
+	KvLockTime                 time.Duration     `config:"kv_lock_time"` // maximum time to keep key-value database locked by process
-	DisableHTTPKeepAlives      bool
+	DisableHTTPKeepAlives      bool              `config:"disable_http_keep_alives"`
-	Metadata                   bool
+	Metadata                   bool              `config:"metadata"`
-	ServerSideAcrossConfigs    bool
+	ServerSideAcrossConfigs    bool              `config:"server_side_across_configs"`
-	TerminalColorMode          TerminalColorMode
+	TerminalColorMode          TerminalColorMode `config:"color"`
-	DefaultTime                Time // time that directories with no time should display
+	DefaultTime                Time              `config:"default_time"` // time that directories with no time should display
-	Inplace                    bool // Download directly to destination file instead of atomic download to temp/rename
+	Inplace                    bool              `config:"inplace"`      // Download directly to destination file instead of atomic download to temp/rename
-	PartialSuffix              string
+	PartialSuffix              string            `config:"partial_suffix"`
-	MetadataMapper             SpaceSepList
+	MetadataMapper             SpaceSepList      `config:"metadata_mapper"`
 }
 
-// NewConfig creates a new config with everything set to the default
+func init() {
-// value.  These are the ultimate defaults and are overridden by the
-// config module.
-func NewConfig() *ConfigInfo {
-	c := new(ConfigInfo)
-
 	// Set any values which aren't the zero for the type
-	c.LogLevel = LogLevelNotice
+	globalConfig.DeleteMode = DeleteModeDefault
-	c.StatsLogLevel = LogLevelInfo
-	c.ModifyWindow = time.Nanosecond
-	c.Checkers = 8
-	c.Transfers = 4
-	c.ConnectTimeout = 60 * time.Second
-	c.Timeout = 5 * 60 * time.Second
-	c.ExpectContinueTimeout = 1 * time.Second
-	c.DeleteMode = DeleteModeDefault
-	c.MaxDelete = -1
-	c.MaxDeleteSize = SizeSuffix(-1)
-	c.Retries = 3
-	c.LowLevelRetries = 10
-	c.MaxDepth = -1
-	c.DataRateUnit = "bytes"
-	c.BufferSize = SizeSuffix(16 << 20)
-	c.UserAgent = "rclone/" + Version
-	c.StreamingUploadCutoff = SizeSuffix(100 * 1024)
-	c.MaxStatsGroups = 1000
-	c.StatsFileNameLength = 45
-	c.AskPassword = true
-	c.TPSLimitBurst = 1
-	c.MaxTransfer = -1
-	c.MaxBacklog = 10000
-	// We do not want to set the default here. We use this variable being empty as part of the fall-through of options.
-	//	c.StatsOneLineDateFormat = "2006/01/02 15:04:05 - "
-	c.MultiThreadCutoff = SizeSuffix(256 * 1024 * 1024)
-	c.MultiThreadStreams = 4
-	c.MultiThreadChunkSize = SizeSuffix(64 * 1024 * 1024)
-	c.MultiThreadWriteBufferSize = SizeSuffix(128 * 1024)
 
-	c.TrackRenamesStrategy = "hash"
+	// Register the config and fill globalConfig with the defaults
-	c.FsCacheExpireDuration = 300 * time.Second
+	RegisterGlobalOptions(OptionsInfo{Name: "main", Opt: globalConfig, Options: ConfigOptionsInfo, Reload: globalConfig.Reload})
-	c.FsCacheExpireInterval = 60 * time.Second
-	c.KvLockTime = 1 * time.Second
-	c.DefaultTime = Time(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC))
-	c.PartialSuffix = ".partial"
 
-	// Perform a simple check for debug flags to enable debug logging during the flag initialization
+	// initial guess at log level from the flags
+	globalConfig.LogLevel = initialLogLevel()
+}
+
+// Reload assumes the config has been edited and does what is necessary to make it live
+func (ci *ConfigInfo) Reload(ctx context.Context) error {
+	// Set -vv if --dump is in use
+	if ci.Dump != 0 && ci.LogLevel != LogLevelDebug {
+		Logf(nil, "Automatically setting -vv as --dump is enabled")
+		ci.LogLevel = LogLevelDebug
+	}
+
+	// If --dry-run or -i then use NOTICE as minimum log level
+	if (ci.DryRun || ci.Interactive) && ci.StatsLogLevel > LogLevelNotice {
+		ci.StatsLogLevel = LogLevelNotice
+	}
+
+	// If --use-json-log then start the JSON logger
+	if ci.UseJSONLog {
+		InstallJSONLogger(ci.LogLevel)
+	}
+
+	// Check --compare-dest and --copy-dest
+	if len(ci.CompareDest) > 0 && len(ci.CopyDest) > 0 {
+		return fmt.Errorf("can't use --compare-dest with --copy-dest")
+	}
+
+	// Check --stats-one-line and dependent flags
+	switch {
+	case len(ci.StatsOneLineDateFormat) > 0:
+		ci.StatsOneLineDate = true
+		ci.StatsOneLine = true
+	case ci.StatsOneLineDate:
+		ci.StatsOneLineDateFormat = "2006/01/02 15:04:05 - "
+		ci.StatsOneLine = true
+	}
+
+	// Check --partial-suffix
+	if len(ci.PartialSuffix) > 16 {
+		return fmt.Errorf("--partial-suffix: Expecting suffix length not greater than %d but got %d", 16, len(ci.PartialSuffix))
+	}
+
+	// Make sure some values are > 0
+	nonZero := func(pi *int) {
+		if *pi <= 0 {
+			*pi = 1
+		}
+	}
+
+	// Check --stats-unit
+	if ci.DataRateUnit != "bits" && ci.DataRateUnit != "bytes" {
+		Errorf(nil, "Unknown unit %q passed to --stats-unit. Defaulting to bytes.", ci.DataRateUnit)
+		ci.DataRateUnit = "bytes"
+	}
+
+	// Check these are all > 0
+	nonZero(&ci.Retries)
+	nonZero(&ci.LowLevelRetries)
+	nonZero(&ci.Transfers)
+	nonZero(&ci.Checkers)
+
+	return nil
+}
+
+// Initial logging level
+//
+// Perform a simple check for debug flags to enable debug logging during the flag initialization
+func initialLogLevel() LogLevel {
+	logLevel := LogLevelNotice
 	for argIndex, arg := range os.Args {
 		if strings.HasPrefix(arg, "-vv") && strings.TrimRight(arg, "v") == "-" {
-			c.LogLevel = LogLevelDebug
+			logLevel = LogLevelDebug
 		}
 		if arg == "--log-level=DEBUG" || (arg == "--log-level" && len(os.Args) > argIndex+1 && os.Args[argIndex+1] == "DEBUG") {
-			c.LogLevel = LogLevelDebug
+			logLevel = LogLevelDebug
 		}
 		if strings.HasPrefix(arg, "--verbose=") {
 			if level, err := strconv.Atoi(arg[10:]); err == nil && level >= 2 {
-				c.LogLevel = LogLevelDebug
+				logLevel = LogLevelDebug
 			}
 		}
 	}
 	envValue, found := os.LookupEnv("RCLONE_LOG_LEVEL")
 	if found && envValue == "DEBUG" {
-		c.LogLevel = LogLevelDebug
+		logLevel = LogLevelDebug
 	}
-
+	return logLevel
-	return c
 }
 
 // TimeoutOrInfinite returns ci.Timeout if > 0 or infinite otherwise
-func (c *ConfigInfo) TimeoutOrInfinite() time.Duration {
+func (ci *ConfigInfo) TimeoutOrInfinite() time.Duration {
-	if c.Timeout > 0 {
+	if ci.Timeout > 0 {
-		return c.Timeout
+		return ci.Timeout
 	}
 	return ModTimeNotSupported
 }

fs/config/configflags/configflags.go

@@ -13,9 +13,6 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/flags"
-	fsLog "github.com/rclone/rclone/fs/log"
-	"github.com/rclone/rclone/fs/rc"
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
 )
 
@@ -38,123 +35,30 @@ var (
 	downloadHeaders []string
 	headers         []string
 	metadataSet     []string
-	partialSuffix   string
 )
 
 // AddFlags adds the non filing system specific flags to the command
 func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
-	rc.AddOption("main", ci)
+	flags.AddFlagsFromOptions(flagSet, "", fs.ConfigOptionsInfo)
-	// NB defaults which aren't the zero for the type should be set in fs/config.go NewConfig
+
+	// Add flags we haven't converted into options yet
 	flags.CountVarP(flagSet, &verbose, "verbose", "v", "Print lots more stuff (repeat for more)", "Logging,Important")
 	flags.BoolVarP(flagSet, &quiet, "quiet", "q", false, "Print as little stuff as possible", "Logging")
-	flags.DurationVarP(flagSet, &ci.ModifyWindow, "modify-window", "", ci.ModifyWindow, "Max time diff to be considered the same", "Copy")
-	flags.IntVarP(flagSet, &ci.Checkers, "checkers", "", ci.Checkers, "Number of checkers to run in parallel", "Performance")
-	flags.IntVarP(flagSet, &ci.Transfers, "transfers", "", ci.Transfers, "Number of file transfers to run in parallel", "Performance")
 	flags.StringVarP(flagSet, &configPath, "config", "", config.GetConfigPath(), "Config file", "Config")
 	flags.StringVarP(flagSet, &cacheDir, "cache-dir", "", config.GetCacheDir(), "Directory rclone will use for caching", "Config")
 	flags.StringVarP(flagSet, &tempDir, "temp-dir", "", os.TempDir(), "Directory rclone will use for temporary files", "Config")
-	flags.BoolVarP(flagSet, &ci.CheckSum, "checksum", "c", ci.CheckSum, "Check for changes with size & checksum (if available, or fallback to size only).", "Copy")
-	flags.BoolVarP(flagSet, &ci.SizeOnly, "size-only", "", ci.SizeOnly, "Skip based on size only, not modtime or checksum", "Copy")
-	flags.BoolVarP(flagSet, &ci.IgnoreTimes, "ignore-times", "I", ci.IgnoreTimes, "Don't skip items that match size and time - transfer all unconditionally", "Copy")
-	flags.BoolVarP(flagSet, &ci.IgnoreExisting, "ignore-existing", "", ci.IgnoreExisting, "Skip all files that exist on destination", "Copy")
-	flags.BoolVarP(flagSet, &ci.IgnoreErrors, "ignore-errors", "", ci.IgnoreErrors, "Delete even if there are I/O errors", "Sync")
-	flags.BoolVarP(flagSet, &ci.DryRun, "dry-run", "n", ci.DryRun, "Do a trial run with no permanent changes", "Config,Important")
-	flags.BoolVarP(flagSet, &ci.Interactive, "interactive", "i", ci.Interactive, "Enable interactive mode", "Config,Important")
-	flags.DurationVarP(flagSet, &ci.ConnectTimeout, "contimeout", "", ci.ConnectTimeout, "Connect timeout", "Networking")
-	flags.DurationVarP(flagSet, &ci.Timeout, "timeout", "", ci.Timeout, "IO idle timeout", "Networking")
-	flags.DurationVarP(flagSet, &ci.ExpectContinueTimeout, "expect-continue-timeout", "", ci.ExpectContinueTimeout, "Timeout when using expect / 100-continue in HTTP", "Networking")
 	flags.BoolVarP(flagSet, &dumpHeaders, "dump-headers", "", false, "Dump HTTP headers - may contain sensitive info", "Debugging")
 	flags.BoolVarP(flagSet, &dumpBodies, "dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info", "Debugging")
-	flags.BoolVarP(flagSet, &ci.InsecureSkipVerify, "no-check-certificate", "", ci.InsecureSkipVerify, "Do not verify the server SSL certificate (insecure)", "Networking")
-	flags.BoolVarP(flagSet, &ci.AskPassword, "ask-password", "", ci.AskPassword, "Allow prompt for password for encrypted configuration", "Config")
-	flags.FVarP(flagSet, &ci.PasswordCommand, "password-command", "", "Command for supplying password for encrypted configuration", "Config")
 	flags.BoolVarP(flagSet, &deleteBefore, "delete-before", "", false, "When synchronizing, delete files on destination before transferring", "Sync")
 	flags.BoolVarP(flagSet, &deleteDuring, "delete-during", "", false, "When synchronizing, delete files during transfer", "Sync")
 	flags.BoolVarP(flagSet, &deleteAfter, "delete-after", "", false, "When synchronizing, delete files on destination after transferring (default)", "Sync")
-	flags.Int64VarP(flagSet, &ci.MaxDelete, "max-delete", "", -1, "When synchronizing, limit the number of deletes", "Sync")
-	flags.FVarP(flagSet, &ci.MaxDeleteSize, "max-delete-size", "", "When synchronizing, limit the total size of deletes", "Sync")
-	flags.BoolVarP(flagSet, &ci.TrackRenames, "track-renames", "", ci.TrackRenames, "When synchronizing, track file renames and do a server-side move if possible", "Sync")
-	flags.StringVarP(flagSet, &ci.TrackRenamesStrategy, "track-renames-strategy", "", ci.TrackRenamesStrategy, "Strategies to use when synchronizing using track-renames hash|modtime|leaf", "Sync")
-	flags.IntVarP(flagSet, &ci.Retries, "retries", "", 3, "Retry operations this many times if they fail", "Config")
-	flags.DurationVarP(flagSet, &ci.RetriesInterval, "retries-sleep", "", 0, "Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)", "Config")
-	flags.IntVarP(flagSet, &ci.LowLevelRetries, "low-level-retries", "", ci.LowLevelRetries, "Number of low level retries to do", "Config")
-	flags.BoolVarP(flagSet, &ci.UpdateOlder, "update", "u", ci.UpdateOlder, "Skip files that are newer on the destination", "Copy")
-	flags.BoolVarP(flagSet, &ci.UseServerModTime, "use-server-modtime", "", ci.UseServerModTime, "Use server modified time instead of object metadata", "Config")
-	flags.BoolVarP(flagSet, &ci.NoGzip, "no-gzip-encoding", "", ci.NoGzip, "Don't set Accept-Encoding: gzip", "Networking")
-	flags.IntVarP(flagSet, &ci.MaxDepth, "max-depth", "", ci.MaxDepth, "If set limits the recursion depth to this", "Filter")
-	flags.BoolVarP(flagSet, &ci.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use modtime or checksum", "Copy")
-	flags.BoolVarP(flagSet, &ci.IgnoreChecksum, "ignore-checksum", "", ci.IgnoreChecksum, "Skip post copy check of checksums", "Copy")
-	flags.BoolVarP(flagSet, &ci.IgnoreCaseSync, "ignore-case-sync", "", ci.IgnoreCaseSync, "Ignore case when synchronizing", "Copy")
-	flags.BoolVarP(flagSet, &ci.FixCase, "fix-case", "", ci.FixCase, "Force rename of case insensitive dest to match source", "Sync")
-	flags.BoolVarP(flagSet, &ci.NoTraverse, "no-traverse", "", ci.NoTraverse, "Don't traverse destination file system on copy", "Copy")
-	flags.BoolVarP(flagSet, &ci.CheckFirst, "check-first", "", ci.CheckFirst, "Do all the checks before starting transfers", "Copy")
-	flags.BoolVarP(flagSet, &ci.NoCheckDest, "no-check-dest", "", ci.NoCheckDest, "Don't check the destination, copy regardless", "Copy")
-	flags.BoolVarP(flagSet, &ci.NoUnicodeNormalization, "no-unicode-normalization", "", ci.NoUnicodeNormalization, "Don't normalize unicode characters in filenames", "Config")
-	flags.BoolVarP(flagSet, &ci.NoUpdateModTime, "no-update-modtime", "", ci.NoUpdateModTime, "Don't update destination modtime if files identical", "Copy")
-	flags.BoolVarP(flagSet, &ci.NoUpdateDirModTime, "no-update-dir-modtime", "", ci.NoUpdateModTime, "Don't update directory modification times", "Copy")
-	flags.StringArrayVarP(flagSet, &ci.CompareDest, "compare-dest", "", nil, "Include additional comma separated server-side paths during comparison", "Copy")
-	flags.StringArrayVarP(flagSet, &ci.CopyDest, "copy-dest", "", nil, "Implies --compare-dest but also copies files from paths into destination", "Copy")
-	flags.StringVarP(flagSet, &ci.BackupDir, "backup-dir", "", ci.BackupDir, "Make backups into hierarchy based in DIR", "Sync")
-	flags.StringVarP(flagSet, &ci.Suffix, "suffix", "", ci.Suffix, "Suffix to add to changed files", "Sync")
-	flags.BoolVarP(flagSet, &ci.SuffixKeepExtension, "suffix-keep-extension", "", ci.SuffixKeepExtension, "Preserve the extension when using --suffix", "Sync")
-	flags.BoolVarP(flagSet, &ci.UseListR, "fast-list", "", ci.UseListR, "Use recursive list if available; uses more memory but fewer transactions", "Listing")
-	flags.Float64VarP(flagSet, &ci.TPSLimit, "tpslimit", "", ci.TPSLimit, "Limit HTTP transactions per second to this", "Networking")
-	flags.IntVarP(flagSet, &ci.TPSLimitBurst, "tpslimit-burst", "", ci.TPSLimitBurst, "Max burst of transactions for --tpslimit", "Networking")
 	flags.StringVarP(flagSet, &bindAddr, "bind", "", "", "Local address to bind to for outgoing connections, IPv4, IPv6 or name", "Networking")
 	flags.StringVarP(flagSet, &disableFeatures, "disable", "", "", "Disable a comma separated list of features (use --disable help to see a list)", "Config")
-	flags.StringVarP(flagSet, &ci.UserAgent, "user-agent", "", ci.UserAgent, "Set the user-agent to a specified string", "Networking")
-	flags.BoolVarP(flagSet, &ci.Immutable, "immutable", "", ci.Immutable, "Do not modify files, fail if existing files have been modified", "Copy")
-	flags.BoolVarP(flagSet, &ci.AutoConfirm, "auto-confirm", "", ci.AutoConfirm, "If enabled, do not request console confirmation", "Config")
-	flags.IntVarP(flagSet, &ci.StatsFileNameLength, "stats-file-name-length", "", ci.StatsFileNameLength, "Max file name length in stats (0 for no limit)", "Logging")
-	flags.FVarP(flagSet, &ci.LogLevel, "log-level", "", "Log level DEBUG|INFO|NOTICE|ERROR", "Logging")
-	flags.FVarP(flagSet, &ci.StatsLogLevel, "stats-log-level", "", "Log level to show --stats output DEBUG|INFO|NOTICE|ERROR", "Logging")
-	flags.FVarP(flagSet, &ci.BwLimit, "bwlimit", "", "Bandwidth limit in KiB/s, or use suffix B|K|M|G|T|P or a full timetable", "Networking")
-	flags.FVarP(flagSet, &ci.BwLimitFile, "bwlimit-file", "", "Bandwidth limit per file in KiB/s, or use suffix B|K|M|G|T|P or a full timetable", "Networking")
-	flags.FVarP(flagSet, &ci.BufferSize, "buffer-size", "", "In memory buffer size when reading files for each --transfer", "Performance")
-	flags.FVarP(flagSet, &ci.StreamingUploadCutoff, "streaming-upload-cutoff", "", "Cutoff for switching to chunked upload if file size is unknown, upload starts after reaching cutoff or when file ends", "Copy")
-	flags.FVarP(flagSet, &ci.Dump, "dump", "", "List of items to dump from: "+fs.DumpFlagsList, "Debugging")
-	flags.FVarP(flagSet, &ci.MaxTransfer, "max-transfer", "", "Maximum size of data to transfer", "Copy")
-	flags.DurationVarP(flagSet, &ci.MaxDuration, "max-duration", "", 0, "Maximum duration rclone will transfer data for", "Copy")
-	flags.FVarP(flagSet, &ci.CutoffMode, "cutoff-mode", "", "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS", "Copy")
-	flags.IntVarP(flagSet, &ci.MaxBacklog, "max-backlog", "", ci.MaxBacklog, "Maximum number of objects in sync or check backlog", "Copy,Check")
-	flags.IntVarP(flagSet, &ci.MaxStatsGroups, "max-stats-groups", "", ci.MaxStatsGroups, "Maximum number of stats groups to keep in memory, on max oldest is discarded", "Logging")
-	flags.BoolVarP(flagSet, &ci.StatsOneLine, "stats-one-line", "", ci.StatsOneLine, "Make the stats fit on one line", "Logging")
-	flags.BoolVarP(flagSet, &ci.StatsOneLineDate, "stats-one-line-date", "", ci.StatsOneLineDate, "Enable --stats-one-line and add current date/time prefix", "Logging")
-	flags.StringVarP(flagSet, &ci.StatsOneLineDateFormat, "stats-one-line-date-format", "", ci.StatsOneLineDateFormat, "Enable --stats-one-line-date and use custom formatted date: Enclose date string in double quotes (\"), see https://golang.org/pkg/time/#Time.Format", "Logging")
-	flags.BoolVarP(flagSet, &ci.ErrorOnNoTransfer, "error-on-no-transfer", "", ci.ErrorOnNoTransfer, "Sets exit code 9 if no files are transferred, useful in scripts", "Config")
-	flags.BoolVarP(flagSet, &ci.Progress, "progress", "P", ci.Progress, "Show progress during transfer", "Logging")
-	flags.BoolVarP(flagSet, &ci.ProgressTerminalTitle, "progress-terminal-title", "", ci.ProgressTerminalTitle, "Show progress on the terminal title (requires -P/--progress)", "Logging")
-	flags.BoolVarP(flagSet, &ci.Cookie, "use-cookies", "", ci.Cookie, "Enable session cookiejar", "Networking")
-	flags.BoolVarP(flagSet, &ci.UseMmap, "use-mmap", "", ci.UseMmap, "Use mmap allocator (see docs)", "Config")
-	flags.StringArrayVarP(flagSet, &ci.CaCert, "ca-cert", "", ci.CaCert, "CA certificate used to verify servers", "Networking")
-	flags.StringVarP(flagSet, &ci.ClientCert, "client-cert", "", ci.ClientCert, "Client SSL certificate (PEM) for mutual TLS auth", "Networking")
-	flags.StringVarP(flagSet, &ci.ClientKey, "client-key", "", ci.ClientKey, "Client SSL private key (PEM) for mutual TLS auth", "Networking")
-	flags.FVarP(flagSet, &ci.MultiThreadCutoff, "multi-thread-cutoff", "", "Use multi-thread downloads for files above this size", "Copy")
-	flags.IntVarP(flagSet, &ci.MultiThreadStreams, "multi-thread-streams", "", ci.MultiThreadStreams, "Number of streams to use for multi-thread downloads", "Copy")
-	flags.FVarP(flagSet, &ci.MultiThreadWriteBufferSize, "multi-thread-write-buffer-size", "", "In memory buffer size for writing when in multi-thread mode", "Copy")
-	flags.FVarP(flagSet, &ci.MultiThreadChunkSize, "multi-thread-chunk-size", "", "Chunk size for multi-thread downloads / uploads, if not set by filesystem", "Copy")
-	flags.BoolVarP(flagSet, &ci.UseJSONLog, "use-json-log", "", ci.UseJSONLog, "Use json log format", "Logging")
-	flags.StringVarP(flagSet, &ci.OrderBy, "order-by", "", ci.OrderBy, "Instructions on how to order the transfers, e.g. 'size,descending'", "Copy")
 	flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions", "Networking")
 	flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions", "Networking")
 	flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions", "Networking")
 	flags.StringArrayVarP(flagSet, &metadataSet, "metadata-set", "", nil, "Add metadata key=value when uploading", "Metadata")
-	flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files", "Copy")
-	flags.BoolVarP(flagSet, &ci.NoConsole, "no-console", "", ci.NoConsole, "Hide console window (supported on Windows only)", "Config")
 	flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21", "Networking")
-	flags.DurationVarP(flagSet, &ci.FsCacheExpireDuration, "fs-cache-expire-duration", "", ci.FsCacheExpireDuration, "Cache remotes for this long (0 to disable caching)", "Config")
-	flags.DurationVarP(flagSet, &ci.FsCacheExpireInterval, "fs-cache-expire-interval", "", ci.FsCacheExpireInterval, "Interval to check for expired remotes", "Config")
-	flags.BoolVarP(flagSet, &ci.DisableHTTP2, "disable-http2", "", ci.DisableHTTP2, "Disable HTTP/2 in the global transport", "Networking")
-	flags.BoolVarP(flagSet, &ci.HumanReadable, "human-readable", "", ci.HumanReadable, "Print numbers in a human-readable format, sizes with suffix Ki|Mi|Gi|Ti|Pi", "Config")
-	flags.DurationVarP(flagSet, &ci.KvLockTime, "kv-lock-time", "", ci.KvLockTime, "Maximum time to keep key-value database locked by process", "Config")
-	flags.BoolVarP(flagSet, &ci.DisableHTTPKeepAlives, "disable-http-keep-alives", "", ci.DisableHTTPKeepAlives, "Disable HTTP keep-alives and use each connection once.", "Networking")
-	flags.BoolVarP(flagSet, &ci.Metadata, "metadata", "M", ci.Metadata, "If set, preserve metadata when copying objects", "Metadata,Copy")
-	flags.BoolVarP(flagSet, &ci.ServerSideAcrossConfigs, "server-side-across-configs", "", ci.ServerSideAcrossConfigs, "Allow server-side operations (e.g. copy) to work across different configs", "Copy")
-	flags.FVarP(flagSet, &ci.TerminalColorMode, "color", "", "When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS", "Config")
-	flags.FVarP(flagSet, &ci.DefaultTime, "default-time", "", "Time to show if modtime is unknown for files and directories", "Config,Listing")
-	flags.BoolVarP(flagSet, &ci.Inplace, "inplace", "", ci.Inplace, "Download directly to destination file instead of atomic download to temp/rename", "Copy")
-	flags.StringVarP(flagSet, &partialSuffix, "partial-suffix", "", ci.PartialSuffix, "Add partial-suffix to temporary file name when --inplace is not used", "Copy")
-	flags.FVarP(flagSet, &ci.MetadataMapper, "metadata-mapper", "", "Program to run to transforming metadata before upload", "Metadata")
 }
 
 // ParseHeaders converts the strings passed in via the header flags into HTTPOptions
@@ -174,8 +78,9 @@ func ParseHeaders(headers []string) []*fs.HTTPOption {
 	return opts
 }
 
-// SetFlags converts any flags into config which weren't straight forward
+// SetFlags sets flags which aren't part of the config system
 func SetFlags(ci *fs.ConfigInfo) {
+	// Process obsolete --dump-headers and --dump-bodies flags
 	if dumpHeaders {
 		ci.Dump |= fs.DumpHeaders
 		fs.Logf(nil, "--dump-headers is obsolete - please use --dump headers instead")
@@ -184,25 +89,23 @@ func SetFlags(ci *fs.ConfigInfo) {
 		ci.Dump |= fs.DumpBodies
 		fs.Logf(nil, "--dump-bodies is obsolete - please use --dump bodies instead")
 	}
-	if ci.Dump != 0 && verbose < 2 && ci.LogLevel != fs.LogLevelDebug {
-		fs.Logf(nil, "Automatically setting -vv as --dump is enabled")
-		verbose = 2
-	}
 
+	// Process -v flag
 	if verbose >= 2 {
 		ci.LogLevel = fs.LogLevelDebug
 	} else if verbose >= 1 {
 		ci.LogLevel = fs.LogLevelInfo
 	}
-	if (ci.DryRun || ci.Interactive) && ci.StatsLogLevel > fs.LogLevelNotice {
+
-		ci.StatsLogLevel = fs.LogLevelNotice
+	// Process -q flag
-	}
 	if quiet {
 		if verbose > 0 {
 			log.Fatalf("Can't set -v and -q")
 		}
 		ci.LogLevel = fs.LogLevelError
 	}
+
+	// Can't set log level, -v, -q
 	logLevelFlag := pflag.Lookup("log-level")
 	if logLevelFlag != nil && logLevelFlag.Changed {
 		if verbose > 0 {
@@ -212,28 +115,8 @@ func SetFlags(ci *fs.ConfigInfo) {
 			log.Fatalf("Can't set -q and --log-level")
 		}
 	}
-	if ci.UseJSONLog {
-		logrus.AddHook(fsLog.NewCallerHook())
-		logrus.SetFormatter(&logrus.JSONFormatter{
-			TimestampFormat: "2006-01-02T15:04:05.999999-07:00",
-		})
-		logrus.SetLevel(logrus.DebugLevel)
-		switch ci.LogLevel {
-		case fs.LogLevelEmergency, fs.LogLevelAlert:
-			logrus.SetLevel(logrus.PanicLevel)
-		case fs.LogLevelCritical:
-			logrus.SetLevel(logrus.FatalLevel)
-		case fs.LogLevelError:
-			logrus.SetLevel(logrus.ErrorLevel)
-		case fs.LogLevelWarning, fs.LogLevelNotice:
-			logrus.SetLevel(logrus.WarnLevel)
-		case fs.LogLevelInfo:
-			logrus.SetLevel(logrus.InfoLevel)
-		case fs.LogLevelDebug:
-			logrus.SetLevel(logrus.DebugLevel)
-		}
-	}
 
+	// Process --delete-before, --delete-during and --delete-after
 	switch {
 	case deleteBefore && (deleteDuring || deleteAfter),
 		deleteDuring && deleteAfter:
@@ -248,19 +131,7 @@ func SetFlags(ci *fs.ConfigInfo) {
 		ci.DeleteMode = fs.DeleteModeDefault
 	}
 
-	if len(ci.CompareDest) > 0 && len(ci.CopyDest) > 0 {
+	// Process --bind into IP address
-		log.Fatalf(`Can't use --compare-dest with --copy-dest.`)
-	}
-
-	switch {
-	case len(ci.StatsOneLineDateFormat) > 0:
-		ci.StatsOneLineDate = true
-		ci.StatsOneLine = true
-	case ci.StatsOneLineDate:
-		ci.StatsOneLineDateFormat = "2006/01/02 15:04:05 - "
-		ci.StatsOneLine = true
-	}
-
 	if bindAddr != "" {
 		addrs, err := net.LookupIP(bindAddr)
 		if err != nil {
@@ -272,6 +143,7 @@ func SetFlags(ci *fs.ConfigInfo) {
 		ci.BindAddr = addrs[0]
 	}
 
+	// Process --disable
 	if disableFeatures != "" {
 		if disableFeatures == "help" {
 			log.Fatalf("Possible backend features are: %s\n", strings.Join(new(fs.Features).List(), ", "))
@@ -279,6 +151,7 @@ func SetFlags(ci *fs.ConfigInfo) {
 		ci.DisableFeatures = strings.Split(disableFeatures, ",")
 	}
 
+	// Process --headers-upload, --headers-download, --headers
 	if len(uploadHeaders) != 0 {
 		ci.UploadHeaders = ParseHeaders(uploadHeaders)
 	}
@@ -288,9 +161,8 @@ func SetFlags(ci *fs.ConfigInfo) {
 	if len(headers) != 0 {
 		ci.Headers = ParseHeaders(headers)
 	}
-	if len(headers) != 0 {
+
-		ci.Headers = ParseHeaders(headers)
+	// Process --metadata-set
-	}
 	if len(metadataSet) != 0 {
 		ci.MetadataSet = make(fs.Metadata, len(metadataSet))
 		for _, kv := range metadataSet {
@@ -302,6 +174,8 @@ func SetFlags(ci *fs.ConfigInfo) {
 		}
 		fs.Debugf(nil, "MetadataUpload %v", ci.MetadataSet)
 	}
+
+	// Process --dscp
 	if len(dscp) != 0 {
 		if value, ok := parseDSCP(dscp); ok {
 			ci.TrafficClass = value << 2
@@ -310,39 +184,24 @@ func SetFlags(ci *fs.ConfigInfo) {
 		}
 	}
 
-	// Set path to configuration file
+	// Process --config path
 	if err := config.SetConfigPath(configPath); err != nil {
 		log.Fatalf("--config: Failed to set %q as config path: %v", configPath, err)
 	}
 
-	// Set path to cache dir
+	// Process --cache-dir path
 	if err := config.SetCacheDir(cacheDir); err != nil {
 		log.Fatalf("--cache-dir: Failed to set %q as cache dir: %v", cacheDir, err)
 	}
 
-	// Set path to temp dir
+	// Process --temp-dir path
 	if err := config.SetTempDir(tempDir); err != nil {
 		log.Fatalf("--temp-dir: Failed to set %q as temp dir: %v", tempDir, err)
 	}
 
-	// Set whether multi-thread-streams was set
+	// Process --multi-thread-streams - set whether multi-thread-streams was set
 	multiThreadStreamsFlag := pflag.Lookup("multi-thread-streams")
 	ci.MultiThreadSet = multiThreadStreamsFlag != nil && multiThreadStreamsFlag.Changed
-
-	if len(partialSuffix) > 16 {
-		log.Fatalf("--partial-suffix: Expecting suffix length not greater than %d but got %d", 16, len(partialSuffix))
-	}
-	ci.PartialSuffix = partialSuffix
-
-	// Make sure some values are > 0
-	nonZero := func(pi *int) {
-		if *pi <= 0 {
-			*pi = 1
-		}
-	}
-	nonZero(&ci.LowLevelRetries)
-	nonZero(&ci.Transfers)
-	nonZero(&ci.Checkers)
 }
 
 // parseHeaders converts DSCP names to value

fs/log.go

@@ -57,6 +57,9 @@ func (logLevelChoices) Type() string {
 // LogPrintPid enables process pid in log
 var LogPrintPid = false
 
+// InstallJSONLogger is a hook that --use-json-log calls
+var InstallJSONLogger = func(logLevel LogLevel) {}
+
 // LogPrint sends the text to the logger of level
 var LogPrint = func(level LogLevel, text string) {
 	text = fmt.Sprintf("%-6s: %s", level, text)

fs/log/caller_hook.go

@@ -5,9 +5,43 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/rclone/rclone/fs"
 	"github.com/sirupsen/logrus"
 )
 
+var loggerInstalled = false
+
+// InstallJSONLogger installs the JSON logger at the specified log level
+func InstallJSONLogger(logLevel fs.LogLevel) {
+	if !loggerInstalled {
+		logrus.AddHook(NewCallerHook())
+		loggerInstalled = true
+	}
+	logrus.SetFormatter(&logrus.JSONFormatter{
+		TimestampFormat: "2006-01-02T15:04:05.999999-07:00",
+	})
+	logrus.SetLevel(logrus.DebugLevel)
+	switch logLevel {
+	case fs.LogLevelEmergency, fs.LogLevelAlert:
+		logrus.SetLevel(logrus.PanicLevel)
+	case fs.LogLevelCritical:
+		logrus.SetLevel(logrus.FatalLevel)
+	case fs.LogLevelError:
+		logrus.SetLevel(logrus.ErrorLevel)
+	case fs.LogLevelWarning, fs.LogLevelNotice:
+		logrus.SetLevel(logrus.WarnLevel)
+	case fs.LogLevelInfo:
+		logrus.SetLevel(logrus.InfoLevel)
+	case fs.LogLevelDebug:
+		logrus.SetLevel(logrus.DebugLevel)
+	}
+}
+
+// install hook in fs to call to avoid circular dependency
+func init() {
+	fs.InstallJSONLogger = InstallJSONLogger
+}
+
 // CallerHook for log the calling file and line of the fine
 type CallerHook struct {
 	Field  string