fs: convert main options to new config system

There are some flags which haven't been converted which could be
converted in the future.
This commit is contained in:
Nick Craig-Wood 2024-07-09 18:42:01 +01:00
parent c2bf300dd8
commit 4d2bc190cc
6 changed files with 726 additions and 329 deletions

View File

@ -14,7 +14,6 @@ import (
"os"
"os/exec"
"path"
"regexp"
"runtime"
"runtime/pprof"
"strconv"
@ -49,7 +48,6 @@ var (
cpuProfile = flags.StringP("cpuprofile", "", "", "Write cpu profile to file", "Debugging")
memProfile = flags.StringP("memprofile", "", "", "Write memory profile to file", "Debugging")
statsInterval = flags.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable)", "Logging")
dataRateUnit = flags.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes' per second", "Logging")
version bool
// Errors
errorCommandNotFound = errors.New("command not found")
@ -472,13 +470,6 @@ func initConfig() {
}
})
}
if m, _ := regexp.MatchString("^(bits|bytes)$", *dataRateUnit); !m {
fs.Errorf(nil, "Invalid unit passed to --stats-unit. Defaulting to bytes.")
ci.DataRateUnit = "bytes"
} else {
ci.DataRateUnit = *dataRateUnit
}
}
func resolveExitCode(err error) {

View File

@ -66,7 +66,7 @@ func TestEnvironmentVariables(t *testing.T) {
assert.NotContains(t, out, "fileAA1.txt") // depth 4
}
// Test of debug logging while initialising flags from environment (tests #5241 Enhance1)
// Test of debug logging while initialising flags from environment (tests #5341 Enhance1)
env = "RCLONE_STATS=173ms"
out, err = rcloneEnv(env, "version", "-vv")
if assert.NoError(t, err) {

View File

@ -3,6 +3,7 @@ package fs
import (
"context"
"errors"
"fmt"
"net"
"os"
"strconv"
@ -13,7 +14,7 @@ import (
// Global
var (
// globalConfig for rclone
globalConfig = NewConfig()
globalConfig = new(ConfigInfo)
// Read a value from the config file
//
@ -49,184 +50,693 @@ var (
ConfigEdit = "config_fs_edit"
)
// ConfigOptionsInfo describes the Options in use
var ConfigOptionsInfo = Options{{
Name: "modify_window",
Default: time.Nanosecond,
Help: "Max time diff to be considered the same",
Groups: "Copy",
}, {
Name: "checkers",
Default: 8,
Help: "Number of checkers to run in parallel",
Groups: "Performance",
}, {
Name: "transfers",
Default: 4,
Help: "Number of file transfers to run in parallel",
Groups: "Performance",
}, {
Name: "checksum",
ShortOpt: "c",
Default: false,
Help: "Check for changes with size & checksum (if available, or fallback to size only).",
Groups: "Copy",
}, {
Name: "size_only",
Default: false,
Help: "Skip based on size only, not modtime or checksum",
Groups: "Copy",
}, {
Name: "ignore_times",
ShortOpt: "I",
Default: false,
Help: "Don't skip items that match size and time - transfer all unconditionally",
Groups: "Copy",
}, {
Name: "ignore_existing",
Default: false,
Help: "Skip all files that exist on destination",
Groups: "Copy",
}, {
Name: "ignore_errors",
Default: false,
Help: "Delete even if there are I/O errors",
Groups: "Sync",
}, {
Name: "dry_run",
ShortOpt: "n",
Default: false,
Help: "Do a trial run with no permanent changes",
Groups: "Config,Important",
}, {
Name: "interactive",
ShortOpt: "i",
Default: false,
Help: "Enable interactive mode",
Groups: "Config,Important",
}, {
Name: "contimeout",
Default: 60 * time.Second,
Help: "Connect timeout",
Groups: "Networking",
}, {
Name: "timeout",
Default: 5 * 60 * time.Second,
Help: "IO idle timeout",
Groups: "Networking",
}, {
Name: "expect_continue_timeout",
Default: 1 * time.Second,
Help: "Timeout when using expect / 100-continue in HTTP",
Groups: "Networking",
}, {
Name: "no_check_certificate",
Default: false,
Help: "Do not verify the server SSL certificate (insecure)",
Groups: "Networking",
}, {
Name: "ask_password",
Default: true,
Help: "Allow prompt for password for encrypted configuration",
Groups: "Config",
}, {
Name: "password_command",
Default: SpaceSepList{},
Help: "Command for supplying password for encrypted configuration",
Groups: "Config",
}, {
Name: "max_delete",
Default: int64(-1),
Help: "When synchronizing, limit the number of deletes",
Groups: "Sync",
}, {
Name: "max_delete_size",
Default: SizeSuffix(-1),
Help: "When synchronizing, limit the total size of deletes",
Groups: "Sync",
}, {
Name: "track_renames",
Default: false,
Help: "When synchronizing, track file renames and do a server-side move if possible",
Groups: "Sync",
}, {
Name: "track_renames_strategy",
Default: "hash",
Help: "Strategies to use when synchronizing using track-renames hash|modtime|leaf",
Groups: "Sync",
}, {
Name: "retries",
Default: 3,
Help: "Retry operations this many times if they fail",
Groups: "Config",
}, {
Name: "retries_sleep",
Default: time.Duration(0),
Help: "Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)",
Groups: "Config",
}, {
Name: "low_level_retries",
Default: 10,
Help: "Number of low level retries to do",
Groups: "Config",
}, {
Name: "update",
ShortOpt: "u",
Default: false,
Help: "Skip files that are newer on the destination",
Groups: "Copy",
}, {
Name: "use_server_modtime",
Default: false,
Help: "Use server modified time instead of object metadata",
Groups: "Config",
}, {
Name: "no_gzip_encoding",
Default: false,
Help: "Don't set Accept-Encoding: gzip",
Groups: "Networking",
}, {
Name: "max_depth",
Default: -1,
Help: "If set limits the recursion depth to this",
Groups: "Filter",
}, {
Name: "ignore_size",
Default: false,
Help: "Ignore size when skipping use modtime or checksum",
Groups: "Copy",
}, {
Name: "ignore_checksum",
Default: false,
Help: "Skip post copy check of checksums",
Groups: "Copy",
}, {
Name: "ignore_case_sync",
Default: false,
Help: "Ignore case when synchronizing",
Groups: "Copy",
}, {
Name: "fix_case",
Default: false,
Help: "Force rename of case insensitive dest to match source",
Groups: "Sync",
}, {
Name: "no_traverse",
Default: false,
Help: "Don't traverse destination file system on copy",
Groups: "Copy",
}, {
Name: "check_first",
Default: false,
Help: "Do all the checks before starting transfers",
Groups: "Copy",
}, {
Name: "no_check_dest",
Default: false,
Help: "Don't check the destination, copy regardless",
Groups: "Copy",
}, {
Name: "no_unicode_normalization",
Default: false,
Help: "Don't normalize unicode characters in filenames",
Groups: "Config",
}, {
Name: "no_update_modtime",
Default: false,
Help: "Don't update destination modtime if files identical",
Groups: "Copy",
}, {
Name: "no_update_dir_modtime",
Default: false,
Help: "Don't update directory modification times",
Groups: "Copy",
}, {
Name: "compare_dest",
Default: []string{},
Help: "Include additional server-side paths during comparison",
Groups: "Copy",
}, {
Name: "copy_dest",
Default: []string{},
Help: "Implies --compare-dest but also copies files from paths into destination",
Groups: "Copy",
}, {
Name: "backup_dir",
Default: "",
Help: "Make backups into hierarchy based in DIR",
Groups: "Sync",
}, {
Name: "suffix",
Default: "",
Help: "Suffix to add to changed files",
Groups: "Sync",
}, {
Name: "suffix_keep_extension",
Default: false,
Help: "Preserve the extension when using --suffix",
Groups: "Sync",
}, {
Name: "fast_list",
Default: false,
Help: "Use recursive list if available; uses more memory but fewer transactions",
Groups: "Listing",
}, {
Name: "tpslimit",
Default: 0.0,
Help: "Limit HTTP transactions per second to this",
Groups: "Networking",
}, {
Name: "tpslimit_burst",
Default: 1,
Help: "Max burst of transactions for --tpslimit",
Groups: "Networking",
}, {
Name: "user_agent",
Default: "rclone/" + Version,
Help: "Set the user-agent to a specified string",
Groups: "Networking",
}, {
Name: "immutable",
Default: false,
Help: "Do not modify files, fail if existing files have been modified",
Groups: "Copy",
}, {
Name: "auto_confirm",
Default: false,
Help: "If enabled, do not request console confirmation",
Groups: "Config",
}, {
Name: "stats_unit",
Default: "bytes",
Help: "Show data rate in stats as either 'bits' or 'bytes' per second",
Groups: "Logging",
}, {
Name: "stats_file_name_length",
Default: 45,
Help: "Max file name length in stats (0 for no limit)",
Groups: "Logging",
}, {
Name: "log_level",
Default: LogLevelNotice,
Help: "Log level DEBUG|INFO|NOTICE|ERROR",
Groups: "Logging",
}, {
Name: "stats_log_level",
Default: LogLevelInfo,
Help: "Log level to show --stats output DEBUG|INFO|NOTICE|ERROR",
Groups: "Logging",
}, {
Name: "bwlimit",
Default: BwTimetable{},
Help: "Bandwidth limit in KiB/s, or use suffix B|K|M|G|T|P or a full timetable",
Groups: "Networking",
}, {
Name: "bwlimit_file",
Default: BwTimetable{},
Help: "Bandwidth limit per file in KiB/s, or use suffix B|K|M|G|T|P or a full timetable",
Groups: "Networking",
}, {
Name: "buffer_size",
Default: SizeSuffix(16 << 20),
Help: "In memory buffer size when reading files for each --transfer",
Groups: "Performance",
}, {
Name: "streaming_upload_cutoff",
Default: SizeSuffix(100 * 1024),
Help: "Cutoff for switching to chunked upload if file size is unknown, upload starts after reaching cutoff or when file ends",
Groups: "Copy",
}, {
Name: "dump",
Default: DumpFlags(0),
Help: "List of items to dump from: " + DumpFlagsList,
Groups: "Debugging",
}, {
Name: "max_transfer",
Default: SizeSuffix(-1),
Help: "Maximum size of data to transfer",
Groups: "Copy",
}, {
Name: "max_duration",
Default: time.Duration(0),
Help: "Maximum duration rclone will transfer data for",
Groups: "Copy",
}, {
Name: "cutoff_mode",
Default: CutoffMode(0),
Help: "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS",
Groups: "Copy",
}, {
Name: "max_backlog",
Default: 10000,
Help: "Maximum number of objects in sync or check backlog",
Groups: "Copy,Check",
}, {
Name: "max_stats_groups",
Default: 1000,
Help: "Maximum number of stats groups to keep in memory, on max oldest is discarded",
Groups: "Logging",
}, {
Name: "stats_one_line",
Default: false,
Help: "Make the stats fit on one line",
Groups: "Logging",
}, {
Name: "stats_one_line_date",
Default: false,
Help: "Enable --stats-one-line and add current date/time prefix",
Groups: "Logging",
}, {
Name: "stats_one_line_date_format",
Default: "",
Help: "Enable --stats-one-line-date and use custom formatted date: Enclose date string in double quotes (\"), see https://golang.org/pkg/time/#Time.Format",
Groups: "Logging",
}, {
Name: "error_on_no_transfer",
Default: false,
Help: "Sets exit code 9 if no files are transferred, useful in scripts",
Groups: "Config",
}, {
Name: "progress",
ShortOpt: "P",
Default: false,
Help: "Show progress during transfer",
Groups: "Logging",
}, {
Name: "progress_terminal_title",
Default: false,
Help: "Show progress on the terminal title (requires -P/--progress)",
Groups: "Logging",
}, {
Name: "use_cookies",
Default: false,
Help: "Enable session cookiejar",
Groups: "Networking",
}, {
Name: "use_mmap",
Default: false,
Help: "Use mmap allocator (see docs)",
Groups: "Config",
}, {
Name: "ca_cert",
Default: []string{},
Help: "CA certificate used to verify servers",
Groups: "Networking",
}, {
Name: "client_cert",
Default: "",
Help: "Client SSL certificate (PEM) for mutual TLS auth",
Groups: "Networking",
}, {
Name: "client_key",
Default: "",
Help: "Client SSL private key (PEM) for mutual TLS auth",
Groups: "Networking",
}, {
Name: "multi_thread_cutoff",
Default: SizeSuffix(256 * 1024 * 1024),
Help: "Use multi-thread downloads for files above this size",
Groups: "Copy",
}, {
Name: "multi_thread_streams",
Default: 4,
Help: "Number of streams to use for multi-thread downloads",
Groups: "Copy",
}, {
Name: "multi_thread_write_buffer_size",
Default: SizeSuffix(128 * 1024),
Help: "In memory buffer size for writing when in multi-thread mode",
Groups: "Copy",
}, {
Name: "multi_thread_chunk_size",
Default: SizeSuffix(64 * 1024 * 1024),
Help: "Chunk size for multi-thread downloads / uploads, if not set by filesystem",
Groups: "Copy",
}, {
Name: "use_json_log",
Default: false,
Help: "Use json log format",
Groups: "Logging",
}, {
Name: "order_by",
Default: "",
Help: "Instructions on how to order the transfers, e.g. 'size,descending'",
Groups: "Copy",
}, {
Name: "refresh_times",
Default: false,
Help: "Refresh the modtime of remote files",
Groups: "Copy",
}, {
Name: "no_console",
Default: false,
Help: "Hide console window (supported on Windows only)",
Groups: "Config",
}, {
Name: "fs_cache_expire_duration",
Default: 300 * time.Second,
Help: "Cache remotes for this long (0 to disable caching)",
Groups: "Config",
}, {
Name: "fs_cache_expire_interval",
Default: 60 * time.Second,
Help: "Interval to check for expired remotes",
Groups: "Config",
}, {
Name: "disable_http2",
Default: false,
Help: "Disable HTTP/2 in the global transport",
Groups: "Networking",
}, {
Name: "human_readable",
Default: false,
Help: "Print numbers in a human-readable format, sizes with suffix Ki|Mi|Gi|Ti|Pi",
Groups: "Config",
}, {
Name: "kv_lock_time",
Default: 1 * time.Second,
Help: "Maximum time to keep key-value database locked by process",
Groups: "Config",
}, {
Name: "disable_http_keep_alives",
Default: false,
Help: "Disable HTTP keep-alives and use each connection once.",
Groups: "Networking",
}, {
Name: "metadata",
ShortOpt: "M",
Default: false,
Help: "If set, preserve metadata when copying objects",
Groups: "Metadata,Copy",
}, {
Name: "server_side_across_configs",
Default: false,
Help: "Allow server-side operations (e.g. copy) to work across different configs",
Groups: "Copy",
}, {
Name: "color",
Default: TerminalColorMode(0),
Help: "When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS",
Groups: "Config",
}, {
Name: "default_time",
Default: Time(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)),
Help: "Time to show if modtime is unknown for files and directories",
Groups: "Config,Listing",
}, {
Name: "inplace",
Default: false,
Help: "Download directly to destination file instead of atomic download to temp/rename",
Groups: "Copy",
}, {
Name: "metadata_mapper",
Default: SpaceSepList{},
Help: "Program to run to transforming metadata before upload",
Groups: "Metadata",
}, {
Name: "partial_suffix",
Default: ".partial",
Help: "Add partial-suffix to temporary file name when --inplace is not used",
Groups: "Copy",
}}
// ConfigInfo is filesystem config options
type ConfigInfo struct {
LogLevel LogLevel
StatsLogLevel LogLevel
UseJSONLog bool
DryRun bool
Interactive bool
CheckSum bool
SizeOnly bool
IgnoreTimes bool
IgnoreExisting bool
IgnoreErrors bool
ModifyWindow time.Duration
Checkers int
Transfers int
ConnectTimeout time.Duration // Connect timeout
Timeout time.Duration // Data channel timeout
ExpectContinueTimeout time.Duration
Dump DumpFlags
InsecureSkipVerify bool // Skip server certificate verification
DeleteMode DeleteMode
MaxDelete int64
MaxDeleteSize SizeSuffix
TrackRenames bool // Track file renames.
TrackRenamesStrategy string // Comma separated list of strategies used to track renames
Retries int // High-level retries
RetriesInterval time.Duration // --retries-sleep
LowLevelRetries int
UpdateOlder bool // Skip files that are newer on the destination
NoGzip bool // Disable compression
MaxDepth int
IgnoreSize bool
IgnoreChecksum bool
IgnoreCaseSync bool
FixCase bool
NoTraverse bool
CheckFirst bool
NoCheckDest bool
NoUnicodeNormalization bool
NoUpdateModTime bool
NoUpdateDirModTime bool
DataRateUnit string
CompareDest []string
CopyDest []string
BackupDir string
Suffix string
SuffixKeepExtension bool
UseListR bool
BufferSize SizeSuffix
BwLimit BwTimetable
BwLimitFile BwTimetable
TPSLimit float64
TPSLimitBurst int
BindAddr net.IP
DisableFeatures []string
UserAgent string
Immutable bool
AutoConfirm bool
StreamingUploadCutoff SizeSuffix
StatsFileNameLength int
AskPassword bool
PasswordCommand SpaceSepList
UseServerModTime bool
MaxTransfer SizeSuffix
MaxDuration time.Duration
CutoffMode CutoffMode
MaxBacklog int
MaxStatsGroups int
StatsOneLine bool
StatsOneLineDate bool // If we want a date prefix at all
StatsOneLineDateFormat string // If we want to customize the prefix
ErrorOnNoTransfer bool // Set appropriate exit code if no files transferred
Progress bool
ProgressTerminalTitle bool
Cookie bool
UseMmap bool
CaCert []string // Client Side CA
ClientCert string // Client Side Cert
ClientKey string // Client Side Key
MultiThreadCutoff SizeSuffix
MultiThreadStreams int
MultiThreadSet bool // whether MultiThreadStreams was set (set in fs/config/configflags)
MultiThreadChunkSize SizeSuffix // Chunk size for multi-thread downloads / uploads, if not set by filesystem
MultiThreadWriteBufferSize SizeSuffix
OrderBy string // instructions on how to order the transfer
UploadHeaders []*HTTPOption
DownloadHeaders []*HTTPOption
Headers []*HTTPOption
MetadataSet Metadata // extra metadata to write when uploading
RefreshTimes bool
NoConsole bool
TrafficClass uint8
FsCacheExpireDuration time.Duration
FsCacheExpireInterval time.Duration
DisableHTTP2 bool
HumanReadable bool
KvLockTime time.Duration // maximum time to keep key-value database locked by process
DisableHTTPKeepAlives bool
Metadata bool
ServerSideAcrossConfigs bool
TerminalColorMode TerminalColorMode
DefaultTime Time // time that directories with no time should display
Inplace bool // Download directly to destination file instead of atomic download to temp/rename
PartialSuffix string
MetadataMapper SpaceSepList
LogLevel LogLevel `config:"log_level"`
StatsLogLevel LogLevel `config:"stats_log_level"`
UseJSONLog bool `config:"use_json_log"`
DryRun bool `config:"dry_run"`
Interactive bool `config:"interactive"`
CheckSum bool `config:"checksum"`
SizeOnly bool `config:"size_only"`
IgnoreTimes bool `config:"ignore_times"`
IgnoreExisting bool `config:"ignore_existing"`
IgnoreErrors bool `config:"ignore_errors"`
ModifyWindow time.Duration `config:"modify_window"`
Checkers int `config:"checkers"`
Transfers int `config:"transfers"`
ConnectTimeout time.Duration `config:"contimeout"` // Connect timeout
Timeout time.Duration `config:"timeout"` // Data channel timeout
ExpectContinueTimeout time.Duration `config:"expect_continue_timeout"`
Dump DumpFlags `config:"dump"`
InsecureSkipVerify bool `config:"no_check_certificate"` // Skip server certificate verification
DeleteMode DeleteMode `config:"delete_mode"`
MaxDelete int64 `config:"max_delete"`
MaxDeleteSize SizeSuffix `config:"max_delete_size"`
TrackRenames bool `config:"track_renames"` // Track file renames.
TrackRenamesStrategy string `config:"track_renames_strategy"` // Comma separated list of strategies used to track renames
Retries int `config:"retries"` // High-level retries
RetriesInterval time.Duration `config:"retries_sleep"`
LowLevelRetries int `config:"low_level_retries"`
UpdateOlder bool `config:"update"` // Skip files that are newer on the destination
NoGzip bool `config:"no_gzip_encoding"` // Disable compression
MaxDepth int `config:"max_depth"`
IgnoreSize bool `config:"ignore_size"`
IgnoreChecksum bool `config:"ignore_checksum"`
IgnoreCaseSync bool `config:"ignore_case_sync"`
FixCase bool `config:"fix_case"`
NoTraverse bool `config:"no_traverse"`
CheckFirst bool `config:"check_first"`
NoCheckDest bool `config:"no_check_dest"`
NoUnicodeNormalization bool `config:"no_unicode_normalization"`
NoUpdateModTime bool `config:"no_update_modtime"`
NoUpdateDirModTime bool `config:"no_update_dir_modtime"`
DataRateUnit string `config:"stats_unit"`
CompareDest []string `config:"compare_dest"`
CopyDest []string `config:"copy_dest"`
BackupDir string `config:"backup_dir"`
Suffix string `config:"suffix"`
SuffixKeepExtension bool `config:"suffix_keep_extension"`
UseListR bool `config:"fast_list"`
BufferSize SizeSuffix `config:"buffer_size"`
BwLimit BwTimetable `config:"bwlimit"`
BwLimitFile BwTimetable `config:"bwlimit_file"`
TPSLimit float64 `config:"tpslimit"`
TPSLimitBurst int `config:"tpslimit_burst"`
BindAddr net.IP `config:"bind_addr"`
DisableFeatures []string `config:"disable"`
UserAgent string `config:"user_agent"`
Immutable bool `config:"immutable"`
AutoConfirm bool `config:"auto_confirm"`
StreamingUploadCutoff SizeSuffix `config:"streaming_upload_cutoff"`
StatsFileNameLength int `config:"stats_file_name_length"`
AskPassword bool `config:"ask_password"`
PasswordCommand SpaceSepList `config:"password_command"`
UseServerModTime bool `config:"use_server_modtime"`
MaxTransfer SizeSuffix `config:"max_transfer"`
MaxDuration time.Duration `config:"max_duration"`
CutoffMode CutoffMode `config:"cutoff_mode"`
MaxBacklog int `config:"max_backlog"`
MaxStatsGroups int `config:"max_stats_groups"`
StatsOneLine bool `config:"stats_one_line"`
StatsOneLineDate bool `config:"stats_one_line_date"` // If we want a date prefix at all
StatsOneLineDateFormat string `config:"stats_one_line_date_format"` // If we want to customize the prefix
ErrorOnNoTransfer bool `config:"error_on_no_transfer"` // Set appropriate exit code if no files transferred
Progress bool `config:"progress"`
ProgressTerminalTitle bool `config:"progress_terminal_title"`
Cookie bool `config:"use_cookies"`
UseMmap bool `config:"use_mmap"`
CaCert []string `config:"ca_cert"` // Client Side CA
ClientCert string `config:"client_cert"` // Client Side Cert
ClientKey string `config:"client_key"` // Client Side Key
MultiThreadCutoff SizeSuffix `config:"multi_thread_cutoff"`
MultiThreadStreams int `config:"multi_thread_streams"`
MultiThreadSet bool `config:"multi_thread_set"` // whether MultiThreadStreams was set (set in fs/config/configflags)
MultiThreadChunkSize SizeSuffix `config:"multi_thread_chunk_size"` // Chunk size for multi-thread downloads / uploads, if not set by filesystem
MultiThreadWriteBufferSize SizeSuffix `config:"multi_thread_write_buffer_size"`
OrderBy string `config:"order_by"` // instructions on how to order the transfer
UploadHeaders []*HTTPOption `config:"upload_headers"`
DownloadHeaders []*HTTPOption `config:"download_headers"`
Headers []*HTTPOption `config:"headers"`
MetadataSet Metadata `config:"metadata_set"` // extra metadata to write when uploading
RefreshTimes bool `config:"refresh_times"`
NoConsole bool `config:"no_console"`
TrafficClass uint8 `config:"traffic_class"`
FsCacheExpireDuration time.Duration `config:"fs_cache_expire_duration"`
FsCacheExpireInterval time.Duration `config:"fs_cache_expire_interval"`
DisableHTTP2 bool `config:"disable_http2"`
HumanReadable bool `config:"human_readable"`
KvLockTime time.Duration `config:"kv_lock_time"` // maximum time to keep key-value database locked by process
DisableHTTPKeepAlives bool `config:"disable_http_keep_alives"`
Metadata bool `config:"metadata"`
ServerSideAcrossConfigs bool `config:"server_side_across_configs"`
TerminalColorMode TerminalColorMode `config:"color"`
DefaultTime Time `config:"default_time"` // time that directories with no time should display
Inplace bool `config:"inplace"` // Download directly to destination file instead of atomic download to temp/rename
PartialSuffix string `config:"partial_suffix"`
MetadataMapper SpaceSepList `config:"metadata_mapper"`
}
// NewConfig creates a new config with everything set to the default
// value. These are the ultimate defaults and are overridden by the
// config module.
func NewConfig() *ConfigInfo {
c := new(ConfigInfo)
func init() {
// Set any values which aren't the zero for the type
c.LogLevel = LogLevelNotice
c.StatsLogLevel = LogLevelInfo
c.ModifyWindow = time.Nanosecond
c.Checkers = 8
c.Transfers = 4
c.ConnectTimeout = 60 * time.Second
c.Timeout = 5 * 60 * time.Second
c.ExpectContinueTimeout = 1 * time.Second
c.DeleteMode = DeleteModeDefault
c.MaxDelete = -1
c.MaxDeleteSize = SizeSuffix(-1)
c.Retries = 3
c.LowLevelRetries = 10
c.MaxDepth = -1
c.DataRateUnit = "bytes"
c.BufferSize = SizeSuffix(16 << 20)
c.UserAgent = "rclone/" + Version
c.StreamingUploadCutoff = SizeSuffix(100 * 1024)
c.MaxStatsGroups = 1000
c.StatsFileNameLength = 45
c.AskPassword = true
c.TPSLimitBurst = 1
c.MaxTransfer = -1
c.MaxBacklog = 10000
// We do not want to set the default here. We use this variable being empty as part of the fall-through of options.
// c.StatsOneLineDateFormat = "2006/01/02 15:04:05 - "
c.MultiThreadCutoff = SizeSuffix(256 * 1024 * 1024)
c.MultiThreadStreams = 4
c.MultiThreadChunkSize = SizeSuffix(64 * 1024 * 1024)
c.MultiThreadWriteBufferSize = SizeSuffix(128 * 1024)
globalConfig.DeleteMode = DeleteModeDefault
c.TrackRenamesStrategy = "hash"
c.FsCacheExpireDuration = 300 * time.Second
c.FsCacheExpireInterval = 60 * time.Second
c.KvLockTime = 1 * time.Second
c.DefaultTime = Time(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC))
c.PartialSuffix = ".partial"
// Register the config and fill globalConfig with the defaults
RegisterGlobalOptions(OptionsInfo{Name: "main", Opt: globalConfig, Options: ConfigOptionsInfo, Reload: globalConfig.Reload})
// Perform a simple check for debug flags to enable debug logging during the flag initialization
// initial guess at log level from the flags
globalConfig.LogLevel = initialLogLevel()
}
// Reload assumes the config has been edited and does what is necessary to make it live
func (ci *ConfigInfo) Reload(ctx context.Context) error {
// Set -vv if --dump is in use
if ci.Dump != 0 && ci.LogLevel != LogLevelDebug {
Logf(nil, "Automatically setting -vv as --dump is enabled")
ci.LogLevel = LogLevelDebug
}
// If --dry-run or -i then use NOTICE as minimum log level
if (ci.DryRun || ci.Interactive) && ci.StatsLogLevel > LogLevelNotice {
ci.StatsLogLevel = LogLevelNotice
}
// If --use-json-log then start the JSON logger
if ci.UseJSONLog {
InstallJSONLogger(ci.LogLevel)
}
// Check --compare-dest and --copy-dest
if len(ci.CompareDest) > 0 && len(ci.CopyDest) > 0 {
return fmt.Errorf("can't use --compare-dest with --copy-dest")
}
// Check --stats-one-line and dependent flags
switch {
case len(ci.StatsOneLineDateFormat) > 0:
ci.StatsOneLineDate = true
ci.StatsOneLine = true
case ci.StatsOneLineDate:
ci.StatsOneLineDateFormat = "2006/01/02 15:04:05 - "
ci.StatsOneLine = true
}
// Check --partial-suffix
if len(ci.PartialSuffix) > 16 {
return fmt.Errorf("--partial-suffix: Expecting suffix length not greater than %d but got %d", 16, len(ci.PartialSuffix))
}
// Make sure some values are > 0
nonZero := func(pi *int) {
if *pi <= 0 {
*pi = 1
}
}
// Check --stats-unit
if ci.DataRateUnit != "bits" && ci.DataRateUnit != "bytes" {
Errorf(nil, "Unknown unit %q passed to --stats-unit. Defaulting to bytes.", ci.DataRateUnit)
ci.DataRateUnit = "bytes"
}
// Check these are all > 0
nonZero(&ci.Retries)
nonZero(&ci.LowLevelRetries)
nonZero(&ci.Transfers)
nonZero(&ci.Checkers)
return nil
}
// Initial logging level
//
// Perform a simple check for debug flags to enable debug logging during the flag initialization
func initialLogLevel() LogLevel {
logLevel := LogLevelNotice
for argIndex, arg := range os.Args {
if strings.HasPrefix(arg, "-vv") && strings.TrimRight(arg, "v") == "-" {
c.LogLevel = LogLevelDebug
logLevel = LogLevelDebug
}
if arg == "--log-level=DEBUG" || (arg == "--log-level" && len(os.Args) > argIndex+1 && os.Args[argIndex+1] == "DEBUG") {
c.LogLevel = LogLevelDebug
logLevel = LogLevelDebug
}
if strings.HasPrefix(arg, "--verbose=") {
if level, err := strconv.Atoi(arg[10:]); err == nil && level >= 2 {
c.LogLevel = LogLevelDebug
logLevel = LogLevelDebug
}
}
}
envValue, found := os.LookupEnv("RCLONE_LOG_LEVEL")
if found && envValue == "DEBUG" {
c.LogLevel = LogLevelDebug
logLevel = LogLevelDebug
}
return c
return logLevel
}
// TimeoutOrInfinite returns ci.Timeout if > 0 or infinite otherwise
func (c *ConfigInfo) TimeoutOrInfinite() time.Duration {
if c.Timeout > 0 {
return c.Timeout
func (ci *ConfigInfo) TimeoutOrInfinite() time.Duration {
if ci.Timeout > 0 {
return ci.Timeout
}
return ModTimeNotSupported
}

View File

@ -13,9 +13,6 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/flags"
fsLog "github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/fs/rc"
"github.com/sirupsen/logrus"
"github.com/spf13/pflag"
)
@ -38,123 +35,30 @@ var (
downloadHeaders []string
headers []string
metadataSet []string
partialSuffix string
)
// AddFlags adds the non filing system specific flags to the command
func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
rc.AddOption("main", ci)
// NB defaults which aren't the zero for the type should be set in fs/config.go NewConfig
flags.AddFlagsFromOptions(flagSet, "", fs.ConfigOptionsInfo)
// Add flags we haven't converted into options yet
flags.CountVarP(flagSet, &verbose, "verbose", "v", "Print lots more stuff (repeat for more)", "Logging,Important")
flags.BoolVarP(flagSet, &quiet, "quiet", "q", false, "Print as little stuff as possible", "Logging")
flags.DurationVarP(flagSet, &ci.ModifyWindow, "modify-window", "", ci.ModifyWindow, "Max time diff to be considered the same", "Copy")
flags.IntVarP(flagSet, &ci.Checkers, "checkers", "", ci.Checkers, "Number of checkers to run in parallel", "Performance")
flags.IntVarP(flagSet, &ci.Transfers, "transfers", "", ci.Transfers, "Number of file transfers to run in parallel", "Performance")
flags.StringVarP(flagSet, &configPath, "config", "", config.GetConfigPath(), "Config file", "Config")
flags.StringVarP(flagSet, &cacheDir, "cache-dir", "", config.GetCacheDir(), "Directory rclone will use for caching", "Config")
flags.StringVarP(flagSet, &tempDir, "temp-dir", "", os.TempDir(), "Directory rclone will use for temporary files", "Config")
flags.BoolVarP(flagSet, &ci.CheckSum, "checksum", "c", ci.CheckSum, "Check for changes with size & checksum (if available, or fallback to size only).", "Copy")
flags.BoolVarP(flagSet, &ci.SizeOnly, "size-only", "", ci.SizeOnly, "Skip based on size only, not modtime or checksum", "Copy")
flags.BoolVarP(flagSet, &ci.IgnoreTimes, "ignore-times", "I", ci.IgnoreTimes, "Don't skip items that match size and time - transfer all unconditionally", "Copy")
flags.BoolVarP(flagSet, &ci.IgnoreExisting, "ignore-existing", "", ci.IgnoreExisting, "Skip all files that exist on destination", "Copy")
flags.BoolVarP(flagSet, &ci.IgnoreErrors, "ignore-errors", "", ci.IgnoreErrors, "Delete even if there are I/O errors", "Sync")
flags.BoolVarP(flagSet, &ci.DryRun, "dry-run", "n", ci.DryRun, "Do a trial run with no permanent changes", "Config,Important")
flags.BoolVarP(flagSet, &ci.Interactive, "interactive", "i", ci.Interactive, "Enable interactive mode", "Config,Important")
flags.DurationVarP(flagSet, &ci.ConnectTimeout, "contimeout", "", ci.ConnectTimeout, "Connect timeout", "Networking")
flags.DurationVarP(flagSet, &ci.Timeout, "timeout", "", ci.Timeout, "IO idle timeout", "Networking")
flags.DurationVarP(flagSet, &ci.ExpectContinueTimeout, "expect-continue-timeout", "", ci.ExpectContinueTimeout, "Timeout when using expect / 100-continue in HTTP", "Networking")
flags.BoolVarP(flagSet, &dumpHeaders, "dump-headers", "", false, "Dump HTTP headers - may contain sensitive info", "Debugging")
flags.BoolVarP(flagSet, &dumpBodies, "dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info", "Debugging")
flags.BoolVarP(flagSet, &ci.InsecureSkipVerify, "no-check-certificate", "", ci.InsecureSkipVerify, "Do not verify the server SSL certificate (insecure)", "Networking")
flags.BoolVarP(flagSet, &ci.AskPassword, "ask-password", "", ci.AskPassword, "Allow prompt for password for encrypted configuration", "Config")
flags.FVarP(flagSet, &ci.PasswordCommand, "password-command", "", "Command for supplying password for encrypted configuration", "Config")
flags.BoolVarP(flagSet, &deleteBefore, "delete-before", "", false, "When synchronizing, delete files on destination before transferring", "Sync")
flags.BoolVarP(flagSet, &deleteDuring, "delete-during", "", false, "When synchronizing, delete files during transfer", "Sync")
flags.BoolVarP(flagSet, &deleteAfter, "delete-after", "", false, "When synchronizing, delete files on destination after transferring (default)", "Sync")
flags.Int64VarP(flagSet, &ci.MaxDelete, "max-delete", "", -1, "When synchronizing, limit the number of deletes", "Sync")
flags.FVarP(flagSet, &ci.MaxDeleteSize, "max-delete-size", "", "When synchronizing, limit the total size of deletes", "Sync")
flags.BoolVarP(flagSet, &ci.TrackRenames, "track-renames", "", ci.TrackRenames, "When synchronizing, track file renames and do a server-side move if possible", "Sync")
flags.StringVarP(flagSet, &ci.TrackRenamesStrategy, "track-renames-strategy", "", ci.TrackRenamesStrategy, "Strategies to use when synchronizing using track-renames hash|modtime|leaf", "Sync")
flags.IntVarP(flagSet, &ci.Retries, "retries", "", 3, "Retry operations this many times if they fail", "Config")
flags.DurationVarP(flagSet, &ci.RetriesInterval, "retries-sleep", "", 0, "Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)", "Config")
flags.IntVarP(flagSet, &ci.LowLevelRetries, "low-level-retries", "", ci.LowLevelRetries, "Number of low level retries to do", "Config")
flags.BoolVarP(flagSet, &ci.UpdateOlder, "update", "u", ci.UpdateOlder, "Skip files that are newer on the destination", "Copy")
flags.BoolVarP(flagSet, &ci.UseServerModTime, "use-server-modtime", "", ci.UseServerModTime, "Use server modified time instead of object metadata", "Config")
flags.BoolVarP(flagSet, &ci.NoGzip, "no-gzip-encoding", "", ci.NoGzip, "Don't set Accept-Encoding: gzip", "Networking")
flags.IntVarP(flagSet, &ci.MaxDepth, "max-depth", "", ci.MaxDepth, "If set limits the recursion depth to this", "Filter")
flags.BoolVarP(flagSet, &ci.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use modtime or checksum", "Copy")
flags.BoolVarP(flagSet, &ci.IgnoreChecksum, "ignore-checksum", "", ci.IgnoreChecksum, "Skip post copy check of checksums", "Copy")
flags.BoolVarP(flagSet, &ci.IgnoreCaseSync, "ignore-case-sync", "", ci.IgnoreCaseSync, "Ignore case when synchronizing", "Copy")
flags.BoolVarP(flagSet, &ci.FixCase, "fix-case", "", ci.FixCase, "Force rename of case insensitive dest to match source", "Sync")
flags.BoolVarP(flagSet, &ci.NoTraverse, "no-traverse", "", ci.NoTraverse, "Don't traverse destination file system on copy", "Copy")
flags.BoolVarP(flagSet, &ci.CheckFirst, "check-first", "", ci.CheckFirst, "Do all the checks before starting transfers", "Copy")
flags.BoolVarP(flagSet, &ci.NoCheckDest, "no-check-dest", "", ci.NoCheckDest, "Don't check the destination, copy regardless", "Copy")
flags.BoolVarP(flagSet, &ci.NoUnicodeNormalization, "no-unicode-normalization", "", ci.NoUnicodeNormalization, "Don't normalize unicode characters in filenames", "Config")
flags.BoolVarP(flagSet, &ci.NoUpdateModTime, "no-update-modtime", "", ci.NoUpdateModTime, "Don't update destination modtime if files identical", "Copy")
flags.BoolVarP(flagSet, &ci.NoUpdateDirModTime, "no-update-dir-modtime", "", ci.NoUpdateModTime, "Don't update directory modification times", "Copy")
flags.StringArrayVarP(flagSet, &ci.CompareDest, "compare-dest", "", nil, "Include additional comma separated server-side paths during comparison", "Copy")
flags.StringArrayVarP(flagSet, &ci.CopyDest, "copy-dest", "", nil, "Implies --compare-dest but also copies files from paths into destination", "Copy")
flags.StringVarP(flagSet, &ci.BackupDir, "backup-dir", "", ci.BackupDir, "Make backups into hierarchy based in DIR", "Sync")
flags.StringVarP(flagSet, &ci.Suffix, "suffix", "", ci.Suffix, "Suffix to add to changed files", "Sync")
flags.BoolVarP(flagSet, &ci.SuffixKeepExtension, "suffix-keep-extension", "", ci.SuffixKeepExtension, "Preserve the extension when using --suffix", "Sync")
flags.BoolVarP(flagSet, &ci.UseListR, "fast-list", "", ci.UseListR, "Use recursive list if available; uses more memory but fewer transactions", "Listing")
flags.Float64VarP(flagSet, &ci.TPSLimit, "tpslimit", "", ci.TPSLimit, "Limit HTTP transactions per second to this", "Networking")
flags.IntVarP(flagSet, &ci.TPSLimitBurst, "tpslimit-burst", "", ci.TPSLimitBurst, "Max burst of transactions for --tpslimit", "Networking")
flags.StringVarP(flagSet, &bindAddr, "bind", "", "", "Local address to bind to for outgoing connections, IPv4, IPv6 or name", "Networking")
flags.StringVarP(flagSet, &disableFeatures, "disable", "", "", "Disable a comma separated list of features (use --disable help to see a list)", "Config")
flags.StringVarP(flagSet, &ci.UserAgent, "user-agent", "", ci.UserAgent, "Set the user-agent to a specified string", "Networking")
flags.BoolVarP(flagSet, &ci.Immutable, "immutable", "", ci.Immutable, "Do not modify files, fail if existing files have been modified", "Copy")
flags.BoolVarP(flagSet, &ci.AutoConfirm, "auto-confirm", "", ci.AutoConfirm, "If enabled, do not request console confirmation", "Config")
flags.IntVarP(flagSet, &ci.StatsFileNameLength, "stats-file-name-length", "", ci.StatsFileNameLength, "Max file name length in stats (0 for no limit)", "Logging")
flags.FVarP(flagSet, &ci.LogLevel, "log-level", "", "Log level DEBUG|INFO|NOTICE|ERROR", "Logging")
flags.FVarP(flagSet, &ci.StatsLogLevel, "stats-log-level", "", "Log level to show --stats output DEBUG|INFO|NOTICE|ERROR", "Logging")
flags.FVarP(flagSet, &ci.BwLimit, "bwlimit", "", "Bandwidth limit in KiB/s, or use suffix B|K|M|G|T|P or a full timetable", "Networking")
flags.FVarP(flagSet, &ci.BwLimitFile, "bwlimit-file", "", "Bandwidth limit per file in KiB/s, or use suffix B|K|M|G|T|P or a full timetable", "Networking")
flags.FVarP(flagSet, &ci.BufferSize, "buffer-size", "", "In memory buffer size when reading files for each --transfer", "Performance")
flags.FVarP(flagSet, &ci.StreamingUploadCutoff, "streaming-upload-cutoff", "", "Cutoff for switching to chunked upload if file size is unknown, upload starts after reaching cutoff or when file ends", "Copy")
flags.FVarP(flagSet, &ci.Dump, "dump", "", "List of items to dump from: "+fs.DumpFlagsList, "Debugging")
flags.FVarP(flagSet, &ci.MaxTransfer, "max-transfer", "", "Maximum size of data to transfer", "Copy")
flags.DurationVarP(flagSet, &ci.MaxDuration, "max-duration", "", 0, "Maximum duration rclone will transfer data for", "Copy")
flags.FVarP(flagSet, &ci.CutoffMode, "cutoff-mode", "", "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS", "Copy")
flags.IntVarP(flagSet, &ci.MaxBacklog, "max-backlog", "", ci.MaxBacklog, "Maximum number of objects in sync or check backlog", "Copy,Check")
flags.IntVarP(flagSet, &ci.MaxStatsGroups, "max-stats-groups", "", ci.MaxStatsGroups, "Maximum number of stats groups to keep in memory, on max oldest is discarded", "Logging")
flags.BoolVarP(flagSet, &ci.StatsOneLine, "stats-one-line", "", ci.StatsOneLine, "Make the stats fit on one line", "Logging")
flags.BoolVarP(flagSet, &ci.StatsOneLineDate, "stats-one-line-date", "", ci.StatsOneLineDate, "Enable --stats-one-line and add current date/time prefix", "Logging")
flags.StringVarP(flagSet, &ci.StatsOneLineDateFormat, "stats-one-line-date-format", "", ci.StatsOneLineDateFormat, "Enable --stats-one-line-date and use custom formatted date: Enclose date string in double quotes (\"), see https://golang.org/pkg/time/#Time.Format", "Logging")
flags.BoolVarP(flagSet, &ci.ErrorOnNoTransfer, "error-on-no-transfer", "", ci.ErrorOnNoTransfer, "Sets exit code 9 if no files are transferred, useful in scripts", "Config")
flags.BoolVarP(flagSet, &ci.Progress, "progress", "P", ci.Progress, "Show progress during transfer", "Logging")
flags.BoolVarP(flagSet, &ci.ProgressTerminalTitle, "progress-terminal-title", "", ci.ProgressTerminalTitle, "Show progress on the terminal title (requires -P/--progress)", "Logging")
flags.BoolVarP(flagSet, &ci.Cookie, "use-cookies", "", ci.Cookie, "Enable session cookiejar", "Networking")
flags.BoolVarP(flagSet, &ci.UseMmap, "use-mmap", "", ci.UseMmap, "Use mmap allocator (see docs)", "Config")
flags.StringArrayVarP(flagSet, &ci.CaCert, "ca-cert", "", ci.CaCert, "CA certificate used to verify servers", "Networking")
flags.StringVarP(flagSet, &ci.ClientCert, "client-cert", "", ci.ClientCert, "Client SSL certificate (PEM) for mutual TLS auth", "Networking")
flags.StringVarP(flagSet, &ci.ClientKey, "client-key", "", ci.ClientKey, "Client SSL private key (PEM) for mutual TLS auth", "Networking")
flags.FVarP(flagSet, &ci.MultiThreadCutoff, "multi-thread-cutoff", "", "Use multi-thread downloads for files above this size", "Copy")
flags.IntVarP(flagSet, &ci.MultiThreadStreams, "multi-thread-streams", "", ci.MultiThreadStreams, "Number of streams to use for multi-thread downloads", "Copy")
flags.FVarP(flagSet, &ci.MultiThreadWriteBufferSize, "multi-thread-write-buffer-size", "", "In memory buffer size for writing when in multi-thread mode", "Copy")
flags.FVarP(flagSet, &ci.MultiThreadChunkSize, "multi-thread-chunk-size", "", "Chunk size for multi-thread downloads / uploads, if not set by filesystem", "Copy")
flags.BoolVarP(flagSet, &ci.UseJSONLog, "use-json-log", "", ci.UseJSONLog, "Use json log format", "Logging")
flags.StringVarP(flagSet, &ci.OrderBy, "order-by", "", ci.OrderBy, "Instructions on how to order the transfers, e.g. 'size,descending'", "Copy")
flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions", "Networking")
flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions", "Networking")
flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions", "Networking")
flags.StringArrayVarP(flagSet, &metadataSet, "metadata-set", "", nil, "Add metadata key=value when uploading", "Metadata")
flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files", "Copy")
flags.BoolVarP(flagSet, &ci.NoConsole, "no-console", "", ci.NoConsole, "Hide console window (supported on Windows only)", "Config")
flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21", "Networking")
flags.DurationVarP(flagSet, &ci.FsCacheExpireDuration, "fs-cache-expire-duration", "", ci.FsCacheExpireDuration, "Cache remotes for this long (0 to disable caching)", "Config")
flags.DurationVarP(flagSet, &ci.FsCacheExpireInterval, "fs-cache-expire-interval", "", ci.FsCacheExpireInterval, "Interval to check for expired remotes", "Config")
flags.BoolVarP(flagSet, &ci.DisableHTTP2, "disable-http2", "", ci.DisableHTTP2, "Disable HTTP/2 in the global transport", "Networking")
flags.BoolVarP(flagSet, &ci.HumanReadable, "human-readable", "", ci.HumanReadable, "Print numbers in a human-readable format, sizes with suffix Ki|Mi|Gi|Ti|Pi", "Config")
flags.DurationVarP(flagSet, &ci.KvLockTime, "kv-lock-time", "", ci.KvLockTime, "Maximum time to keep key-value database locked by process", "Config")
flags.BoolVarP(flagSet, &ci.DisableHTTPKeepAlives, "disable-http-keep-alives", "", ci.DisableHTTPKeepAlives, "Disable HTTP keep-alives and use each connection once.", "Networking")
flags.BoolVarP(flagSet, &ci.Metadata, "metadata", "M", ci.Metadata, "If set, preserve metadata when copying objects", "Metadata,Copy")
flags.BoolVarP(flagSet, &ci.ServerSideAcrossConfigs, "server-side-across-configs", "", ci.ServerSideAcrossConfigs, "Allow server-side operations (e.g. copy) to work across different configs", "Copy")
flags.FVarP(flagSet, &ci.TerminalColorMode, "color", "", "When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS", "Config")
flags.FVarP(flagSet, &ci.DefaultTime, "default-time", "", "Time to show if modtime is unknown for files and directories", "Config,Listing")
flags.BoolVarP(flagSet, &ci.Inplace, "inplace", "", ci.Inplace, "Download directly to destination file instead of atomic download to temp/rename", "Copy")
flags.StringVarP(flagSet, &partialSuffix, "partial-suffix", "", ci.PartialSuffix, "Add partial-suffix to temporary file name when --inplace is not used", "Copy")
flags.FVarP(flagSet, &ci.MetadataMapper, "metadata-mapper", "", "Program to run to transforming metadata before upload", "Metadata")
}
// ParseHeaders converts the strings passed in via the header flags into HTTPOptions
@ -174,8 +78,9 @@ func ParseHeaders(headers []string) []*fs.HTTPOption {
return opts
}
// SetFlags converts any flags into config which weren't straight forward
// SetFlags sets flags which aren't part of the config system
func SetFlags(ci *fs.ConfigInfo) {
// Process obsolete --dump-headers and --dump-bodies flags
if dumpHeaders {
ci.Dump |= fs.DumpHeaders
fs.Logf(nil, "--dump-headers is obsolete - please use --dump headers instead")
@ -184,25 +89,23 @@ func SetFlags(ci *fs.ConfigInfo) {
ci.Dump |= fs.DumpBodies
fs.Logf(nil, "--dump-bodies is obsolete - please use --dump bodies instead")
}
if ci.Dump != 0 && verbose < 2 && ci.LogLevel != fs.LogLevelDebug {
fs.Logf(nil, "Automatically setting -vv as --dump is enabled")
verbose = 2
}
// Process -v flag
if verbose >= 2 {
ci.LogLevel = fs.LogLevelDebug
} else if verbose >= 1 {
ci.LogLevel = fs.LogLevelInfo
}
if (ci.DryRun || ci.Interactive) && ci.StatsLogLevel > fs.LogLevelNotice {
ci.StatsLogLevel = fs.LogLevelNotice
}
// Process -q flag
if quiet {
if verbose > 0 {
log.Fatalf("Can't set -v and -q")
}
ci.LogLevel = fs.LogLevelError
}
// Can't set log level, -v, -q
logLevelFlag := pflag.Lookup("log-level")
if logLevelFlag != nil && logLevelFlag.Changed {
if verbose > 0 {
@ -212,28 +115,8 @@ func SetFlags(ci *fs.ConfigInfo) {
log.Fatalf("Can't set -q and --log-level")
}
}
if ci.UseJSONLog {
logrus.AddHook(fsLog.NewCallerHook())
logrus.SetFormatter(&logrus.JSONFormatter{
TimestampFormat: "2006-01-02T15:04:05.999999-07:00",
})
logrus.SetLevel(logrus.DebugLevel)
switch ci.LogLevel {
case fs.LogLevelEmergency, fs.LogLevelAlert:
logrus.SetLevel(logrus.PanicLevel)
case fs.LogLevelCritical:
logrus.SetLevel(logrus.FatalLevel)
case fs.LogLevelError:
logrus.SetLevel(logrus.ErrorLevel)
case fs.LogLevelWarning, fs.LogLevelNotice:
logrus.SetLevel(logrus.WarnLevel)
case fs.LogLevelInfo:
logrus.SetLevel(logrus.InfoLevel)
case fs.LogLevelDebug:
logrus.SetLevel(logrus.DebugLevel)
}
}
// Process --delete-before, --delete-during and --delete-after
switch {
case deleteBefore && (deleteDuring || deleteAfter),
deleteDuring && deleteAfter:
@ -248,19 +131,7 @@ func SetFlags(ci *fs.ConfigInfo) {
ci.DeleteMode = fs.DeleteModeDefault
}
if len(ci.CompareDest) > 0 && len(ci.CopyDest) > 0 {
log.Fatalf(`Can't use --compare-dest with --copy-dest.`)
}
switch {
case len(ci.StatsOneLineDateFormat) > 0:
ci.StatsOneLineDate = true
ci.StatsOneLine = true
case ci.StatsOneLineDate:
ci.StatsOneLineDateFormat = "2006/01/02 15:04:05 - "
ci.StatsOneLine = true
}
// Process --bind into IP address
if bindAddr != "" {
addrs, err := net.LookupIP(bindAddr)
if err != nil {
@ -272,6 +143,7 @@ func SetFlags(ci *fs.ConfigInfo) {
ci.BindAddr = addrs[0]
}
// Process --disable
if disableFeatures != "" {
if disableFeatures == "help" {
log.Fatalf("Possible backend features are: %s\n", strings.Join(new(fs.Features).List(), ", "))
@ -279,6 +151,7 @@ func SetFlags(ci *fs.ConfigInfo) {
ci.DisableFeatures = strings.Split(disableFeatures, ",")
}
// Process --headers-upload, --headers-download, --headers
if len(uploadHeaders) != 0 {
ci.UploadHeaders = ParseHeaders(uploadHeaders)
}
@ -288,9 +161,8 @@ func SetFlags(ci *fs.ConfigInfo) {
if len(headers) != 0 {
ci.Headers = ParseHeaders(headers)
}
if len(headers) != 0 {
ci.Headers = ParseHeaders(headers)
}
// Process --metadata-set
if len(metadataSet) != 0 {
ci.MetadataSet = make(fs.Metadata, len(metadataSet))
for _, kv := range metadataSet {
@ -302,6 +174,8 @@ func SetFlags(ci *fs.ConfigInfo) {
}
fs.Debugf(nil, "MetadataUpload %v", ci.MetadataSet)
}
// Process --dscp
if len(dscp) != 0 {
if value, ok := parseDSCP(dscp); ok {
ci.TrafficClass = value << 2
@ -310,39 +184,24 @@ func SetFlags(ci *fs.ConfigInfo) {
}
}
// Set path to configuration file
// Process --config path
if err := config.SetConfigPath(configPath); err != nil {
log.Fatalf("--config: Failed to set %q as config path: %v", configPath, err)
}
// Set path to cache dir
// Process --cache-dir path
if err := config.SetCacheDir(cacheDir); err != nil {
log.Fatalf("--cache-dir: Failed to set %q as cache dir: %v", cacheDir, err)
}
// Set path to temp dir
// Process --temp-dir path
if err := config.SetTempDir(tempDir); err != nil {
log.Fatalf("--temp-dir: Failed to set %q as temp dir: %v", tempDir, err)
}
// Set whether multi-thread-streams was set
// Process --multi-thread-streams - set whether multi-thread-streams was set
multiThreadStreamsFlag := pflag.Lookup("multi-thread-streams")
ci.MultiThreadSet = multiThreadStreamsFlag != nil && multiThreadStreamsFlag.Changed
if len(partialSuffix) > 16 {
log.Fatalf("--partial-suffix: Expecting suffix length not greater than %d but got %d", 16, len(partialSuffix))
}
ci.PartialSuffix = partialSuffix
// Make sure some values are > 0
nonZero := func(pi *int) {
if *pi <= 0 {
*pi = 1
}
}
nonZero(&ci.LowLevelRetries)
nonZero(&ci.Transfers)
nonZero(&ci.Checkers)
}
// parseHeaders converts DSCP names to value

View File

@ -57,6 +57,9 @@ func (logLevelChoices) Type() string {
// LogPrintPid enables process pid in log
var LogPrintPid = false
// InstallJSONLogger is a hook that --use-json-log calls
var InstallJSONLogger = func(logLevel LogLevel) {}
// LogPrint sends the text to the logger of level
var LogPrint = func(level LogLevel, text string) {
text = fmt.Sprintf("%-6s: %s", level, text)

View File

@ -5,9 +5,43 @@ import (
"runtime"
"strings"
"github.com/rclone/rclone/fs"
"github.com/sirupsen/logrus"
)
var loggerInstalled = false
// InstallJSONLogger installs the JSON logger at the specified log level
func InstallJSONLogger(logLevel fs.LogLevel) {
if !loggerInstalled {
logrus.AddHook(NewCallerHook())
loggerInstalled = true
}
logrus.SetFormatter(&logrus.JSONFormatter{
TimestampFormat: "2006-01-02T15:04:05.999999-07:00",
})
logrus.SetLevel(logrus.DebugLevel)
switch logLevel {
case fs.LogLevelEmergency, fs.LogLevelAlert:
logrus.SetLevel(logrus.PanicLevel)
case fs.LogLevelCritical:
logrus.SetLevel(logrus.FatalLevel)
case fs.LogLevelError:
logrus.SetLevel(logrus.ErrorLevel)
case fs.LogLevelWarning, fs.LogLevelNotice:
logrus.SetLevel(logrus.WarnLevel)
case fs.LogLevelInfo:
logrus.SetLevel(logrus.InfoLevel)
case fs.LogLevelDebug:
logrus.SetLevel(logrus.DebugLevel)
}
}
// install hook in fs to call to avoid circular dependency
func init() {
fs.InstallJSONLogger = InstallJSONLogger
}
// CallerHook for log the calling file and line of the fine
type CallerHook struct {
Field string