From c954c397d9c7b9c363cd4bc4570c2239f44e9277 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 4 Jul 2024 17:29:50 +0100 Subject: [PATCH] filter: convert options to new style --- fs/filter/filter.go | 156 ++++++++++++++++++++++++--- fs/filter/filter_test.go | 10 +- fs/filter/filterflags/filterflags.go | 52 +-------- fs/filter/rules.go | 12 +-- 4 files changed, 152 insertions(+), 78 deletions(-) diff --git a/fs/filter/filter.go b/fs/filter/filter.go index 277623600..408165080 100644 --- a/fs/filter/filter.go +++ b/fs/filter/filter.go @@ -19,24 +19,137 @@ import ( // This is accessed through GetConfig and AddConfig var globalConfig = mustNewFilter(nil) +// OptionsInfo describes the Options in use +var OptionsInfo = fs.Options{{ + Name: "delete_excluded", + Default: false, + Help: "Delete files on dest excluded from sync", + Groups: "Filter", +}, { + Name: "exclude_if_present", + Default: []string{}, + Help: "Exclude directories if filename is present", + Groups: "Filter", +}, { + Name: "files_from", + Default: []string{}, + Help: "Read list of source-file names from file (use - to read from stdin)", + Groups: "Filter", +}, { + Name: "files_from_raw", + Default: []string{}, + Help: "Read list of source-file names from file without any processing of lines (use - to read from stdin)", + Groups: "Filter", +}, { + Name: "min_age", + Default: fs.DurationOff, + Help: "Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y", + Groups: "Filter", +}, { + Name: "max_age", + Default: fs.DurationOff, + Help: "Only transfer files younger than this in s or suffix ms|s|m|h|d|w|M|y", + Groups: "Filter", +}, { + Name: "min_size", + Default: fs.SizeSuffix(-1), + Help: "Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P", + Groups: "Filter", +}, { + Name: "max_size", + Default: fs.SizeSuffix(-1), + Help: "Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P", + Groups: "Filter", +}, { + Name: "ignore_case", + Default: false, + Help: "Ignore case in filters (case insensitive)", + Groups: "Filter", +}, { + Name: "filter", + Default: []string{}, + ShortOpt: "f", + Help: "Add a file filtering rule", + Groups: "Filter", +}, { + Name: "filter_from", + Default: []string{}, + Help: "Read file filtering patterns from a file (use - to read from stdin)", + Groups: "Filter", +}, { + Name: "exclude", + Default: []string{}, + Help: "Exclude files matching pattern", + Groups: "Filter", +}, { + Name: "exclude_from", + Default: []string{}, + Help: "Read file exclude patterns from file (use - to read from stdin)", + Groups: "Filter", +}, { + Name: "include", + Default: []string{}, + Help: "Include files matching pattern", + Groups: "Filter", +}, { + Name: "include_from", + Default: []string{}, + Help: "Read file include patterns from file (use - to read from stdin)", + Groups: "Filter", +}, { + Name: "metadata_filter", + Default: []string{}, + Help: "Add a metadata filtering rule", + Groups: "Filter,Metadata", +}, { + Name: "metadata_filter_from", + Default: []string{}, + Help: "Read metadata filtering patterns from a file (use - to read from stdin)", + Groups: "Filter,Metadata", +}, { + Name: "metadata_exclude", + Default: []string{}, + Help: "Exclude metadatas matching pattern", + Groups: "Filter,Metadata", +}, { + Name: "metadata_exclude_from", + Default: []string{}, + Help: "Read metadata exclude patterns from file (use - to read from stdin)", + Groups: "Filter,Metadata", +}, { + Name: "metadata_include", + Default: []string{}, + Help: "Include metadatas matching pattern", + Groups: "Filter,Metadata", +}, { + Name: "metadata_include_from", + Default: []string{}, + Help: "Read metadata include patterns from file (use - to read from stdin)", + Groups: "Filter,Metadata", +}} + // Options configures the filter type Options struct { - DeleteExcluded bool - RulesOpt // embedded so we don't change the JSON API - ExcludeFile []string - FilesFrom []string - FilesFromRaw []string - MetaRules RulesOpt - MinAge fs.Duration - MaxAge fs.Duration - MinSize fs.SizeSuffix - MaxSize fs.SizeSuffix - IgnoreCase bool + DeleteExcluded bool `config:"delete_excluded"` + RulesOpt // embedded so we don't change the JSON API + ExcludeFile []string `config:"exclude_if_present"` + FilesFrom []string `config:"files_from"` + FilesFromRaw []string `config:"files_from_raw"` + MetaRules RulesOpt `config:"metadata"` + MinAge fs.Duration `config:"min_age"` + MaxAge fs.Duration `config:"max_age"` + MinSize fs.SizeSuffix `config:"min_size"` + MaxSize fs.SizeSuffix `config:"max_size"` + IgnoreCase bool `config:"ignore_case"` } -// DefaultOpt is the default config for the filter -var DefaultOpt = Options{ - MinAge: fs.DurationOff, +func init() { + fs.RegisterGlobalOptions(fs.OptionsInfo{Name: "filter", Opt: &Opt, Options: OptionsInfo, Reload: Reload}) +} + +// Opt is the default config for the filter +var Opt = Options{ + MinAge: fs.DurationOff, // These have to be set here as the options are parsed once before the defaults are set MaxAge: fs.DurationOff, MinSize: fs.SizeSuffix(-1), MaxSize: fs.SizeSuffix(-1), @@ -66,7 +179,7 @@ func NewFilter(opt *Options) (f *Filter, err error) { if opt != nil { f.Opt = *opt } else { - f.Opt = DefaultOpt + f.Opt = Opt } // Filter flags @@ -77,7 +190,7 @@ func NewFilter(opt *Options) (f *Filter, err error) { if f.Opt.MaxAge.IsSet() { f.ModTimeFrom = time.Now().Add(-time.Duration(f.Opt.MaxAge)) if !f.ModTimeTo.IsZero() && f.ModTimeTo.Before(f.ModTimeFrom) { - log.Fatal("filter: --min-age can't be larger than --max-age") + log.Fatalf("filter: --min-age %q can't be larger than --max-age %q", opt.MinAge, opt.MaxAge) } fs.Debugf(nil, "--max-age %v to %v", f.Opt.MaxAge, f.ModTimeFrom) } @@ -568,3 +681,14 @@ func SetUseFilter(ctx context.Context, useFilter bool) context.Context { *pVal = useFilter return context.WithValue(ctx, useFlagContextKey, pVal) } + +// Reload the filters from the flags +func Reload(ctx context.Context) (err error) { + fi := GetConfig(ctx) + newFilter, err := NewFilter(&Opt) + if err != nil { + return err + } + *fi = *newFilter + return nil +} diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.go index 7821cec74..0b5f934da 100644 --- a/fs/filter/filter_test.go +++ b/fs/filter/filter_test.go @@ -43,7 +43,7 @@ func testFile(t *testing.T, contents string) string { } func TestNewFilterForbiddenMixOfFilesFromAndFilterRule(t *testing.T) { - Opt := DefaultOpt + Opt := Opt // Set up the input Opt.FilterRule = []string{"- filter1", "- filter1b"} @@ -66,7 +66,7 @@ func TestNewFilterForbiddenMixOfFilesFromAndFilterRule(t *testing.T) { } func TestNewFilterForbiddenMixOfFilesFromRawAndFilterRule(t *testing.T) { - Opt := DefaultOpt + Opt := Opt // Set up the input Opt.FilterRule = []string{"- filter1", "- filter1b"} @@ -89,7 +89,7 @@ func TestNewFilterForbiddenMixOfFilesFromRawAndFilterRule(t *testing.T) { } func TestNewFilterWithFilesFromAlone(t *testing.T) { - Opt := DefaultOpt + Opt := Opt // Set up the input Opt.FilesFrom = []string{testFile(t, "#comment\nfiles1\nfiles2\n")} @@ -117,7 +117,7 @@ func TestNewFilterWithFilesFromAlone(t *testing.T) { } func TestNewFilterWithFilesFromRaw(t *testing.T) { - Opt := DefaultOpt + Opt := Opt // Set up the input Opt.FilesFromRaw = []string{testFile(t, "#comment\nfiles1\nfiles2\n")} @@ -145,7 +145,7 @@ func TestNewFilterWithFilesFromRaw(t *testing.T) { } func TestNewFilterFullExceptFilesFromOpt(t *testing.T) { - Opt := DefaultOpt + Opt := Opt mins := fs.SizeSuffix(100 * 1024) maxs := fs.SizeSuffix(1000 * 1024) diff --git a/fs/filter/filterflags/filterflags.go b/fs/filter/filterflags/filterflags.go index e5202bfc1..eaead5e79 100644 --- a/fs/filter/filterflags/filterflags.go +++ b/fs/filter/filterflags/filterflags.go @@ -2,62 +2,12 @@ package filterflags import ( - "context" - "fmt" - "github.com/rclone/rclone/fs/config/flags" "github.com/rclone/rclone/fs/filter" - "github.com/rclone/rclone/fs/rc" "github.com/spf13/pflag" ) -// Options set by command line flags -var ( - Opt = filter.DefaultOpt -) - -// Reload the filters from the flags -func Reload(ctx context.Context) (err error) { - fi := filter.GetConfig(ctx) - newFilter, err := filter.NewFilter(&Opt) - if err != nil { - return err - } - *fi = *newFilter - return nil -} - -// AddRuleFlags add a set of rules flags with prefix -func AddRuleFlags(flagSet *pflag.FlagSet, Opt *filter.RulesOpt, what, prefix string) { - shortFilter := "" - if prefix == "" { - shortFilter = "f" - } - group := "Filter" - if prefix == "metadata-" { - group += ",Metadata" - } - flags.StringArrayVarP(flagSet, &Opt.FilterRule, prefix+"filter", shortFilter, nil, fmt.Sprintf("Add a %s filtering rule", what), group) - flags.StringArrayVarP(flagSet, &Opt.FilterFrom, prefix+"filter-from", "", nil, fmt.Sprintf("Read %s filtering patterns from a file (use - to read from stdin)", what), group) - flags.StringArrayVarP(flagSet, &Opt.ExcludeRule, prefix+"exclude", "", nil, fmt.Sprintf("Exclude %ss matching pattern", what), group) - flags.StringArrayVarP(flagSet, &Opt.ExcludeFrom, prefix+"exclude-from", "", nil, fmt.Sprintf("Read %s exclude patterns from file (use - to read from stdin)", what), group) - flags.StringArrayVarP(flagSet, &Opt.IncludeRule, prefix+"include", "", nil, fmt.Sprintf("Include %ss matching pattern", what), group) - flags.StringArrayVarP(flagSet, &Opt.IncludeFrom, prefix+"include-from", "", nil, fmt.Sprintf("Read %s include patterns from file (use - to read from stdin)", what), group) -} - // AddFlags adds the non filing system specific flags to the command func AddFlags(flagSet *pflag.FlagSet) { - rc.AddOptionReload("filter", &Opt, Reload) - flags.BoolVarP(flagSet, &Opt.DeleteExcluded, "delete-excluded", "", false, "Delete files on dest excluded from sync", "Filter") - AddRuleFlags(flagSet, &Opt.RulesOpt, "file", "") - AddRuleFlags(flagSet, &Opt.MetaRules, "metadata", "metadata-") - flags.StringArrayVarP(flagSet, &Opt.ExcludeFile, "exclude-if-present", "", nil, "Exclude directories if filename is present", "Filter") - flags.StringArrayVarP(flagSet, &Opt.FilesFrom, "files-from", "", nil, "Read list of source-file names from file (use - to read from stdin)", "Filter") - flags.StringArrayVarP(flagSet, &Opt.FilesFromRaw, "files-from-raw", "", nil, "Read list of source-file names from file without any processing of lines (use - to read from stdin)", "Filter") - flags.FVarP(flagSet, &Opt.MinAge, "min-age", "", "Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y", "Filter") - flags.FVarP(flagSet, &Opt.MaxAge, "max-age", "", "Only transfer files younger than this in s or suffix ms|s|m|h|d|w|M|y", "Filter") - flags.FVarP(flagSet, &Opt.MinSize, "min-size", "", "Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P", "Filter") - flags.FVarP(flagSet, &Opt.MaxSize, "max-size", "", "Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P", "Filter") - flags.BoolVarP(flagSet, &Opt.IgnoreCase, "ignore-case", "", false, "Ignore case in filters (case insensitive)", "Filter") - //cvsExclude = BoolP("cvs-exclude", "C", false, "Exclude files in the same way CVS does") + flags.AddFlagsFromOptions(flagSet, "", filter.OptionsInfo) } diff --git a/fs/filter/rules.go b/fs/filter/rules.go index 683cc7037..dce5851a6 100644 --- a/fs/filter/rules.go +++ b/fs/filter/rules.go @@ -12,12 +12,12 @@ import ( // RulesOpt is configuration for a rule set type RulesOpt struct { - FilterRule []string - FilterFrom []string - ExcludeRule []string - ExcludeFrom []string - IncludeRule []string - IncludeFrom []string + FilterRule []string `config:"filter"` + FilterFrom []string `config:"filter_from"` + ExcludeRule []string `config:"exclude"` + ExcludeFrom []string `config:"exclude_from"` + IncludeRule []string `config:"include"` + IncludeFrom []string `config:"include_from"` } // rule is one filter rule