From dcf53a1d1210ea06da4f0b70a8776121181d1b63 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 7 Dec 2016 13:37:40 +0000 Subject: [PATCH] Allows multiple --include/--exclude/--filter options - fixes #875 --- docs/content/filtering.md | 47 +++++++++++++++-- fs/filter.go | 108 +++++++++++++++++++++----------------- fs/filter_test.go | 51 ++++++++---------- 3 files changed, 127 insertions(+), 79 deletions(-) diff --git a/docs/content/filtering.md b/docs/content/filtering.md index ebb292253..088bc08a2 100644 --- a/docs/content/filtering.md +++ b/docs/content/filtering.md @@ -19,10 +19,6 @@ and exclude rules like `--include`, `--exclude`, `--include-from`, try them out is using the `ls` command, or `--dry-run` together with `-v`. -**Important** Due to limitations of the command line parser you can -only use any of these options once - if you duplicate them then rclone -will use the last one only. - ## Patterns ## The patterns used to match files for inclusion or exclusion are based @@ -159,16 +155,44 @@ based remotes (eg s3, swift, google compute storage, b2). Filtering rules are added with the following command line flags. +### Repeating options ## + +You can repeat the following options to add more than one rule of that +type. + + * `--include` + * `--include-from` + * `--exclude` + * `--exclude-from` + * `--filter` + * `--filter-from` + +Note that all the options of the same type are processed together in +the order above, regardless of what order they were placed on the +command line. + +So all `--include` options are processed first in the order they +appeared on the command line, then all `--include-from` options etc. + +To mix up the order includes and excludes, the `--filter` flag can be +used. + ### `--exclude` - Exclude files matching pattern ### Add a single exclude rule with `--exclude`. +This flag can be repeated. See above for the order the flags are +processed in. + Eg `--exclude *.bak` to exclude all bak files from the sync. ### `--exclude-from` - Read exclude patterns from file ### Add exclude rules from a file. +This flag can be repeated. See above for the order the flags are +processed in. + Prepare a file like this `exclude-file.txt` # a sample exclude rule file @@ -184,6 +208,9 @@ This is useful if you have a lot of rules. Add a single include rule with `--include`. +This flag can be repeated. See above for the order the flags are +processed in. + Eg `--include *.{png,jpg}` to include all `png` and `jpg` files in the backup and no others. @@ -197,6 +224,9 @@ flexibility then you must use `--filter-from`. Add include rules from a file. +This flag can be repeated. See above for the order the flags are +processed in. + Prepare a file like this `include-file.txt` # a sample include rule file @@ -221,12 +251,18 @@ This can be used to add a single include or exclude rule. Include rules start with `+ ` and exclude rules start with `- `. A special rule called `!` can be used to clear the existing rules. +This flag can be repeated. See above for the order the flags are +processed in. + Eg `--filter "- *.bak"` to exclude all bak files from the sync. ### `--filter-from` - Read filtering patterns from a file ### Add include/exclude rules from a file. +This flag can be repeated. See above for the order the flags are +processed in. + Prepare a file like this `filter-file.txt` # a sample exclude rule file @@ -250,6 +286,9 @@ This reads a list of file names from the file passed in and **only** these files are transferred. The filtering rules are ignored completely if you use this option. +This option can be repeated to read from more than one file. These +are read in the order that they are placed on the command line. + Prepare a file like this `files-from.txt` # comment diff --git a/fs/filter.go b/fs/filter.go index e093cb879..404a9d8c6 100644 --- a/fs/filter.go +++ b/fs/filter.go @@ -20,13 +20,13 @@ import ( var ( // Flags deleteExcluded = pflag.BoolP("delete-excluded", "", false, "Delete files on dest excluded from sync") - filterRule = pflag.StringP("filter", "f", "", "Add a file-filtering rule") - filterFrom = pflag.StringP("filter-from", "", "", "Read filtering patterns from a file") - excludeRule = pflag.StringP("exclude", "", "", "Exclude files matching pattern") - excludeFrom = pflag.StringP("exclude-from", "", "", "Read exclude patterns from file") - includeRule = pflag.StringP("include", "", "", "Include files matching pattern") - includeFrom = pflag.StringP("include-from", "", "", "Read include patterns from file") - filesFrom = pflag.StringP("files-from", "", "", "Read list of source-file names from file") + filterRule = pflag.StringArrayP("filter", "f", nil, "Add a file-filtering rule") + filterFrom = pflag.StringArrayP("filter-from", "", nil, "Read filtering patterns from a file") + excludeRule = pflag.StringArrayP("exclude", "", nil, "Exclude files matching pattern") + excludeFrom = pflag.StringArrayP("exclude-from", "", nil, "Read exclude patterns from file") + includeRule = pflag.StringArrayP("include", "", nil, "Include files matching pattern") + includeFrom = pflag.StringArrayP("include-from", "", nil, "Read include patterns from file") + filesFrom = pflag.StringArrayP("files-from", "", nil, "Read list of source-file names from file") minAge = pflag.StringP("min-age", "", "", "Don't transfer any file younger than this in s or suffix ms|s|m|h|d|w|M|y") maxAge = pflag.StringP("max-age", "", "", "Don't transfer any file older than this in s or suffix ms|s|m|h|d|w|M|y") minSize = SizeSuffix(-1) @@ -157,54 +157,68 @@ func NewFilter() (f *Filter, err error) { } addImplicitExclude := false - if *includeRule != "" { - err = f.Add(true, *includeRule) - if err != nil { - return nil, err - } - addImplicitExclude = true - } - if *includeFrom != "" { - err := forEachLine(*includeFrom, func(line string) error { - return f.Add(true, line) - }) - if err != nil { - return nil, err - } - addImplicitExclude = true - } - if *excludeRule != "" { - err = f.Add(false, *excludeRule) - if err != nil { - return nil, err + if includeRule != nil { + for _, rule := range *includeRule { + err = f.Add(true, rule) + if err != nil { + return nil, err + } + addImplicitExclude = true } } - if *excludeFrom != "" { - err := forEachLine(*excludeFrom, func(line string) error { - return f.Add(false, line) - }) - if err != nil { - return nil, err + if includeFrom != nil { + for _, rule := range *includeFrom { + err := forEachLine(rule, func(line string) error { + return f.Add(true, line) + }) + if err != nil { + return nil, err + } + addImplicitExclude = true } } - if *filterRule != "" { - err = f.AddRule(*filterRule) - if err != nil { - return nil, err + if excludeRule != nil { + for _, rule := range *excludeRule { + err = f.Add(false, rule) + if err != nil { + return nil, err + } } } - if *filterFrom != "" { - err := forEachLine(*filterFrom, f.AddRule) - if err != nil { - return nil, err + if excludeFrom != nil { + for _, rule := range *excludeFrom { + err := forEachLine(rule, func(line string) error { + return f.Add(false, line) + }) + if err != nil { + return nil, err + } } } - if *filesFrom != "" { - err := forEachLine(*filesFrom, func(line string) error { - return f.AddFile(line) - }) - if err != nil { - return nil, err + if filterRule != nil { + for _, rule := range *filterRule { + err = f.AddRule(rule) + if err != nil { + return nil, err + } + } + } + if filterFrom != nil { + for _, rule := range *filterFrom { + err := forEachLine(rule, f.AddRule) + if err != nil { + return nil, err + } + } + } + if filesFrom != nil { + for _, rule := range *filesFrom { + err := forEachLine(rule, func(line string) error { + return f.AddFile(line) + }) + if err != nil { + return nil, err + } } } if addImplicitExclude { diff --git a/fs/filter_test.go b/fs/filter_test.go index e582a3ecf..b0fe17bc8 100644 --- a/fs/filter_test.go +++ b/fs/filter_test.go @@ -54,13 +54,8 @@ func TestNewFilterDefault(t *testing.T) { assert.True(t, f.InActive()) } -// return a pointer to the string -func stringP(s string) *string { - return &s -} - // testFile creates a temp file with the contents -func testFile(t *testing.T, contents string) *string { +func testFile(t *testing.T, contents string) string { out, err := ioutil.TempFile("", "filter_test") require.NoError(t, err) defer func() { @@ -70,25 +65,24 @@ func testFile(t *testing.T, contents string) *string { _, err = out.Write([]byte(contents)) require.NoError(t, err) s := out.Name() - return &s + return s } func TestNewFilterFull(t *testing.T) { mins := int64(100 * 1024) maxs := int64(1000 * 1024) - emptyString := "" isFalse := false isTrue := true // Set up the input deleteExcluded = &isTrue - filterRule = stringP("- filter1") - filterFrom = testFile(t, "#comment\n+ filter2\n- filter3\n") - excludeRule = stringP("exclude1") - excludeFrom = testFile(t, "#comment\nexclude2\nexclude3\n") - includeRule = stringP("include1") - includeFrom = testFile(t, "#comment\ninclude2\ninclude3\n") - filesFrom = testFile(t, "#comment\nfiles1\nfiles2\n") + filterRule = &[]string{"- filter1", "- filter1b"} + filterFrom = &[]string{testFile(t, "#comment\n+ filter2\n- filter3\n")} + excludeRule = &[]string{"exclude1"} + excludeFrom = &[]string{testFile(t, "#comment\nexclude2\nexclude3\n")} + includeRule = &[]string{"include1"} + includeFrom = &[]string{testFile(t, "#comment\ninclude2\ninclude3\n")} + filesFrom = &[]string{testFile(t, "#comment\nfiles1\nfiles2\n")} minSize = SizeSuffix(mins) maxSize = SizeSuffix(maxs) @@ -100,20 +94,20 @@ func TestNewFilterFull(t *testing.T) { } // Reset the input defer func() { - rm(*filterFrom) - rm(*excludeFrom) - rm(*includeFrom) - rm(*filesFrom) + rm((*filterFrom)[0]) + rm((*excludeFrom)[0]) + rm((*includeFrom)[0]) + rm((*filesFrom)[0]) minSize = -1 maxSize = -1 deleteExcluded = &isFalse - filterRule = &emptyString - filterFrom = &emptyString - excludeRule = &emptyString - excludeFrom = &emptyString - includeRule = &emptyString - includeFrom = &emptyString - filesFrom = &emptyString + filterRule = nil + filterFrom = nil + excludeRule = nil + excludeFrom = nil + includeRule = nil + includeFrom = nil + filesFrom = nil }() f, err := NewFilter() @@ -130,6 +124,7 @@ func TestNewFilterFull(t *testing.T) { - (^|/)exclude2$ - (^|/)exclude3$ - (^|/)filter1$ +- (^|/)filter1b$ + (^|/)filter2$ - (^|/)filter3$ - ^.*$ @@ -356,11 +351,11 @@ four five six `) defer func() { - err := os.Remove(*file) + err := os.Remove(file) require.NoError(t, err) }() lines := []string{} - err := forEachLine(*file, func(s string) error { + err := forEachLine(file, func(s string) error { lines = append(lines, s) return nil })