From f4f0e444bf60b7bf6ebac9fa1e8a94d6821bdd40 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Wed, 8 Jun 2022 09:29:01 +0200 Subject: [PATCH] filter: allow multiple --exclude-if-present flags - fixes #6219 --- docs/content/docs.md | 1 + docs/content/filtering.md | 6 +++--- docs/content/flags.md | 2 +- fs/filter/filter.go | 22 +++++++++++++--------- fs/filter/filterflags/filterflags.go | 2 +- fs/operations/listdirsorted_test.go | 4 ++-- fs/operations/operations_test.go | 2 +- fs/sync/sync_test.go | 2 +- fs/walk/walk.go | 10 ++++++---- fs/walk/walk_test.go | 4 ++-- 10 files changed, 31 insertions(+), 24 deletions(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index 9be5038b8..50bc83cfa 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -2136,6 +2136,7 @@ For the filtering options * `--filter-from` * `--exclude` * `--exclude-from` + * `--exclude-if-present` * `--include` * `--include-from` * `--files-from` diff --git a/docs/content/filtering.md b/docs/content/filtering.md index f306accf5..de191ddc2 100644 --- a/docs/content/filtering.md +++ b/docs/content/filtering.md @@ -750,7 +750,9 @@ Useful for debugging. The `--exclude-if-present` flag controls whether a directory is within the scope of an rclone command based on the presence of a -named file within it. +named file within it. The flag can be repeated to check for +multiple file names, presence of any of them will exclude the +directory. This flag has a priority over other filter flags. @@ -764,8 +766,6 @@ E.g. for the following directory structure: The command `rclone ls --exclude-if-present .ignore dir1` does not list `dir3`, `file3` or `.ignore`. -`--exclude-if-present` can only be used once in an rclone command. - ## Common pitfalls The most frequent filter support issues on diff --git a/docs/content/flags.md b/docs/content/flags.md index 69a0ea380..bfc367e63 100644 --- a/docs/content/flags.md +++ b/docs/content/flags.md @@ -47,7 +47,7 @@ These flags are available for every command. --error-on-no-transfer Sets exit code 9 if no files are transferred, useful in scripts --exclude stringArray Exclude files matching pattern --exclude-from stringArray Read exclude patterns from file (use - to read from stdin) - --exclude-if-present string Exclude directories if filename is present + --exclude-if-present stringArray Exclude directories if filename is present --expect-continue-timeout duration Timeout when using expect / 100-continue in HTTP (default 1s) --fast-list Use recursive list if available; uses more memory but fewer transactions --files-from stringArray Read list of source-file names from file (use - to read from stdin) diff --git a/fs/filter/filter.go b/fs/filter/filter.go index 72a25462f..37071f9d2 100644 --- a/fs/filter/filter.go +++ b/fs/filter/filter.go @@ -86,7 +86,7 @@ type Opt struct { FilterFrom []string ExcludeRule []string ExcludeFrom []string - ExcludeFile string + ExcludeFile []string IncludeRule []string IncludeFrom []string FilesFrom []string @@ -392,8 +392,10 @@ func (f *Filter) ListContainsExcludeFile(entries fs.DirEntries) bool { obj, ok := entry.(fs.Object) if ok { basename := path.Base(obj.Remote()) - if basename == f.Opt.ExcludeFile { - return true + for _, excludeFile := range f.Opt.ExcludeFile { + if basename == excludeFile { + return true + } } } } @@ -436,12 +438,14 @@ func (f *Filter) IncludeDirectory(ctx context.Context, fs fs.Fs) func(string) (b // empty string (for testing). func (f *Filter) DirContainsExcludeFile(ctx context.Context, fremote fs.Fs, remote string) (bool, error) { if len(f.Opt.ExcludeFile) > 0 { - exists, err := fs.FileExists(ctx, fremote, path.Join(remote, f.Opt.ExcludeFile)) - if err != nil { - return false, err - } - if exists { - return true, nil + for _, excludeFile := range f.Opt.ExcludeFile { + exists, err := fs.FileExists(ctx, fremote, path.Join(remote, excludeFile)) + if err != nil { + return false, err + } + if exists { + return true, nil + } } } return false, nil diff --git a/fs/filter/filterflags/filterflags.go b/fs/filter/filterflags/filterflags.go index e926fd894..3963e0c04 100644 --- a/fs/filter/filterflags/filterflags.go +++ b/fs/filter/filterflags/filterflags.go @@ -34,7 +34,7 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.StringArrayVarP(flagSet, &Opt.FilterFrom, "filter-from", "", nil, "Read filtering patterns from a file (use - to read from stdin)") flags.StringArrayVarP(flagSet, &Opt.ExcludeRule, "exclude", "", nil, "Exclude files matching pattern") flags.StringArrayVarP(flagSet, &Opt.ExcludeFrom, "exclude-from", "", nil, "Read exclude patterns from file (use - to read from stdin)") - flags.StringVarP(flagSet, &Opt.ExcludeFile, "exclude-if-present", "", "", "Exclude directories if filename is present") + flags.StringArrayVarP(flagSet, &Opt.ExcludeFile, "exclude-if-present", "", nil, "Exclude directories if filename is present") flags.StringArrayVarP(flagSet, &Opt.IncludeRule, "include", "", nil, "Include files matching pattern") flags.StringArrayVarP(flagSet, &Opt.IncludeFrom, "include-from", "", nil, "Read include patterns from file (use - to read from stdin)") flags.StringArrayVarP(flagSet, &Opt.FilesFrom, "files-from", "", nil, "Read list of source-file names from file (use - to read from stdin)") diff --git a/fs/operations/listdirsorted_test.go b/fs/operations/listdirsorted_test.go index ae09ab4c9..5a4e3eeea 100644 --- a/fs/operations/listdirsorted_test.go +++ b/fs/operations/listdirsorted_test.go @@ -81,7 +81,7 @@ func TestListDirSorted(t *testing.T) { assert.Equal(t, "sub dir/sub sub dir/", str(1)) // testing ignore file - fi.Opt.ExcludeFile = ".ignore" + fi.Opt.ExcludeFile = []string{".ignore"} items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir") require.NoError(t, err) @@ -98,7 +98,7 @@ func TestListDirSorted(t *testing.T) { assert.Equal(t, "sub dir/ignore dir/.ignore", str(0)) assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1)) - fi.Opt.ExcludeFile = "" + fi.Opt.ExcludeFile = nil items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir") require.NoError(t, err) require.Len(t, items, 2) diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 363d88f59..83df26398 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -1309,7 +1309,7 @@ func TestOverlappingFilterCheckWithFilter(t *testing.T) { fi, err := filter.NewFilter(nil) require.NoError(t, err) require.NoError(t, fi.Add(false, "*/exclude/")) - fi.Opt.ExcludeFile = ".ignore" + fi.Opt.ExcludeFile = []string{".ignore"} ctx = filter.ReplaceConfig(ctx, fi) src := &testFs{testFsInfo{name: "name", root: "root"}} diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index bd0772bd6..19e225fd9 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -1452,7 +1452,7 @@ func TestSyncOverlapWithFilter(t *testing.T) { require.NoError(t, err) require.NoError(t, fi.Add(false, "/rclone-sync-test/")) require.NoError(t, fi.Add(false, "*/layer2/")) - fi.Opt.ExcludeFile = ".ignore" + fi.Opt.ExcludeFile = []string{".ignore"} ctx = filter.ReplaceConfig(ctx, fi) subRemoteName := r.FremoteName + "/rclone-sync-test" diff --git a/fs/walk/walk.go b/fs/walk/walk.go index 5f93cca49..b6ca99c32 100644 --- a/fs/walk/walk.go +++ b/fs/walk/walk.go @@ -507,10 +507,12 @@ func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll boo // Check if we need to prune a directory later. if !includeAll && len(fi.Opt.ExcludeFile) > 0 { basename := path.Base(x.Remote()) - if basename == fi.Opt.ExcludeFile { - excludeDir := parentDir(x.Remote()) - toPrune[excludeDir] = true - fs.Debugf(basename, "Excluded from sync (and deletion) based on exclude file") + for _, excludeFile := range fi.Opt.ExcludeFile { + if basename == excludeFile { + excludeDir := parentDir(x.Remote()) + toPrune[excludeDir] = true + fs.Debugf(basename, "Excluded from sync (and deletion) based on exclude file") + } } } case fs.Directory: diff --git a/fs/walk/walk_test.go b/fs/walk/walk_test.go index ecfa3ed7b..cbfa82ce8 100644 --- a/fs/walk/walk_test.go +++ b/fs/walk/walk_test.go @@ -736,13 +736,13 @@ b/c/d/ e `, nil, "", -1, "ign", true}, } { - fi.Opt.ExcludeFile = test.excludeFile + fi.Opt.ExcludeFile = []string{test.excludeFile} r, err := walkRDirTree(context.Background(), nil, test.root, test.includeAll, test.level, makeListRCallback(test.entries, test.err)) assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test)) assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test)) } // Set to default value, to avoid side effects - fi.Opt.ExcludeFile = "" + fi.Opt.ExcludeFile = nil } func TestListType(t *testing.T) {