From 538246f6c3c28f72b7f7cc19d0527f76aa8bb437 Mon Sep 17 00:00:00 2001 From: Iakov Davydov Date: Thu, 9 Nov 2017 10:28:36 +0100 Subject: [PATCH] support exclude file in --fast-list mode --- fs/filter.go | 55 ++++++++++++++++++++++++++-------- fs/filter_test.go | 3 +- fs/operations.go | 21 +++++++++---- fs/operations_internal_test.go | 4 +-- fs/walk.go | 23 +++++++++++++- 5 files changed, 83 insertions(+), 23 deletions(-) diff --git a/fs/filter.go b/fs/filter.go index cc36d7dad..4e74adedf 100644 --- a/fs/filter.go +++ b/fs/filter.go @@ -406,22 +406,51 @@ func (f *Filter) ListContainsExcludeFile(entries DirEntries) bool { return false } -// IncludeDirectory returns whether this directory should be included -// in the sync or not. -func (f *Filter) IncludeDirectory(remote string) bool { - remote = strings.Trim(remote, "/") - // filesFrom takes precedence - if f.files != nil { - _, include := f.dirs[remote] - return include +// IncludeDirectory returns a function which checks whether this +// directory should be included in the sync or not. +func (f *Filter) IncludeDirectory(fs Fs) func(string) (bool, error) { + return func(remote string) (bool, error) { + remote = strings.Trim(remote, "/") + // first check if we need to remove directory based on + // the exclude file + excl, err := f.DirContainsExcludeFile(fs, remote) + if err != nil { + return false, err + } + if excl { + return false, nil + } + + // filesFrom takes precedence + if f.files != nil { + _, include := f.dirs[remote] + return include, nil + } + remote += "/" + for _, rule := range f.dirRules.rules { + if rule.Match(remote) { + return rule.Include, nil + } + } + + return true, nil } - remote += "/" - for _, rule := range f.dirRules.rules { - if rule.Match(remote) { - return rule.Include +} + +// DirContainsExcludeFile checks if exclude file is present in a +// directroy. If fs is nil, it works properly if ExcludeFile is an +// empty string (for testing). +func (f *Filter) DirContainsExcludeFile(fs Fs, remote string) (bool, error) { + if len(Config.Filter.ExcludeFile) > 0 { + exists, err := FileExists(fs, path.Join(remote, Config.Filter.ExcludeFile)) + if err != nil { + return false, err + } + if exists { + return true, nil } } - return true + return false, nil } // Include returns whether this object should be included into the diff --git a/fs/filter_test.go b/fs/filter_test.go index 3e24b6b84..363448a9a 100644 --- a/fs/filter_test.go +++ b/fs/filter_test.go @@ -164,7 +164,8 @@ type includeDirTest struct { func testDirInclude(t *testing.T, f *Filter, tests []includeDirTest) { for _, test := range tests { - got := f.IncludeDirectory(test.in) + got, err := f.IncludeDirectory(nil)(test.in) + require.NoError(t, err) assert.Equal(t, test.want, got, test.in) } } diff --git a/fs/operations.go b/fs/operations.go index b3595c7f2..93d971b66 100644 --- a/fs/operations.go +++ b/fs/operations.go @@ -622,17 +622,20 @@ func ListDirSorted(fs Fs, includeAll bool, dir string) (entries DirEntries, err if err != nil { return nil, err } + // This should happen only if exclude files lives in the + // starting directory, otherwise ListDirSorted should not be + // called. if !includeAll && Config.Filter.ListContainsExcludeFile(entries) { - Debugf(dir, "Excluded from sync (and deletion) based on exclude file") + Debugf(dir, "Excluded from sync (and deletion)") return nil, nil } - return filterAndSortDir(entries, includeAll, dir, Config.Filter.IncludeObject, Config.Filter.IncludeDirectory) + return filterAndSortDir(entries, includeAll, dir, Config.Filter.IncludeObject, Config.Filter.IncludeDirectory(fs)) } // filter (if required) and check the entries, then sort them func filterAndSortDir(entries DirEntries, includeAll bool, dir string, IncludeObject func(o Object) bool, - IncludeDirectory func(remote string) bool) (newEntries DirEntries, err error) { + IncludeDirectory func(remote string) (bool, error)) (newEntries DirEntries, err error) { newEntries = entries[:0] // in place filter prefix := "" if dir != "" { @@ -649,9 +652,15 @@ func filterAndSortDir(entries DirEntries, includeAll bool, dir string, Debugf(x, "Excluded from sync (and deletion)") } case Directory: - if !includeAll && !IncludeDirectory(x.Remote()) { - ok = false - Debugf(x, "Excluded from sync (and deletion)") + if !includeAll { + include, err := IncludeDirectory(x.Remote()) + if err != nil { + return nil, err + } + if !include { + ok = false + Debugf(x, "Excluded from sync (and deletion)") + } } default: return nil, errors.Errorf("unknown object type %T", entry) diff --git a/fs/operations_internal_test.go b/fs/operations_internal_test.go index e51ea44b4..d9745ca6f 100644 --- a/fs/operations_internal_test.go +++ b/fs/operations_internal_test.go @@ -22,8 +22,8 @@ func TestFilterAndSortIncludeAll(t *testing.T) { includeObject := func(o Object) bool { return o != oB } - includeDirectory := func(remote string) bool { - return remote != "c" + includeDirectory := func(remote string) (bool, error) { + return remote != "c", nil } // no filter newEntries, err := filterAndSortDir(entries, true, "", includeObject, includeDirectory) diff --git a/fs/walk.go b/fs/walk.go index 8a2fab4f5..dc4e092a0 100644 --- a/fs/walk.go +++ b/fs/walk.go @@ -349,6 +349,10 @@ func (dt DirTree) String() string { func walkRDirTree(f Fs, startPath string, includeAll bool, maxLevel int, listR ListRFn) (DirTree, error) { dirs := make(DirTree) + // Entries can come in arbitrary order. We use toPrune to keep + // all directories to exclude later. + toPrune := make(map[string]bool) + includeDirectory := Config.Filter.IncludeDirectory(f) var mu sync.Mutex err := listR(startPath, func(entries DirEntries) error { mu.Lock() @@ -372,8 +376,21 @@ func walkRDirTree(f Fs, startPath string, includeAll bool, maxLevel int, listR L } else { Debugf(x, "Excluded from sync (and deletion)") } + // Check if we need to prune a directory later. + if !includeAll && len(Config.Filter.ExcludeFile) > 0 { + basename := path.Base(x.Remote()) + if basename == Config.Filter.ExcludeFile { + excludeDir := parentDir(x.Remote()) + toPrune[excludeDir] = true + Debugf(basename, "Excluded from sync (and deletion) based on exclude file") + } + } case Directory: - if includeAll || Config.Filter.IncludeDirectory(x.Remote()) { + inc, err := includeDirectory(x.Remote()) + if err != nil { + return err + } + if includeAll || inc { if maxLevel < 0 || slashes <= maxLevel-1 { if slashes == maxLevel-1 { // Just add the object if at maxLevel @@ -398,6 +415,10 @@ func walkRDirTree(f Fs, startPath string, includeAll bool, maxLevel int, listR L if len(dirs) == 0 { dirs[startPath] = nil } + err = dirs.Prune(toPrune) + if err != nil { + return nil, err + } dirs.Sort() return dirs, nil }