mirror of
https://github.com/rclone/rclone.git
synced 2024-11-07 09:04:52 +01:00
filter: Fix incorrect filtering with UseFilter context flag and wrapping backends
In this commit
8d1fff9a82
local: obey file filters in listing to fix errors on excluded files
We started using filters in the local backend so the user could short
circuit troublesome files/directories at a low level.
However this caused a number of integration tests to fail. This turned
out to be in backends wrapping the local backend. For example the
combine backend test failed because it changes the paths passed to the
local backend so they no longer match the paths in the current filter.
To fix this, a new feature flag `FilterAware` was added and the
UseFilter context flag is only passed to backends which support it. As
the wrapping backends don't support the flag, this fixes the problems
in the integration tests.
In future the wrapping backends could modify the active filters to
match the path modifications and then they could set the FilterAware
flag.
See #6376
This commit is contained in:
parent
3cb7734eac
commit
bd787e8f45
@ -1210,6 +1210,7 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
|
||||
WriteMimeType: true,
|
||||
CanHaveEmptyDirectories: true,
|
||||
ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
|
||||
FilterAware: true,
|
||||
}).Fill(ctx, f)
|
||||
|
||||
// Create a new authorized Drive client.
|
||||
|
@ -518,6 +518,9 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
|
||||
|
||||
// TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery
|
||||
func (f *Fs) InternalTestAgeQuery(t *testing.T) {
|
||||
// Check set up for filtering
|
||||
assert.True(t, f.Features().FilterAware)
|
||||
|
||||
opt := &filter.Opt{}
|
||||
err := opt.MaxAge.Set("1h")
|
||||
assert.NoError(t, err)
|
||||
|
@ -300,6 +300,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
ReadMetadata: true,
|
||||
WriteMetadata: true,
|
||||
UserMetadata: xattrSupported, // can only R/W general purpose metadata if xattrs are supported
|
||||
FilterAware: true,
|
||||
}).Fill(ctx, f)
|
||||
if opt.FollowSymlinks {
|
||||
f.lstat = os.Stat
|
||||
|
@ -378,6 +378,9 @@ func TestFilter(t *testing.T) {
|
||||
r.WriteFile("excluded", "excluded file", when)
|
||||
f := r.Flocal.(*Fs)
|
||||
|
||||
// Check set up for filtering
|
||||
assert.True(t, f.Features().FilterAware)
|
||||
|
||||
// Add a filter
|
||||
ctx, fi := filter.AddConfig(ctx)
|
||||
require.NoError(t, fi.AddRule("+ included"))
|
||||
|
@ -29,6 +29,7 @@ type Features struct {
|
||||
ReadMetadata bool // can read metadata from objects
|
||||
WriteMetadata bool // can write metadata to objects
|
||||
UserMetadata bool // can read/write general purpose metadata
|
||||
FilterAware bool // can make use of filters if provided for listing
|
||||
|
||||
// Purge all files in the directory specified
|
||||
//
|
||||
@ -320,6 +321,7 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features {
|
||||
// ft.IsLocal = ft.IsLocal && mask.IsLocal Don't propagate IsLocal
|
||||
ft.SlowModTime = ft.SlowModTime && mask.SlowModTime
|
||||
ft.SlowHash = ft.SlowHash && mask.SlowHash
|
||||
ft.FilterAware = ft.FilterAware && mask.FilterAware
|
||||
|
||||
if mask.Purge == nil {
|
||||
ft.Purge = nil
|
||||
|
@ -83,7 +83,7 @@ func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listD
|
||||
if !(ci.UseListR && f.Features().ListR != nil) && // !--fast-list active and
|
||||
!(ci.NoTraverse && fi.HaveFilesFrom()) { // !(--files-from and --no-traverse)
|
||||
return func(dir string) (entries fs.DirEntries, err error) {
|
||||
dirCtx := filter.SetUseFilter(m.Ctx, !includeAll) // make filter-aware backends constrain List
|
||||
dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
|
||||
return list.DirSorted(dirCtx, f, includeAll, dir)
|
||||
}
|
||||
}
|
||||
@ -100,7 +100,7 @@ func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listD
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !started {
|
||||
dirCtx := filter.SetUseFilter(m.Ctx, !includeAll) // make filter-aware backends constrain List
|
||||
dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
|
||||
dirs, dirsErr = walk.NewDirTree(dirCtx, f, m.Dir, includeAll, ci.MaxDepth)
|
||||
started = true
|
||||
}
|
||||
|
@ -64,7 +64,7 @@ type Func func(path string, entries fs.DirEntries, err error) error
|
||||
func Walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
|
||||
ci := fs.GetConfig(ctx)
|
||||
fi := filter.GetConfig(ctx)
|
||||
ctx = filter.SetUseFilter(ctx, !includeAll) // make filter-aware backends constrain List
|
||||
ctx = filter.SetUseFilter(ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
|
||||
if ci.NoTraverse && fi.HaveFilesFrom() {
|
||||
return walkR(ctx, f, path, includeAll, maxLevel, fn, fi.MakeListR(ctx, f.NewObject))
|
||||
}
|
||||
@ -158,7 +158,7 @@ func ListR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel
|
||||
fi.UsesDirectoryFilters() { // ...using any directory filters
|
||||
return listRwalk(ctx, f, path, includeAll, maxLevel, listType, fn)
|
||||
}
|
||||
ctx = filter.SetUseFilter(ctx, !includeAll) // make filter-aware backends constrain List
|
||||
ctx = filter.SetUseFilter(ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
|
||||
return listR(ctx, f, path, includeAll, listType, fn, doListR, listType.Dirs() && f.Features().BucketBased)
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user