mirror of
https://github.com/rclone/rclone.git
synced 2024-12-22 23:22:08 +01:00
rmdirs: remove directories concurrently controlled by --checkers
See: https://forum.rclone.org/t/how-to-list-empty-directories-recursively/40995
This commit is contained in:
parent
dc803b572c
commit
03aab1a123
@ -35,7 +35,10 @@ empty directories in. For example the [delete](/commands/rclone_delete/)
|
||||
command will delete files but leave the directory structure (unless
|
||||
used with option ` + "`--rmdirs`" + `).
|
||||
|
||||
To delete a path and any objects in it, use [purge](/commands/rclone_purge/)
|
||||
This will delete ` + "`--checkers`" + ` directories concurrently so
|
||||
if you have thousands of empty directories consider increasing this number.
|
||||
|
||||
To delete a path and any objects in it, use the [purge](/commands/rclone_purge/)
|
||||
command.
|
||||
`,
|
||||
Annotations: map[string]string{
|
||||
|
@ -1551,28 +1551,69 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to rmdirs: %w", err)
|
||||
}
|
||||
// Now delete the empty directories, starting from the longest path
|
||||
var toDelete []string
|
||||
|
||||
// Group directories to delete by level
|
||||
var toDelete [][]string
|
||||
for dir, empty := range dirEmpty {
|
||||
if empty {
|
||||
toDelete = append(toDelete, dir)
|
||||
}
|
||||
}
|
||||
sort.Strings(toDelete)
|
||||
for i := len(toDelete) - 1; i >= 0; i-- {
|
||||
dir := toDelete[i]
|
||||
// If a filter matches the directory then that
|
||||
// directory is a candidate for deletion
|
||||
if !fi.IncludeRemote(dir + "/") {
|
||||
if fi.IncludeRemote(dir + "/") {
|
||||
level := strings.Count(dir, "/") + 1
|
||||
// The root directory "" is at the top level
|
||||
if dir == "" {
|
||||
level = 0
|
||||
}
|
||||
if len(toDelete) < level+1 {
|
||||
toDelete = append(toDelete, make([][]string, level+1-len(toDelete))...)
|
||||
}
|
||||
toDelete[level] = append(toDelete[level], dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
errMu sync.Mutex
|
||||
errCount int
|
||||
lastError error
|
||||
)
|
||||
// Delete all directories at the same level in parallel
|
||||
for level := len(toDelete) - 1; level >= 0; level-- {
|
||||
dirs := toDelete[level]
|
||||
if len(dirs) == 0 {
|
||||
continue
|
||||
}
|
||||
err = TryRmdir(ctx, f, dir)
|
||||
fs.Debugf(nil, "removing %d level %d directories", len(dirs), level)
|
||||
sort.Strings(dirs)
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(ci.Checkers)
|
||||
for _, dir := range dirs {
|
||||
// End early if error
|
||||
if gCtx.Err() != nil {
|
||||
break
|
||||
}
|
||||
dir := dir
|
||||
g.Go(func() error {
|
||||
err := TryRmdir(gCtx, f, dir)
|
||||
if err != nil {
|
||||
err = fs.CountError(err)
|
||||
fs.Errorf(dir, "Failed to rmdir: %v", err)
|
||||
errMu.Lock()
|
||||
lastError = err
|
||||
errCount += 1
|
||||
errMu.Unlock()
|
||||
}
|
||||
return nil // don't return errors, just count them
|
||||
})
|
||||
}
|
||||
err := g.Wait()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if lastError != nil {
|
||||
return fmt.Errorf("failed to remove %d directories: last error: %w", errCount, lastError)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -702,6 +702,22 @@ func TestRmdirsNoLeaveRoot(t *testing.T) {
|
||||
fs.GetModifyWindow(ctx, r.Fremote),
|
||||
)
|
||||
|
||||
// Delete the files so we can remove everything including the root
|
||||
for _, file := range []fstest.Item{file1, file2} {
|
||||
o, err := r.Fremote.NewObject(ctx, file.Path)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, o.Remove(ctx))
|
||||
}
|
||||
|
||||
require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "", false))
|
||||
|
||||
fstest.CheckListingWithPrecision(
|
||||
t,
|
||||
r.Fremote,
|
||||
[]fstest.Item{},
|
||||
[]string{},
|
||||
fs.GetModifyWindow(ctx, r.Fremote),
|
||||
)
|
||||
}
|
||||
|
||||
func TestRmdirsLeaveRoot(t *testing.T) {
|
||||
|
Loading…
Reference in New Issue
Block a user