bisync: Add support for --create-empty-src-dirs - Fixes #6109

Sync creation and deletion of empty directories.
https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=3.%20Bisync%20should%20create/delete%20empty%20directories%20as%20sync%20does%2C%20when%20%2D%2Dcreate%2Dempty%2Dsrc%2Ddirs%20is%20passed

Also fixed an issue causing --resync to erroneously delete empty folders and duplicate files unique to Path2
https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=2.%20%2D%2Dresync%20deletes%20data%2C%20contrary%20to%20docs

cmd/bisync/bisync_test.go

@@ -614,6 +614,8 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
 			opt.DryRun = true
 		case "force":
 			opt.Force = true
+		case "create-empty-src-dirs":
+			opt.CreateEmptySrcDirs = true
 		case "remove-empty-dirs":
 			opt.RemoveEmptyDirs = true
 		case "check-sync-only":

cmd/bisync/cmd.go

@@ -31,6 +31,7 @@ type Options struct {
 	CheckAccess           bool
 	CheckFilename         string
 	CheckSync             CheckSyncMode
+	CreateEmptySrcDirs    bool
 	RemoveEmptyDirs       bool
 	MaxDelete             int // percentage from 0 to 100
 	Force                 bool
@@ -105,7 +106,8 @@ func init() {
 	flags.StringVarP(cmdFlags, &Opt.CheckFilename, "check-filename", "", Opt.CheckFilename, makeHelp("Filename for --check-access (default: {CHECKFILE})"), "")
 	flags.BoolVarP(cmdFlags, &Opt.Force, "force", "", Opt.Force, "Bypass --max-delete safety check and run the sync. Consider using with --verbose", "")
 	flags.FVarP(cmdFlags, &Opt.CheckSync, "check-sync", "", "Controls comparison of final listings: true|false|only (default: true)", "")
-	flags.BoolVarP(cmdFlags, &Opt.RemoveEmptyDirs, "remove-empty-dirs", "", Opt.RemoveEmptyDirs, "Remove empty directories at the final cleanup step.", "")
+	flags.BoolVarP(cmdFlags, &Opt.CreateEmptySrcDirs, "create-empty-src-dirs", "", Opt.CreateEmptySrcDirs, "Sync creation and deletion of empty directories. (Not compatible with --remove-empty-dirs)", "")
+	flags.BoolVarP(cmdFlags, &Opt.RemoveEmptyDirs, "remove-empty-dirs", "", Opt.RemoveEmptyDirs, "Remove ALL empty directories at the final cleanup step.", "")
 	flags.StringVarP(cmdFlags, &Opt.FiltersFile, "filters-file", "", Opt.FiltersFile, "Read filtering patterns from a file", "")
 	flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, makeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"), "")
 	flags.BoolVarP(cmdFlags, &tzLocal, "localtime", "", tzLocal, "Use local time in listings (default: UTC)", "")

cmd/bisync/deltas.go

@@ -229,6 +229,24 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 
 	ctxMove := b.opt.setDryRun(ctx)
 
+	// efficient isDir check
+	// we load the listing just once and store only the dirs
+	dirs1, dirs1Err := b.listDirsOnly(1)
+	if dirs1Err != nil {
+		b.critical = true
+		b.retryable = true
+		fs.Debugf(nil, "Error generating dirsonly list for path1: %v", dirs1Err)
+		return
+	}
+
+	dirs2, dirs2Err := b.listDirsOnly(2)
+	if dirs2Err != nil {
+		b.critical = true
+		b.retryable = true
+		fs.Debugf(nil, "Error generating dirsonly list for path2: %v", dirs2Err)
+		return
+	}
+
 	// build a list of only the "deltaOther"s so we don't have to check more files than necessary
 	// this is essentially the same as running rclone check with a --files-from filter, then exempting the --match results from being renamed
 	// we therefore avoid having to list the same directory more than once.
@@ -275,28 +293,32 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 				b.indent("!WARNING", file, "New or changed in both paths")
 
 				//if files are identical, leave them alone instead of renaming
+				if dirs1.has(file) && dirs2.has(file) {
+					fs.Debugf(nil, "This is a directory, not a file. Skipping equality check and will not rename: %s", file)
+				} else {
 					equal := matches.Has(file)
 					if equal {
 						fs.Infof(nil, "Files are equal! Skipping: %s", file)
 					} else {
 						fs.Debugf(nil, "Files are NOT equal: %s", file)
-				b.indent("!Path1", p1+"..path1", "Renaming Path1 copy")
-				if err = operations.MoveFile(ctxMove, b.fs1, b.fs1, file+"..path1", file); err != nil {
-					err = fmt.Errorf("path1 rename failed for %s: %w", p1, err)
-					b.critical = true
-					return
-				}
-				b.indent("!Path1", p2+"..path1", "Queue copy to Path2")
-				copy1to2.Add(file + "..path1")
+						b.indent("!Path1", p1+"..path1", "Renaming Path1 copy")
+						if err = operations.MoveFile(ctxMove, b.fs1, b.fs1, file+"..path1", file); err != nil {
+							err = fmt.Errorf("path1 rename failed for %s: %w", p1, err)
+							b.critical = true
+							return
+						}
+						b.indent("!Path1", p2+"..path1", "Queue copy to Path2")
+						copy1to2.Add(file + "..path1")
 
-				b.indent("!Path2", p2+"..path2", "Renaming Path2 copy")
-				if err = operations.MoveFile(ctxMove, b.fs2, b.fs2, file+"..path2", file); err != nil {
-					err = fmt.Errorf("path2 rename failed for %s: %w", file, err)
-					return
-				}
-				b.indent("!Path2", p1+"..path2", "Queue copy to Path1")
-				copy2to1.Add(file + "..path2")
+						b.indent("!Path2", p2+"..path2", "Renaming Path2 copy")
+						if err = operations.MoveFile(ctxMove, b.fs2, b.fs2, file+"..path2", file); err != nil {
+							err = fmt.Errorf("path2 rename failed for %s: %w", file, err)
+							return
+						}
+						b.indent("!Path2", p1+"..path2", "Queue copy to Path1")
+						copy2to1.Add(file + "..path2")
 					}
+				}
 				handled.Add(file)
 			}
 		} else {
@@ -340,6 +362,9 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 		if err != nil {
 			return
 		}
+
+		//copy empty dirs from path2 to path1 (if --create-empty-src-dirs)
+		b.syncEmptyDirs(ctx, b.fs1, copy2to1, dirs2, "make")
 	}
 
 	if copy1to2.NotEmpty() {
@@ -349,6 +374,9 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 		if err != nil {
 			return
 		}
+
+		//copy empty dirs from path1 to path2 (if --create-empty-src-dirs)
+		b.syncEmptyDirs(ctx, b.fs2, copy1to2, dirs1, "make")
 	}
 
 	if delete1.NotEmpty() {
@@ -358,6 +386,9 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 		if err != nil {
 			return
 		}
+
+		//propagate deletions of empty dirs from path2 to path1 (if --create-empty-src-dirs)
+		b.syncEmptyDirs(ctx, b.fs1, delete1, dirs1, "remove")
 	}
 
 	if delete2.NotEmpty() {
@@ -367,6 +398,9 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 		if err != nil {
 			return
 		}
+
+		//propagate deletions of empty dirs from path1 to path2 (if --create-empty-src-dirs)
+		b.syncEmptyDirs(ctx, b.fs2, delete2, dirs2, "remove")
 	}
 
 	return

cmd/bisync/listing.go

@@ -43,10 +43,11 @@ var tzLocal = false
 
 // fileInfo describes a file
 type fileInfo struct {
-	size int64
-	time time.Time
-	hash string
-	id   string
+	size  int64
+	time  time.Time
+	hash  string
+	id    string
+	flags string
 }
 
 // fileList represents a listing
@@ -76,17 +77,18 @@ func (ls *fileList) get(file string) *fileInfo {
 	return ls.info[file]
 }
 
-func (ls *fileList) put(file string, size int64, time time.Time, hash, id string) {
+func (ls *fileList) put(file string, size int64, time time.Time, hash, id string, flags string) {
 	fi := ls.get(file)
 	if fi != nil {
 		fi.size = size
 		fi.time = time
 	} else {
 		fi = &fileInfo{
-			size: size,
-			time: time,
-			hash: hash,
-			id:   id,
+			size:  size,
+			time:  time,
+			hash:  hash,
+			id:    id,
+			flags: flags,
 		}
 		ls.info[file] = fi
 		ls.list = append(ls.list, file)
@@ -152,7 +154,11 @@ func (ls *fileList) save(ctx context.Context, listing string) error {
 			id = "-"
 		}
 
-		flags := "-"
+		flags := fi.flags
+		if flags == "" {
+			flags = "-"
+		}
+
 		_, err = fmt.Fprintf(file, lineFormat, flags, fi.size, hash, id, time, remote)
 		if err != nil {
 			_ = file.Close()
@@ -217,7 +223,7 @@ func (b *bisyncRun) loadListing(listing string) (*fileList, error) {
 			}
 		}
 
-		if flags != "-" || id != "-" || sizeErr != nil || timeErr != nil || hashErr != nil || nameErr != nil {
+		if (flags != "-" && flags != "d") || id != "-" || sizeErr != nil || timeErr != nil || hashErr != nil || nameErr != nil {
 			fs.Logf(listing, "Ignoring incorrect line: %q", line)
 			continue
 		}
@@ -229,7 +235,7 @@ func (b *bisyncRun) loadListing(listing string) (*fileList, error) {
 			}
 		}
 
-		ls.put(nameVal, sizeVal, timeVal.In(TZ), hashVal, id)
+		ls.put(nameVal, sizeVal, timeVal.In(TZ), hashVal, id, flags)
 	}
 
 	return ls, nil
@@ -262,7 +268,11 @@ func (b *bisyncRun) makeListing(ctx context.Context, f fs.Fs, listing string) (l
 	ls = newFileList()
 	ls.hash = hashType
 	var lock sync.Mutex
-	err = walk.ListR(ctx, f, "", false, depth, walk.ListObjects, func(entries fs.DirEntries) error {
+	listType := walk.ListObjects
+	if b.opt.CreateEmptySrcDirs {
+		listType = walk.ListAll
+	}
+	err = walk.ListR(ctx, f, "", false, depth, listType, func(entries fs.DirEntries) error {
 		var firstErr error
 		entries.ForObject(func(o fs.Object) {
 			//tr := accounting.Stats(ctx).NewCheckingTransfer(o) // TODO
@@ -277,12 +287,27 @@ func (b *bisyncRun) makeListing(ctx context.Context, f fs.Fs, listing string) (l
 				}
 			}
 			time := o.ModTime(ctx).In(TZ)
-			id := "" // TODO
+			id := ""     // TODO
+			flags := "-" // "-" for a file and "d" for a directory
 			lock.Lock()
-			ls.put(o.Remote(), o.Size(), time, hashVal, id)
+			ls.put(o.Remote(), o.Size(), time, hashVal, id, flags)
 			lock.Unlock()
 			//tr.Done(ctx, nil) // TODO
 		})
+		if b.opt.CreateEmptySrcDirs {
+			entries.ForDir(func(o fs.Directory) {
+				var (
+					hashVal string
+				)
+				time := o.ModTime(ctx).In(TZ)
+				id := ""     // TODO
+				flags := "d" // "-" for a file and "d" for a directory
+				lock.Lock()
+				//record size as 0 instead of -1, so bisync doesn't think it's a google doc
+				ls.put(o.Remote(), 0, time, hashVal, id, flags)
+				lock.Unlock()
+			})
+		}
 		return firstErr
 	})
 	if err == nil {
@@ -304,3 +329,50 @@ func (b *bisyncRun) checkListing(ls *fileList, listing, msg string) error {
 	b.retryable = true
 	return fmt.Errorf("empty %s listing: %s", msg, listing)
 }
+
+// listingNum should be 1 for path1 or 2 for path2
+func (b *bisyncRun) loadListingNum(listingNum int) (*fileList, error) {
+	listingpath := b.basePath + ".path1.lst-new"
+	if listingNum == 2 {
+		listingpath = b.basePath + ".path2.lst-new"
+	}
+
+	if b.opt.DryRun {
+		listingpath = strings.Replace(listingpath, ".lst-", ".lst-dry-", 1)
+	}
+
+	fs.Debugf(nil, "loading listing for path %d at: %s", listingNum, listingpath)
+	return b.loadListing(listingpath)
+}
+
+func (b *bisyncRun) listDirsOnly(listingNum int) (*fileList, error) {
+	var fulllisting *fileList
+	var dirsonly = newFileList()
+	var err error
+
+	if !b.opt.CreateEmptySrcDirs {
+		return dirsonly, err
+	}
+
+	fulllisting, err = b.loadListingNum(listingNum)
+
+	if err != nil {
+		b.critical = true
+		b.retryable = true
+		fs.Debugf(nil, "Error loading listing to generate dirsonly list: %v", err)
+		return dirsonly, err
+	}
+
+	for _, obj := range fulllisting.list {
+		info := fulllisting.get(obj)
+
+		if info.flags == "d" {
+			fs.Debugf(nil, "found a dir: %s", obj)
+			dirsonly.put(obj, info.size, info.time, info.hash, info.id, info.flags)
+		} else {
+			fs.Debugf(nil, "not a dir: %s", obj)
+		}
+	}
+
+	return dirsonly, err
+}

cmd/bisync/operations.go

@@ -128,14 +128,14 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
 			fs.Errorf(nil, "Bisync critical error: %v", err)
 			fs.Errorf(nil, "Bisync aborted. Error is retryable without --resync due to --resilient mode.")
 		} else {
-		if bilib.FileExists(listing1) {
-			_ = os.Rename(listing1, listing1+"-err")
-		}
-		if bilib.FileExists(listing2) {
-			_ = os.Rename(listing2, listing2+"-err")
-		}
-		fs.Errorf(nil, "Bisync critical error: %v", err)
-		fs.Errorf(nil, "Bisync aborted. Must run --resync to recover.")
+			if bilib.FileExists(listing1) {
+				_ = os.Rename(listing1, listing1+"-err")
+			}
+			if bilib.FileExists(listing2) {
+				_ = os.Rename(listing2, listing2+"-err")
+			}
+			fs.Errorf(nil, "Bisync critical error: %v", err)
+			fs.Errorf(nil, "Bisync aborted. Must run --resync to recover.")
 		}
 		return ErrBisyncAborted
 	}
@@ -413,11 +413,34 @@ func (b *bisyncRun) resync(octx, fctx context.Context, listing1, listing2 string
 		// prevent overwriting Google Doc files (their size is -1)
 		filterSync.Opt.MinSize = 0
 	}
-	if err = sync.Sync(ctxSync, b.fs2, b.fs1, false); err != nil {
+	if err = sync.CopyDir(ctxSync, b.fs2, b.fs1, b.opt.CreateEmptySrcDirs); err != nil {
 		b.critical = true
 		return err
 	}
 
+	if b.opt.CreateEmptySrcDirs {
+		// copy Path2 back to Path1, for empty dirs
+		// the fastCopy above cannot include directories, because it relies on --files-from for filtering,
+		// so instead we'll copy them here, relying on fctx for our filtering.
+
+		// This preserves the original resync order for backward compatibility. It is essentially:
+		// rclone copy Path2 Path1 --ignore-existing
+		// rclone copy Path1 Path2 --create-empty-src-dirs
+		// rclone copy Path2 Path1 --create-empty-src-dirs
+
+		// although if we were starting from scratch, it might be cleaner and faster to just do:
+		// rclone copy Path2 Path1 --create-empty-src-dirs
+		// rclone copy Path1 Path2 --create-empty-src-dirs
+
+		fs.Infof(nil, "Resynching Path2 to Path1 (for empty dirs)")
+
+		//note copy (not sync) and dst comes before src
+		if err = sync.CopyDir(ctxSync, b.fs1, b.fs2, b.opt.CreateEmptySrcDirs); err != nil {
+			b.critical = true
+			return err
+		}
+	}
+
 	fs.Infof(nil, "Resync updating listings")
 	if _, err = b.makeListing(fctx, b.fs1, listing1); err != nil {
 		b.critical = true

cmd/bisync/queue.go

@@ -3,6 +3,7 @@ package bisync
 import (
 	"context"
 	"fmt"
+	"sort"
 
 	"github.com/rclone/rclone/cmd/bisync/bilib"
 	"github.com/rclone/rclone/fs"
@@ -23,7 +24,7 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.
 		}
 	}
 
-	return sync.CopyDir(ctxCopy, fdst, fsrc, false)
+	return sync.CopyDir(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)
 }
 
 func (b *bisyncRun) fastDelete(ctx context.Context, f fs.Fs, files bilib.Names, queueName string) error {
@@ -60,6 +61,36 @@ func (b *bisyncRun) fastDelete(ctx context.Context, f fs.Fs, files bilib.Names,
 	return err
 }
 
+// operation should be "make" or "remove"
+func (b *bisyncRun) syncEmptyDirs(ctx context.Context, dst fs.Fs, candidates bilib.Names, dirsList *fileList, operation string) {
+	if b.opt.CreateEmptySrcDirs && (!b.opt.Resync || operation == "make") {
+
+		candidatesList := candidates.ToList()
+		if operation == "remove" {
+			// reverse the sort order to ensure we remove subdirs before parent dirs
+			sort.Sort(sort.Reverse(sort.StringSlice(candidatesList)))
+		}
+
+		for _, s := range candidatesList {
+			var direrr error
+			if dirsList.has(s) { //make sure it's a dir, not a file
+				if operation == "remove" {
+					//note: we need to use Rmdirs instead of Rmdir because directories will fail to delete if they have other empty dirs inside of them.
+					direrr = operations.Rmdirs(ctx, dst, s, false)
+				} else if operation == "make" {
+					direrr = operations.Mkdir(ctx, dst, s)
+				} else {
+					direrr = fmt.Errorf("invalid operation. Expected 'make' or 'remove', received '%q'", operation)
+				}
+
+				if direrr != nil {
+					fs.Debugf(nil, "Error syncing directory: %v", direrr)
+				}
+			}
+		}
+	}
+}
+
 func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error {
 	if !b.opt.SaveQueues {
 		return nil

cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que

@@ -0,0 +1 @@
+"subdir"

cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.delete2.que

@@ -0,0 +1 @@
+"subdir"

cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path1.lst

@@ -0,0 +1,7 @@
+# bisync listing v1 from test
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy1.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy2.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy3.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy4.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy5.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.txt"

cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path1.lst-new

@@ -0,0 +1,7 @@
+# bisync listing v1 from test
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy1.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy2.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy3.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy4.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy5.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.txt"

cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path2.lst

@@ -0,0 +1,7 @@
+# bisync listing v1 from test
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy1.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy2.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy3.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy4.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy5.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.txt"

cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path2.lst-new

@@ -0,0 +1,7 @@
+# bisync listing v1 from test
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy1.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy2.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy3.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy4.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy5.txt"
+-        0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.txt"

cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que

@@ -0,0 +1 @@
+"subdir"

cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log

@@ -0,0 +1,142 @@
+(01)  : test createemptysrcdirs
+
+
+(02)  : test initial bisync
+(03)  : touch-glob 2001-01-02 {datadir/} placeholder.txt
+(04)  : copy-as {datadir/}placeholder.txt {path1/} file1.txt
+(05)  : copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt
+(06)  : copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt
+(07)  : copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt
+(08)  : copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt
+(09)  : copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt
+(10)  : bisync resync
+INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO  : Copying unique Path2 files to Path1
+INFO  : Resynching Path1 to Path2
+INFO  : Resync updating listings
+INFO  : Bisync successful
+
+(11)  : test 1. Create an empty dir on Path1 by creating subdir/placeholder.txt and then deleting the placeholder
+(12)  : copy-as {datadir/}placeholder.txt {path1/} subdir/placeholder.txt
+(13)  : touch-glob 2001-01-02 {path1/} subdir
+(14)  : delete-file {path1/}subdir/placeholder.txt
+
+(15)  : test 2. Run bisync without --create-empty-src-dirs
+(16)  : bisync
+INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO  : Path1 checking for diffs
+INFO  : Path2 checking for diffs
+INFO  : No changes found
+INFO  : Updating listings
+INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
+INFO  : Bisync successful
+
+(17)  : test 3. Confirm the subdir exists only on Path1 and not Path2
+(18)  : list-dirs {path1/}
+subdir/
+(19)  : list-dirs {path2/}
+
+(20)  : test 4.Run bisync WITH --create-empty-src-dirs
+(21)  : bisync create-empty-src-dirs
+INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO  : Path1 checking for diffs
+INFO  : - Path1    File is new                         - subdir
+INFO  : Path1:    1 changes:    1 new,    0 newer,    0 older,    0 deleted
+INFO  : Path2 checking for diffs
+INFO  : Applying changes
+INFO  : - Path1    Queue copy to Path2                 - {path2/}subdir
+INFO  : - Path1    Do queued copies to                 - Path2
+INFO  : Updating listings
+INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
+INFO  : Bisync successful
+
+(22)  : test 5. Confirm the subdir exists on both paths
+(23)  : list-dirs {path1/}
+subdir/
+(24)  : list-dirs {path2/}
+subdir/
+
+(25)  : test 6. Delete the empty dir on Path1 using purge-children (and also add files so the path isn't empty)
+(26)  : purge-children {path1/}
+(27)  : copy-as {datadir/}placeholder.txt {path1/} file1.txt
+(28)  : copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt
+(29)  : copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt
+(30)  : copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt
+(31)  : copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt
+(32)  : copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt
+
+(33)  : test 7. Run bisync without --create-empty-src-dirs
+(34)  : bisync
+INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO  : Path1 checking for diffs
+INFO  : - Path1    File was deleted                    - RCLONE_TEST
+INFO  : - Path1    File was deleted                    - subdir
+INFO  : Path1:    2 changes:    0 new,    0 newer,    0 older,    2 deleted
+INFO  : Path2 checking for diffs
+INFO  : - Path2    File was deleted                    - subdir
+INFO  : Path2:    1 changes:    0 new,    0 newer,    0 older,    1 deleted
+INFO  : Applying changes
+INFO  : - Path2    Queue delete                        - {path2/}RCLONE_TEST
+INFO  : -          Do queued deletes on                - Path2
+INFO  : Updating listings
+INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
+INFO  : Bisync successful
+
+(35)  : test 8. Confirm the subdir exists only on Path2 and not Path1
+(36)  : list-dirs {path1/}
+(37)  : list-dirs {path2/}
+subdir/
+
+(38)  : test 9. Reset, do the delete again, and run bisync WITH --create-empty-src-dirs
+(39)  : bisync resync create-empty-src-dirs
+INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO  : Copying unique Path2 files to Path1
+INFO  : - Path2    Resync will copy to Path1           - subdir
+INFO  : - Path2    Resync is doing queued copies to    - Path1
+INFO  : Resynching Path1 to Path2
+INFO  : Resynching Path2 to Path1 (for empty dirs)
+INFO  : Resync updating listings
+INFO  : Bisync successful
+(40)  : list-dirs {path1/}
+subdir/
+(41)  : list-dirs {path2/}
+subdir/
+
+(42)  : purge-children {path1/}
+(43)  : copy-as {datadir/}placeholder.txt {path1/} file1.txt
+(44)  : copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt
+(45)  : copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt
+(46)  : copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt
+(47)  : copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt
+(48)  : copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt
+(49)  : list-dirs {path1/}
+(50)  : list-dirs {path2/}
+subdir/
+
+(51)  : bisync create-empty-src-dirs
+INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO  : Path1 checking for diffs
+INFO  : - Path1    File was deleted                    - subdir
+INFO  : Path1:    1 changes:    0 new,    0 newer,    0 older,    1 deleted
+INFO  : Path2 checking for diffs
+INFO  : Applying changes
+INFO  : - Path2    Queue delete                        - {path2/}subdir
+INFO  : -          Do queued deletes on                - Path2
+INFO  : subdir: Removing directory
+INFO  : Updating listings
+INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
+INFO  : Bisync successful
+
+(52)  : test 10. Confirm the subdir has been removed on both paths
+(53)  : list-dirs {path1/}
+(54)  : list-dirs {path2/}
+
+(55)  : test 11. bisync again (because if we leave subdir in listings, test will fail due to mismatched modtime)
+(56)  : bisync create-empty-src-dirs
+INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO  : Path1 checking for diffs
+INFO  : Path2 checking for diffs
+INFO  : No changes found
+INFO  : Updating listings
+INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
+INFO  : Bisync successful

cmd/bisync/testdata/test_createemptysrcdirs/initial/RCLONE_TEST

@@ -0,0 +1 @@
+This file is used for testing the health of rclone accesses to the local/remote file system.  Do not delete.

cmd/bisync/testdata/test_createemptysrcdirs/scenario.txt

@@ -0,0 +1,87 @@
+test createemptysrcdirs
+# Test the --create-empty-src-dirs logic.
+# Should behave the same way as rclone sync.
+# Without this flag, empty directories created/deleted on one side are NOT created/deleted on the other side
+# With this flag, empty directories created/deleted on one side are created/deleted on the other side; the result should be an exact mirror.
+#
+# Placeholders are necessary to ensure that git does not lose our empty folders
+# After the initial setup sync:
+#  1. Create an empty dir on Path1 by creating subdir/placeholder.txt and then deleting the placeholder
+#  2. Run bisync without --create-empty-src-dirs
+#  3. Confirm the subdir exists only on Path1 and not Path2
+#  4. Run bisync WITH --create-empty-src-dirs
+#  5. Confirm the subdir exists on both paths
+#  6. Delete the empty dir on Path1 using purge-children (and also add files so the path isn't empty)
+#  7. Run bisync without --create-empty-src-dirs
+#  8. Confirm the subdir exists only on Path2 and not Path1
+#  9. Reset, do the delete again, and run bisync WITH --create-empty-src-dirs
+#  10. Confirm the subdir has been removed on both paths
+
+test initial bisync
+touch-glob 2001-01-02 {datadir/} placeholder.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt
+bisync resync
+
+test 1. Create an empty dir on Path1 by creating subdir/placeholder.txt and then deleting the placeholder
+copy-as {datadir/}placeholder.txt {path1/} subdir/placeholder.txt
+touch-glob 2001-01-02 {path1/} subdir
+delete-file {path1/}subdir/placeholder.txt
+
+test 2. Run bisync without --create-empty-src-dirs
+bisync
+
+test 3. Confirm the subdir exists only on Path1 and not Path2
+list-dirs {path1/}
+list-dirs {path2/}
+
+test 4.Run bisync WITH --create-empty-src-dirs
+bisync create-empty-src-dirs
+
+test 5. Confirm the subdir exists on both paths
+list-dirs {path1/}
+list-dirs {path2/}
+
+test 6. Delete the empty dir on Path1 using purge-children (and also add files so the path isn't empty)
+purge-children {path1/}
+copy-as {datadir/}placeholder.txt {path1/} file1.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt
+
+test 7. Run bisync without --create-empty-src-dirs
+bisync
+
+test 8. Confirm the subdir exists only on Path2 and not Path1
+list-dirs {path1/}
+list-dirs {path2/}
+
+test 9. Reset, do the delete again, and run bisync WITH --create-empty-src-dirs
+bisync resync create-empty-src-dirs
+list-dirs {path1/}
+list-dirs {path2/}
+
+purge-children {path1/}
+copy-as {datadir/}placeholder.txt {path1/} file1.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt
+copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt
+list-dirs {path1/}
+list-dirs {path2/}
+
+bisync create-empty-src-dirs
+
+test 10. Confirm the subdir has been removed on both paths
+list-dirs {path1/}
+list-dirs {path2/}
+
+test 11. bisync again (because if we leave subdir in listings, test will fail due to mismatched modtime)
+bisync create-empty-src-dirs

cmd/bisync/testdata/test_dry_run/golden/test.log

@@ -54,13 +54,10 @@ NOTICE: file4.txt: Skipped copy as --dry-run is set (size 0)
 NOTICE: file6.txt: Skipped copy as --dry-run is set (size 19)
 INFO  : Resynching Path1 to Path2
 NOTICE: file1.txt: Skipped copy as --dry-run is set (size 0)
-NOTICE: file10.txt: Skipped delete as --dry-run is set (size 19)
 NOTICE: file11.txt: Skipped copy as --dry-run is set (size 19)
 NOTICE: file2.txt: Skipped copy as --dry-run is set (size 13)
 NOTICE: file3.txt: Skipped copy as --dry-run is set (size 0)
-NOTICE: file4.txt: Skipped delete as --dry-run is set (size 0)
 NOTICE: file5.txt: Skipped copy (or update modification time) as --dry-run is set (size 39)
-NOTICE: file6.txt: Skipped delete as --dry-run is set (size 19)
 NOTICE: file7.txt: Skipped copy as --dry-run is set (size 19)
 INFO  : Resync updating listings
 INFO  : Bisync successful

docs/content/bisync.md

@@ -91,6 +91,8 @@ Optional Flags:
                                 If exceeded, the bisync run will abort. (default: 50%)
       --force                   Bypass `--max-delete` safety check and run the sync.
                                 Consider using with `--verbose`
+      --create-empty-src-dirs   Sync creation and deletion of empty directories. 
+                                  (Not compatible with --remove-empty-dirs)
       --remove-empty-dirs       Remove empty directories at the final cleanup step.
   -1, --resync                  Performs the resync run.
                                 Warning: Path1 files may overwrite Path2 versions.
@@ -125,7 +127,7 @@ Cloud references are distinguished by having a `:` in the argument
 (see [Windows support](#windows) below).
 
 Path1 and Path2 are treated equally, in that neither has priority for
-file changes, and access efficiency does not change whether a remote
+file changes (except during [`--resync`](#resync)), and access efficiency does not change whether a remote
 is on Path1 or Path2.
 
 The listings in bisync working directory (default: `~/.cache/rclone/bisync`)
@@ -134,8 +136,8 @@ to individual directories within the tree may be set up, e.g.:
 `path_to_local_tree..dropbox_subdir.lst`.
 
 Any empty directories after the sync on both the Path1 and Path2
-filesystems are not deleted by default. If the `--remove-empty-dirs`
-flag is specified, then both paths will have any empty directories purged
+filesystems are not deleted by default, unless `--create-empty-src-dirs` is specified. 
+If the `--remove-empty-dirs` flag is specified, then both paths will have ALL empty directories purged
 as the last step in the process.
 
 ## Command-line flags
@@ -144,15 +146,31 @@ as the last step in the process.
 
 This will effectively make both Path1 and Path2 filesystems contain a
 matching superset of all files. Path2 files that do not exist in Path1 will
-be copied to Path1, and the process will then sync the Path1 tree to Path2.
+be copied to Path1, and the process will then copy the Path1 tree to Path2.
 
-The base directories on the both Path1 and Path2 filesystems must exist
+The `--resync` sequence is roughly equivalent to:
+```
+rclone copy Path2 Path1 --ignore-existing
+rclone copy Path1 Path2
+```
+Or, if using `--create-empty-src-dirs`:
+```
+rclone copy Path2 Path1 --ignore-existing
+rclone copy Path1 Path2 --create-empty-src-dirs
+rclone copy Path2 Path1 --create-empty-src-dirs
+```
+
+The base directories on both Path1 and Path2 filesystems must exist
 or bisync will fail. This is required for safety - that bisync can verify
 that both paths are valid.
 
-When using `--resync`, a newer version of a file either on Path1 or Path2
-filesystem, will overwrite the file on the other path (only the last version
-will be kept). Carefully evaluate deltas using [--dry-run](/flags/#non-backend-flags).
+When using `--resync`, a newer version of a file on the Path2 filesystem
+will be overwritten by the Path1 filesystem version. 
+(Note that this is [NOT entirely symmetrical](https://github.com/rclone/rclone/issues/5681#issuecomment-938761815).)
+Carefully evaluate deltas using [--dry-run](/flags/#non-backend-flags).
+
+[//]: # (I reverted a recent change in the above paragraph, as it was incorrect.
+https://github.com/rclone/rclone/commit/dd72aff98a46c6e20848ac7ae5f7b19d45802493 )
 
 For a resync run, one of the paths may be empty (no files in the path tree).
 The resync run should result in files on both paths, else a normal non-resync
@@ -493,6 +511,22 @@ rclone copy PATH1 PATH2 --filter "+ */" --filter "- **" --create-empty-src-dirs
 rclone copy PATH2 PATH2 --filter "+ */" --filter "- **" --create-empty-src-dirs
 ```
 
+### Empty directories
+
+By default, new/deleted empty directories on one path are _not_ propagated to the other side.
+This is because bisync (and rclone) natively works on files, not directories.
+However, this can be changed with the `--create-empty-src-dirs` flag, which works in
+much the same way as in [`sync`](/commands/rclone_sync/) and [`copy`](/commands/rclone_copy/).
+When used, empty directories created or deleted on one side will also be created or deleted on the other side.
+The following should be noted:
+* `--create-empty-src-dirs` is not compatible with `--remove-empty-dirs`. Use only one or the other (or neither).
+* It is not recommended to switch back and forth between `--create-empty-src-dirs` 
+and the default (no `--create-empty-src-dirs`) without running `--resync`. 
+This is because it may appear as though all directories (not just the empty ones) were created/deleted,
+when actually you've just toggled between making them visible/invisible to bisync. 
+It looks scarier than it is, but it's still probably best to stick to one or the other, 
+and use `--resync` when you need to switch.
+
 ### Renamed directories
 
 Renaming a folder on the Path1 side results in deleting all files on
@@ -1187,11 +1221,15 @@ about _Unison_ and synchronization in general.
 ### `v1.64`
 * Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry) 
 causing dry runs to inadvertently commit filter changes
+* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=2.%20%2D%2Dresync%20deletes%20data%2C%20contrary%20to%20docs) 
+causing `--resync` to erroneously delete empty folders and duplicate files unique to Path2
 * `--check-access` is now enforced during `--resync`, preventing data loss in [certain user error scenarios](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=%2D%2Dcheck%2Daccess%20doesn%27t%20always%20fail%20when%20it%20should)
 * Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=5.%20Bisync%20reads%20files%20in%20excluded%20directories%20during%20delete%20operations) 
 causing bisync to consider more files than necessary due to overbroad filters during delete operations
 * [Improved detection of false positive change conflicts](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Identical%20files%20should%20be%20left%20alone%2C%20even%20if%20new/newer/changed%20on%20both%20sides) 
 (identical files are now left alone instead of renamed)
+* Added [support for `--create-empty-src-dirs`](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=3.%20Bisync%20should%20create/delete%20empty%20directories%20as%20sync%20does%2C%20when%20%2D%2Dcreate%2Dempty%2Dsrc%2Ddirs%20is%20passed)
 * Added experimental `--resilient` mode to allow [recovery from self-correctable errors](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=2.%20Bisync%20should%20be%20more%20resilient%20to%20self%2Dcorrectable%20errors)
 * Added [new `--ignore-listing-checksum` flag](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=6.%20%2D%2Dignore%2Dchecksum%20should%20be%20split%20into%20two%20flags%20for%20separate%20purposes) 
 to distinguish from `--ignore-checksum`
+* [Performance improvements](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=6.%20Deletes%20take%20several%20times%20longer%20than%20copies) for large remotes