check and cryptcheck: report directory differences with --report-dirs - fixes #6440

Before this change, check and cryptcheck would not inform the user of directory
differences (for example, an empty directory that exists only on the src.)

After this change, a new --report-dirs flag allows alerting users of such
differences.

Note that directories will be reported regardless of whether they are empty.
They will be included in the total error count, but counted and summarized
separately, and not included in output files. --report-dirs is ignored when
--checkfile is in use.

cmd/check/check.go

@@ -21,6 +21,7 @@ import (
 var (
 	download          = false
 	oneway            = false
+	ReportDirs        = false
 	combined          = ""
 	missingOnSrc      = ""
 	missingOnDst      = ""
@@ -41,6 +42,7 @@ func init() {
 // AddFlags adds the check flags to the cmdFlags command
 func AddFlags(cmdFlags *pflag.FlagSet) {
 	flags.BoolVarP(cmdFlags, &oneway, "one-way", "", oneway, "Check one way only, source files must exist on remote", "")
+	flags.BoolVarP(cmdFlags, &ReportDirs, "report-dirs", "", ReportDirs, "Report directory differences in addition to files", "")
 	flags.StringVarP(cmdFlags, &combined, "combined", "", combined, "Make a combined report of changes to this file", "")
 	flags.StringVarP(cmdFlags, &missingOnSrc, "missing-on-src", "", missingOnSrc, "Report all files missing from the source to this file", "")
 	flags.StringVarP(cmdFlags, &missingOnDst, "missing-on-dst", "", missingOnDst, "Report all files missing from the destination to this file", "")
@@ -85,6 +87,7 @@ func GetCheckOpt(fsrc, fdst fs.Fs) (opt *operations.CheckOpt, close func(), err
 		Fsrc:       fsrc,
 		Fdst:       fdst,
 		OneWay:     oneway,
+		ReportDirs: ReportDirs,
 	}
 
 	open := func(name string, pout *io.Writer) error {

cmd/cryptcheck/cryptcheck.go

@@ -80,6 +80,8 @@ func cryptCheck(ctx context.Context, fdst, fsrc fs.Fs) error {
 	}
 	defer close()
 
+	opt.ReportDirs = check.ReportDirs
+
 	// checkIdentical checks to see if dst and src are identical
 	//
 	// it returns true if differences were found

fs/operations/check.go

@@ -38,6 +38,7 @@ type CheckOpt struct {
 	Fdst, Fsrc   fs.Fs     // fses to check
 	Check        checkFn   // function to use for checking
 	OneWay       bool      // one way only?
+	ReportDirs   bool      // report dir differences in addition to files
 	Combined     io.Writer // a file with file names with leading sigils
 	MissingOnSrc io.Writer // files only in the destination
 	MissingOnDst io.Writer // files only in the source
@@ -56,6 +57,8 @@ type checkMarch struct {
 	noHashes        atomic.Int32
 	srcFilesMissing atomic.Int32
 	dstFilesMissing atomic.Int32
+	srcDirsMissing  atomic.Int32
+	dstDirsMissing  atomic.Int32
 	matches         atomic.Int32
 	opt             CheckOpt
 }
@@ -92,6 +95,13 @@ func (c *checkMarch) DstOnly(dst fs.DirEntry) (recurse bool) {
 		if c.opt.OneWay {
 			return false
 		}
+		if c.opt.ReportDirs {
+			err := fmt.Errorf("directory not in %v", c.opt.Fsrc)
+			fs.Errorf(dst, "%v", err)
+			_ = fs.CountError(err)
+			c.differences.Add(1)
+			c.srcDirsMissing.Add(1)
+		}
 		return true
 	default:
 		panic("Bad object in DirEntries")
@@ -111,6 +121,13 @@ func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {
 		c.report(src, c.opt.MissingOnDst, '+')
 	case fs.Directory:
 		// Do the same thing to the entire contents of the directory
+		if c.opt.ReportDirs {
+			err := fmt.Errorf("directory not in %v", c.opt.Fdst)
+			fs.Errorf(src, "%v", err)
+			_ = fs.CountError(err)
+			c.differences.Add(1)
+			c.dstDirsMissing.Add(1)
+		}
 		return true
 	default:
 		panic("Bad object in DirEntries")
@@ -184,8 +201,11 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b
 		}
 	case fs.Directory:
 		// Do the same thing to the entire contents of the directory
-		_, ok := dst.(fs.Directory)
+		dstX, ok := dst.(fs.Directory)
 		if ok {
+			if c.opt.ReportDirs {
+				fs.Debugf(dstX, "OK (no hash to check for directories)")
+			}
 			return true
 		}
 		err := fmt.Errorf("is file on %v but directory on %v", c.opt.Fdst, c.opt.Fsrc)
@@ -246,6 +266,14 @@ func (c *checkMarch) reportResults(ctx context.Context, err error) error {
 		}
 		fs.Logf(c.opt.Fsrc, "%d %s missing", c.srcFilesMissing.Load(), entity)
 	}
+	if c.opt.ReportDirs {
+		if c.dstDirsMissing.Load() > 0 {
+			fs.Logf(c.opt.Fdst, "%d directories missing", c.dstDirsMissing.Load())
+		}
+		if c.srcDirsMissing.Load() > 0 {
+			fs.Logf(c.opt.Fsrc, "%d directories missing", c.srcDirsMissing.Load())
+		}
+	}
 
 	fs.Logf(c.opt.Fdst, "%d differences found", accounting.Stats(ctx).GetErrors())
 	if errs := accounting.Stats(ctx).GetErrors(); errs > 0 {
@@ -416,6 +444,10 @@ func CheckSum(ctx context.Context, fsrc, fsum fs.Fs, sumFile string, hashType ha
 		return fmt.Errorf("%s: hash type is not supported by file system: %s", hashType, opt.Fdst)
 	}
 
+	if options.ReportDirs {
+		fs.Logf(nil, "ignoring --report-dirs as --checkfile is in use.")
+	}
+
 	if sumFile == "" {
 		return fmt.Errorf("not a sum file: %s", fsum)
 	}