check and cryptcheck: report directory differences with --report-dirs - fixes #6440

Before this change, check and cryptcheck would not inform the user of directory
differences (for example, an empty directory that exists only on the src.)

After this change, a new --report-dirs flag allows alerting users of such
differences.

Note that directories will be reported regardless of whether they are empty.
They will be included in the total error count, but counted and summarized
separately, and not included in output files. --report-dirs is ignored when
--checkfile is in use.
This commit is contained in:
nielash 2023-12-18 11:47:46 -05:00
parent 88141928f2
commit d19ee91960
3 changed files with 41 additions and 4 deletions

View File

@ -21,6 +21,7 @@ import (
var (
download = false
oneway = false
ReportDirs = false
combined = ""
missingOnSrc = ""
missingOnDst = ""
@ -41,6 +42,7 @@ func init() {
// AddFlags adds the check flags to the cmdFlags command
func AddFlags(cmdFlags *pflag.FlagSet) {
flags.BoolVarP(cmdFlags, &oneway, "one-way", "", oneway, "Check one way only, source files must exist on remote", "")
flags.BoolVarP(cmdFlags, &ReportDirs, "report-dirs", "", ReportDirs, "Report directory differences in addition to files", "")
flags.StringVarP(cmdFlags, &combined, "combined", "", combined, "Make a combined report of changes to this file", "")
flags.StringVarP(cmdFlags, &missingOnSrc, "missing-on-src", "", missingOnSrc, "Report all files missing from the source to this file", "")
flags.StringVarP(cmdFlags, &missingOnDst, "missing-on-dst", "", missingOnDst, "Report all files missing from the destination to this file", "")
@ -82,9 +84,10 @@ func GetCheckOpt(fsrc, fdst fs.Fs) (opt *operations.CheckOpt, close func(), err
closers := []io.Closer{}
opt = &operations.CheckOpt{
Fsrc: fsrc,
Fdst: fdst,
OneWay: oneway,
Fsrc: fsrc,
Fdst: fdst,
OneWay: oneway,
ReportDirs: ReportDirs,
}
open := func(name string, pout *io.Writer) error {

View File

@ -80,6 +80,8 @@ func cryptCheck(ctx context.Context, fdst, fsrc fs.Fs) error {
}
defer close()
opt.ReportDirs = check.ReportDirs
// checkIdentical checks to see if dst and src are identical
//
// it returns true if differences were found

View File

@ -38,6 +38,7 @@ type CheckOpt struct {
Fdst, Fsrc fs.Fs // fses to check
Check checkFn // function to use for checking
OneWay bool // one way only?
ReportDirs bool // report dir differences in addition to files
Combined io.Writer // a file with file names with leading sigils
MissingOnSrc io.Writer // files only in the destination
MissingOnDst io.Writer // files only in the source
@ -56,6 +57,8 @@ type checkMarch struct {
noHashes atomic.Int32
srcFilesMissing atomic.Int32
dstFilesMissing atomic.Int32
srcDirsMissing atomic.Int32
dstDirsMissing atomic.Int32
matches atomic.Int32
opt CheckOpt
}
@ -92,6 +95,13 @@ func (c *checkMarch) DstOnly(dst fs.DirEntry) (recurse bool) {
if c.opt.OneWay {
return false
}
if c.opt.ReportDirs {
err := fmt.Errorf("directory not in %v", c.opt.Fsrc)
fs.Errorf(dst, "%v", err)
_ = fs.CountError(err)
c.differences.Add(1)
c.srcDirsMissing.Add(1)
}
return true
default:
panic("Bad object in DirEntries")
@ -111,6 +121,13 @@ func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {
c.report(src, c.opt.MissingOnDst, '+')
case fs.Directory:
// Do the same thing to the entire contents of the directory
if c.opt.ReportDirs {
err := fmt.Errorf("directory not in %v", c.opt.Fdst)
fs.Errorf(src, "%v", err)
_ = fs.CountError(err)
c.differences.Add(1)
c.dstDirsMissing.Add(1)
}
return true
default:
panic("Bad object in DirEntries")
@ -184,8 +201,11 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b
}
case fs.Directory:
// Do the same thing to the entire contents of the directory
_, ok := dst.(fs.Directory)
dstX, ok := dst.(fs.Directory)
if ok {
if c.opt.ReportDirs {
fs.Debugf(dstX, "OK (no hash to check for directories)")
}
return true
}
err := fmt.Errorf("is file on %v but directory on %v", c.opt.Fdst, c.opt.Fsrc)
@ -246,6 +266,14 @@ func (c *checkMarch) reportResults(ctx context.Context, err error) error {
}
fs.Logf(c.opt.Fsrc, "%d %s missing", c.srcFilesMissing.Load(), entity)
}
if c.opt.ReportDirs {
if c.dstDirsMissing.Load() > 0 {
fs.Logf(c.opt.Fdst, "%d directories missing", c.dstDirsMissing.Load())
}
if c.srcDirsMissing.Load() > 0 {
fs.Logf(c.opt.Fsrc, "%d directories missing", c.srcDirsMissing.Load())
}
}
fs.Logf(c.opt.Fdst, "%d differences found", accounting.Stats(ctx).GetErrors())
if errs := accounting.Stats(ctx).GetErrors(); errs > 0 {
@ -416,6 +444,10 @@ func CheckSum(ctx context.Context, fsrc, fsum fs.Fs, sumFile string, hashType ha
return fmt.Errorf("%s: hash type is not supported by file system: %s", hashType, opt.Fdst)
}
if options.ReportDirs {
fs.Logf(nil, "ignoring --report-dirs as --checkfile is in use.")
}
if sumFile == "" {
return fmt.Errorf("not a sum file: %s", fsum)
}