From 3fcff32524f9d4593bc325d24ac4e8ec2c89aab3 Mon Sep 17 00:00:00 2001 From: Leonid Shalupov Date: Sat, 8 Aug 2015 21:10:31 +0200 Subject: [PATCH] do not print stats in quiet mode - fixes #70 ...unless had some errors or stats interval requested. Add fs.ErrorLog to differentiate between Logs which should be suppressed and errors which shouldn't. --- drive/drive.go | 12 +++++------ dropbox/dropbox.go | 14 ++++++------- dropbox/nametree.go | 12 +++++------ fs/fs.go | 6 ++++++ fs/operations.go | 26 ++++++++++++------------ googlecloudstorage/googlecloudstorage.go | 10 ++++----- local/local.go | 18 ++++++++-------- rclone.go | 2 +- s3/s3.go | 10 ++++----- swift/swift.go | 10 ++++----- 10 files changed, 63 insertions(+), 57 deletions(-) diff --git a/drive/drive.go b/drive/drive.go index e1e50fb86..d73705058 100644 --- a/drive/drive.go +++ b/drive/drive.go @@ -681,7 +681,7 @@ func (f *FsDrive) List() fs.ObjectsChan { err := f.findRoot(false) if err != nil { fs.Stats.Error() - fs.Log(f, "Couldn't find root: %s", err) + fs.ErrorLog(f, "Couldn't find root: %s", err) } else { if f.root == "" && *driveFullList { err = f.listDirFull(f.rootId, "", out) @@ -690,7 +690,7 @@ func (f *FsDrive) List() fs.ObjectsChan { } if err != nil { fs.Stats.Error() - fs.Log(f, "List failed: %s", err) + fs.ErrorLog(f, "List failed: %s", err) } } }() @@ -705,7 +705,7 @@ func (f *FsDrive) ListDir() fs.DirChan { err := f.findRoot(false) if err != nil { fs.Stats.Error() - fs.Log(f, "Couldn't find root: %s", err) + fs.ErrorLog(f, "Couldn't find root: %s", err) } else { _, err := f.listAll(f.rootId, "", true, false, func(item *drive.File) bool { dir := &fs.Dir{ @@ -719,7 +719,7 @@ func (f *FsDrive) ListDir() fs.DirChan { }) if err != nil { fs.Stats.Error() - fs.Log(f, "ListDir failed: %s", err) + fs.ErrorLog(f, "ListDir failed: %s", err) } } }() @@ -934,7 +934,7 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) { err := o.readMetaData() if err != nil { fs.Stats.Error() - fs.Log(o, "Failed to read metadata: %s", err) + fs.ErrorLog(o, "Failed to read metadata: %s", err) return } // New metadata @@ -948,7 +948,7 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) { }) if err != nil { fs.Stats.Error() - fs.Log(o, "Failed to update remote mtime: %s", err) + fs.ErrorLog(o, "Failed to update remote mtime: %s", err) return } // Update info from read data diff --git a/dropbox/dropbox.go b/dropbox/dropbox.go index 0b0fa6d2d..547eb2906 100644 --- a/dropbox/dropbox.go +++ b/dropbox/dropbox.go @@ -262,7 +262,7 @@ func (f *FsDropbox) stripRoot(path string) *string { if !strings.HasPrefix(lowercase, f.slashRootSlash) { fs.Stats.Error() - fs.Log(f, "Path '%s' is not under root '%s'", path, f.slashRootSlash) + fs.ErrorLog(f, "Path '%s' is not under root '%s'", path, f.slashRootSlash) return nil } @@ -281,11 +281,11 @@ func (f *FsDropbox) list(out fs.ObjectsChan) { deltaPage, err := f.db.Delta(cursor, f.slashRoot) if err != nil { fs.Stats.Error() - fs.Log(f, "Couldn't list: %s", err) + fs.ErrorLog(f, "Couldn't list: %s", err) break } else { if deltaPage.Reset && cursor != "" { - fs.Log(f, "Unexpected reset during listing - try again") + fs.ErrorLog(f, "Unexpected reset during listing - try again") fs.Stats.Error() break } @@ -305,7 +305,7 @@ func (f *FsDropbox) list(out fs.ObjectsChan) { } else { if len(entry.Path) <= 1 || entry.Path[0] != '/' { fs.Stats.Error() - fs.Log(f, "dropbox API inconsistency: a path should always start with a slash and be at least 2 characters: %s", entry.Path) + fs.ErrorLog(f, "dropbox API inconsistency: a path should always start with a slash and be at least 2 characters: %s", entry.Path) continue } @@ -374,7 +374,7 @@ func (f *FsDropbox) ListDir() fs.DirChan { entry, err := f.db.Metadata(f.root, true, false, "", "", metadataLimit) if err != nil { fs.Stats.Error() - fs.Log(f, "Couldn't list directories in root: %s", err) + fs.ErrorLog(f, "Couldn't list directories in root: %s", err) } else { for i := range entry.Contents { entry := &entry.Contents[i] @@ -750,7 +750,7 @@ func (o *FsObjectDropbox) deleteMetadata() { fs.Debug(o, "Deleting metadata from datastore") err := o.dropbox.deleteMetadata(o.metadataKey()) if err != nil { - fs.Log(o, "Error deleting metadata: %v", err) + fs.ErrorLog(o, "Error deleting metadata: %v", err) fs.Stats.Error() } } @@ -762,7 +762,7 @@ func (o *FsObjectDropbox) SetModTime(modTime time.Time) { err := o.setModTimeAndMd5sum(modTime, "") if err != nil { fs.Stats.Error() - fs.Log(o, err.Error()) + fs.ErrorLog(o, err.Error()) } } diff --git a/dropbox/nametree.go b/dropbox/nametree.go index b51ff6b45..da68ca691 100644 --- a/dropbox/nametree.go +++ b/dropbox/nametree.go @@ -51,7 +51,7 @@ func (tree *NameTreeNode) getTreeNode(path string) *NameTreeNode { for _, component := range strings.Split(path, "/") { if len(component) == 0 { fs.Stats.Error() - fs.Log(tree, "getTreeNode: path component is empty (full path %q)", path) + fs.ErrorLog(tree, "getTreeNode: path component is empty (full path %q)", path) return nil } @@ -72,7 +72,7 @@ func (tree *NameTreeNode) getTreeNode(path string) *NameTreeNode { func (tree *NameTreeNode) PutCaseCorrectDirectoryName(parentPath string, caseCorrectDirectoryName string) { if len(caseCorrectDirectoryName) == 0 { fs.Stats.Error() - fs.Log(tree, "PutCaseCorrectDirectoryName: empty caseCorrectDirectoryName is not allowed (parentPath: %q)", parentPath) + fs.ErrorLog(tree, "PutCaseCorrectDirectoryName: empty caseCorrectDirectoryName is not allowed (parentPath: %q)", parentPath) return } @@ -89,7 +89,7 @@ func (tree *NameTreeNode) PutCaseCorrectDirectoryName(parentPath string, caseCor } else { if len(directory.CaseCorrectName) > 0 { fs.Stats.Error() - fs.Log(tree, "PutCaseCorrectDirectoryName: directory %q is already exists under parent path %q", caseCorrectDirectoryName, parentPath) + fs.ErrorLog(tree, "PutCaseCorrectDirectoryName: directory %q is already exists under parent path %q", caseCorrectDirectoryName, parentPath) return } @@ -105,7 +105,7 @@ func (tree *NameTreeNode) PutFile(parentPath string, caseCorrectFileName string, if node.Files[caseCorrectFileName] != nil { fs.Stats.Error() - fs.Log(tree, "PutFile: file %q is already exists at %q", caseCorrectFileName, parentPath) + fs.ErrorLog(tree, "PutFile: file %q is already exists at %q", caseCorrectFileName, parentPath) return } @@ -124,7 +124,7 @@ func (tree *NameTreeNode) GetPathWithCorrectCase(path string) *string { for _, component := range strings.Split(path, "/") { if component == "" { fs.Stats.Error() - fs.Log(tree, "GetPathWithCorrectCase: path component is empty (full path %q)", path) + fs.ErrorLog(tree, "GetPathWithCorrectCase: path component is empty (full path %q)", path) return nil } @@ -161,7 +161,7 @@ func (tree *NameTreeNode) walkFilesRec(currentPath string, walkFunc NameTreeFile caseCorrectName := directory.CaseCorrectName if caseCorrectName == "" { fs.Stats.Error() - fs.Log(tree, "WalkFiles: exact name of the directory %q is unknown (parent path: %q)", lowerCaseName, currentPath) + fs.ErrorLog(tree, "WalkFiles: exact name of the directory %q is unknown (parent path: %q)", lowerCaseName, currentPath) continue } diff --git a/fs/fs.go b/fs/fs.go index ca6318609..2180a06f7 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -264,6 +264,12 @@ func Log(o interface{}, text string, args ...interface{}) { } } +// Write error log output for this Object or Fs +// Unconditionally logs a message regardless of Config.Quiet or Config.Verbose +func ErrorLog(o interface{}, text string, args ...interface{}) { + OutputLog(o, text, args...) +} + // checkClose is a utility function used to check the return from // Close in a defer statement. func checkClose(c io.Closer, err *error) { diff --git a/fs/operations.go b/fs/operations.go index efc7e3ff4..dd697b7bb 100644 --- a/fs/operations.go +++ b/fs/operations.go @@ -36,13 +36,13 @@ func CheckMd5sums(src, dst Object) (bool, error) { srcMd5, err := src.Md5sum() if err != nil { Stats.Error() - Log(src, "Failed to calculate src md5: %s", err) + ErrorLog(src, "Failed to calculate src md5: %s", err) return false, err } dstMd5, err := dst.Md5sum() if err != nil { Stats.Error() - Log(dst, "Failed to calculate dst md5: %s", err) + ErrorLog(dst, "Failed to calculate dst md5: %s", err) return false, err } // Debug("Src MD5 %s", srcMd5) @@ -148,7 +148,7 @@ tryAgain: in0, err := src.Open() if err != nil { Stats.Error() - Log(src, "Failed to open: %s", err) + ErrorLog(src, "Failed to open: %s", err) return } in := NewAccount(in0) // account the transfer @@ -178,7 +178,7 @@ tryAgain: } if err != nil { Stats.Error() - Log(src, "Failed to copy: %s", err) + ErrorLog(src, "Failed to copy: %s", err) removeFailedCopy(dst) return } @@ -187,7 +187,7 @@ tryAgain: if src.Size() != dst.Size() { Stats.Error() err = fmt.Errorf("Corrupted on transfer: sizes differ %d vs %d", src.Size(), dst.Size()) - Log(dst, "%s", err) + ErrorLog(dst, "%s", err) removeFailedCopy(dst) return } @@ -197,16 +197,16 @@ tryAgain: srcMd5sum, md5sumErr := src.Md5sum() if md5sumErr != nil { Stats.Error() - Log(src, "Failed to read md5sum: %s", md5sumErr) + ErrorLog(src, "Failed to read md5sum: %s", md5sumErr) } else if srcMd5sum != "" { dstMd5sum, md5sumErr := dst.Md5sum() if md5sumErr != nil { Stats.Error() - Log(dst, "Failed to read md5sum: %s", md5sumErr) + ErrorLog(dst, "Failed to read md5sum: %s", md5sumErr) } else if dstMd5sum != "" && srcMd5sum != dstMd5sum { Stats.Error() err = fmt.Errorf("Corrupted on transfer: md5sums differ %q vs %q", srcMd5sum, dstMd5sum) - Log(dst, "%s", err) + ErrorLog(dst, "%s", err) removeFailedCopy(dst) return } @@ -280,7 +280,7 @@ func DeleteFiles(to_be_deleted ObjectsChan) { Stats.DoneChecking(dst) if err != nil { Stats.Error() - Log(dst, "Couldn't delete: %s", err) + ErrorLog(dst, "Couldn't delete: %s", err) } else { Debug(dst, "Deleted") } @@ -405,13 +405,13 @@ func Check(fdst, fsrc Fs) error { Log(fdst, "%d files not in %v", len(dstFiles), fsrc) for _, dst := range dstFiles { Stats.Error() - Log(dst, "File not in %v", fsrc) + ErrorLog(dst, "File not in %v", fsrc) } Log(fsrc, "%d files not in %s", len(srcFiles), fdst) for _, src := range srcFiles { Stats.Error() - Log(src, "File not in %v", fdst) + ErrorLog(src, "File not in %v", fdst) } checks := make(chan []Object, Config.Transfers) @@ -433,7 +433,7 @@ func Check(fdst, fsrc Fs) error { if src.Size() != dst.Size() { Stats.DoneChecking(src) Stats.Error() - Log(src, "Sizes differ") + ErrorLog(src, "Sizes differ") continue } same, err := CheckMd5sums(src, dst) @@ -443,7 +443,7 @@ func Check(fdst, fsrc Fs) error { } if !same { Stats.Error() - Log(src, "Md5sums differ") + ErrorLog(src, "Md5sums differ") } Debug(src, "OK") } diff --git a/googlecloudstorage/googlecloudstorage.go b/googlecloudstorage/googlecloudstorage.go index 1b06b5a93..a8ce8aef8 100644 --- a/googlecloudstorage/googlecloudstorage.go +++ b/googlecloudstorage/googlecloudstorage.go @@ -251,7 +251,7 @@ func (f *FsStorage) list(directories bool, fn func(string, *storage.Object)) { objects, err := list.Do() if err != nil { fs.Stats.Error() - fs.Log(f, "Couldn't read bucket %q: %s", f.bucket, err) + fs.ErrorLog(f, "Couldn't read bucket %q: %s", f.bucket, err) return } if !directories { @@ -286,7 +286,7 @@ func (f *FsStorage) List() fs.ObjectsChan { // Return no objects at top level list close(out) fs.Stats.Error() - fs.Log(f, "Can't list objects at root - choose a bucket using lsd") + fs.ErrorLog(f, "Can't list objects at root - choose a bucket using lsd") } else { // List the objects go func() { @@ -310,7 +310,7 @@ func (f *FsStorage) ListDir() fs.DirChan { defer close(out) if f.projectNumber == "" { fs.Stats.Error() - fs.Log(f, "Can't list buckets without project number") + fs.ErrorLog(f, "Can't list buckets without project number") return } listBuckets := f.svc.Buckets.List(f.projectNumber).MaxResults(listChunks) @@ -318,7 +318,7 @@ func (f *FsStorage) ListDir() fs.DirChan { buckets, err := listBuckets.Do() if err != nil { fs.Stats.Error() - fs.Log(f, "Couldn't list buckets: %v", err) + fs.ErrorLog(f, "Couldn't list buckets: %v", err) break } else { for _, bucket := range buckets.Items { @@ -505,7 +505,7 @@ func (o *FsObjectStorage) SetModTime(modTime time.Time) { newObject, err := o.storage.svc.Objects.Patch(o.storage.bucket, o.storage.root+o.remote, &object).Do() if err != nil { fs.Stats.Error() - fs.Log(o, "Failed to update remote mtime: %s", err) + fs.ErrorLog(o, "Failed to update remote mtime: %s", err) } o.setMetaData(newObject) } diff --git a/local/local.go b/local/local.go index d6909e4a2..50cd7e08f 100644 --- a/local/local.go +++ b/local/local.go @@ -110,12 +110,12 @@ func (f *FsLocal) List() fs.ObjectsChan { err := filepath.Walk(f.root, func(path string, fi os.FileInfo, err error) error { if err != nil { fs.Stats.Error() - fs.Log(f, "Failed to open directory: %s: %s", path, err) + fs.ErrorLog(f, "Failed to open directory: %s: %s", path, err) } else { remote, err := filepath.Rel(f.root, path) if err != nil { fs.Stats.Error() - fs.Log(f, "Failed to get relative path %s: %s", path, err) + fs.ErrorLog(f, "Failed to get relative path %s: %s", path, err) return nil } if remote == "." { @@ -132,7 +132,7 @@ func (f *FsLocal) List() fs.ObjectsChan { }) if err != nil { fs.Stats.Error() - fs.Log(f, "Failed to open directory: %s: %s", f.root, err) + fs.ErrorLog(f, "Failed to open directory: %s: %s", f.root, err) } close(out) }() @@ -161,7 +161,7 @@ func (f *FsLocal) ListDir() fs.DirChan { items, err := ioutil.ReadDir(f.root) if err != nil { fs.Stats.Error() - fs.Log(f, "Couldn't find read directory: %s", err) + fs.ErrorLog(f, "Couldn't find read directory: %s", err) } else { for _, item := range items { if item.IsDir() { @@ -176,7 +176,7 @@ func (f *FsLocal) ListDir() fs.DirChan { err := filepath.Walk(dirpath, func(path string, fi os.FileInfo, err error) error { if err != nil { fs.Stats.Error() - fs.Log(f, "Failed to open directory: %s: %s", path, err) + fs.ErrorLog(f, "Failed to open directory: %s: %s", path, err) } else { dir.Count += 1 dir.Bytes += fi.Size() @@ -185,7 +185,7 @@ func (f *FsLocal) ListDir() fs.DirChan { }) if err != nil { fs.Stats.Error() - fs.Log(f, "Failed to open directory: %s: %s", dirpath, err) + fs.ErrorLog(f, "Failed to open directory: %s: %s", dirpath, err) } out <- dir } @@ -324,7 +324,7 @@ func (o *FsObjectLocal) Md5sum() (string, error) { in, err := os.Open(o.path) if err != nil { fs.Stats.Error() - fs.Log(o, "Failed to open: %s", err) + fs.ErrorLog(o, "Failed to open: %s", err) return "", err } hash := md5.New() @@ -332,12 +332,12 @@ func (o *FsObjectLocal) Md5sum() (string, error) { closeErr := in.Close() if err != nil { fs.Stats.Error() - fs.Log(o, "Failed to read: %s", err) + fs.ErrorLog(o, "Failed to read: %s", err) return "", err } if closeErr != nil { fs.Stats.Error() - fs.Log(o, "Failed to close: %s", closeErr) + fs.ErrorLog(o, "Failed to close: %s", closeErr) return "", closeErr } o.md5sum = hex.EncodeToString(hash.Sum(nil)) diff --git a/rclone.go b/rclone.go index 2822bcb02..35756e5da 100644 --- a/rclone.go +++ b/rclone.go @@ -377,7 +377,7 @@ func main() { // Run the actual command if command.Run != nil { command.Run(fdst, fsrc) - if !command.NoStats { + if !command.NoStats && (!fs.Config.Quiet || fs.Stats.Errored() || *statsInterval > 0) { fmt.Fprintln(os.Stderr, fs.Stats) } if fs.Config.Verbose { diff --git a/s3/s3.go b/s3/s3.go index d5fb1cc08..f69142354 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -273,7 +273,7 @@ func (f *FsS3) list(directories bool, fn func(string, *s3.Key)) { objects, err := f.b.List(f.root, delimiter, marker, listChunkSize) if err != nil { fs.Stats.Error() - fs.Log(f, "Couldn't read bucket %q: %s", f.bucket, err) + fs.ErrorLog(f, "Couldn't read bucket %q: %s", f.bucket, err) } else { rootLength := len(f.root) if directories { @@ -318,7 +318,7 @@ func (f *FsS3) List() fs.ObjectsChan { // Return no objects at top level list close(out) fs.Stats.Error() - fs.Log(f, "Can't list objects at root - choose a bucket using lsd") + fs.ErrorLog(f, "Can't list objects at root - choose a bucket using lsd") } else { go func() { defer close(out) @@ -342,7 +342,7 @@ func (f *FsS3) ListDir() fs.DirChan { buckets, err := f.c.ListBuckets() if err != nil { fs.Stats.Error() - fs.Log(f, "Couldn't list buckets: %s", err) + fs.ErrorLog(f, "Couldn't list buckets: %s", err) } else { for _, bucket := range buckets { out <- &fs.Dir{ @@ -515,14 +515,14 @@ func (o *FsObjectS3) SetModTime(modTime time.Time) { err := o.readMetaData() if err != nil { fs.Stats.Error() - fs.Log(o, "Failed to read metadata: %s", err) + fs.ErrorLog(o, "Failed to read metadata: %s", err) return } o.meta[metaMtime] = swift.TimeToFloatString(modTime) _, err = o.s3.b.Update(o.s3.root+o.remote, o.s3.perm, o.meta) if err != nil { fs.Stats.Error() - fs.Log(o, "Failed to update remote mtime: %s", err) + fs.ErrorLog(o, "Failed to update remote mtime: %s", err) } } diff --git a/swift/swift.go b/swift/swift.go index d299bcb23..b9faeb0e3 100644 --- a/swift/swift.go +++ b/swift/swift.go @@ -230,7 +230,7 @@ func (f *FsSwift) list(directories bool, fn func(string, *swift.Object)) { }) if err != nil { fs.Stats.Error() - fs.Log(f, "Couldn't read container %q: %s", f.container, err) + fs.ErrorLog(f, "Couldn't read container %q: %s", f.container, err) } } @@ -241,7 +241,7 @@ func (f *FsSwift) List() fs.ObjectsChan { // Return no objects at top level list close(out) fs.Stats.Error() - fs.Log(f, "Can't list objects at root - choose a container using lsd") + fs.ErrorLog(f, "Can't list objects at root - choose a container using lsd") } else { // List the objects go func() { @@ -266,7 +266,7 @@ func (f *FsSwift) ListDir() fs.DirChan { containers, err := f.c.ContainersAll(nil) if err != nil { fs.Stats.Error() - fs.Log(f, "Couldn't list containers: %v", err) + fs.ErrorLog(f, "Couldn't list containers: %v", err) } else { for _, container := range containers { out <- &fs.Dir{ @@ -393,14 +393,14 @@ func (o *FsObjectSwift) SetModTime(modTime time.Time) { err := o.readMetaData() if err != nil { fs.Stats.Error() - fs.Log(o, "Failed to read metadata: %s", err) + fs.ErrorLog(o, "Failed to read metadata: %s", err) return } o.meta.SetModTime(modTime) err = o.swift.c.ObjectUpdate(o.swift.container, o.swift.root+o.remote, o.meta.ObjectHeaders()) if err != nil { fs.Stats.Error() - fs.Log(o, "Failed to update remote mtime: %s", err) + fs.ErrorLog(o, "Failed to update remote mtime: %s", err) } }