ncdu: fix issue where dir size is summed when file sizes are -1

Some backends may not provide size for all objects, and instead
return -1. Existing version included these in directory sums,
with strange results. With this commit rclone ncdu will consider
negative sizes as zero, but add a new prefix flag '~' with a
description that indicates the shown size is inaccurate.

Fixes #6084

cmd/ncdu/ncdu.go

@@ -283,7 +283,7 @@ func (u *UI) biggestEntry() (biggest int64) {
 		return
 	}
 	for i := range u.entries {
-		size, _, _, _, _, _ := u.d.AttrI(u.sortPerm[i])
+		size, _, _, _, _, _, _ := u.d.AttrI(u.sortPerm[i])
 		if size > biggest {
 			biggest = size
 		}
@@ -297,7 +297,7 @@ func (u *UI) hasEmptyDir() bool {
 		return false
 	}
 	for i := range u.entries {
-		_, count, isDir, _, _, _ := u.d.AttrI(u.sortPerm[i])
+		_, count, _, isDir, _, _, _ := u.d.AttrI(u.sortPerm[i])
 		if isDir && count == 0 {
 			return true
 		}
@@ -343,7 +343,7 @@ func (u *UI) Draw() error {
 			if y >= h-1 {
 				break
 			}
-			size, count, isDir, readable, entriesHaveErrors, err := u.d.AttrI(u.sortPerm[n])
+			size, count, countUnknownSize, isDir, readable, entriesHaveErrors, err := u.d.AttrI(u.sortPerm[n])
 			fg := termbox.ColorWhite
 			if entriesHaveErrors {
 				fg = termbox.ColorYellow
@@ -364,6 +364,10 @@ func (u *UI) Draw() error {
 			if !readable {
 				message = " [not read yet]"
 			}
+			if countUnknownSize > 0 {
+				message = fmt.Sprintf(" [%d of %d files have unknown size, size may be underestimated]", countUnknownSize, count)
+				fileFlag = '~'
+			}
 			if entriesHaveErrors {
 				message = " [some subdirectories could not be read, size may be underestimated]"
 				fileFlag = '.'
@@ -383,7 +387,10 @@ func (u *UI) Draw() error {
 			}
 			var averageSize float64
 			if count > 0 {
-				averageSize = float64(size) / float64(count)
+				countForAverage := count - countUnknownSize
+				if countForAverage > 0 {
+					averageSize = float64(size) / float64(countForAverage)
+				}
 			}
 			if u.showDirAverageSize {
 				ss := operations.SizeStringField(int64(averageSize), u.humanReadable, 9) + " "
@@ -559,8 +566,8 @@ type ncduSort struct {
 // Less is part of sort.Interface.
 func (ds *ncduSort) Less(i, j int) bool {
 	var iAvgSize, jAvgSize float64
-	isize, icount, _, _, _, _ := ds.d.AttrI(ds.sortPerm[i])
+	isize, icount, _, _, _, _, _ := ds.d.AttrI(ds.sortPerm[i])
-	jsize, jcount, _, _, _, _ := ds.d.AttrI(ds.sortPerm[j])
+	jsize, jcount, _, _, _, _, _ := ds.d.AttrI(ds.sortPerm[j])
 	iname, jname := ds.entries[ds.sortPerm[i]].Remote(), ds.entries[ds.sortPerm[j]].Remote()
 	if icount > 0 {
 		iAvgSize = float64(isize / icount)

cmd/ncdu/scan/scan.go

@@ -16,8 +16,9 @@ type Dir struct {
 	parent            *Dir
 	path              string
 	mu                sync.Mutex
-	count             int64
 	size              int64
+	count             int64
+	countUnknownSize  int64
 	entries           fs.DirEntries
 	dirs              map[string]*Dir
 	readError         error
@@ -49,7 +50,13 @@ func newDir(parent *Dir, dirPath string, entries fs.DirEntries, err error) *Dir
 	for _, entry := range entries {
 		if o, ok := entry.(fs.Object); ok {
 			d.count++
-			d.size += o.Size()
+			size := o.Size()
+			if size < 0 {
+				// Some backends may return -1 because size of object is not known
+				d.countUnknownSize++
+			} else {
+				d.size += size
+			}
 		}
 	}
 	// Set my directory entry in parent
@@ -62,8 +69,9 @@ func newDir(parent *Dir, dirPath string, entries fs.DirEntries, err error) *Dir
 	// Accumulate counts in parents
 	for ; parent != nil; parent = parent.parent {
 		parent.mu.Lock()
-		parent.count += d.count
 		parent.size += d.size
+		parent.count += d.count
+		parent.countUnknownSize += d.countUnknownSize
 		if d.readError != nil {
 			parent.entriesHaveErrors = true
 		}
@@ -91,17 +99,24 @@ func (d *Dir) Remove(i int) {
 // Call with d.mu held
 func (d *Dir) remove(i int) {
 	size := d.entries[i].Size()
+	countUnknownSize := int64(0)
+	if size < 0 {
+		size = 0
+		countUnknownSize = 1
+	}
 	count := int64(1)
 
 	subDir, ok := d.getDir(i)
 	if ok {
 		size = subDir.size
 		count = subDir.count
+		countUnknownSize = subDir.countUnknownSize
 		delete(d.dirs, path.Base(subDir.path))
 	}
 
 	d.size -= size
 	d.count -= count
+	d.countUnknownSize -= countUnknownSize
 	d.entries = append(d.entries[:i], d.entries[i+1:]...)
 
 	dir := d
@@ -111,6 +126,7 @@ func (d *Dir) remove(i int) {
 		parent.dirs[path.Base(dir.path)] = dir
 		parent.size -= size
 		parent.count -= count
+		parent.countUnknownSize -= countUnknownSize
 		dir = parent
 		parent.mu.Unlock()
 	}
@@ -151,19 +167,19 @@ func (d *Dir) Attr() (size int64, count int64) {
 }
 
 // AttrI returns the size, count and flags for the i-th directory entry
-func (d *Dir) AttrI(i int) (size int64, count int64, isDir bool, readable bool, entriesHaveErrors bool, err error) {
+func (d *Dir) AttrI(i int) (size int64, count int64, countUnknownSize int64, isDir bool, readable bool, entriesHaveErrors bool, err error) {
 	d.mu.Lock()
 	defer d.mu.Unlock()
 	subDir, isDir := d.getDir(i)
 
 	if !isDir {
-		return d.entries[i].Size(), 0, false, true, d.entriesHaveErrors, d.readError
+		return d.entries[i].Size(), 0, 0, false, true, d.entriesHaveErrors, d.readError
 	}
 	if subDir == nil {
-		return 0, 0, true, false, false, nil
+		return 0, 0, 0, true, false, false, nil
 	}
 	size, count = subDir.Attr()
-	return size, count, true, true, subDir.entriesHaveErrors, subDir.readError
+	return size, count, subDir.countUnknownSize, true, true, subDir.entriesHaveErrors, subDir.readError
 }
 
 // Scan the Fs passed in, returning a root directory channel and an