From 86bd5f69228e896ccabb14a2898e9acaf5fdfdbb Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Wed, 6 Apr 2022 14:15:07 +0200
Subject: [PATCH] size: warn about inaccurate results when objects with unknown
 size

---
 cmd/size/size.go                 | 14 ++++++++++----
 cmd/test/memory/memory.go        |  2 +-
 fs/operations/dedupe_test.go     |  2 +-
 fs/operations/operations.go      |  6 ++++--
 fs/operations/operations_test.go |  3 ++-
 fs/operations/rc.go              |  3 ++-
 fs/operations/rc_test.go         |  5 +++--
 7 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/cmd/size/size.go b/cmd/size/size.go
index be4ef2d5e..1d9cf625b 100644
--- a/cmd/size/size.go
+++ b/cmd/size/size.go
@@ -30,20 +30,26 @@ var commandDefinition = &cobra.Command{
 		cmd.Run(false, false, command, func() error {
 			var err error
 			var results struct {
-				Count int64 `json:"count"`
-				Bytes int64 `json:"bytes"`
+				Count    int64 `json:"count"`
+				Bytes    int64 `json:"bytes"`
+				Sizeless int64 `json:"sizeless"`
 			}
 
-			results.Count, results.Bytes, err = operations.Count(context.Background(), fsrc)
+			results.Count, results.Bytes, results.Sizeless, err = operations.Count(context.Background(), fsrc)
 			if err != nil {
 				return err
 			}
-
+			if results.Sizeless > 0 {
+				fs.Logf(fsrc, "Size may be underestimated due to %d objects with unknown size", results.Sizeless)
+			}
 			if jsonOutput {
 				return json.NewEncoder(os.Stdout).Encode(results)
 			}
 			fmt.Printf("Total objects: %s (%d)\n", fs.CountSuffix(results.Count), results.Count)
 			fmt.Printf("Total size: %s (%d Byte)\n", fs.SizeSuffix(results.Bytes).ByteUnit(), results.Bytes)
+			if results.Sizeless > 0 {
+				fmt.Printf("Total objects with unknown size: %s (%d)\n", fs.CountSuffix(results.Sizeless), results.Sizeless)
+			}
 			return nil
 		})
 	},
diff --git a/cmd/test/memory/memory.go b/cmd/test/memory/memory.go
index 823cade14..551723258 100644
--- a/cmd/test/memory/memory.go
+++ b/cmd/test/memory/memory.go
@@ -25,7 +25,7 @@ var commandDefinition = &cobra.Command{
 		cmd.Run(false, false, command, func() error {
 			ctx := context.Background()
 			ci := fs.GetConfig(context.Background())
-			objects, _, err := operations.Count(ctx, fsrc)
+			objects, _, _, err := operations.Count(ctx, fsrc)
 			if err != nil {
 				return err
 			}
diff --git a/fs/operations/dedupe_test.go b/fs/operations/dedupe_test.go
index a4391ee54..60c960510 100644
--- a/fs/operations/dedupe_test.go
+++ b/fs/operations/dedupe_test.go
@@ -121,7 +121,7 @@ func TestDeduplicateFirst(t *testing.T) {
 	// list until we get one object
 	var objects, size int64
 	for try := 1; try <= *fstest.ListRetries; try++ {
-		objects, size, err = operations.Count(context.Background(), r.Fremote)
+		objects, size, _, err = operations.Count(context.Background(), r.Fremote)
 		require.NoError(t, err)
 		if objects == 1 {
 			break
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 6066eda29..2807657e4 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -1070,11 +1070,13 @@ func HashSumStream(ht hash.Type, outputBase64 bool, in io.ReadCloser, w io.Write
 // Count counts the objects and their sizes in the Fs
 //
 // Obeys includes and excludes
-func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, err error) {
+func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, sizelessObjects int64, err error) {
 	err = ListFn(ctx, f, func(o fs.Object) {
 		atomic.AddInt64(&objects, 1)
 		objectSize := o.Size()
-		if objectSize > 0 {
+		if objectSize < 0 {
+			atomic.AddInt64(&sizelessObjects, 1)
+		} else if objectSize > 0 {
 			atomic.AddInt64(&size, objectSize)
 		}
 	})
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 55e91850d..309c31efb 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -401,10 +401,11 @@ func TestCount(t *testing.T) {
 	// Check the MaxDepth too
 	ci.MaxDepth = 1
 
-	objects, size, err := operations.Count(ctx, r.Fremote)
+	objects, size, sizeless, err := operations.Count(ctx, r.Fremote)
 	require.NoError(t, err)
 	assert.Equal(t, int64(2), objects)
 	assert.Equal(t, int64(61), size)
+	assert.Equal(t, int64(0), sizeless)
 }
 
 func TestDelete(t *testing.T) {
diff --git a/fs/operations/rc.go b/fs/operations/rc.go
index fabb3c7f2..de77dae09 100644
--- a/fs/operations/rc.go
+++ b/fs/operations/rc.go
@@ -344,13 +344,14 @@ func rcSize(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 	if err != nil {
 		return nil, err
 	}
-	count, bytes, err := Count(ctx, f)
+	count, bytes, sizeless, err := Count(ctx, f)
 	if err != nil {
 		return nil, err
 	}
 	out = make(rc.Params)
 	out["count"] = count
 	out["bytes"] = bytes
+	out["sizeless"] = sizeless
 	return out, nil
 }
 
diff --git a/fs/operations/rc_test.go b/fs/operations/rc_test.go
index 4bbc96ffe..8b4cf1bb6 100644
--- a/fs/operations/rc_test.go
+++ b/fs/operations/rc_test.go
@@ -451,8 +451,9 @@ func TestRcSize(t *testing.T) {
 	out, err := call.Fn(context.Background(), in)
 	require.NoError(t, err)
 	assert.Equal(t, rc.Params{
-		"count": int64(3),
-		"bytes": int64(120),
+		"count":    int64(3),
+		"bytes":    int64(120),
+		"sizeless": int64(0),
 	}, out)
 }