extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-02-02 03:29:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

size: warn about inaccurate results when objects with unknown size

Browse Source
This commit is contained in:
albertony 2022-04-06 14:15:07 +02:00
parent fe271a4e35
commit 86bd5f6922
7 changed files with 23 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/size/size.go
View File

@ -30,20 +30,26 @@ var commandDefinition = &cobra.Command{
cmd.Run(false, false, command, func() error {
var err error
var results struct {
Count int64 `json:"count"`
Bytes int64 `json:"bytes"`
Count int64 `json:"count"`
Bytes int64 `json:"bytes"`
Sizeless int64 `json:"sizeless"`
}
results.Count, results.Bytes, err = operations.Count(context.Background(), fsrc)
results.Count, results.Bytes, results.Sizeless, err = operations.Count(context.Background(), fsrc)
if err != nil {
return err
}
if results.Sizeless > 0 {
fs.Logf(fsrc, "Size may be underestimated due to %d objects with unknown size", results.Sizeless)
}
if jsonOutput {
return json.NewEncoder(os.Stdout).Encode(results)
}
fmt.Printf("Total objects: %s (%d)\n", fs.CountSuffix(results.Count), results.Count)
fmt.Printf("Total size: %s (%d Byte)\n", fs.SizeSuffix(results.Bytes).ByteUnit(), results.Bytes)
if results.Sizeless > 0 {
fmt.Printf("Total objects with unknown size: %s (%d)\n", fs.CountSuffix(results.Sizeless), results.Sizeless)
}
return nil
})
},

2
cmd/test/memory/memory.go
View File

@ -25,7 +25,7 @@ var commandDefinition = &cobra.Command{
cmd.Run(false, false, command, func() error {
ctx := context.Background()
ci := fs.GetConfig(context.Background())
objects, _, err := operations.Count(ctx, fsrc)
objects, _, _, err := operations.Count(ctx, fsrc)
if err != nil {
return err
}

2
fs/operations/dedupe_test.go
View File

@ -121,7 +121,7 @@ func TestDeduplicateFirst(t *testing.T) {
// list until we get one object
var objects, size int64
for try := 1; try <= *fstest.ListRetries; try++ {
objects, size, err = operations.Count(context.Background(), r.Fremote)
objects, size, _, err = operations.Count(context.Background(), r.Fremote)
require.NoError(t, err)
if objects == 1 {
break

6
fs/operations/operations.go
View File

@ -1070,11 +1070,13 @@ func HashSumStream(ht hash.Type, outputBase64 bool, in io.ReadCloser, w io.Write
// Count counts the objects and their sizes in the Fs
//
// Obeys includes and excludes
func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, err error) {
func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, sizelessObjects int64, err error) {
err = ListFn(ctx, f, func(o fs.Object) {
atomic.AddInt64(&objects, 1)
objectSize := o.Size()
if objectSize > 0 {
if objectSize < 0 {
atomic.AddInt64(&sizelessObjects, 1)
} else if objectSize > 0 {
atomic.AddInt64(&size, objectSize)
}
})

3
fs/operations/operations_test.go
View File

@ -401,10 +401,11 @@ func TestCount(t *testing.T) {
// Check the MaxDepth too
ci.MaxDepth = 1
objects, size, err := operations.Count(ctx, r.Fremote)
objects, size, sizeless, err := operations.Count(ctx, r.Fremote)
require.NoError(t, err)
assert.Equal(t, int64(2), objects)
assert.Equal(t, int64(61), size)
assert.Equal(t, int64(0), sizeless)
}
func TestDelete(t *testing.T) {

3
fs/operations/rc.go
View File

@ -344,13 +344,14 @@ func rcSize(ctx context.Context, in rc.Params) (out rc.Params, err error) {
if err != nil {
return nil, err
}
count, bytes, err := Count(ctx, f)
count, bytes, sizeless, err := Count(ctx, f)
if err != nil {
return nil, err
}
out = make(rc.Params)
out["count"] = count
out["bytes"] = bytes
out["sizeless"] = sizeless
return out, nil
}

5
fs/operations/rc_test.go
View File

@ -451,8 +451,9 @@ func TestRcSize(t *testing.T) {
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params{
"count": int64(3),
"bytes": int64(120),
"count": int64(3),
"bytes": int64(120),
"sizeless": int64(0),
}, out)
}