hashsum: Add flag --base64 flag - fixes #3663

This flag can be used to output QuickXorHash in the same format as MS
Graph API.

cmd/hashsum/hashsum.go

@@ -7,13 +7,20 @@ import (
 	"os"
 
 	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/spf13/cobra"
 )
 
+var (
+	outputBase64 = false
+)
+
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
+	cmdFlags := commandDefinition.Flags()
+	flags.BoolVarP(cmdFlags, &outputBase64, "base64", "", outputBase64, "Output base64 encoded hashsum")
 }
 
 var commandDefinition = &cobra.Command{
@@ -55,6 +62,9 @@ Then
 		}
 		fsrc := cmd.NewFsSrc(args[1:])
 		cmd.Run(false, false, command, func() error {
+			if outputBase64 {
+				return operations.HashListerBase64(context.Background(), ht, fsrc, os.Stdout)
+			}
 			return operations.HashLister(context.Background(), ht, fsrc, os.Stdout)
 		})
 		return nil

fs/operations/operations.go

@@ -4,7 +4,9 @@ package operations
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"encoding/csv"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -1042,8 +1044,9 @@ func Sha1sum(ctx context.Context, f fs.Fs, w io.Writer) error {
 }
 
 // hashSum returns the human readable hash for ht passed in.  This may
-// be UNSUPPORTED or ERROR.
+// be UNSUPPORTED or ERROR. If it isn't returning a valid hash it will
-func hashSum(ctx context.Context, ht hash.Type, o fs.Object) string {
+// return an error.
+func hashSum(ctx context.Context, ht hash.Type, o fs.Object) (string, error) {
 	var err error
 	tr := accounting.Stats(ctx).NewCheckingTransfer(o)
 	defer func() {
@@ -1056,17 +1059,30 @@ func hashSum(ctx context.Context, ht hash.Type, o fs.Object) string {
 		fs.Debugf(o, "Failed to read %v: %v", ht, err)
 		sum = "ERROR"
 	}
-	return sum
+	return sum, err
 }
 
 // HashLister does a md5sum equivalent for the hash type passed in
 func HashLister(ctx context.Context, ht hash.Type, f fs.Fs, w io.Writer) error {
 	return ListFn(ctx, f, func(o fs.Object) {
-		sum := hashSum(ctx, ht, o)
+		sum, _ := hashSum(ctx, ht, o)
 		syncFprintf(w, "%*s  %s\n", hash.Width(ht), sum, o.Remote())
 	})
 }
 
+// HashListerBase64 does a md5sum equivalent for the hash type passed in with base64 encoded
+func HashListerBase64(ctx context.Context, ht hash.Type, f fs.Fs, w io.Writer) error {
+	return ListFn(ctx, f, func(o fs.Object) {
+		sum, err := hashSum(ctx, ht, o)
+		if err == nil {
+			hexBytes, _ := hex.DecodeString(sum)
+			sum = base64.URLEncoding.EncodeToString(hexBytes)
+		}
+		width := base64.URLEncoding.EncodedLen(hash.Width(ht) / 2)
+		syncFprintf(w, "%*s  %s\n", width, sum, o.Remote())
+	})
+}
+
 // Count counts the objects and their sizes in the Fs
 //
 // Obeys includes and excludes

fs/operations/operations_test.go

@@ -225,6 +225,43 @@ func TestHashSums(t *testing.T) {
 		!strings.Contains(res, "                                          potato2\n") {
 		t.Errorf("potato2 missing: %q", res)
 	}
+
+	// QuickXorHash Sum
+
+	buf.Reset()
+	var ht hash.Type
+	err = ht.Set("QuickXorHash")
+	require.NoError(t, err)
+	err = operations.HashLister(context.Background(), ht, r.Fremote, &buf)
+	require.NoError(t, err)
+	res = buf.String()
+	if !strings.Contains(res, "2d00000000000000000000000100000000000000  empty space\n") &&
+		!strings.Contains(res, "                             UNSUPPORTED  empty space\n") &&
+		!strings.Contains(res, "                                          empty space\n") {
+		t.Errorf("empty space missing: %q", res)
+	}
+	if !strings.Contains(res, "4001dad296b6b4a52d6d694b67dad296b6b4a52d  potato2\n") &&
+		!strings.Contains(res, "                             UNSUPPORTED  potato2\n") &&
+		!strings.Contains(res, "                                          potato2\n") {
+		t.Errorf("potato2 missing: %q", res)
+	}
+
+	// QuickXorHash Sum with Base64 Encoded
+
+	buf.Reset()
+	err = operations.HashListerBase64(context.Background(), ht, r.Fremote, &buf)
+	require.NoError(t, err)
+	res = buf.String()
+	if !strings.Contains(res, "LQAAAAAAAAAAAAAAAQAAAAAAAAA=  empty space\n") &&
+		!strings.Contains(res, "                 UNSUPPORTED  empty space\n") &&
+		!strings.Contains(res, "                              empty space\n") {
+		t.Errorf("empty space missing: %q", res)
+	}
+	if !strings.Contains(res, "QAHa0pa2tKUtbWlLZ9rSlra0pS0=  potato2\n") &&
+		!strings.Contains(res, "                 UNSUPPORTED  potato2\n") &&
+		!strings.Contains(res, "                              potato2\n") {
+		t.Errorf("potato2 missing: %q", res)
+	}
 }
 
 func TestSuffixName(t *testing.T) {