selfupdate: fix "invalid hashsum signature" error

This was caused by a change to the upstream library
ProtonMail/go-crypto checking the flags on the keys more strictly.

However the signing key for rclone is very old and does not have those
flags. Adding those flags using `gpg --edit-key` and then the
`change-usage` subcommand to remove, save, quite then re-add, save
quit the signing capabilities caused the key to work.

This also adds tests for the verification and adds the selfupdate
tests into the integration test harness as they had been disabled on
CI because they rely on external sources and are sometimes unreliable.

Fixes #7373

cmd/selfupdate/selfupdate_test.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	_ "github.com/rclone/rclone/fstest" // needed to run under integration tests
 	"github.com/rclone/rclone/fstest/testy"
 	"github.com/stretchr/testify/assert"
 )

cmd/selfupdate/testdata/verify/SHA256SUMS

@@ -0,0 +1,10 @@
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+
+b20b47f579a2c790ca752fb5d8e5651fade7d5867cbac0a4f71e805fc5c468d0  archive.zip
+-----BEGIN PGP SIGNATURE-----
+
+iF0EARECAB0WIQT79zfs6firGGBL0qyTk14C/ztU+gUCZS+oVQAKCRCTk14C/ztU
++lNsAJ9XRiODlM4fIW9yqiltO3N+lLeucwCfRzD3cXk6BCB5wdz7pTgnItk9N74=
+=1GTr
+-----END PGP SIGNATURE-----

cmd/selfupdate/verify.go

@@ -26,24 +26,37 @@ QbogRGodbKhqY4v+cMNkKiemBuTQiWPkpKjifwNsD1fNjNKfDP3pJ64Yz7a4fuzV
 X1YwBACpKVuEen34lmcX6ziY4jq8rKibKBs4JjQCRO24kYoHDULVe+RS9krQWY5b
 e0foDhru4dsKccefK099G+WEzKVCKxupstWkTT/iJwajR8mIqd4AhD0wO9W3MCfV
 Ov8ykMDZ7qBWk1DHc87Ep3W1o8t8wq74ifV+HjhhWg8QAylXg7QlTmljayBDcmFp
-Zy1Xb29kIDxuaWNrQGNyYWlnLXdvb2QuY29tPohxBBMRCAAxBQsHCgMEAxUDAgMW
+Zy1Xb29kIDxuaWNrQGNyYWlnLXdvb2QuY29tPoh0BBMRCAA0BQsHCgMEAxUDAgMW
-AgECF4AWIQT79zfs6firGGBL0qyTk14C/ztU+gUCXjg2UgIZAQAKCRCTk14C/ztU
+AgECF4ACGQEWIQT79zfs6firGGBL0qyTk14C/ztU+gUCZS/mXAIbIwAKCRCTk14C
-+lmmAJ4jH5FyULzStjisuTvHLTVz6G44eQCfaR5QGZFPseenE5ic2WeQcBcmtoG5
+/ztU+tX+AJ9CUAnPvT4w5yRAPRfDiwWIPUqBOgCgiTelkzvUxvLWnYmpowwzKmsx
-Ag0EO7LdgRAIAI6QdFBg3/xa1gFKPYy1ihV9eSdGqwWZGJvokWsfCvHy5180tj/v
+qaSJAjMEEAEIAB0WIQTjs1jchY+zB/SBcLnLDb68XzLIHQUCZPRnNAAKCRDLDb68
-UNOLAJrdqglMSvevNTXe8bT65D6423AAsLhch9wq/aNqrHolTYABzxRigjcS1//T
+XzLIHZSAD/oCk9Z0xJfbpriphTBxFy7bWyPKF1lM1GZZaLKkktGfunf1i0Q7rhwp
-yln5naGUzlVQXDVfrDk3Md/NrkdOFj7r/YyMF0+iWwpFz2qAjL95i5wfVZ1kWGrT
+Nu+u1launlOTp6ZoY36Ce2Qa1eSxWAQdjVajw9kOHXCAewrTREOMY/mb7RVGjajo
-2AmivE1wD1sWT/Ja3FDI0NRkU0Nbz/a0TKe4ml8iLVtZXpTRbxxCCPdkHXXgSyu1
+0Egl8T9iD3JRyaxu2iVtbpZYuqehtGG28CaCzmtqE+EJcx1cGqAGSuuaDWRYlVX8
-eZ4NrF/wTJuvwGn12TJ1EF95aVkHxAUw0+KmLGdcyBG+IKuHamrsjWIAXGXV///K
+KDip44GQB5Lut30vwSIoZG1CPCR6VE82u4cl3mYZUfcJkCHsiLzoeadVzb+fOd+2
-AxPgUthccQ03HMjltFsrdmen5Q034YM3eOsAAwUH/jAKiIAA8LpZmZPnt9GZ4+Ol
+ybzBn8Y77ifGgM+dSFSHe03mFfcHPdp0QImF9HQR7XI0UMZmEJsw7c2vDrRa+kRY
-Zp22VAfyfDOFl4Ol+cWjkLAgjAFsm5gnOKcRSE/9XPxnQqkhw7+ZygYuUMgTDJ99
+2A4/amGn4Tahuazq8g2yqgGm3yAj49qGNarAau849lDr7R49j73ESnNVBGJ9ShzU
-/5IM1UQL3ooS+oFrDaE99S8bLeOe17skcdXcA/K83VqD9m93rQRnbtD+75zqKkZn
+4Ls+S1A5gohZVu2s1fkE3mbAmoTfU4JCrpRydOuL9xRJk5gbL44sKeuGODNshyTP
-9WNFyKCXg5P6PFPdNYRtlQKOcwFR9mHRLUmapQSAM8Y2pCgALZ7GViKQca8/TT1T
+JzG9DmRHpLsBn59v8mg5tqSfBIGqcqBxxnYHJnkK801MkaLW2m7wDmtz6P3TW86g
-gZk9fJMZYGez+IlOPxTJxjn80+vywk4/wdIWSiQj+8u5RzT9sjmm77wbMVNGRqYd
+GukzfIN3/OufLjnpN3Nx376JwWDDIyif7sn6/q+ZMwGz9uLKZkAeM5c3Dh4ygpgl
-W/EemW9Zz9vi0CIvJGgbPMqcuxw8e/5lnuQ6Mi3uDR0P2RNIAhFrdZpVSME8xQaI
+iSLoV2bZzDz0iLxKWW7QOVVdWHmlEqbTldpQ7gUEPG7mxpzVo0xd6nHncSq0M91x
-RgQYEQIABgUCO7LdgQAKCRCTk14C/ztU+mLBAKC2cdFy7eLaQAvyzcE2VK6HVIjn
+29It4B3fATx/iJB2eardMzSsbzHiwTg0eswhYYGpSKZLgp4RShnVAbkCDQQ7st2B
-JACguA00bxLQuJ4+RCJrLFZP8ZlN2sc=
+EAgAjpB0UGDf/FrWAUo9jLWKFX15J0arBZkYm+iRax8K8fLnXzS2P+9Q04sAmt2q
-=TtR5
+CUxK9681Nd7xtPrkPrjbcACwuFyH3Cr9o2qseiVNgAHPFGKCNxLX/9PKWfmdoZTO
------END PGP PUBLIC KEY BLOCK-----`
+VVBcNV+sOTcx382uR04WPuv9jIwXT6JbCkXPaoCMv3mLnB9VnWRYatPYCaK8TXAP
+WxZP8lrcUMjQ1GRTQ1vP9rRMp7iaXyItW1lelNFvHEII92QddeBLK7V5ng2sX/BM
+m6/AafXZMnUQX3lpWQfEBTDT4qYsZ1zIEb4gq4dqauyNYgBcZdX//8oDE+BS2Fxx
+DTccyOW0Wyt2Z6flDTfhgzd46wADBQf+MAqIgADwulmZk+e30Znj46VmnbZUB/J8
+M4WXg6X5xaOQsCCMAWybmCc4pxFIT/1c/GdCqSHDv5nKBi5QyBMMn33/kgzVRAve
+ihL6gWsNoT31Lxst457XuyRx1dwD8rzdWoP2b3etBGdu0P7vnOoqRmf1Y0XIoJeD
+k/o8U901hG2VAo5zAVH2YdEtSZqlBIAzxjakKAAtnsZWIpBxrz9NPVOBmT18kxlg
+Z7P4iU4/FMnGOfzT6/LCTj/B0hZKJCP7y7lHNP2yOabvvBsxU0ZGph1b8R6Zb1nP
+2+LQIi8kaBs8ypy7HDx7/mWe5DoyLe4NHQ/ZE0gCEWt1mlVIwTzFBohGBBgRAgAG
+BQI7st2BAAoJEJOTXgL/O1T6YsEAoLZx0XLt4tpAC/LNwTZUrodUiOckAKC4DTRv
+EtC4nj5EImssVk/xmU3axw==
+=VUqh
+-----END PGP PUBLIC KEY BLOCK-----
+`
 
 func verifyHashsum(ctx context.Context, siteURL, version, archive string, hash []byte) error {
 	sumsURL := fmt.Sprintf("%s/%s/SHA256SUMS", siteURL, version)
@@ -52,16 +65,26 @@ func verifyHashsum(ctx context.Context, siteURL, version, archive string, hash [
 		return err
 	}
 	fs.Debugf(nil, "downloaded hashsum list: %s", sumsURL)
+	return verifyHashsumDownloaded(ctx, sumsBuf, archive, hash)
+}
 
+func verifyHashsumDownloaded(ctx context.Context, sumsBuf []byte, archive string, hash []byte) error {
 	keyRing, err := openpgp.ReadArmoredKeyRing(strings.NewReader(ncwPublicKeyPGP))
 	if err != nil {
-		return errors.New("unsupported signing key")
+		return fmt.Errorf("unsupported signing key: %w", err)
 	}
+
 	block, rest := clearsign.Decode(sumsBuf)
-	// block.Bytes = block.Bytes[1:] // uncomment to test invalid signature
+	if block == nil {
+		return errors.New("invalid hashsum signature: couldn't find detached signature")
+	}
+	if len(rest) > 0 {
+		return fmt.Errorf("invalid hashsum signature: %d bytes of unsigned data", len(rest))
+	}
+
 	_, err = openpgp.CheckDetachedSignature(keyRing, bytes.NewReader(block.Bytes), block.ArmoredSignature.Body, nil)
-	if err != nil || len(rest) > 0 {
+	if err != nil {
-		return errors.New("invalid hashsum signature")
+		return fmt.Errorf("invalid hashsum signature: %w", err)
 	}
 
 	wantHash, err := findFileHash(sumsBuf, archive)

cmd/selfupdate/verify_test.go

@@ -0,0 +1,40 @@
+package selfupdate
+
+import (
+	"context"
+	"encoding/hex"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVerify(t *testing.T) {
+	ctx := context.Background()
+	sumsBuf, err := os.ReadFile("testdata/verify/SHA256SUMS")
+	require.NoError(t, err)
+	hash, err := hex.DecodeString("b20b47f579a2c790ca752fb5d8e5651fade7d5867cbac0a4f71e805fc5c468d0")
+	require.NoError(t, err)
+
+	t.Run("NoError", func(t *testing.T) {
+		err = verifyHashsumDownloaded(ctx, sumsBuf, "archive.zip", hash)
+		require.NoError(t, err)
+	})
+	t.Run("BadSig", func(t *testing.T) {
+		sumsBuf[0x60] ^= 1 // change the signature by one bit
+		err = verifyHashsumDownloaded(ctx, sumsBuf, "archive.zip", hash)
+		assert.ErrorContains(t, err, "invalid signature")
+		sumsBuf[0x60] ^= 1 // undo the change
+	})
+	t.Run("BadSum", func(t *testing.T) {
+		hash[0] ^= 1 // change the SHA256 by one bit
+		err = verifyHashsumDownloaded(ctx, sumsBuf, "archive.zip", hash)
+		assert.ErrorContains(t, err, "archive hash mismatch")
+		hash[0] ^= 1 // undo the change
+	})
+	t.Run("BadName", func(t *testing.T) {
+		err = verifyHashsumDownloaded(ctx, sumsBuf, "archive.zipX", hash)
+		assert.ErrorContains(t, err, "unable to find hash")
+	})
+}

fstest/test_all/config.yaml

@@ -10,6 +10,8 @@ tests:
  - path: vfs
  - path: cmd/serve/restic
    localonly: true
+ - path: cmd/selfupdate
+   localonly: true
 backends:
  # - backend:  "amazonclouddrive"
  #   remote:   "TestAmazonCloudDrive:"