crypt: Provide migration path to V2 cipher; Make cipher version handling more strict and explicit; Introduce --crypt-exact-size parameter to support in-place migration from V1 to V2; Update documentation

backend/crypt/cipher.go

@@ -208,6 +208,7 @@ type Cipher struct {
 	passBadBlocks   bool // if set passed bad blocks as zeroed blocks
 	encryptedSuffix string
 	version         string
+	exactSize       bool
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
@@ -252,29 +253,33 @@ func (c *Cipher) setCipherVersion(cipherVersion string) {
 	c.version = cipherVersion
 }
 
-func (c *Cipher) getFileHeaderSize() int {
+func (c *Cipher) setExactSize(exact bool) {
-	if c.version == CipherVersionV2 {
+	c.exactSize = exact
+}
+
+func getFileHeaderSize(cipherVersion string) int {
+	if cipherVersion == CipherVersionV2 {
 		return fileHeaderSizeV2
 	}
 	return fileHeaderSize
 }
 
-func (c *Cipher) getFileMagicSize() int {
+func getFileMagicSize(cipherVersion string) int {
-	if c.version == CipherVersionV2 {
+	if cipherVersion == CipherVersionV2 {
 		return fileMagicSizeV2
 	}
 	return fileMagicSize
 }
 
-func (c *Cipher) getFileMagicBytes() []byte {
+func getFileMagicBytes(cipherVersion string) []byte {
-	if c.version == CipherVersionV2 {
+	if cipherVersion == CipherVersionV2 {
 		return fileMagicBytesV2
 	}
 	return fileMagicBytes
 }
 
-func (c *Cipher) getFileNonceSize() int { // Effective nonce size. Both V1 and V2 use 24 bytes, but V2 consists 23+1 bytes nonce, where last byte is calculated dynamically depending if the processing block is last (truncation protection)
+func getFileNonceSize(cipherVersion string) int { // Effective nonce size. Both V1 and V2 use 24 bytes, but V2 consists 23+1 bytes nonce, where last byte is calculated dynamically depending if the processing block is last (truncation protection)
-	if c.version == CipherVersionV2 {
+	if cipherVersion == CipherVersionV2 {
 		return fileNonceSizeV2
 	}
 	return fileNonceSize
@@ -775,22 +780,22 @@ func (c *Cipher) newEncrypter(in io.Reader, nonce *nonce, cek *cek) (*encrypter,
 		c:       c,
 		buf:     c.getBlock(),
 		readBuf: c.getBlock(),
-		bufSize: c.getFileHeaderSize(),
+		bufSize: getFileHeaderSize(c.version),
 	}
 	// Initialise nonce
 	if nonce != nil {
 		fh.nonce = *nonce
 	} else {
-		err := fh.nonce.fromReader(c.cryptoRand, c.getFileNonceSize())
+		err := fh.nonce.fromReader(c.cryptoRand, getFileNonceSize(c.version))
 		if err != nil {
 			return nil, err
 		}
 	}
 	// Copy magic into buffer
-	copy((*fh.buf)[:], c.getFileMagicBytes())
+	copy((*fh.buf)[:], getFileMagicBytes(c.version))
 
 	// Copy nonce into buffer
-	copy((*fh.buf)[c.getFileMagicSize():], fh.nonce[:c.getFileNonceSize()])
+	copy((*fh.buf)[getFileMagicSize(c.version):], fh.nonce[:getFileNonceSize(c.version)])
 
 	if c.version == CipherVersionV1 {
 		fh.cek = fh.c.dataKey
@@ -820,14 +825,14 @@ func (c *Cipher) newEncrypter(in io.Reader, nonce *nonce, cek *cek) (*encrypter,
 		// WRAP KEY - END
 
 		// Copy file encryption key
-		copy((*fh.buf)[c.getFileMagicSize()+c.getFileNonceSize():], wrappedCek)
+		copy((*fh.buf)[getFileMagicSize(c.version)+getFileNonceSize(c.version):], wrappedCek)
 
 		// Copy 4 reserved bytes:
 		//	- cipher type (XSalsa20, AES-GCM etc.),
 		//	- not used,
 		//	- not used,
 		//	- not used
-		copy((*fh.buf)[c.getFileMagicSize()+c.getFileNonceSize()+fileWrappedCekSize:], fileReservedBytesV2)
+		copy((*fh.buf)[getFileMagicSize(c.version)+getFileNonceSize(c.version)+fileWrappedCekSize:], fileReservedBytesV2)
 	}
 
 	return fh, nil
@@ -913,6 +918,7 @@ type decrypter struct {
 	err           error
 	limit         int64 // limit of bytes to read, -1 for unlimited
 	open          OpenRangeSeek
+	cipherVersion string
 }
 
 // newDecrypter creates a new file handle decrypting on the fly
@@ -938,20 +944,20 @@ func (c *Cipher) newDecrypter(rc io.ReadCloser, customCek *cek) (*decrypter, err
 	isMagicHeaderV2 := bytes.Equal(readBuf[:fileMagicSizeV2], fileMagicBytesV2)
 
 	if isMagicHeader {
-		c.setCipherVersion(CipherVersionV1)
+		fh.cipherVersion = CipherVersionV1
 		fh.cek = fh.c.dataKey
 	} else if isMagicHeaderV2 {
-		c.setCipherVersion(CipherVersionV2)
+		fh.cipherVersion = CipherVersionV2
 	} else {
 		return nil, fh.finishAndClose(ErrorEncryptedBadMagic)
 	}
 
-	offsetStart := c.getFileMagicSize()
+	offsetStart := getFileMagicSize(fh.cipherVersion)
-	offsetEnd := offsetStart + c.getFileNonceSize()
+	offsetEnd := offsetStart + getFileNonceSize(fh.cipherVersion)
-	fh.nonce.fromBuf(readBuf[offsetStart:offsetEnd], c.getFileNonceSize())
+	fh.nonce.fromBuf(readBuf[offsetStart:offsetEnd], getFileNonceSize(fh.cipherVersion))
 	fh.initialNonce = fh.nonce
 
-	if c.version == CipherVersionV2 { // V2 cipher has a longer header, so after reading V1 header, we need to read remaining bytes from the reader to finialize V2 cipher initialization
+	if fh.cipherVersion == CipherVersionV2 { // V2 cipher has a longer header, so after reading V1 header, we need to read remaining bytes from the reader to finialize V2 cipher initialization
 		remainingBytesFromV1Buffer := fileNonceSize - fileNonceSizeV2 // V2 nonce is 1 byte shorter, so by reading the V1 header size (fileHeaderSize - 32 bytes), we've actually read 1 byte of wrapped CEK. We need to preserve that byte, so we prepend to next: `combinedBuffer`
 		lastByte := readBuf[len(readBuf)-remainingBytesFromV1Buffer:]
 
@@ -1006,12 +1012,12 @@ func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offse
 		rc, err = open(ctx, 0, -1)
 	} else if offset == 0 {
 		// If no offset open the header + limit worth of the file
-		_, underlyingLimit, _, _ := calculateUnderlying(offset, limit, c.getFileHeaderSize())
+		_, underlyingLimit, _, _ := calculateUnderlying(offset, limit, getFileHeaderSize(c.version)) // Is `c.version` (config) right value here? We get the actual value from decrypter couple lines below: `fh.cipherVersion`
-		rc, err = open(ctx, 0, int64(c.getFileHeaderSize())+underlyingLimit)
+		rc, err = open(ctx, 0, int64(getFileHeaderSize(c.version))+underlyingLimit)                  // Check `c.version` vs `fh.cipherVersion`
 		setLimit = true
 	} else {
 		// Otherwise just read the header to start with
-		rc, err = open(ctx, 0, int64(c.getFileHeaderSize()))
+		rc, err = open(ctx, 0, int64(getFileHeaderSize(c.version))) // Check `c.version` vs `fh.cipherVersion`
 		doRangeSeek = true
 	}
 	if err != nil {
@@ -1057,7 +1063,7 @@ func (fh *decrypter) fillBuffer() (err error) {
 
 	isLastBlock := n < blockDataSize
 
-	if fh.c.version == CipherVersionV2 && isLastBlock { // last block
+	if fh.cipherVersion == CipherVersionV2 && isLastBlock { // last block
 		fh.nonce[len(fh.nonce)-1] = lastBlockFlag // Set last block flag at the last byte
 		n -= int(HashEncryptedSizeWithHeader)     // Skip last bytes with encrypted hash
 
@@ -1177,7 +1183,7 @@ func (fh *decrypter) RangeSeek(ctx context.Context, offset int64, whence int, li
 		return 0, fh.err
 	}
 
-	underlyingOffset, underlyingLimit, discard, blocks := calculateUnderlying(offset, limit, fh.c.getFileHeaderSize())
+	underlyingOffset, underlyingLimit, discard, blocks := calculateUnderlying(offset, limit, getFileHeaderSize(fh.cipherVersion))
 
 	// Move the nonce on the correct number of blocks from the start
 	fh.nonce = fh.initialNonce
@@ -1310,7 +1316,7 @@ func (c *Cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset
 // EncryptedSize calculates the size of the data when encrypted
 func (c *Cipher) EncryptedSize(size int64) int64 {
 	blocks, residue := size/blockDataSize, size%blockDataSize
-	encryptedSize := int64(c.getFileHeaderSize()) + blocks*(blockHeaderSize+blockDataSize)
+	encryptedSize := int64(getFileHeaderSize(c.version)) + blocks*(blockHeaderSize+blockDataSize)
 
 	if c.version == CipherVersionV2 {
 		encryptedSize += HashEncryptedSizeWithHeader
@@ -1323,28 +1329,24 @@ func (c *Cipher) EncryptedSize(size int64) int64 {
 }
 
 // DecryptedSize calculates the size of the data when decrypted
-func (c *Cipher) DecryptedSize(size int64) (int64, error) {
+func (c *Cipher) DecryptedSize(size int64, cipherVersion string) (int64, error) {
-	size -= int64(c.getFileHeaderSize()) // WARNING: DecryptedSize might return invalid value before `newDecrypter()` is called if V2 cipher is enabled and user tries to read V1 object. Eventually `newDecrypter()` will be called, which will then call: `setCipherVersion()` which would then configure: `getFileHeaderSize()` effectively making subsequent calls to `DecryptedSize()` return exact value.
+
-	if c.version == CipherVersionV1 && size < 0 {
+	var headerSize int
-		return 0, ErrorEncryptedFileTooShort
+	if cipherVersion == CipherVersionV2 {
+		headerSize = fileHeaderSizeV2
+	} else {
+		headerSize = fileHeaderSize
 	}
 
-	if c.version == CipherVersionV2 {
+	size -= int64(headerSize)
+
+	if cipherVersion == CipherVersionV2 { // Footer
 		size -= HashEncryptedSizeWithHeader
 	}
 
-	v1CipherOverhead := int64(fileHeaderSize)
+	if size < 0 {
-	v2CipherOverhead := int64(fileHeaderSizeV2) + HashEncryptedSizeWithHeader
-	sizeDiff := v1CipherOverhead - v2CipherOverhead
-	// Below check was "loosened" to prevent code from return ErrorEncryptedFileTooShort when reading small `CipherVersionV1` encrypted object when `CipherVersionV2` is enabled as a config.
-	// If V2 cipher is enabled then we assume file was encrypted using V2 header (but this is just assumption we can't be sure until `setCipherVersion()` is called).
-	// V2 format has longer header and introduces concept of a footer. If we by any chance read small V1 object and assume it's a V2 header we may end up getting the negative decrypted size and trigger this check - preventing reading the file.
-	// The `setCipherVersion()` is called once we actually read the file contents.
-	// @TODO Don't use: "loosened" approach once: `setCipherVersion()` is called. Distinct implicit (assumed from config) cipher version assumption and explicit (set by setCipherVersion())
-	if c.version == CipherVersionV2 && size < sizeDiff {
 		return 0, ErrorEncryptedFileTooShort
 	}
-
 	blocks, residue := size/blockSize, size%blockSize
 	decryptedSize := blocks * blockDataSize
 	if residue != 0 {
@@ -1357,6 +1359,32 @@ func (c *Cipher) DecryptedSize(size int64) (int64, error) {
 	return decryptedSize, nil
 }
 
+// DecryptedSizeExact calculates the size of the data when decrypted by issuing HTTP request
+func (c *Cipher) DecryptedSizeExact(o *Object) (int64, error) {
+	ctx := context.Background() // @TODO Can we use this context or do we need to pass somehow the context from the top
+
+	// Return cached
+	if o.decryptedSize != -1 {
+		return o.decryptedSize, nil
+	}
+
+	// Get cipher version
+	d, err := o.f.getDecrypter(ctx, o, nil)
+	if err != nil {
+		return 0, err
+	}
+
+	encryptedSize := o.Object.Size()
+	decryptedSize, err := c.DecryptedSize(encryptedSize, d.cipherVersion)
+	if err != nil {
+		return 0, err
+	}
+
+	// Cache decrypted size
+	o.decryptedSize = decryptedSize
+	return decryptedSize, nil
+}
+
 // check interfaces
 var (
 	_ io.ReadCloser  = (*decrypter)(nil)

backend/crypt/cipher_test.go

@@ -701,7 +701,7 @@ func TestEncryptedSize(t *testing.T) {
 	} {
 		actual := c.EncryptedSize(test.in)
 		assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %d", test.in))
-		recovered, err := c.DecryptedSize(test.expected)
+		recovered, err := c.DecryptedSize(test.expected, CipherVersionV1)
 		assert.NoError(t, err, fmt.Sprintf("Testing reverse %d", test.expected))
 		assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %d", test.expected))
 	}
@@ -723,7 +723,7 @@ func TestDecryptedSize(t *testing.T) {
 		{32 + 16 + 65536 + 1, ErrorEncryptedFileBadHeader},
 		{32 + 16 + 65536 + 16, ErrorEncryptedFileBadHeader},
 	} {
-		_, actualErr := c.DecryptedSize(test.in)
+		_, actualErr := c.DecryptedSize(test.in, CipherVersionV1)
 		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("Testing %d", test.in))
 	}
 }

backend/crypt/crypt.go

@@ -206,6 +206,22 @@ when the path length is critical.`,
 				},
 				Advanced: true,
 			},
+			{
+				Name:    "exact_size",
+				Help:    `Detect object decrypted size by reading a header. This isn't normally needed except rare cases where user would like to migrate cipher versions.'`,
+				Default: false,
+				Examples: []fs.OptionExample{
+					{
+						Value: "true",
+						Help:  "Get size according to object's cipher version. Requires HTTP call for every object",
+					},
+					{
+						Value: "false",
+						Help:  "Get size according to cipher version configuration.",
+					},
+				},
+				Advanced: true,
+			},
 		},
 	})
 }
@@ -241,6 +257,7 @@ func newCipherForConfig(opt *Options) (*Cipher, error) {
 	cipher.setEncryptedSuffix(opt.Suffix)
 	cipher.setPassBadBlocks(opt.PassBadBlocks)
 	cipher.setCipherVersion(opt.CipherVersion)
+	cipher.setExactSize(opt.ExactSize)
 	return cipher, nil
 }
 
@@ -347,6 +364,7 @@ type Options struct {
 	Suffix                  string `config:"suffix"`
 	StrictNames             bool   `config:"strict_names"`
 	CipherVersion           string `config:"cipher_version"`
+	ExactSize               bool   `config:"exact_size"`
 }
 
 // Fs represents a wrapped fs.Fs
@@ -1127,12 +1145,14 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 type Object struct {
 	fs.Object
 	f             *Fs
+	decryptedSize int64
 }
 
 func (f *Fs) newObject(o fs.Object) *Object {
 	return &Object{
 		Object:        o,
 		f:             f,
+		decryptedSize: -1,
 	}
 }
 
@@ -1165,7 +1185,12 @@ func (o *Object) Size() int64 {
 	size := o.Object.Size()
 	if !o.f.opt.NoDataEncryption {
 		var err error
-		size, err = o.f.cipher.DecryptedSize(size)
+		if o.f.opt.ExactSize {
+			size, err = o.f.cipher.DecryptedSizeExact(o)
+		} else {
+			size, err = o.f.cipher.DecryptedSize(size, o.f.cipher.version)
+		}
+
 		if err != nil {
 			fs.Debugf(o, "Bad size for decrypt: %v", err)
 		}
@@ -1188,7 +1213,7 @@ func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
 	}
 
 	// Objects encrypted with V1 doesn't support hash. Even if V2 cipher is enabled in the config, that doesn't make existing V1 objects support hashing, so we return "" to skip hash verification.
-	if d.c.version == CipherVersionV1 {
+	if d.cipherVersion == CipherVersionV1 {
 		return "", nil
 	}
 
@@ -1213,7 +1238,13 @@ func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
 	}
 
 	// We need to get the decrypted size, to workout the amount of blocks, so we can workout the nonce that was used to encrypt the hash.
-	decryptedSize, err := o.f.cipher.DecryptedSize(encryptedSize)
+	var decryptedSize int64
+	if d.c.exactSize {
+		decryptedSize, err = o.f.cipher.DecryptedSizeExact(o)
+	} else {
+		decryptedSize, err = o.f.cipher.DecryptedSize(encryptedSize, d.cipherVersion)
+	}
+
 	if err != nil {
 		return "", err
 	}

docs/content/crypt.md

@@ -868,6 +868,37 @@ then rclone uses an internal one.
 encrypted data.  For full protection against this you should always use
 a salt.
 
+### V2 cipher
+In `v1.69.0` Rclone introduced new cipher version (V2) which supports secure file sharing,
+protection against file truncation and support for MD5 hashes which are now calculated 
+and stored encrypted in the file footer.
+
+#### Migration
+Using two versions simultaneously in a single location isn't supported
+and may lead to unintended consequences.
+
+It is recommended to migrate data to a different remote, so in case something goes wrong, source remote
+remains unaffected and acts as a backup.
+
+Before V2 cipher is enabled in the config (`cipher_version` flag), user should migrate all of their
+existing V1 encrypted data.
+
+##### In-place migration
+It should only be used for data that's already backed up or as a last resort. There isn't much room
+for error and if something goes wrong, data loss may happen.
+
+In order to migrate objects from V1 to V2 within a same `crypt` back-end (e.g. `cryptA`):
+- clone configuration of your `cryptA` to `cryptA_clone`,
+- set `cipher_version = 2` in your `cryptA_clone`,
+- run `copy --no-check-dest --crypt-exact-size cryptA: cryptA_clone:`
+
+Command: `--no-check-dest` will make sure that file gets copied even if it's seemingly the same.
+
+Command: `--crypt-exact-size` issues calculates size based on object's cipher version detection.
+It shouldn't be used on a daily basis as it requires additional HTTP call for every single object.
+Normally cipher version is assumed from the config, but this isn't reliable during migration which 
+may already have mixed cipher versions temporarily.
+
 ## SEE ALSO
 
 * [rclone cryptdecode](/commands/rclone_cryptdecode/)    - Show forward/reverse mapping of encrypted filenames