extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-02-18 03:21:11 +01:00
Code Issues Packages Projects Releases Wiki Activity

crypt: Introduce cipherVersion on the object level

Browse Source 
Introduce cipherVersion on the object level, so it can then be used in Size() function to determine the correct decrypted size based on explicit object version; Modify DecryptDataSeek and newDecrypterSeek, to include 'o *Object' parameter, so it can then be passed to newDecrypter where version detection happens based on magic bytes.
This commit is contained in:
Tomasz Raganowicz 2024-10-07 17:12:43 +02:00
parent 0538517cee
commit c1ad0291c6
3 changed files with 40 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
backend/crypt/cipher.go
View File

@ -922,7 +922,7 @@ type decrypter struct {
} }
// newDecrypter creates a new file handle decrypting on the fly // newDecrypter creates a new file handle decrypting on the fly
func (c *Cipher) newDecrypter(rc io.ReadCloser, customCek *cek) (*decrypter, error) { func (c *Cipher) newDecrypter(rc io.ReadCloser, customCek *cek, o *Object) (*decrypter, error) {
fh := &decrypter{ fh := &decrypter{
rc: rc, rc: rc,
c: c, c: c,
@ -952,6 +952,10 @@ func (c *Cipher) newDecrypter(rc io.ReadCloser, customCek *cek) (*decrypter, err
return nil, fh.finishAndClose(ErrorEncryptedBadMagic) return nil, fh.finishAndClose(ErrorEncryptedBadMagic)
} }
if o != nil { // Set cipher version on the object level
o.cipherVersion = fh.cipherVersion
}
offsetStart := getFileMagicSize(fh.cipherVersion) offsetStart := getFileMagicSize(fh.cipherVersion)
offsetEnd := offsetStart + getFileNonceSize(fh.cipherVersion) offsetEnd := offsetStart + getFileNonceSize(fh.cipherVersion)
fh.nonce.fromBuf(readBuf[offsetStart:offsetEnd], getFileNonceSize(fh.cipherVersion)) fh.nonce.fromBuf(readBuf[offsetStart:offsetEnd], getFileNonceSize(fh.cipherVersion))
@ -1002,7 +1006,7 @@ func (c *Cipher) newDecrypter(rc io.ReadCloser, customCek *cek) (*decrypter, err
} }
// newDecrypterSeek creates a new file handle decrypting on the fly // newDecrypterSeek creates a new file handle decrypting on the fly
func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64, customCek *cek) (fh *decrypter, err error) { func (c *Cipher) newDecrypterSeek(ctx context.Context, o *Object, open OpenRangeSeek, offset, limit int64, customCek *cek) (fh *decrypter, err error) {
var rc io.ReadCloser var rc io.ReadCloser
doRangeSeek := false doRangeSeek := false
setLimit := false setLimit := false
@ -1024,7 +1028,7 @@ func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offse
return nil, err return nil, err
} }
// Open the stream which fills in the nonce // Open the stream which fills in the nonce
fh, err = c.newDecrypter(rc, customCek) fh, err = c.newDecrypter(rc, customCek, o)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1293,7 +1297,7 @@ func (fh *decrypter) finishAndClose(err error) error {
// DecryptData decrypts the data stream // DecryptData decrypts the data stream
func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) { func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
out, err := c.newDecrypter(rc, nil) out, err := c.newDecrypter(rc, nil, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1305,8 +1309,8 @@ func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
// The open function must return a ReadCloser opened to the offset supplied. // The open function must return a ReadCloser opened to the offset supplied.
// //
// You must use this form of DecryptData if you might want to Seek the file handle // You must use this form of DecryptData if you might want to Seek the file handle
func (c *Cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64, customCek *cek) (ReadSeekCloser, error) { func (c *Cipher) DecryptDataSeek(ctx context.Context, o *Object, open OpenRangeSeek, offset, limit int64, customCek *cek) (ReadSeekCloser, error) {
out, err := c.newDecrypterSeek(ctx, open, offset, limit, customCek) out, err := c.newDecrypterSeek(ctx, o, open, offset, limit, customCek)
if err != nil { if err != nil {
return nil, err return nil, err
} }

30
backend/crypt/cipher_test.go
View File

@ -1087,7 +1087,7 @@ func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
source := newRandomSource(copySize) source := newRandomSource(copySize)
encrypted, err := c.newEncrypter(source, nil, nil) encrypted, err := c.newEncrypter(source, nil, nil)
assert.NoError(t, err) assert.NoError(t, err)
decrypted, err := c.newDecrypter(io.NopCloser(encrypted), nil) decrypted, err := c.newDecrypter(io.NopCloser(encrypted), nil, nil)
assert.NoError(t, err) assert.NoError(t, err)
sink := newRandomSource(copySize) sink := newRandomSource(copySize)
n, err := io.CopyBuffer(sink, decrypted, buf) n, err := io.CopyBuffer(sink, decrypted, buf)
@ -1300,7 +1300,7 @@ func TestNewDecrypter(t *testing.T) {
c.cryptoRand = newRandomSource(1e8) // nodge the crypto rand generator c.cryptoRand = newRandomSource(1e8) // nodge the crypto rand generator
cd := newCloseDetector(bytes.NewBuffer(file0)) cd := newCloseDetector(bytes.NewBuffer(file0))
fh, err := c.newDecrypter(cd, nil) fh, err := c.newDecrypter(cd, nil, nil)
assert.NoError(t, err) assert.NoError(t, err)
// check nonce is in place // check nonce is in place
assert.Equal(t, file0[8:32], fh.nonce[:]) assert.Equal(t, file0[8:32], fh.nonce[:])
@ -1309,7 +1309,7 @@ func TestNewDecrypter(t *testing.T) {
// Test error paths // Test error paths
for i := range file0 { for i := range file0 {
cd := newCloseDetector(bytes.NewBuffer(file0[:i])) cd := newCloseDetector(bytes.NewBuffer(file0[:i]))
fh, err = c.newDecrypter(cd, nil) fh, err = c.newDecrypter(cd, nil, nil)
assert.Nil(t, fh) assert.Nil(t, fh)
assert.EqualError(t, err, ErrorEncryptedFileTooShort.Error()) assert.EqualError(t, err, ErrorEncryptedFileTooShort.Error())
assert.Equal(t, 1, cd.closed) assert.Equal(t, 1, cd.closed)
@ -1317,7 +1317,7 @@ func TestNewDecrypter(t *testing.T) {
er := &readers.ErrorReader{Err: errors.New("potato")} er := &readers.ErrorReader{Err: errors.New("potato")}
cd = newCloseDetector(er) cd = newCloseDetector(er)
fh, err = c.newDecrypter(cd, nil) fh, err = c.newDecrypter(cd, nil, nil)
assert.Nil(t, fh) assert.Nil(t, fh)
assert.EqualError(t, err, "potato") assert.EqualError(t, err, "potato")
assert.Equal(t, 1, cd.closed) assert.Equal(t, 1, cd.closed)
@ -1328,7 +1328,7 @@ func TestNewDecrypter(t *testing.T) {
for i := range fileMagic { for i := range fileMagic {
file0copy[i] ^= 0x1 file0copy[i] ^= 0x1
cd := newCloseDetector(bytes.NewBuffer(file0copy)) cd := newCloseDetector(bytes.NewBuffer(file0copy))
fh, err := c.newDecrypter(cd, nil) fh, err := c.newDecrypter(cd, nil, nil)
assert.Nil(t, fh) assert.Nil(t, fh)
if i == 7 { // This test accidentally swaps last byte and converts `fileMagic` (RCLONE\x00\x00") into `fileMagicV2` ("RCLONE\x00\x01") resulting in a different than: "ErrorEncryptedBadMagic" error. if i == 7 { // This test accidentally swaps last byte and converts `fileMagic` (RCLONE\x00\x00") into `fileMagicV2` ("RCLONE\x00\x01") resulting in a different than: "ErrorEncryptedBadMagic" error.
assert.EqualError(t, err, ErrorEncryptedFileTooShort.Error()) assert.EqualError(t, err, ErrorEncryptedFileTooShort.Error())
@ -1349,7 +1349,7 @@ func TestNewDecrypterErrUnexpectedEOF(t *testing.T) {
in1 := bytes.NewBuffer(file16) in1 := bytes.NewBuffer(file16)
in := io.NopCloser(io.MultiReader(in1, in2)) in := io.NopCloser(io.MultiReader(in1, in2))
fh, err := c.newDecrypter(in, nil) fh, err := c.newDecrypter(in, nil, nil)
assert.NoError(t, err) assert.NoError(t, err)
n, err := io.CopyN(io.Discard, fh, 1e6) n, err := io.CopyN(io.Discard, fh, 1e6)
@ -1423,7 +1423,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
if offset+limit > len(plaintext) { if offset+limit > len(plaintext) {
continue continue
} }
rc, err := c.DecryptDataSeek(context.Background(), open, int64(offset), int64(limit), nil) rc, err := c.DecryptDataSeek(context.Background(), nil, open, int64(offset), int64(limit), nil)
assert.NoError(t, err) assert.NoError(t, err)
check(rc, offset, limit) check(rc, offset, limit)
@ -1431,7 +1431,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
} }
// Try decoding it with a single open and lots of seeks // Try decoding it with a single open and lots of seeks
fh, err := c.DecryptDataSeek(context.Background(), open, 0, -1, nil) fh, err := c.DecryptDataSeek(context.Background(), nil, open, 0, -1, nil)
assert.NoError(t, err) assert.NoError(t, err)
for _, offset := range trials { for _, offset := range trials {
for _, limit := range limits { for _, limit := range limits {
@ -1503,7 +1503,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
callCount++ callCount++
return open(ctx, underlyingOffset, underlyingLimit) return open(ctx, underlyingOffset, underlyingLimit)
} }
fh, err := c.DecryptDataSeek(context.Background(), testOpen, 0, -1, nil) fh, err := c.DecryptDataSeek(context.Background(), nil, testOpen, 0, -1, nil)
assert.NoError(t, err) assert.NoError(t, err)
gotOffset, err := fh.RangeSeek(context.Background(), test.offset, io.SeekStart, test.limit) gotOffset, err := fh.RangeSeek(context.Background(), test.offset, io.SeekStart, test.limit)
assert.NoError(t, err) assert.NoError(t, err)
@ -1571,7 +1571,7 @@ func TestDecrypterRead(t *testing.T) {
for i := 0; i < len(file16)-1; i++ { for i := 0; i < len(file16)-1; i++ {
what := fmt.Sprintf("truncating to %d/%d", i, len(file16)) what := fmt.Sprintf("truncating to %d/%d", i, len(file16))
cd := newCloseDetector(bytes.NewBuffer(file16[:i])) cd := newCloseDetector(bytes.NewBuffer(file16[:i]))
fh, err := c.newDecrypter(cd, nil) fh, err := c.newDecrypter(cd, nil, nil)
if i < fileHeaderSize { if i < fileHeaderSize {
assert.EqualError(t, err, ErrorEncryptedFileTooShort.Error(), what) assert.EqualError(t, err, ErrorEncryptedFileTooShort.Error(), what)
continue continue
@ -1604,7 +1604,7 @@ func TestDecrypterRead(t *testing.T) {
in2 := &readers.ErrorReader{Err: errors.New("potato")} in2 := &readers.ErrorReader{Err: errors.New("potato")}
in := io.MultiReader(in1, in2) in := io.MultiReader(in1, in2)
cd := newCloseDetector(in) cd := newCloseDetector(in)
fh, err := c.newDecrypter(cd, nil) fh, err := c.newDecrypter(cd, nil, nil)
assert.NoError(t, err) assert.NoError(t, err)
_, err = io.ReadAll(fh) _, err = io.ReadAll(fh)
assert.EqualError(t, err, "potato") assert.EqualError(t, err, "potato")
@ -1616,7 +1616,7 @@ func TestDecrypterRead(t *testing.T) {
copy(file16copy, file16) copy(file16copy, file16)
for i := range file16copy { for i := range file16copy {
file16copy[i] ^= 0xFF file16copy[i] ^= 0xFF
fh, err := c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)), nil) fh, err := c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)), nil, nil)
if i < fileMagicSize { if i < fileMagicSize {
assert.EqualError(t, err, ErrorEncryptedBadMagic.Error()) assert.EqualError(t, err, ErrorEncryptedBadMagic.Error())
assert.Nil(t, fh) assert.Nil(t, fh)
@ -1633,7 +1633,7 @@ func TestDecrypterRead(t *testing.T) {
copy(file16copy, file16) copy(file16copy, file16)
file16copy[len(file16copy)-1] ^= 0xFF file16copy[len(file16copy)-1] ^= 0xFF
c.passBadBlocks = true c.passBadBlocks = true
fh, err = c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)), nil) fh, err = c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)), nil, nil)
assert.NoError(t, err) assert.NoError(t, err)
buf, err := io.ReadAll(fh) buf, err := io.ReadAll(fh)
assert.NoError(t, err) assert.NoError(t, err)
@ -1645,7 +1645,7 @@ func TestDecrypterClose(t *testing.T) {
assert.NoError(t, err) assert.NoError(t, err)
cd := newCloseDetector(bytes.NewBuffer(file16)) cd := newCloseDetector(bytes.NewBuffer(file16))
fh, err := c.newDecrypter(cd, nil) fh, err := c.newDecrypter(cd, nil, nil)
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, 0, cd.closed) assert.Equal(t, 0, cd.closed)
@ -1663,7 +1663,7 @@ func TestDecrypterClose(t *testing.T) {
// try again reading the file this time // try again reading the file this time
cd = newCloseDetector(bytes.NewBuffer(file1)) cd = newCloseDetector(bytes.NewBuffer(file1))
fh, err = c.newDecrypter(cd, nil) fh, err = c.newDecrypter(cd, nil, nil)
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, 0, cd.closed) assert.Equal(t, 0, cd.closed)

19
backend/crypt/crypt.go
View File

@ -916,7 +916,7 @@ func (f *Fs) getDecrypter(ctx context.Context, o *Object, overrideCek *cek) (*de
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to open object to read nonce: %w", err) return nil, fmt.Errorf("failed to open object to read nonce: %w", err)
} }
d, err := f.cipher.newDecrypter(in, overrideCek) d, err := f.cipher.newDecrypter(in, overrideCek, o)
if err != nil { if err != nil {
_ = in.Close() _ = in.Close()
return nil, fmt.Errorf("failed to open object to read nonce: %w", err) return nil, fmt.Errorf("failed to open object to read nonce: %w", err)
@ -1146,6 +1146,7 @@ type Object struct {
fs.Object fs.Object
f *Fs f *Fs
decryptedSize int64 decryptedSize int64
cipherVersion string
} }
func (f *Fs) newObject(o fs.Object) *Object { func (f *Fs) newObject(o fs.Object) *Object {
@ -1153,6 +1154,7 @@ func (f *Fs) newObject(o fs.Object) *Object {
Object: o, Object: o,
f: f, f: f,
decryptedSize: -1, decryptedSize: -1,
cipherVersion: "",
} }
} }
@ -1185,10 +1187,18 @@ func (o *Object) Size() int64 {
size := o.Object.Size() size := o.Object.Size()
if !o.f.opt.NoDataEncryption { if !o.f.opt.NoDataEncryption {
var err error var err error
if o.f.opt.ExactSize { if o.f.opt.ExactSize && o.cipherVersion == "" { // Use `ExactSize` setting if cipherVersion isn't explicitly set on the object level. If cipherVersion is explicitly set, we deduce size correctly without reading file header.
size, err = o.f.cipher.DecryptedSizeExact(o) size, err = o.f.cipher.DecryptedSizeExact(o)
} else { } else {
size, err = o.f.cipher.DecryptedSize(size, o.f.cipher.version)
var cipherVersion string
if o.cipherVersion != "" { // Explicit cipher version (set by newDecrypter) detected from magic bytes
cipherVersion = o.cipherVersion
} else { // Assume cipher version based on config. Might not be correct if existing object was encrypted using different cipher version than currently configured.
cipherVersion = o.f.cipher.version
}
size, err = o.f.cipher.DecryptedSize(size, cipherVersion)
} }
if err != nil { if err != nil {
@ -1296,7 +1306,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
openOptions = append(openOptions, option) openOptions = append(openOptions, option)
} }
} }
rc, err = o.f.cipher.DecryptDataSeek(ctx, func(ctx context.Context, underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) {
rc, err = o.f.cipher.DecryptDataSeek(ctx, o, func(ctx context.Context, underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) {
if underlyingOffset == 0 && underlyingLimit < 0 { if underlyingOffset == 0 && underlyingLimit < 0 {
// Open with no seek // Open with no seek
return o.Object.Open(ctx, openOptions...) return o.Object.Open(ctx, openOptions...)