extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-25 09:54:44 +01:00
Code Issues Packages Projects Releases Wiki Activity

crypt: Introduce cipherVersion on the object level

Browse Source 
Introduce cipherVersion on the object level, so it can then be used in Size() function to determine the correct decrypted size based on explicit object version; Modify DecryptDataSeek and newDecrypterSeek, to include 'o *Object' parameter, so it can then be passed to newDecrypter where version detection happens based on magic bytes.
This commit is contained in:
Tomasz Raganowicz 2024-10-07 17:12:43 +02:00
parent 0538517cee
commit c1ad0291c6
3 changed files with 40 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
backend/crypt/cipher.go
View File

@ -922,7 +922,7 @@ type decrypter struct {
}
// newDecrypter creates a new file handle decrypting on the fly
func (c *Cipher) newDecrypter(rc io.ReadCloser, customCek *cek) (*decrypter, error) {
func (c *Cipher) newDecrypter(rc io.ReadCloser, customCek *cek, o *Object) (*decrypter, error) {
fh := &decrypter{
rc: rc,
c: c,
@ -952,6 +952,10 @@ func (c *Cipher) newDecrypter(rc io.ReadCloser, customCek *cek) (*decrypter, err
return nil, fh.finishAndClose(ErrorEncryptedBadMagic)
}
if o != nil { // Set cipher version on the object level
o.cipherVersion = fh.cipherVersion
}
offsetStart := getFileMagicSize(fh.cipherVersion)
offsetEnd := offsetStart + getFileNonceSize(fh.cipherVersion)
fh.nonce.fromBuf(readBuf[offsetStart:offsetEnd], getFileNonceSize(fh.cipherVersion))
@ -1002,7 +1006,7 @@ func (c *Cipher) newDecrypter(rc io.ReadCloser, customCek *cek) (*decrypter, err
}
// newDecrypterSeek creates a new file handle decrypting on the fly
func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64, customCek *cek) (fh *decrypter, err error) {
func (c *Cipher) newDecrypterSeek(ctx context.Context, o *Object, open OpenRangeSeek, offset, limit int64, customCek *cek) (fh *decrypter, err error) {
var rc io.ReadCloser
doRangeSeek := false
setLimit := false
@ -1024,7 +1028,7 @@ func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offse
return nil, err
}
// Open the stream which fills in the nonce
fh, err = c.newDecrypter(rc, customCek)
fh, err = c.newDecrypter(rc, customCek, o)
if err != nil {
return nil, err
}
@ -1293,7 +1297,7 @@ func (fh *decrypter) finishAndClose(err error) error {
// DecryptData decrypts the data stream
func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
out, err := c.newDecrypter(rc, nil)
out, err := c.newDecrypter(rc, nil, nil)
if err != nil {
return nil, err
}
@ -1305,8 +1309,8 @@ func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
// The open function must return a ReadCloser opened to the offset supplied.
//
// You must use this form of DecryptData if you might want to Seek the file handle
func (c *Cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64, customCek *cek) (ReadSeekCloser, error) {
out, err := c.newDecrypterSeek(ctx, open, offset, limit, customCek)
func (c *Cipher) DecryptDataSeek(ctx context.Context, o *Object, open OpenRangeSeek, offset, limit int64, customCek *cek) (ReadSeekCloser, error) {
out, err := c.newDecrypterSeek(ctx, o, open, offset, limit, customCek)
if err != nil {
return nil, err
}

30
backend/crypt/cipher_test.go
View File

@ -1087,7 +1087,7 @@ func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
source := newRandomSource(copySize)
encrypted, err := c.newEncrypter(source, nil, nil)
assert.NoError(t, err)
decrypted, err := c.newDecrypter(io.NopCloser(encrypted), nil)
decrypted, err := c.newDecrypter(io.NopCloser(encrypted), nil, nil)
assert.NoError(t, err)
sink := newRandomSource(copySize)
n, err := io.CopyBuffer(sink, decrypted, buf)
@ -1300,7 +1300,7 @@ func TestNewDecrypter(t *testing.T) {
c.cryptoRand = newRandomSource(1e8) // nodge the crypto rand generator
cd := newCloseDetector(bytes.NewBuffer(file0))
fh, err := c.newDecrypter(cd, nil)
fh, err := c.newDecrypter(cd, nil, nil)
assert.NoError(t, err)
// check nonce is in place
assert.Equal(t, file0[8:32], fh.nonce[:])
@ -1309,7 +1309,7 @@ func TestNewDecrypter(t *testing.T) {
// Test error paths
for i := range file0 {
cd := newCloseDetector(bytes.NewBuffer(file0[:i]))
fh, err = c.newDecrypter(cd, nil)
fh, err = c.newDecrypter(cd, nil, nil)
assert.Nil(t, fh)
assert.EqualError(t, err, ErrorEncryptedFileTooShort.Error())
assert.Equal(t, 1, cd.closed)
@ -1317,7 +1317,7 @@ func TestNewDecrypter(t *testing.T) {
er := &readers.ErrorReader{Err: errors.New("potato")}
cd = newCloseDetector(er)
fh, err = c.newDecrypter(cd, nil)
fh, err = c.newDecrypter(cd, nil, nil)
assert.Nil(t, fh)
assert.EqualError(t, err, "potato")
assert.Equal(t, 1, cd.closed)
@ -1328,7 +1328,7 @@ func TestNewDecrypter(t *testing.T) {
for i := range fileMagic {
file0copy[i] ^= 0x1
cd := newCloseDetector(bytes.NewBuffer(file0copy))
fh, err := c.newDecrypter(cd, nil)
fh, err := c.newDecrypter(cd, nil, nil)
assert.Nil(t, fh)
if i == 7 { // This test accidentally swaps last byte and converts `fileMagic` (RCLONE\x00\x00") into `fileMagicV2` ("RCLONE\x00\x01") resulting in a different than: "ErrorEncryptedBadMagic" error.
assert.EqualError(t, err, ErrorEncryptedFileTooShort.Error())
@ -1349,7 +1349,7 @@ func TestNewDecrypterErrUnexpectedEOF(t *testing.T) {
in1 := bytes.NewBuffer(file16)
in := io.NopCloser(io.MultiReader(in1, in2))
fh, err := c.newDecrypter(in, nil)
fh, err := c.newDecrypter(in, nil, nil)
assert.NoError(t, err)
n, err := io.CopyN(io.Discard, fh, 1e6)
@ -1423,7 +1423,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
if offset+limit > len(plaintext) {
continue
}
rc, err := c.DecryptDataSeek(context.Background(), open, int64(offset), int64(limit), nil)
rc, err := c.DecryptDataSeek(context.Background(), nil, open, int64(offset), int64(limit), nil)
assert.NoError(t, err)
check(rc, offset, limit)
@ -1431,7 +1431,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
}
// Try decoding it with a single open and lots of seeks
fh, err := c.DecryptDataSeek(context.Background(), open, 0, -1, nil)
fh, err := c.DecryptDataSeek(context.Background(), nil, open, 0, -1, nil)
assert.NoError(t, err)
for _, offset := range trials {
for _, limit := range limits {
@ -1503,7 +1503,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
callCount++
return open(ctx, underlyingOffset, underlyingLimit)
}
fh, err := c.DecryptDataSeek(context.Background(), testOpen, 0, -1, nil)
fh, err := c.DecryptDataSeek(context.Background(), nil, testOpen, 0, -1, nil)
assert.NoError(t, err)
gotOffset, err := fh.RangeSeek(context.Background(), test.offset, io.SeekStart, test.limit)
assert.NoError(t, err)
@ -1571,7 +1571,7 @@ func TestDecrypterRead(t *testing.T) {
for i := 0; i < len(file16)-1; i++ {
what := fmt.Sprintf("truncating to %d/%d", i, len(file16))
cd := newCloseDetector(bytes.NewBuffer(file16[:i]))
fh, err := c.newDecrypter(cd, nil)
fh, err := c.newDecrypter(cd, nil, nil)
if i < fileHeaderSize {
assert.EqualError(t, err, ErrorEncryptedFileTooShort.Error(), what)
continue
@ -1604,7 +1604,7 @@ func TestDecrypterRead(t *testing.T) {
in2 := &readers.ErrorReader{Err: errors.New("potato")}
in := io.MultiReader(in1, in2)
cd := newCloseDetector(in)
fh, err := c.newDecrypter(cd, nil)
fh, err := c.newDecrypter(cd, nil, nil)
assert.NoError(t, err)
_, err = io.ReadAll(fh)
assert.EqualError(t, err, "potato")
@ -1616,7 +1616,7 @@ func TestDecrypterRead(t *testing.T) {
copy(file16copy, file16)
for i := range file16copy {
file16copy[i] ^= 0xFF
fh, err := c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)), nil)
fh, err := c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)), nil, nil)
if i < fileMagicSize {
assert.EqualError(t, err, ErrorEncryptedBadMagic.Error())
assert.Nil(t, fh)
@ -1633,7 +1633,7 @@ func TestDecrypterRead(t *testing.T) {
copy(file16copy, file16)
file16copy[len(file16copy)-1] ^= 0xFF
c.passBadBlocks = true
fh, err = c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)), nil)
fh, err = c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)), nil, nil)
assert.NoError(t, err)
buf, err := io.ReadAll(fh)
assert.NoError(t, err)
@ -1645,7 +1645,7 @@ func TestDecrypterClose(t *testing.T) {
assert.NoError(t, err)
cd := newCloseDetector(bytes.NewBuffer(file16))
fh, err := c.newDecrypter(cd, nil)
fh, err := c.newDecrypter(cd, nil, nil)
assert.NoError(t, err)
assert.Equal(t, 0, cd.closed)
@ -1663,7 +1663,7 @@ func TestDecrypterClose(t *testing.T) {
// try again reading the file this time
cd = newCloseDetector(bytes.NewBuffer(file1))
fh, err = c.newDecrypter(cd, nil)
fh, err = c.newDecrypter(cd, nil, nil)
assert.NoError(t, err)
assert.Equal(t, 0, cd.closed)

19
backend/crypt/crypt.go
View File

@ -916,7 +916,7 @@ func (f *Fs) getDecrypter(ctx context.Context, o *Object, overrideCek *cek) (*de
if err != nil {
return nil, fmt.Errorf("failed to open object to read nonce: %w", err)
}
d, err := f.cipher.newDecrypter(in, overrideCek)
d, err := f.cipher.newDecrypter(in, overrideCek, o)
if err != nil {
_ = in.Close()
return nil, fmt.Errorf("failed to open object to read nonce: %w", err)
@ -1146,6 +1146,7 @@ type Object struct {
fs.Object
f *Fs
decryptedSize int64
cipherVersion string
}
func (f *Fs) newObject(o fs.Object) *Object {
@ -1153,6 +1154,7 @@ func (f *Fs) newObject(o fs.Object) *Object {
Object: o,
f: f,
decryptedSize: -1,
cipherVersion: "",
}
}
@ -1185,10 +1187,18 @@ func (o *Object) Size() int64 {
size := o.Object.Size()
if !o.f.opt.NoDataEncryption {
var err error
if o.f.opt.ExactSize {
if o.f.opt.ExactSize && o.cipherVersion == "" { // Use `ExactSize` setting if cipherVersion isn't explicitly set on the object level. If cipherVersion is explicitly set, we deduce size correctly without reading file header.
size, err = o.f.cipher.DecryptedSizeExact(o)
} else {
size, err = o.f.cipher.DecryptedSize(size, o.f.cipher.version)
var cipherVersion string
if o.cipherVersion != "" { // Explicit cipher version (set by newDecrypter) detected from magic bytes
cipherVersion = o.cipherVersion
} else { // Assume cipher version based on config. Might not be correct if existing object was encrypted using different cipher version than currently configured.
cipherVersion = o.f.cipher.version
}
size, err = o.f.cipher.DecryptedSize(size, cipherVersion)
}
if err != nil {
@ -1296,7 +1306,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
openOptions = append(openOptions, option)
}
}
rc, err = o.f.cipher.DecryptDataSeek(ctx, func(ctx context.Context, underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) {
rc, err = o.f.cipher.DecryptDataSeek(ctx, o, func(ctx context.Context, underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) {
if underlyingOffset == 0 && underlyingLimit < 0 {
// Open with no seek
return o.Object.Open(ctx, openOptions...)