Add options for Open and implement Range for all remotes

This commit is contained in:
Nick Craig-Wood 2016-09-10 11:29:57 +01:00
parent 75e5e59385
commit aef2ac5c04
33 changed files with 547 additions and 78 deletions

View File

@ -786,18 +786,19 @@ func (o *Object) Storable() bool {
} }
// Open an object for read // Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) { func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
bigObject := o.Size() >= int64(tempLinkThreshold) bigObject := o.Size() >= int64(tempLinkThreshold)
if bigObject { if bigObject {
fs.Debug(o, "Dowloading large object via tempLink") fs.Debug(o, "Dowloading large object via tempLink")
} }
file := acd.File{Node: o.info} file := acd.File{Node: o.info}
var resp *http.Response var resp *http.Response
headers := fs.OpenOptionHeaders(options)
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
if !bigObject { if !bigObject {
in, resp, err = file.Open() in, resp, err = file.OpenHeaders(headers)
} else { } else {
in, resp, err = file.OpenTempURL(o.fs.noAuthClient) in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers)
} }
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(resp, err)
}) })

View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

View File

@ -1082,11 +1082,12 @@ func (file *openFile) Close() (err error) {
var _ io.ReadCloser = &openFile{} var _ io.ReadCloser = &openFile{}
// Open an object for read // Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) { func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "GET", Method: "GET",
Absolute: true, Absolute: true,
Path: o.fs.info.DownloadURL, Path: o.fs.info.DownloadURL,
Options: options,
} }
// Download by id if set otherwise by name // Download by id if set otherwise by name
if o.id != "" { if o.id != "" {

View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

View File

@ -355,9 +355,9 @@ func (n *nonce) fromBuf(buf []byte) {
} }
} }
// increment to add 1 to the nonce // carry 1 up the nonce from position i
func (n *nonce) increment() { func (n *nonce) carry(i int) {
for i := 0; i < len(*n); i++ { for ; i < len(*n); i++ {
digit := (*n)[i] digit := (*n)[i]
newDigit := digit + 1 newDigit := digit + 1
(*n)[i] = newDigit (*n)[i] = newDigit
@ -368,6 +368,27 @@ func (n *nonce) increment() {
} }
} }
// increment to add 1 to the nonce
func (n *nonce) increment() {
n.carry(0)
}
// add an uint64 to the nonce
func (n *nonce) add(x uint64) {
carry := uint16(0)
for i := 0; i < 8; i++ {
digit := (*n)[i]
xDigit := byte(x)
x >>= 8
carry += uint16(digit) + uint16(xDigit)
(*n)[i] = byte(carry)
carry >>= 8
}
if carry != 0 {
n.carry(8)
}
}
// encrypter encrypts an io.Reader on the fly // encrypter encrypts an io.Reader on the fly
type encrypter struct { type encrypter struct {
in io.Reader in io.Reader
@ -528,6 +549,17 @@ func (fh *decrypter) Read(p []byte) (n int, err error) {
return n, nil return n, nil
} }
// seek the decryption forwards the amount given
//
// returns an offset for the underlying rc to be seeked and the number
// of bytes to be discarded
func (fh *decrypter) seek(offset int64) (underlyingOffset int64, discard int64) {
blocks, discard := offset/blockDataSize, offset%blockDataSize
underlyingOffset = int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize)
fh.nonce.add(uint64(blocks))
return
}
// finish sets the final error and tidies up // finish sets the final error and tidies up
func (fh *decrypter) finish(err error) error { func (fh *decrypter) finish(err error) error {
if fh.err != nil { if fh.err != nil {

View File

@ -464,6 +464,144 @@ func TestNonceIncrement(t *testing.T) {
} }
} }
func TestNonceAdd(t *testing.T) {
for _, test := range []struct {
add uint64
in nonce
out nonce
}{
{
0x01,
nonce{0x00},
nonce{0x01},
},
{
0xFF,
nonce{0xFF},
nonce{0xFE, 0x01},
},
{
0xFFFF,
nonce{0xFF, 0xFF},
nonce{0xFE, 0xFF, 0x01},
},
{
0xFFFFFF,
nonce{0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0x01},
},
{
0xFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFe, 0xFF, 0xFF, 0xFF, 0x01},
},
{
0xFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0x01},
},
{
0xFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01},
},
{
0xFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
},
} {
x := test.in
x.add(test.add)
assert.Equal(t, test.out, x)
}
}
// randomSource can read or write a random sequence // randomSource can read or write a random sequence
type randomSource struct { type randomSource struct {
counter int64 counter int64

View File

@ -4,6 +4,7 @@ package crypt
import ( import (
"fmt" "fmt"
"io" "io"
"io/ioutil"
"path" "path"
"sync" "sync"
@ -297,12 +298,59 @@ func (o *Object) Hash(hash fs.HashType) (string, error) {
} }
// Open opens the file for read. Call Close() on the returned io.ReadCloser // Open opens the file for read. Call Close() on the returned io.ReadCloser
func (o *Object) Open() (io.ReadCloser, error) { func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
var offset int64
for _, option := range options {
switch x := option.(type) {
case *fs.SeekOption:
offset = x.Offset
default:
if option.Mandatory() {
fs.Log(o, "Unsupported mandatory option: %v", option)
}
}
}
in, err := o.Object.Open() in, err := o.Object.Open()
if err != nil { if err != nil {
return in, err return nil, err
} }
return o.f.cipher.DecryptData(in)
// This reads the header and checks it is OK
rc, err := o.f.cipher.DecryptData(in)
if err != nil {
return nil, err
}
// If seeking required, then...
if offset != 0 {
// FIXME could cache the unseeked decrypter as we re-read the header on every seek
decrypter := rc.(*decrypter)
// Seek the decrypter and work out where to seek the
// underlying file and how many bytes to discard
underlyingOffset, discard := decrypter.seek(offset)
// Re-open stream with a seek of underlyingOffset
err = in.Close()
if err != nil {
return nil, err
}
in, err := o.Object.Open(&fs.SeekOption{Offset: underlyingOffset})
if err != nil {
return nil, err
}
// Update the stream
decrypter.rc = in
// Discard the bytes
_, err = io.CopyN(ioutil.Discard, decrypter, discard)
if err != nil {
return nil, err
}
}
return rc, err
} }
// Update in to the object with the modTime given of the given size // Update in to the object with the modTime given of the given size

View File

@ -51,6 +51,7 @@ func TestObjectMimeType2(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime2(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSetModTime2(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize2(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectSize2(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen2(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectOpen2(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek2(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate2(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectUpdate2(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable2(t *testing.T) { fstests.TestObjectStorable(t) } func TestObjectStorable2(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile2(t *testing.T) { fstests.TestFsIsFile(t) } func TestFsIsFile2(t *testing.T) { fstests.TestFsIsFile(t) }

View File

@ -51,6 +51,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

View File

@ -827,7 +827,7 @@ func (o *Object) Size() int64 {
if o.isDocument && o.bytes < 0 { if o.isDocument && o.bytes < 0 {
// If it is a google doc then we must HEAD it to see // If it is a google doc then we must HEAD it to see
// how big it is // how big it is
res, err := o.httpResponse("HEAD") _, res, err := o.httpResponse("HEAD", nil)
if err != nil { if err != nil {
fs.ErrorLog(o, "Error reading size: %v", err) fs.ErrorLog(o, "Error reading size: %v", err)
return 0 return 0
@ -929,22 +929,23 @@ func (o *Object) Storable() bool {
// httpResponse gets an http.Response object for the object o.url // httpResponse gets an http.Response object for the object o.url
// using the method passed in // using the method passed in
func (o *Object) httpResponse(method string) (res *http.Response, err error) { func (o *Object) httpResponse(method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) {
if o.url == "" { if o.url == "" {
return nil, errors.New("forbidden to download - check sharing permission") return nil, nil, errors.New("forbidden to download - check sharing permission")
} }
req, err := http.NewRequest(method, o.url, nil) req, err = http.NewRequest(method, o.url, nil)
if err != nil { if err != nil {
return nil, err return req, nil, err
} }
fs.OpenOptionAddHTTPHeaders(req.Header, options)
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
res, err = o.fs.client.Do(req) res, err = o.fs.client.Do(req)
return shouldRetry(err) return shouldRetry(err)
}) })
if err != nil { if err != nil {
return nil, err return req, nil, err
} }
return res, nil return req, res, nil
} }
// openFile represents an Object open for reading // openFile represents an Object open for reading
@ -979,12 +980,13 @@ func (file *openFile) Close() (err error) {
var _ io.ReadCloser = &openFile{} var _ io.ReadCloser = &openFile{}
// Open an object for read // Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) { func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
res, err := o.httpResponse("GET") req, res, err := o.httpResponse("GET", options)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if res.StatusCode != 200 { _, isRanging := req.Header["Range"]
if !(res.StatusCode == http.StatusOK || (isRanging && res.StatusCode == http.StatusPartialContent)) {
_ = res.Body.Close() // ignore error _ = res.Body.Close() // ignore error
return nil, errors.Errorf("bad response: %d: %s", res.StatusCode, res.Status) return nil, errors.Errorf("bad response: %d: %s", res.StatusCode, res.Status)
} }

View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

View File

@ -710,8 +710,21 @@ func (o *Object) Storable() bool {
} }
// Open an object for read // Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) { func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
in, _, err = o.fs.db.Download(o.remotePath(), "", 0) // FIXME should send a patch for dropbox module which allow setting headers
var offset int64
for _, option := range options {
switch x := option.(type) {
case *fs.SeekOption:
offset = x.Offset
default:
if option.Mandatory() {
fs.Log(o, "Unsupported mandatory option: %v", option)
}
}
}
in, _, err = o.fs.db.Download(o.remotePath(), "", offset)
if dropboxErr, ok := err.(*dropbox.Error); ok { if dropboxErr, ok := err.(*dropbox.Error); ok {
// Dropbox return 461 for copyright violation so don't // Dropbox return 461 for copyright violation so don't
// attempt to retry this error // attempt to retry this error

View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

View File

@ -172,7 +172,7 @@ type Object interface {
SetModTime(time.Time) error SetModTime(time.Time) error
// Open opens the file for read. Call Close() on the returned io.ReadCloser // Open opens the file for read. Call Close() on the returned io.ReadCloser
Open() (io.ReadCloser, error) Open(options ...OpenOption) (io.ReadCloser, error)
// Update in to the object with the modTime given of the given size // Update in to the object with the modTime given of the given size
Update(in io.Reader, src ObjectInfo) error Update(in io.Reader, src ObjectInfo) error

View File

@ -29,7 +29,7 @@ func (o mockObject) ModTime() (t time.Time) { return t }
func (o mockObject) Size() int64 { return 0 } func (o mockObject) Size() int64 { return 0 }
func (o mockObject) Storable() bool { return true } func (o mockObject) Storable() bool { return true }
func (o mockObject) SetModTime(time.Time) error { return errNotImpl } func (o mockObject) SetModTime(time.Time) error { return errNotImpl }
func (o mockObject) Open() (io.ReadCloser, error) { return nil, errNotImpl } func (o mockObject) Open(options ...OpenOption) (io.ReadCloser, error) { return nil, errNotImpl }
func (o mockObject) Update(in io.Reader, src ObjectInfo) error { return errNotImpl } func (o mockObject) Update(in io.Reader, src ObjectInfo) error { return errNotImpl }
func (o mockObject) Remove() error { return errNotImpl } func (o mockObject) Remove() error { return errNotImpl }

137
fs/options.go Normal file
View File

@ -0,0 +1,137 @@
// Define the options for Open
package fs
import (
"fmt"
"net/http"
"strconv"
)
// OpenOption is an interface describing options for Open
type OpenOption interface {
fmt.Stringer
// Header returns the option as an HTTP header
Header() (key string, value string)
// Mandatory returns whether this option can be ignored or not
Mandatory() bool
}
// RangeOption defines an HTTP Range option with start and end. If
// either start or end are < 0 then they will be omitted.
type RangeOption struct {
Start int64
End int64
}
// Header formats the option as an http header
func (o *RangeOption) Header() (key string, value string) {
key = "Range"
value = "bytes="
if o.Start >= 0 {
value += strconv.FormatInt(o.Start, 64)
}
value += "-"
if o.End >= 0 {
value += strconv.FormatInt(o.End, 64)
}
return key, value
}
// String formats the option into human readable form
func (o *RangeOption) String() string {
return fmt.Sprintf("RangeOption(%d,%d)", o.Start, o.End)
}
// Mandatory returns whether the option must be parsed or can be ignored
func (o *RangeOption) Mandatory() bool {
return false
}
// SeekOption defines an HTTP Range option with start only.
type SeekOption struct {
Offset int64
}
// Header formats the option as an http header
func (o *SeekOption) Header() (key string, value string) {
key = "Range"
value = fmt.Sprintf("bytes=%d-", o.Offset)
return key, value
}
// String formats the option into human readable form
func (o *SeekOption) String() string {
return fmt.Sprintf("SeekOption(%d)", o.Offset)
}
// Mandatory returns whether the option must be parsed or can be ignored
func (o *SeekOption) Mandatory() bool {
return true
}
// HTTPOption defines a general purpose HTTP option
type HTTPOption struct {
Key string
Value string
}
// Header formats the option as an http header
func (o *HTTPOption) Header() (key string, value string) {
return o.Key, o.Value
}
// String formats the option into human readable form
func (o *HTTPOption) String() string {
return fmt.Sprintf("HTTPOption(%q,%q)", o.Key, o.Value)
}
// Mandatory returns whether the option must be parsed or can be ignored
func (o *HTTPOption) Mandatory() bool {
return false
}
// OpenOptionAddHeaders adds each header found in options to the
// headers map provided the key was non empty.
func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) {
for _, option := range options {
key, value := option.Header()
if key != "" && value != "" {
headers[key] = value
}
}
}
// OpenOptionHeaders adds each header found in options to the
// headers map provided the key was non empty.
//
// It returns a nil map if options was empty
func OpenOptionHeaders(options []OpenOption) (headers map[string]string) {
if len(options) == 0 {
return nil
}
headers = make(map[string]string, len(options))
OpenOptionAddHeaders(options, headers)
return headers
}
// OpenOptionAddHTTPHeaders Sets each header found in options to the
// http.Header map provided the key was non empty.
func OpenOptionAddHTTPHeaders(headers http.Header, options []OpenOption) {
for _, option := range options {
key, value := option.Header()
if key != "" && value != "" {
headers.Set(key, value)
}
}
}
// check interface
var (
_ OpenOption = (*RangeOption)(nil)
_ OpenOption = (*SeekOption)(nil)
_ OpenOption = (*HTTPOption)(nil)
)

View File

@ -10,6 +10,7 @@ import (
"flag" "flag"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"os" "os"
"path" "path"
"sort" "sort"
@ -37,11 +38,13 @@ var (
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
Path: "file name.txt", Path: "file name.txt",
} }
file1Contents = ""
file2 = fstest.Item{ file2 = fstest.Item{
ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"), ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`, Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`,
WinPath: `hello_ sausage/êé/Hello, 世界/ _ ' @ _ _ & _ + ≠/z.txt`, WinPath: `hello_ sausage/êé/Hello, 世界/ _ ' @ _ _ & _ + ≠/z.txt`,
} }
file2Contents = ""
verbose = flag.Bool("verbose", false, "Set to enable logging") verbose = flag.Bool("verbose", false, "Set to enable logging")
dumpHeaders = flag.Bool("dump-headers", false, "Dump HTTP headers - may contain sensitive info") dumpHeaders = flag.Bool("dump-headers", false, "Dump HTTP headers - may contain sensitive info")
dumpBodies = flag.Bool("dump-bodies", false, "Dump HTTP headers and bodies - may contain sensitive info") dumpBodies = flag.Bool("dump-bodies", false, "Dump HTTP headers and bodies - may contain sensitive info")
@ -195,9 +198,10 @@ func findObject(t *testing.T, Name string) fs.Object {
return obj return obj
} }
func testPut(t *testing.T, file *fstest.Item) { func testPut(t *testing.T, file *fstest.Item) string {
again: again:
buf := bytes.NewBufferString(fstest.RandomString(100)) contents := fstest.RandomString(100)
buf := bytes.NewBufferString(contents)
hash := fs.NewMultiHasher() hash := fs.NewMultiHasher()
in := io.TeeReader(buf, hash) in := io.TeeReader(buf, hash)
@ -222,24 +226,25 @@ again:
// Re-read the object and check again // Re-read the object and check again
obj = findObject(t, file.Path) obj = findObject(t, file.Path)
file.Check(t, obj, remote.Precision()) file.Check(t, obj, remote.Precision())
return contents
} }
// TestFsPutFile1 tests putting a file // TestFsPutFile1 tests putting a file
func TestFsPutFile1(t *testing.T) { func TestFsPutFile1(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
testPut(t, &file1) file1Contents = testPut(t, &file1)
} }
// TestFsPutFile2 tests putting a file into a subdirectory // TestFsPutFile2 tests putting a file into a subdirectory
func TestFsPutFile2(t *testing.T) { func TestFsPutFile2(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
testPut(t, &file2) file2Contents = testPut(t, &file2)
} }
// TestFsUpdateFile1 tests updating file1 with new contents // TestFsUpdateFile1 tests updating file1 with new contents
func TestFsUpdateFile1(t *testing.T) { func TestFsUpdateFile1(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
testPut(t, &file1) file1Contents = testPut(t, &file1)
// Note that the next test will check there are no duplicated file names // Note that the next test will check there are no duplicated file names
} }
@ -541,42 +546,56 @@ func TestObjectSize(t *testing.T) {
assert.Equal(t, file1.Size, obj.Size()) assert.Equal(t, file1.Size, obj.Size())
} }
// read the contents of an object as a string
func readObject(t *testing.T, obj fs.Object, options ...fs.OpenOption) string {
in, err := obj.Open(options...)
require.NoError(t, err)
contents, err := ioutil.ReadAll(in)
require.NoError(t, err)
err = in.Close()
require.NoError(t, err)
return string(contents)
}
// TestObjectOpen tests that Open works // TestObjectOpen tests that Open works
func TestObjectOpen(t *testing.T) { func TestObjectOpen(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
in, err := obj.Open() assert.Equal(t, file1Contents, readObject(t, obj), "contents of file1 differ")
require.NoError(t, err)
hasher := fs.NewMultiHasher()
n, err := io.Copy(hasher, in)
require.NoError(t, err, fmt.Sprintf("hasher copy error: %v", err))
require.Equal(t, file1.Size, n, "Read wrong number of bytes")
err = in.Close()
require.NoError(t, err)
// Check content of file by comparing the calculated hashes
for hashType, got := range hasher.Sums() {
assert.Equal(t, file1.Hashes[hashType], got)
} }
// TestObjectOpenSeek tests that Open works with Seek
func TestObjectOpenSeek(t *testing.T) {
skipIfNotOk(t)
obj := findObject(t, file1.Path)
assert.Equal(t, file1Contents[50:], readObject(t, obj, &fs.SeekOption{Offset: 50}), "contents of file1 differ after seek")
} }
// TestObjectUpdate tests that Update works // TestObjectUpdate tests that Update works
func TestObjectUpdate(t *testing.T) { func TestObjectUpdate(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
buf := bytes.NewBufferString(fstest.RandomString(200)) contents := fstest.RandomString(200)
buf := bytes.NewBufferString(contents)
hash := fs.NewMultiHasher() hash := fs.NewMultiHasher()
in := io.TeeReader(buf, hash) in := io.TeeReader(buf, hash)
file1.Size = int64(buf.Len()) file1.Size = int64(buf.Len())
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
obji := fs.NewStaticObjectInfo(file1.Path, file1.ModTime, file1.Size, true, nil, obj.Fs()) obji := fs.NewStaticObjectInfo(file1.Path, file1.ModTime, int64(len(contents)), true, nil, obj.Fs())
err := obj.Update(in, obji) err := obj.Update(in, obji)
require.NoError(t, err) require.NoError(t, err)
file1.Hashes = hash.Sums() file1.Hashes = hash.Sums()
// check the object has been updated
file1.Check(t, obj, remote.Precision()) file1.Check(t, obj, remote.Precision())
// Re-read the object and check again // Re-read the object and check again
obj = findObject(t, file1.Path) obj = findObject(t, file1.Path)
file1.Check(t, obj, remote.Precision()) file1.Check(t, obj, remote.Precision())
// check contents correct
assert.Equal(t, contents, readObject(t, obj), "contents of updated file1 differ")
file1Contents = contents
} }
// TestObjectStorable tests that Storable works // TestObjectStorable tests that Storable works

View File

@ -651,16 +651,18 @@ func (o *Object) Storable() bool {
} }
// Open an object for read // Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) { func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
req, err := http.NewRequest("GET", o.url, nil) req, err := http.NewRequest("GET", o.url, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
fs.OpenOptionAddHTTPHeaders(req.Header, options)
res, err := o.fs.client.Do(req) res, err := o.fs.client.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if res.StatusCode != 200 { _, isRanging := req.Header["Range"]
if !(res.StatusCode == http.StatusOK || (isRanging && res.StatusCode == http.StatusPartialContent)) {
_ = res.Body.Close() // ignore error _ = res.Body.Close() // ignore error
return nil, errors.Errorf("bad response: %d: %s", res.StatusCode, res.Status) return nil, errors.Errorf("bad response: %d: %s", res.StatusCode, res.Status)
} }

View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

View File

@ -585,18 +585,36 @@ func (file *localOpenFile) Close() (err error) {
} }
// Open an object for read // Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) { func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
in, err = os.Open(o.path) var offset int64
for _, option := range options {
switch x := option.(type) {
case *fs.SeekOption:
offset = x.Offset
default:
if option.Mandatory() {
fs.Log(o, "Unsupported mandatory option: %v", option)
}
}
}
fd, err := os.Open(o.path)
if err != nil { if err != nil {
return return
} }
if offset != 0 {
// seek the object
_, err = fd.Seek(offset, 0)
// don't attempt to make checksums
return fd, err
}
// Update the md5sum as we go along // Update the md5sum as we go along
in = &localOpenFile{ in = &localOpenFile{
o: o, o: o,
in: in, in: fd,
hash: fs.NewMultiHasher(), hash: fs.NewMultiHasher(),
} }
return return in, nil
} }
// mkdirAll makes all the directories needed to store the object // mkdirAll makes all the directories needed to store the object

View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

View File

@ -775,7 +775,7 @@ func (o *Object) Storable() bool {
} }
// Open an object for read // Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) { func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
if o.id == "" { if o.id == "" {
return nil, errors.New("can't download - no id") return nil, errors.New("can't download - no id")
} }
@ -783,6 +783,7 @@ func (o *Object) Open() (in io.ReadCloser, err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "GET", Method: "GET",
Path: "/drive/items/" + o.id + "/content", Path: "/drive/items/" + o.id + "/content",
Options: options,
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(&opts)

View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

View File

@ -83,6 +83,7 @@ type Opts struct {
ExtraHeaders map[string]string ExtraHeaders map[string]string
UserName string // username for Basic Auth UserName string // username for Basic Auth
Password string // password for Basic Auth Password string // password for Basic Auth
Options []fs.OpenOption
} }
// DecodeJSON decodes resp.Body into result // DecodeJSON decodes resp.Body into result
@ -92,6 +93,27 @@ func DecodeJSON(resp *http.Response, result interface{}) (err error) {
return decoder.Decode(result) return decoder.Decode(result)
} }
// Make a new http client which resets the headers passed in on redirect
func clientWithHeaderReset(c *http.Client, headers map[string]string) *http.Client {
if len(headers) == 0 {
return c
}
clientCopy := *c
clientCopy.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
// Reset the headers in the new request
for k, v := range headers {
if v != "" {
req.Header.Add(k, v)
}
}
return nil
}
return &clientCopy
}
// Call makes the call and returns the http.Response // Call makes the call and returns the http.Response
// //
// if err != nil then resp.Body will need to be closed // if err != nil then resp.Body will need to be closed
@ -136,6 +158,8 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) {
headers[k] = v headers[k] = v
} }
} }
// add any options to the headers
fs.OpenOptionAddHeaders(opts.Options, headers)
// Now set the headers // Now set the headers
for k, v := range headers { for k, v := range headers {
if v != "" { if v != "" {
@ -145,8 +169,9 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) {
if opts.UserName != "" || opts.Password != "" { if opts.UserName != "" || opts.Password != "" {
req.SetBasicAuth(opts.UserName, opts.Password) req.SetBasicAuth(opts.UserName, opts.Password)
} }
c := clientWithHeaderReset(api.c, headers)
api.mu.RUnlock() api.mu.RUnlock()
resp, err = api.c.Do(req) resp, err = c.Do(req)
api.mu.RLock() api.mu.RLock()
if err != nil { if err != nil {
return nil, err return nil, err

View File

@ -845,12 +845,23 @@ func (o *Object) Storable() bool {
} }
// Open an object for read // Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) { func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
key := o.fs.root + o.remote key := o.fs.root + o.remote
req := s3.GetObjectInput{ req := s3.GetObjectInput{
Bucket: &o.fs.bucket, Bucket: &o.fs.bucket,
Key: &key, Key: &key,
} }
for _, option := range options {
switch option.(type) {
case *fs.RangeOption, *fs.SeekOption:
_, value := option.Header()
req.Range = &value
default:
if option.Mandatory() {
fs.Log(o, "Unsupported mandatory option: %v", option)
}
}
}
resp, err := o.fs.c.GetObject(&req) resp, err := o.fs.c.GetObject(&req)
if err != nil { if err != nil {
return nil, err return nil, err

View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

View File

@ -629,8 +629,10 @@ func (o *Object) Storable() bool {
} }
// Open an object for read // Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) { func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, true, nil) headers := fs.OpenOptionHeaders(options)
_, isRanging := headers["Range"]
in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, !isRanging, headers)
return return
} }

View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

View File

@ -13,13 +13,13 @@ type DownloadResponse struct {
Templated bool `json:"templated"` Templated bool `json:"templated"`
} }
// Download will get specified data from Yandex.Disk. // Download will get specified data from Yandex.Disk supplying the extra headers
func (c *Client) Download(remotePath string) (io.ReadCloser, error) { //io.Writer func (c *Client) Download(remotePath string, headers map[string]string) (io.ReadCloser, error) { //io.Writer
ur, err := c.DownloadRequest(remotePath) ur, err := c.DownloadRequest(remotePath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return c.PerformDownload(ur.HRef) return c.PerformDownload(ur.HRef, headers)
} }
// DownloadRequest will make an download request and return a URL to download data to. // DownloadRequest will make an download request and return a URL to download data to.

View File

@ -8,13 +8,18 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
) )
// PerformDownload does the actual download via unscoped PUT request. // PerformDownload does the actual download via unscoped GET request.
func (c *Client) PerformDownload(url string) (out io.ReadCloser, err error) { func (c *Client) PerformDownload(url string, headers map[string]string) (out io.ReadCloser, err error) {
req, err := http.NewRequest("GET", url, nil) req, err := http.NewRequest("GET", url, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// Set any extra headers
for k, v := range headers {
req.Header.Set(k, v)
}
//c.setRequestScope(req) //c.setRequestScope(req)
resp, err := c.HTTPClient.Do(req) resp, err := c.HTTPClient.Do(req)
@ -22,7 +27,8 @@ func (c *Client) PerformDownload(url string) (out io.ReadCloser, err error) {
return nil, err return nil, err
} }
if resp.StatusCode != 200 { _, isRanging := req.Header["Range"]
if !(resp.StatusCode == http.StatusOK || (isRanging && resp.StatusCode == http.StatusPartialContent)) {
defer CheckClose(resp.Body, &err) defer CheckClose(resp.Body, &err)
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {

View File

@ -487,8 +487,8 @@ func (o *Object) ModTime() time.Time {
} }
// Open an object for read // Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) { func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
return o.fs.yd.Download(o.remotePath()) return o.fs.yd.Download(o.remotePath(), fs.OpenOptionHeaders(options))
} }
// Remove an object // Remove an object

View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }