mirror of
https://github.com/rclone/rclone.git
synced 2024-11-25 18:04:55 +01:00
Add options for Open and implement Range for all remotes
This commit is contained in:
parent
75e5e59385
commit
aef2ac5c04
@ -786,18 +786,19 @@ func (o *Object) Storable() bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Open an object for read
|
// Open an object for read
|
||||||
func (o *Object) Open() (in io.ReadCloser, err error) {
|
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||||
bigObject := o.Size() >= int64(tempLinkThreshold)
|
bigObject := o.Size() >= int64(tempLinkThreshold)
|
||||||
if bigObject {
|
if bigObject {
|
||||||
fs.Debug(o, "Dowloading large object via tempLink")
|
fs.Debug(o, "Dowloading large object via tempLink")
|
||||||
}
|
}
|
||||||
file := acd.File{Node: o.info}
|
file := acd.File{Node: o.info}
|
||||||
var resp *http.Response
|
var resp *http.Response
|
||||||
|
headers := fs.OpenOptionHeaders(options)
|
||||||
err = o.fs.pacer.Call(func() (bool, error) {
|
err = o.fs.pacer.Call(func() (bool, error) {
|
||||||
if !bigObject {
|
if !bigObject {
|
||||||
in, resp, err = file.Open()
|
in, resp, err = file.OpenHeaders(headers)
|
||||||
} else {
|
} else {
|
||||||
in, resp, err = file.OpenTempURL(o.fs.noAuthClient)
|
in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers)
|
||||||
}
|
}
|
||||||
return o.fs.shouldRetry(resp, err)
|
return o.fs.shouldRetry(resp, err)
|
||||||
})
|
})
|
||||||
|
@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
|
|||||||
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
||||||
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
|
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
|
3
b2/b2.go
3
b2/b2.go
@ -1082,11 +1082,12 @@ func (file *openFile) Close() (err error) {
|
|||||||
var _ io.ReadCloser = &openFile{}
|
var _ io.ReadCloser = &openFile{}
|
||||||
|
|
||||||
// Open an object for read
|
// Open an object for read
|
||||||
func (o *Object) Open() (in io.ReadCloser, err error) {
|
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||||
opts := rest.Opts{
|
opts := rest.Opts{
|
||||||
Method: "GET",
|
Method: "GET",
|
||||||
Absolute: true,
|
Absolute: true,
|
||||||
Path: o.fs.info.DownloadURL,
|
Path: o.fs.info.DownloadURL,
|
||||||
|
Options: options,
|
||||||
}
|
}
|
||||||
// Download by id if set otherwise by name
|
// Download by id if set otherwise by name
|
||||||
if o.id != "" {
|
if o.id != "" {
|
||||||
|
@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
|
|||||||
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
||||||
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
|
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
|
@ -355,9 +355,9 @@ func (n *nonce) fromBuf(buf []byte) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// increment to add 1 to the nonce
|
// carry 1 up the nonce from position i
|
||||||
func (n *nonce) increment() {
|
func (n *nonce) carry(i int) {
|
||||||
for i := 0; i < len(*n); i++ {
|
for ; i < len(*n); i++ {
|
||||||
digit := (*n)[i]
|
digit := (*n)[i]
|
||||||
newDigit := digit + 1
|
newDigit := digit + 1
|
||||||
(*n)[i] = newDigit
|
(*n)[i] = newDigit
|
||||||
@ -368,6 +368,27 @@ func (n *nonce) increment() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// increment to add 1 to the nonce
|
||||||
|
func (n *nonce) increment() {
|
||||||
|
n.carry(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
// add an uint64 to the nonce
|
||||||
|
func (n *nonce) add(x uint64) {
|
||||||
|
carry := uint16(0)
|
||||||
|
for i := 0; i < 8; i++ {
|
||||||
|
digit := (*n)[i]
|
||||||
|
xDigit := byte(x)
|
||||||
|
x >>= 8
|
||||||
|
carry += uint16(digit) + uint16(xDigit)
|
||||||
|
(*n)[i] = byte(carry)
|
||||||
|
carry >>= 8
|
||||||
|
}
|
||||||
|
if carry != 0 {
|
||||||
|
n.carry(8)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// encrypter encrypts an io.Reader on the fly
|
// encrypter encrypts an io.Reader on the fly
|
||||||
type encrypter struct {
|
type encrypter struct {
|
||||||
in io.Reader
|
in io.Reader
|
||||||
@ -528,6 +549,17 @@ func (fh *decrypter) Read(p []byte) (n int, err error) {
|
|||||||
return n, nil
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// seek the decryption forwards the amount given
|
||||||
|
//
|
||||||
|
// returns an offset for the underlying rc to be seeked and the number
|
||||||
|
// of bytes to be discarded
|
||||||
|
func (fh *decrypter) seek(offset int64) (underlyingOffset int64, discard int64) {
|
||||||
|
blocks, discard := offset/blockDataSize, offset%blockDataSize
|
||||||
|
underlyingOffset = int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize)
|
||||||
|
fh.nonce.add(uint64(blocks))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// finish sets the final error and tidies up
|
// finish sets the final error and tidies up
|
||||||
func (fh *decrypter) finish(err error) error {
|
func (fh *decrypter) finish(err error) error {
|
||||||
if fh.err != nil {
|
if fh.err != nil {
|
||||||
|
@ -464,6 +464,144 @@ func TestNonceIncrement(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNonceAdd(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
add uint64
|
||||||
|
in nonce
|
||||||
|
out nonce
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
0x01,
|
||||||
|
nonce{0x00},
|
||||||
|
nonce{0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFF,
|
||||||
|
nonce{0xFF},
|
||||||
|
nonce{0xFE, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFF,
|
||||||
|
nonce{0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFe, 0xFF, 0xFF, 0xFF, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
0xFFFFFFFFFFFFFFFF,
|
||||||
|
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
|
||||||
|
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
x := test.in
|
||||||
|
x.add(test.add)
|
||||||
|
assert.Equal(t, test.out, x)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// randomSource can read or write a random sequence
|
// randomSource can read or write a random sequence
|
||||||
type randomSource struct {
|
type randomSource struct {
|
||||||
counter int64
|
counter int64
|
||||||
|
@ -4,6 +4,7 @@ package crypt
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
"path"
|
"path"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
@ -297,12 +298,59 @@ func (o *Object) Hash(hash fs.HashType) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
||||||
func (o *Object) Open() (io.ReadCloser, error) {
|
func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
|
||||||
|
var offset int64
|
||||||
|
for _, option := range options {
|
||||||
|
switch x := option.(type) {
|
||||||
|
case *fs.SeekOption:
|
||||||
|
offset = x.Offset
|
||||||
|
default:
|
||||||
|
if option.Mandatory() {
|
||||||
|
fs.Log(o, "Unsupported mandatory option: %v", option)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
in, err := o.Object.Open()
|
in, err := o.Object.Open()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return in, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return o.f.cipher.DecryptData(in)
|
|
||||||
|
// This reads the header and checks it is OK
|
||||||
|
rc, err := o.f.cipher.DecryptData(in)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// If seeking required, then...
|
||||||
|
if offset != 0 {
|
||||||
|
// FIXME could cache the unseeked decrypter as we re-read the header on every seek
|
||||||
|
decrypter := rc.(*decrypter)
|
||||||
|
|
||||||
|
// Seek the decrypter and work out where to seek the
|
||||||
|
// underlying file and how many bytes to discard
|
||||||
|
underlyingOffset, discard := decrypter.seek(offset)
|
||||||
|
|
||||||
|
// Re-open stream with a seek of underlyingOffset
|
||||||
|
err = in.Close()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
in, err := o.Object.Open(&fs.SeekOption{Offset: underlyingOffset})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update the stream
|
||||||
|
decrypter.rc = in
|
||||||
|
|
||||||
|
// Discard the bytes
|
||||||
|
_, err = io.CopyN(ioutil.Discard, decrypter, discard)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return rc, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update in to the object with the modTime given of the given size
|
// Update in to the object with the modTime given of the given size
|
||||||
|
@ -51,6 +51,7 @@ func TestObjectMimeType2(t *testing.T) { fstests.TestObjectMimeType(t) }
|
|||||||
func TestObjectSetModTime2(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
func TestObjectSetModTime2(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
||||||
func TestObjectSize2(t *testing.T) { fstests.TestObjectSize(t) }
|
func TestObjectSize2(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen2(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen2(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
|
func TestObjectOpenSeek2(t *testing.T) { fstests.TestObjectOpenSeek(t) }
|
||||||
func TestObjectUpdate2(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate2(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable2(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable2(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestFsIsFile2(t *testing.T) { fstests.TestFsIsFile(t) }
|
func TestFsIsFile2(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
|
@ -51,6 +51,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
|
|||||||
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
||||||
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
|
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
|
@ -827,7 +827,7 @@ func (o *Object) Size() int64 {
|
|||||||
if o.isDocument && o.bytes < 0 {
|
if o.isDocument && o.bytes < 0 {
|
||||||
// If it is a google doc then we must HEAD it to see
|
// If it is a google doc then we must HEAD it to see
|
||||||
// how big it is
|
// how big it is
|
||||||
res, err := o.httpResponse("HEAD")
|
_, res, err := o.httpResponse("HEAD", nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fs.ErrorLog(o, "Error reading size: %v", err)
|
fs.ErrorLog(o, "Error reading size: %v", err)
|
||||||
return 0
|
return 0
|
||||||
@ -929,22 +929,23 @@ func (o *Object) Storable() bool {
|
|||||||
|
|
||||||
// httpResponse gets an http.Response object for the object o.url
|
// httpResponse gets an http.Response object for the object o.url
|
||||||
// using the method passed in
|
// using the method passed in
|
||||||
func (o *Object) httpResponse(method string) (res *http.Response, err error) {
|
func (o *Object) httpResponse(method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) {
|
||||||
if o.url == "" {
|
if o.url == "" {
|
||||||
return nil, errors.New("forbidden to download - check sharing permission")
|
return nil, nil, errors.New("forbidden to download - check sharing permission")
|
||||||
}
|
}
|
||||||
req, err := http.NewRequest(method, o.url, nil)
|
req, err = http.NewRequest(method, o.url, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return req, nil, err
|
||||||
}
|
}
|
||||||
|
fs.OpenOptionAddHTTPHeaders(req.Header, options)
|
||||||
err = o.fs.pacer.Call(func() (bool, error) {
|
err = o.fs.pacer.Call(func() (bool, error) {
|
||||||
res, err = o.fs.client.Do(req)
|
res, err = o.fs.client.Do(req)
|
||||||
return shouldRetry(err)
|
return shouldRetry(err)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return req, nil, err
|
||||||
}
|
}
|
||||||
return res, nil
|
return req, res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// openFile represents an Object open for reading
|
// openFile represents an Object open for reading
|
||||||
@ -979,12 +980,13 @@ func (file *openFile) Close() (err error) {
|
|||||||
var _ io.ReadCloser = &openFile{}
|
var _ io.ReadCloser = &openFile{}
|
||||||
|
|
||||||
// Open an object for read
|
// Open an object for read
|
||||||
func (o *Object) Open() (in io.ReadCloser, err error) {
|
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||||
res, err := o.httpResponse("GET")
|
req, res, err := o.httpResponse("GET", options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if res.StatusCode != 200 {
|
_, isRanging := req.Header["Range"]
|
||||||
|
if !(res.StatusCode == http.StatusOK || (isRanging && res.StatusCode == http.StatusPartialContent)) {
|
||||||
_ = res.Body.Close() // ignore error
|
_ = res.Body.Close() // ignore error
|
||||||
return nil, errors.Errorf("bad response: %d: %s", res.StatusCode, res.Status)
|
return nil, errors.Errorf("bad response: %d: %s", res.StatusCode, res.Status)
|
||||||
}
|
}
|
||||||
|
@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
|
|||||||
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
||||||
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
|
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
|
@ -710,8 +710,21 @@ func (o *Object) Storable() bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Open an object for read
|
// Open an object for read
|
||||||
func (o *Object) Open() (in io.ReadCloser, err error) {
|
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||||
in, _, err = o.fs.db.Download(o.remotePath(), "", 0)
|
// FIXME should send a patch for dropbox module which allow setting headers
|
||||||
|
var offset int64
|
||||||
|
for _, option := range options {
|
||||||
|
switch x := option.(type) {
|
||||||
|
case *fs.SeekOption:
|
||||||
|
offset = x.Offset
|
||||||
|
default:
|
||||||
|
if option.Mandatory() {
|
||||||
|
fs.Log(o, "Unsupported mandatory option: %v", option)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
in, _, err = o.fs.db.Download(o.remotePath(), "", offset)
|
||||||
if dropboxErr, ok := err.(*dropbox.Error); ok {
|
if dropboxErr, ok := err.(*dropbox.Error); ok {
|
||||||
// Dropbox return 461 for copyright violation so don't
|
// Dropbox return 461 for copyright violation so don't
|
||||||
// attempt to retry this error
|
// attempt to retry this error
|
||||||
|
@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
|
|||||||
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
||||||
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
|
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
|
2
fs/fs.go
2
fs/fs.go
@ -172,7 +172,7 @@ type Object interface {
|
|||||||
SetModTime(time.Time) error
|
SetModTime(time.Time) error
|
||||||
|
|
||||||
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
||||||
Open() (io.ReadCloser, error)
|
Open(options ...OpenOption) (io.ReadCloser, error)
|
||||||
|
|
||||||
// Update in to the object with the modTime given of the given size
|
// Update in to the object with the modTime given of the given size
|
||||||
Update(in io.Reader, src ObjectInfo) error
|
Update(in io.Reader, src ObjectInfo) error
|
||||||
|
@ -29,7 +29,7 @@ func (o mockObject) ModTime() (t time.Time) { return t }
|
|||||||
func (o mockObject) Size() int64 { return 0 }
|
func (o mockObject) Size() int64 { return 0 }
|
||||||
func (o mockObject) Storable() bool { return true }
|
func (o mockObject) Storable() bool { return true }
|
||||||
func (o mockObject) SetModTime(time.Time) error { return errNotImpl }
|
func (o mockObject) SetModTime(time.Time) error { return errNotImpl }
|
||||||
func (o mockObject) Open() (io.ReadCloser, error) { return nil, errNotImpl }
|
func (o mockObject) Open(options ...OpenOption) (io.ReadCloser, error) { return nil, errNotImpl }
|
||||||
func (o mockObject) Update(in io.Reader, src ObjectInfo) error { return errNotImpl }
|
func (o mockObject) Update(in io.Reader, src ObjectInfo) error { return errNotImpl }
|
||||||
func (o mockObject) Remove() error { return errNotImpl }
|
func (o mockObject) Remove() error { return errNotImpl }
|
||||||
|
|
||||||
|
137
fs/options.go
Normal file
137
fs/options.go
Normal file
@ -0,0 +1,137 @@
|
|||||||
|
// Define the options for Open
|
||||||
|
|
||||||
|
package fs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
)
|
||||||
|
|
||||||
|
// OpenOption is an interface describing options for Open
|
||||||
|
type OpenOption interface {
|
||||||
|
fmt.Stringer
|
||||||
|
|
||||||
|
// Header returns the option as an HTTP header
|
||||||
|
Header() (key string, value string)
|
||||||
|
|
||||||
|
// Mandatory returns whether this option can be ignored or not
|
||||||
|
Mandatory() bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// RangeOption defines an HTTP Range option with start and end. If
|
||||||
|
// either start or end are < 0 then they will be omitted.
|
||||||
|
type RangeOption struct {
|
||||||
|
Start int64
|
||||||
|
End int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Header formats the option as an http header
|
||||||
|
func (o *RangeOption) Header() (key string, value string) {
|
||||||
|
key = "Range"
|
||||||
|
value = "bytes="
|
||||||
|
if o.Start >= 0 {
|
||||||
|
value += strconv.FormatInt(o.Start, 64)
|
||||||
|
|
||||||
|
}
|
||||||
|
value += "-"
|
||||||
|
if o.End >= 0 {
|
||||||
|
value += strconv.FormatInt(o.End, 64)
|
||||||
|
}
|
||||||
|
return key, value
|
||||||
|
}
|
||||||
|
|
||||||
|
// String formats the option into human readable form
|
||||||
|
func (o *RangeOption) String() string {
|
||||||
|
return fmt.Sprintf("RangeOption(%d,%d)", o.Start, o.End)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mandatory returns whether the option must be parsed or can be ignored
|
||||||
|
func (o *RangeOption) Mandatory() bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// SeekOption defines an HTTP Range option with start only.
|
||||||
|
type SeekOption struct {
|
||||||
|
Offset int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Header formats the option as an http header
|
||||||
|
func (o *SeekOption) Header() (key string, value string) {
|
||||||
|
key = "Range"
|
||||||
|
value = fmt.Sprintf("bytes=%d-", o.Offset)
|
||||||
|
return key, value
|
||||||
|
}
|
||||||
|
|
||||||
|
// String formats the option into human readable form
|
||||||
|
func (o *SeekOption) String() string {
|
||||||
|
return fmt.Sprintf("SeekOption(%d)", o.Offset)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mandatory returns whether the option must be parsed or can be ignored
|
||||||
|
func (o *SeekOption) Mandatory() bool {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// HTTPOption defines a general purpose HTTP option
|
||||||
|
type HTTPOption struct {
|
||||||
|
Key string
|
||||||
|
Value string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Header formats the option as an http header
|
||||||
|
func (o *HTTPOption) Header() (key string, value string) {
|
||||||
|
return o.Key, o.Value
|
||||||
|
}
|
||||||
|
|
||||||
|
// String formats the option into human readable form
|
||||||
|
func (o *HTTPOption) String() string {
|
||||||
|
return fmt.Sprintf("HTTPOption(%q,%q)", o.Key, o.Value)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mandatory returns whether the option must be parsed or can be ignored
|
||||||
|
func (o *HTTPOption) Mandatory() bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenOptionAddHeaders adds each header found in options to the
|
||||||
|
// headers map provided the key was non empty.
|
||||||
|
func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) {
|
||||||
|
for _, option := range options {
|
||||||
|
key, value := option.Header()
|
||||||
|
if key != "" && value != "" {
|
||||||
|
headers[key] = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenOptionHeaders adds each header found in options to the
|
||||||
|
// headers map provided the key was non empty.
|
||||||
|
//
|
||||||
|
// It returns a nil map if options was empty
|
||||||
|
func OpenOptionHeaders(options []OpenOption) (headers map[string]string) {
|
||||||
|
if len(options) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
headers = make(map[string]string, len(options))
|
||||||
|
OpenOptionAddHeaders(options, headers)
|
||||||
|
return headers
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenOptionAddHTTPHeaders Sets each header found in options to the
|
||||||
|
// http.Header map provided the key was non empty.
|
||||||
|
func OpenOptionAddHTTPHeaders(headers http.Header, options []OpenOption) {
|
||||||
|
for _, option := range options {
|
||||||
|
key, value := option.Header()
|
||||||
|
if key != "" && value != "" {
|
||||||
|
headers.Set(key, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// check interface
|
||||||
|
var (
|
||||||
|
_ OpenOption = (*RangeOption)(nil)
|
||||||
|
_ OpenOption = (*SeekOption)(nil)
|
||||||
|
_ OpenOption = (*HTTPOption)(nil)
|
||||||
|
)
|
@ -10,6 +10,7 @@ import (
|
|||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"sort"
|
"sort"
|
||||||
@ -37,11 +38,13 @@ var (
|
|||||||
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
|
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
|
||||||
Path: "file name.txt",
|
Path: "file name.txt",
|
||||||
}
|
}
|
||||||
|
file1Contents = ""
|
||||||
file2 = fstest.Item{
|
file2 = fstest.Item{
|
||||||
ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
|
ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
|
||||||
Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`,
|
Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`,
|
||||||
WinPath: `hello_ sausage/êé/Hello, 世界/ _ ' @ _ _ & _ + ≠/z.txt`,
|
WinPath: `hello_ sausage/êé/Hello, 世界/ _ ' @ _ _ & _ + ≠/z.txt`,
|
||||||
}
|
}
|
||||||
|
file2Contents = ""
|
||||||
verbose = flag.Bool("verbose", false, "Set to enable logging")
|
verbose = flag.Bool("verbose", false, "Set to enable logging")
|
||||||
dumpHeaders = flag.Bool("dump-headers", false, "Dump HTTP headers - may contain sensitive info")
|
dumpHeaders = flag.Bool("dump-headers", false, "Dump HTTP headers - may contain sensitive info")
|
||||||
dumpBodies = flag.Bool("dump-bodies", false, "Dump HTTP headers and bodies - may contain sensitive info")
|
dumpBodies = flag.Bool("dump-bodies", false, "Dump HTTP headers and bodies - may contain sensitive info")
|
||||||
@ -195,9 +198,10 @@ func findObject(t *testing.T, Name string) fs.Object {
|
|||||||
return obj
|
return obj
|
||||||
}
|
}
|
||||||
|
|
||||||
func testPut(t *testing.T, file *fstest.Item) {
|
func testPut(t *testing.T, file *fstest.Item) string {
|
||||||
again:
|
again:
|
||||||
buf := bytes.NewBufferString(fstest.RandomString(100))
|
contents := fstest.RandomString(100)
|
||||||
|
buf := bytes.NewBufferString(contents)
|
||||||
hash := fs.NewMultiHasher()
|
hash := fs.NewMultiHasher()
|
||||||
in := io.TeeReader(buf, hash)
|
in := io.TeeReader(buf, hash)
|
||||||
|
|
||||||
@ -222,24 +226,25 @@ again:
|
|||||||
// Re-read the object and check again
|
// Re-read the object and check again
|
||||||
obj = findObject(t, file.Path)
|
obj = findObject(t, file.Path)
|
||||||
file.Check(t, obj, remote.Precision())
|
file.Check(t, obj, remote.Precision())
|
||||||
|
return contents
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestFsPutFile1 tests putting a file
|
// TestFsPutFile1 tests putting a file
|
||||||
func TestFsPutFile1(t *testing.T) {
|
func TestFsPutFile1(t *testing.T) {
|
||||||
skipIfNotOk(t)
|
skipIfNotOk(t)
|
||||||
testPut(t, &file1)
|
file1Contents = testPut(t, &file1)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestFsPutFile2 tests putting a file into a subdirectory
|
// TestFsPutFile2 tests putting a file into a subdirectory
|
||||||
func TestFsPutFile2(t *testing.T) {
|
func TestFsPutFile2(t *testing.T) {
|
||||||
skipIfNotOk(t)
|
skipIfNotOk(t)
|
||||||
testPut(t, &file2)
|
file2Contents = testPut(t, &file2)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestFsUpdateFile1 tests updating file1 with new contents
|
// TestFsUpdateFile1 tests updating file1 with new contents
|
||||||
func TestFsUpdateFile1(t *testing.T) {
|
func TestFsUpdateFile1(t *testing.T) {
|
||||||
skipIfNotOk(t)
|
skipIfNotOk(t)
|
||||||
testPut(t, &file1)
|
file1Contents = testPut(t, &file1)
|
||||||
// Note that the next test will check there are no duplicated file names
|
// Note that the next test will check there are no duplicated file names
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -541,42 +546,56 @@ func TestObjectSize(t *testing.T) {
|
|||||||
assert.Equal(t, file1.Size, obj.Size())
|
assert.Equal(t, file1.Size, obj.Size())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// read the contents of an object as a string
|
||||||
|
func readObject(t *testing.T, obj fs.Object, options ...fs.OpenOption) string {
|
||||||
|
in, err := obj.Open(options...)
|
||||||
|
require.NoError(t, err)
|
||||||
|
contents, err := ioutil.ReadAll(in)
|
||||||
|
require.NoError(t, err)
|
||||||
|
err = in.Close()
|
||||||
|
require.NoError(t, err)
|
||||||
|
return string(contents)
|
||||||
|
}
|
||||||
|
|
||||||
// TestObjectOpen tests that Open works
|
// TestObjectOpen tests that Open works
|
||||||
func TestObjectOpen(t *testing.T) {
|
func TestObjectOpen(t *testing.T) {
|
||||||
skipIfNotOk(t)
|
skipIfNotOk(t)
|
||||||
obj := findObject(t, file1.Path)
|
obj := findObject(t, file1.Path)
|
||||||
in, err := obj.Open()
|
assert.Equal(t, file1Contents, readObject(t, obj), "contents of file1 differ")
|
||||||
require.NoError(t, err)
|
|
||||||
hasher := fs.NewMultiHasher()
|
|
||||||
n, err := io.Copy(hasher, in)
|
|
||||||
require.NoError(t, err, fmt.Sprintf("hasher copy error: %v", err))
|
|
||||||
require.Equal(t, file1.Size, n, "Read wrong number of bytes")
|
|
||||||
err = in.Close()
|
|
||||||
require.NoError(t, err)
|
|
||||||
// Check content of file by comparing the calculated hashes
|
|
||||||
for hashType, got := range hasher.Sums() {
|
|
||||||
assert.Equal(t, file1.Hashes[hashType], got)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestObjectOpenSeek tests that Open works with Seek
|
||||||
|
func TestObjectOpenSeek(t *testing.T) {
|
||||||
|
skipIfNotOk(t)
|
||||||
|
obj := findObject(t, file1.Path)
|
||||||
|
assert.Equal(t, file1Contents[50:], readObject(t, obj, &fs.SeekOption{Offset: 50}), "contents of file1 differ after seek")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestObjectUpdate tests that Update works
|
// TestObjectUpdate tests that Update works
|
||||||
func TestObjectUpdate(t *testing.T) {
|
func TestObjectUpdate(t *testing.T) {
|
||||||
skipIfNotOk(t)
|
skipIfNotOk(t)
|
||||||
buf := bytes.NewBufferString(fstest.RandomString(200))
|
contents := fstest.RandomString(200)
|
||||||
|
buf := bytes.NewBufferString(contents)
|
||||||
hash := fs.NewMultiHasher()
|
hash := fs.NewMultiHasher()
|
||||||
in := io.TeeReader(buf, hash)
|
in := io.TeeReader(buf, hash)
|
||||||
|
|
||||||
file1.Size = int64(buf.Len())
|
file1.Size = int64(buf.Len())
|
||||||
obj := findObject(t, file1.Path)
|
obj := findObject(t, file1.Path)
|
||||||
obji := fs.NewStaticObjectInfo(file1.Path, file1.ModTime, file1.Size, true, nil, obj.Fs())
|
obji := fs.NewStaticObjectInfo(file1.Path, file1.ModTime, int64(len(contents)), true, nil, obj.Fs())
|
||||||
err := obj.Update(in, obji)
|
err := obj.Update(in, obji)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
file1.Hashes = hash.Sums()
|
file1.Hashes = hash.Sums()
|
||||||
|
|
||||||
|
// check the object has been updated
|
||||||
file1.Check(t, obj, remote.Precision())
|
file1.Check(t, obj, remote.Precision())
|
||||||
|
|
||||||
// Re-read the object and check again
|
// Re-read the object and check again
|
||||||
obj = findObject(t, file1.Path)
|
obj = findObject(t, file1.Path)
|
||||||
file1.Check(t, obj, remote.Precision())
|
file1.Check(t, obj, remote.Precision())
|
||||||
|
|
||||||
|
// check contents correct
|
||||||
|
assert.Equal(t, contents, readObject(t, obj), "contents of updated file1 differ")
|
||||||
|
file1Contents = contents
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestObjectStorable tests that Storable works
|
// TestObjectStorable tests that Storable works
|
||||||
|
@ -651,16 +651,18 @@ func (o *Object) Storable() bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Open an object for read
|
// Open an object for read
|
||||||
func (o *Object) Open() (in io.ReadCloser, err error) {
|
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||||
req, err := http.NewRequest("GET", o.url, nil)
|
req, err := http.NewRequest("GET", o.url, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
fs.OpenOptionAddHTTPHeaders(req.Header, options)
|
||||||
res, err := o.fs.client.Do(req)
|
res, err := o.fs.client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if res.StatusCode != 200 {
|
_, isRanging := req.Header["Range"]
|
||||||
|
if !(res.StatusCode == http.StatusOK || (isRanging && res.StatusCode == http.StatusPartialContent)) {
|
||||||
_ = res.Body.Close() // ignore error
|
_ = res.Body.Close() // ignore error
|
||||||
return nil, errors.Errorf("bad response: %d: %s", res.StatusCode, res.Status)
|
return nil, errors.Errorf("bad response: %d: %s", res.StatusCode, res.Status)
|
||||||
}
|
}
|
||||||
|
@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
|
|||||||
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
||||||
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
|
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
|
@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
|
|||||||
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
||||||
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
|
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
|
@ -585,18 +585,36 @@ func (file *localOpenFile) Close() (err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Open an object for read
|
// Open an object for read
|
||||||
func (o *Object) Open() (in io.ReadCloser, err error) {
|
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||||
in, err = os.Open(o.path)
|
var offset int64
|
||||||
|
for _, option := range options {
|
||||||
|
switch x := option.(type) {
|
||||||
|
case *fs.SeekOption:
|
||||||
|
offset = x.Offset
|
||||||
|
default:
|
||||||
|
if option.Mandatory() {
|
||||||
|
fs.Log(o, "Unsupported mandatory option: %v", option)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fd, err := os.Open(o.path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if offset != 0 {
|
||||||
|
// seek the object
|
||||||
|
_, err = fd.Seek(offset, 0)
|
||||||
|
// don't attempt to make checksums
|
||||||
|
return fd, err
|
||||||
|
}
|
||||||
// Update the md5sum as we go along
|
// Update the md5sum as we go along
|
||||||
in = &localOpenFile{
|
in = &localOpenFile{
|
||||||
o: o,
|
o: o,
|
||||||
in: in,
|
in: fd,
|
||||||
hash: fs.NewMultiHasher(),
|
hash: fs.NewMultiHasher(),
|
||||||
}
|
}
|
||||||
return
|
return in, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// mkdirAll makes all the directories needed to store the object
|
// mkdirAll makes all the directories needed to store the object
|
||||||
|
@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
|
|||||||
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
||||||
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
|
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
|
@ -775,7 +775,7 @@ func (o *Object) Storable() bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Open an object for read
|
// Open an object for read
|
||||||
func (o *Object) Open() (in io.ReadCloser, err error) {
|
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||||
if o.id == "" {
|
if o.id == "" {
|
||||||
return nil, errors.New("can't download - no id")
|
return nil, errors.New("can't download - no id")
|
||||||
}
|
}
|
||||||
@ -783,6 +783,7 @@ func (o *Object) Open() (in io.ReadCloser, err error) {
|
|||||||
opts := rest.Opts{
|
opts := rest.Opts{
|
||||||
Method: "GET",
|
Method: "GET",
|
||||||
Path: "/drive/items/" + o.id + "/content",
|
Path: "/drive/items/" + o.id + "/content",
|
||||||
|
Options: options,
|
||||||
}
|
}
|
||||||
err = o.fs.pacer.Call(func() (bool, error) {
|
err = o.fs.pacer.Call(func() (bool, error) {
|
||||||
resp, err = o.fs.srv.Call(&opts)
|
resp, err = o.fs.srv.Call(&opts)
|
||||||
|
@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
|
|||||||
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
||||||
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
|
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
|
27
rest/rest.go
27
rest/rest.go
@ -83,6 +83,7 @@ type Opts struct {
|
|||||||
ExtraHeaders map[string]string
|
ExtraHeaders map[string]string
|
||||||
UserName string // username for Basic Auth
|
UserName string // username for Basic Auth
|
||||||
Password string // password for Basic Auth
|
Password string // password for Basic Auth
|
||||||
|
Options []fs.OpenOption
|
||||||
}
|
}
|
||||||
|
|
||||||
// DecodeJSON decodes resp.Body into result
|
// DecodeJSON decodes resp.Body into result
|
||||||
@ -92,6 +93,27 @@ func DecodeJSON(resp *http.Response, result interface{}) (err error) {
|
|||||||
return decoder.Decode(result)
|
return decoder.Decode(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Make a new http client which resets the headers passed in on redirect
|
||||||
|
func clientWithHeaderReset(c *http.Client, headers map[string]string) *http.Client {
|
||||||
|
if len(headers) == 0 {
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
clientCopy := *c
|
||||||
|
clientCopy.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||||
|
if len(via) >= 10 {
|
||||||
|
return errors.New("stopped after 10 redirects")
|
||||||
|
}
|
||||||
|
// Reset the headers in the new request
|
||||||
|
for k, v := range headers {
|
||||||
|
if v != "" {
|
||||||
|
req.Header.Add(k, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &clientCopy
|
||||||
|
}
|
||||||
|
|
||||||
// Call makes the call and returns the http.Response
|
// Call makes the call and returns the http.Response
|
||||||
//
|
//
|
||||||
// if err != nil then resp.Body will need to be closed
|
// if err != nil then resp.Body will need to be closed
|
||||||
@ -136,6 +158,8 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) {
|
|||||||
headers[k] = v
|
headers[k] = v
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// add any options to the headers
|
||||||
|
fs.OpenOptionAddHeaders(opts.Options, headers)
|
||||||
// Now set the headers
|
// Now set the headers
|
||||||
for k, v := range headers {
|
for k, v := range headers {
|
||||||
if v != "" {
|
if v != "" {
|
||||||
@ -145,8 +169,9 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) {
|
|||||||
if opts.UserName != "" || opts.Password != "" {
|
if opts.UserName != "" || opts.Password != "" {
|
||||||
req.SetBasicAuth(opts.UserName, opts.Password)
|
req.SetBasicAuth(opts.UserName, opts.Password)
|
||||||
}
|
}
|
||||||
|
c := clientWithHeaderReset(api.c, headers)
|
||||||
api.mu.RUnlock()
|
api.mu.RUnlock()
|
||||||
resp, err = api.c.Do(req)
|
resp, err = c.Do(req)
|
||||||
api.mu.RLock()
|
api.mu.RLock()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
13
s3/s3.go
13
s3/s3.go
@ -845,12 +845,23 @@ func (o *Object) Storable() bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Open an object for read
|
// Open an object for read
|
||||||
func (o *Object) Open() (in io.ReadCloser, err error) {
|
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||||
key := o.fs.root + o.remote
|
key := o.fs.root + o.remote
|
||||||
req := s3.GetObjectInput{
|
req := s3.GetObjectInput{
|
||||||
Bucket: &o.fs.bucket,
|
Bucket: &o.fs.bucket,
|
||||||
Key: &key,
|
Key: &key,
|
||||||
}
|
}
|
||||||
|
for _, option := range options {
|
||||||
|
switch option.(type) {
|
||||||
|
case *fs.RangeOption, *fs.SeekOption:
|
||||||
|
_, value := option.Header()
|
||||||
|
req.Range = &value
|
||||||
|
default:
|
||||||
|
if option.Mandatory() {
|
||||||
|
fs.Log(o, "Unsupported mandatory option: %v", option)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
resp, err := o.fs.c.GetObject(&req)
|
resp, err := o.fs.c.GetObject(&req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
|
|||||||
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
||||||
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
|
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
|
@ -629,8 +629,10 @@ func (o *Object) Storable() bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Open an object for read
|
// Open an object for read
|
||||||
func (o *Object) Open() (in io.ReadCloser, err error) {
|
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||||
in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, true, nil)
|
headers := fs.OpenOptionHeaders(options)
|
||||||
|
_, isRanging := headers["Range"]
|
||||||
|
in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, !isRanging, headers)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
|
|||||||
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
||||||
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
|
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
|
@ -13,13 +13,13 @@ type DownloadResponse struct {
|
|||||||
Templated bool `json:"templated"`
|
Templated bool `json:"templated"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Download will get specified data from Yandex.Disk.
|
// Download will get specified data from Yandex.Disk supplying the extra headers
|
||||||
func (c *Client) Download(remotePath string) (io.ReadCloser, error) { //io.Writer
|
func (c *Client) Download(remotePath string, headers map[string]string) (io.ReadCloser, error) { //io.Writer
|
||||||
ur, err := c.DownloadRequest(remotePath)
|
ur, err := c.DownloadRequest(remotePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return c.PerformDownload(ur.HRef)
|
return c.PerformDownload(ur.HRef, headers)
|
||||||
}
|
}
|
||||||
|
|
||||||
// DownloadRequest will make an download request and return a URL to download data to.
|
// DownloadRequest will make an download request and return a URL to download data to.
|
||||||
|
@ -8,13 +8,18 @@ import (
|
|||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
// PerformDownload does the actual download via unscoped PUT request.
|
// PerformDownload does the actual download via unscoped GET request.
|
||||||
func (c *Client) PerformDownload(url string) (out io.ReadCloser, err error) {
|
func (c *Client) PerformDownload(url string, headers map[string]string) (out io.ReadCloser, err error) {
|
||||||
req, err := http.NewRequest("GET", url, nil)
|
req, err := http.NewRequest("GET", url, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Set any extra headers
|
||||||
|
for k, v := range headers {
|
||||||
|
req.Header.Set(k, v)
|
||||||
|
}
|
||||||
|
|
||||||
//c.setRequestScope(req)
|
//c.setRequestScope(req)
|
||||||
|
|
||||||
resp, err := c.HTTPClient.Do(req)
|
resp, err := c.HTTPClient.Do(req)
|
||||||
@ -22,7 +27,8 @@ func (c *Client) PerformDownload(url string) (out io.ReadCloser, err error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if resp.StatusCode != 200 {
|
_, isRanging := req.Header["Range"]
|
||||||
|
if !(resp.StatusCode == http.StatusOK || (isRanging && resp.StatusCode == http.StatusPartialContent)) {
|
||||||
defer CheckClose(resp.Body, &err)
|
defer CheckClose(resp.Body, &err)
|
||||||
body, err := ioutil.ReadAll(resp.Body)
|
body, err := ioutil.ReadAll(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -487,8 +487,8 @@ func (o *Object) ModTime() time.Time {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Open an object for read
|
// Open an object for read
|
||||||
func (o *Object) Open() (in io.ReadCloser, err error) {
|
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||||
return o.fs.yd.Download(o.remotePath())
|
return o.fs.yd.Download(o.remotePath(), fs.OpenOptionHeaders(options))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove an object
|
// Remove an object
|
||||||
|
@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
|
|||||||
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
||||||
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||||
|
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
|
||||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||||
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
|
||||||
|
Loading…
Reference in New Issue
Block a user