mirror of
https://github.com/rclone/rclone.git
synced 2025-01-03 04:49:47 +01:00
Create separate interface for object information.
Take out read-only information about a Fs in a separate struct to limit access. See discussion at #282.
This commit is contained in:
parent
85a0f25b95
commit
ef06371c93
@ -62,7 +62,7 @@ var (
|
|||||||
|
|
||||||
// Register with Fs
|
// Register with Fs
|
||||||
func init() {
|
func init() {
|
||||||
fs.Register(&fs.Info{
|
fs.Register(&fs.RegInfo{
|
||||||
Name: "amazon cloud drive",
|
Name: "amazon cloud drive",
|
||||||
NewFs: NewFs,
|
NewFs: NewFs,
|
||||||
Config: func(name string) {
|
Config: func(name string) {
|
||||||
@ -521,7 +521,9 @@ func (f *Fs) ListDir() fs.DirChan {
|
|||||||
// Copy the reader in to the new object which is returned
|
// Copy the reader in to the new object which is returned
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
|
||||||
|
remote := src.Remote()
|
||||||
|
size := src.Size()
|
||||||
// Temporary Object under construction
|
// Temporary Object under construction
|
||||||
o := &Object{
|
o := &Object{
|
||||||
fs: f,
|
fs: f,
|
||||||
@ -538,7 +540,7 @@ func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs
|
|||||||
var info *acd.File
|
var info *acd.File
|
||||||
var resp *http.Response
|
var resp *http.Response
|
||||||
err = f.pacer.CallNoRetry(func() (bool, error) {
|
err = f.pacer.CallNoRetry(func() (bool, error) {
|
||||||
if size != 0 {
|
if src.Size() != 0 {
|
||||||
info, resp, err = folder.Put(in, leaf)
|
info, resp, err = folder.Put(in, leaf)
|
||||||
} else {
|
} else {
|
||||||
info, resp, err = folder.PutSized(in, size, leaf)
|
info, resp, err = folder.PutSized(in, size, leaf)
|
||||||
@ -663,7 +665,7 @@ func (f *Fs) Purge() error {
|
|||||||
// ------------------------------------------------------------
|
// ------------------------------------------------------------
|
||||||
|
|
||||||
// Fs returns the parent Fs
|
// Fs returns the parent Fs
|
||||||
func (o *Object) Fs() fs.Fs {
|
func (o *Object) Fs() fs.Info {
|
||||||
return o.fs
|
return o.fs
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -774,7 +776,8 @@ func (o *Object) Open() (in io.ReadCloser, err error) {
|
|||||||
// Update the object with the contents of the io.Reader, modTime and size
|
// Update the object with the contents of the io.Reader, modTime and size
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
|
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
|
||||||
|
size := src.Size()
|
||||||
file := acd.File{Node: o.info}
|
file := acd.File{Node: o.info}
|
||||||
var info *acd.File
|
var info *acd.File
|
||||||
var resp *http.Response
|
var resp *http.Response
|
||||||
|
15
b2/b2.go
15
b2/b2.go
@ -39,7 +39,7 @@ const (
|
|||||||
|
|
||||||
// Register with Fs
|
// Register with Fs
|
||||||
func init() {
|
func init() {
|
||||||
fs.Register(&fs.Info{
|
fs.Register(&fs.RegInfo{
|
||||||
Name: "b2",
|
Name: "b2",
|
||||||
NewFs: NewFs,
|
NewFs: NewFs,
|
||||||
Options: []fs.Option{{
|
Options: []fs.Option{{
|
||||||
@ -462,13 +462,13 @@ func (f *Fs) ListDir() fs.DirChan {
|
|||||||
// Copy the reader in to the new object which is returned
|
// Copy the reader in to the new object which is returned
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
|
||||||
// Temporary Object under construction
|
// Temporary Object under construction
|
||||||
fs := &Object{
|
fs := &Object{
|
||||||
fs: f,
|
fs: f,
|
||||||
remote: remote,
|
remote: src.Remote(),
|
||||||
}
|
}
|
||||||
return fs, fs.Update(in, modTime, size)
|
return fs, fs.Update(in, src)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mkdir creates the bucket if it doesn't exist
|
// Mkdir creates the bucket if it doesn't exist
|
||||||
@ -599,7 +599,7 @@ func (f *Fs) Hashes() fs.HashSet {
|
|||||||
// ------------------------------------------------------------
|
// ------------------------------------------------------------
|
||||||
|
|
||||||
// Fs returns the parent Fs
|
// Fs returns the parent Fs
|
||||||
func (o *Object) Fs() fs.Fs {
|
func (o *Object) Fs() fs.Info {
|
||||||
return o.fs
|
return o.fs
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -875,7 +875,10 @@ func urlEncode(in string) string {
|
|||||||
// Update the object with the contents of the io.Reader, modTime and size
|
// Update the object with the contents of the io.Reader, modTime and size
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) (err error) {
|
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) (err error) {
|
||||||
|
size := src.Size()
|
||||||
|
modTime := src.ModTime()
|
||||||
|
|
||||||
// Open a temp file to copy the input
|
// Open a temp file to copy the input
|
||||||
fd, err := ioutil.TempFile("", "rclone-b2-")
|
fd, err := ioutil.TempFile("", "rclone-b2-")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -84,7 +84,7 @@ var (
|
|||||||
|
|
||||||
// Register with Fs
|
// Register with Fs
|
||||||
func init() {
|
func init() {
|
||||||
fs.Register(&fs.Info{
|
fs.Register(&fs.RegInfo{
|
||||||
Name: "drive",
|
Name: "drive",
|
||||||
NewFs: NewFs,
|
NewFs: NewFs,
|
||||||
Config: func(name string) {
|
Config: func(name string) {
|
||||||
@ -600,7 +600,11 @@ func (f *Fs) createFileInfo(remote string, modTime time.Time, size int64) (*Obje
|
|||||||
// Copy the reader in to the new object which is returned
|
// Copy the reader in to the new object which is returned
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
|
||||||
|
remote := src.Remote()
|
||||||
|
size := src.Size()
|
||||||
|
modTime := src.ModTime()
|
||||||
|
|
||||||
o, createInfo, err := f.createFileInfo(remote, modTime, size)
|
o, createInfo, err := f.createFileInfo(remote, modTime, size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -818,7 +822,7 @@ func (f *Fs) Hashes() fs.HashSet {
|
|||||||
// ------------------------------------------------------------
|
// ------------------------------------------------------------
|
||||||
|
|
||||||
// Fs returns the parent Fs
|
// Fs returns the parent Fs
|
||||||
func (o *Object) Fs() fs.Fs {
|
func (o *Object) Fs() fs.Info {
|
||||||
return o.fs
|
return o.fs
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1025,7 +1029,9 @@ func (o *Object) Open() (in io.ReadCloser, err error) {
|
|||||||
// Copy the reader into the object updating modTime and size
|
// Copy the reader into the object updating modTime and size
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
|
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
|
||||||
|
size := src.Size()
|
||||||
|
modTime := src.ModTime()
|
||||||
if o.isDocument {
|
if o.isDocument {
|
||||||
return fmt.Errorf("Can't update a google document")
|
return fmt.Errorf("Can't update a google document")
|
||||||
}
|
}
|
||||||
|
@ -44,7 +44,7 @@ var (
|
|||||||
|
|
||||||
// Register with Fs
|
// Register with Fs
|
||||||
func init() {
|
func init() {
|
||||||
fs.Register(&fs.Info{
|
fs.Register(&fs.RegInfo{
|
||||||
Name: "dropbox",
|
Name: "dropbox",
|
||||||
NewFs: NewFs,
|
NewFs: NewFs,
|
||||||
Config: configHelper,
|
Config: configHelper,
|
||||||
@ -379,13 +379,13 @@ func (rc *readCloser) Close() error {
|
|||||||
// Copy the reader in to the new object which is returned
|
// Copy the reader in to the new object which is returned
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
|
||||||
// Temporary Object under construction
|
// Temporary Object under construction
|
||||||
o := &Object{
|
o := &Object{
|
||||||
fs: f,
|
fs: f,
|
||||||
remote: remote,
|
remote: src.Remote(),
|
||||||
}
|
}
|
||||||
return o, o.Update(in, modTime, size)
|
return o, o.Update(in, src)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mkdir creates the container if it doesn't exist
|
// Mkdir creates the container if it doesn't exist
|
||||||
@ -531,7 +531,7 @@ func (f *Fs) Hashes() fs.HashSet {
|
|||||||
// ------------------------------------------------------------
|
// ------------------------------------------------------------
|
||||||
|
|
||||||
// Fs returns the parent Fs
|
// Fs returns the parent Fs
|
||||||
func (o *Object) Fs() fs.Fs {
|
func (o *Object) Fs() fs.Info {
|
||||||
return o.fs
|
return o.fs
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -656,7 +656,7 @@ func (o *Object) Open() (in io.ReadCloser, err error) {
|
|||||||
// Copy the reader into the object updating modTime and size
|
// Copy the reader into the object updating modTime and size
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
|
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
|
||||||
remote := o.remotePath()
|
remote := o.remotePath()
|
||||||
if ignoredFiles.MatchString(remote) {
|
if ignoredFiles.MatchString(remote) {
|
||||||
fs.Log(o, "File name disallowed - not uploading")
|
fs.Log(o, "File name disallowed - not uploading")
|
||||||
|
116
fs/fs.go
116
fs/fs.go
@ -22,7 +22,7 @@ const (
|
|||||||
// Globals
|
// Globals
|
||||||
var (
|
var (
|
||||||
// Filesystem registry
|
// Filesystem registry
|
||||||
fsRegistry []*Info
|
fsRegistry []*RegInfo
|
||||||
// ErrorNotFoundInConfigFile is returned by NewFs if not found in config file
|
// ErrorNotFoundInConfigFile is returned by NewFs if not found in config file
|
||||||
ErrorNotFoundInConfigFile = fmt.Errorf("Didn't find section in config file")
|
ErrorNotFoundInConfigFile = fmt.Errorf("Didn't find section in config file")
|
||||||
ErrorCantPurge = fmt.Errorf("Can't purge directory")
|
ErrorCantPurge = fmt.Errorf("Can't purge directory")
|
||||||
@ -32,8 +32,8 @@ var (
|
|||||||
ErrorDirExists = fmt.Errorf("Can't copy directory - destination already exists")
|
ErrorDirExists = fmt.Errorf("Can't copy directory - destination already exists")
|
||||||
)
|
)
|
||||||
|
|
||||||
// Info information about a filesystem
|
// RegInfo provides information about a filesystem
|
||||||
type Info struct {
|
type RegInfo struct {
|
||||||
// Name of this fs
|
// Name of this fs
|
||||||
Name string
|
Name string
|
||||||
// Create a new file system. If root refers to an existing
|
// Create a new file system. If root refers to an existing
|
||||||
@ -63,20 +63,13 @@ type OptionExample struct {
|
|||||||
// Register a filesystem
|
// Register a filesystem
|
||||||
//
|
//
|
||||||
// Fs modules should use this in an init() function
|
// Fs modules should use this in an init() function
|
||||||
func Register(info *Info) {
|
func Register(info *RegInfo) {
|
||||||
fsRegistry = append(fsRegistry, info)
|
fsRegistry = append(fsRegistry, info)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fs is the interface a cloud storage system must provide
|
// Fs is the interface a cloud storage system must provide
|
||||||
type Fs interface {
|
type Fs interface {
|
||||||
// Name of the remote (as passed into NewFs)
|
Info
|
||||||
Name() string
|
|
||||||
|
|
||||||
// Root of the remote (as passed into NewFs)
|
|
||||||
Root() string
|
|
||||||
|
|
||||||
// String returns a description of the FS
|
|
||||||
String() string
|
|
||||||
|
|
||||||
// List the Fs into a channel
|
// List the Fs into a channel
|
||||||
List() ObjectsChan
|
List() ObjectsChan
|
||||||
@ -92,7 +85,7 @@ type Fs interface {
|
|||||||
// May create the object even if it returns an error - if so
|
// May create the object even if it returns an error - if so
|
||||||
// will return the object and the error, otherwise will return
|
// will return the object and the error, otherwise will return
|
||||||
// nil and the error
|
// nil and the error
|
||||||
Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error)
|
Put(in io.Reader, src ObjectInfo) (Object, error)
|
||||||
|
|
||||||
// Mkdir makes the directory (container, bucket)
|
// Mkdir makes the directory (container, bucket)
|
||||||
//
|
//
|
||||||
@ -103,6 +96,18 @@ type Fs interface {
|
|||||||
//
|
//
|
||||||
// Return an error if it doesn't exist or isn't empty
|
// Return an error if it doesn't exist or isn't empty
|
||||||
Rmdir() error
|
Rmdir() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Info provides an interface to reading information about a filesystem.
|
||||||
|
type Info interface {
|
||||||
|
// Name of the remote (as passed into NewFs)
|
||||||
|
Name() string
|
||||||
|
|
||||||
|
// Root of the remote (as passed into NewFs)
|
||||||
|
Root() string
|
||||||
|
|
||||||
|
// String returns a description of the FS
|
||||||
|
String() string
|
||||||
|
|
||||||
// Precision of the ModTimes in this Fs
|
// Precision of the ModTimes in this Fs
|
||||||
Precision() time.Duration
|
Precision() time.Duration
|
||||||
@ -113,40 +118,45 @@ type Fs interface {
|
|||||||
|
|
||||||
// Object is a filesystem like object provided by an Fs
|
// Object is a filesystem like object provided by an Fs
|
||||||
type Object interface {
|
type Object interface {
|
||||||
|
ObjectInfo
|
||||||
|
|
||||||
// String returns a description of the Object
|
// String returns a description of the Object
|
||||||
String() string
|
String() string
|
||||||
|
|
||||||
// Fs returns the Fs that this object is part of
|
// SetModTime sets the metadata on the object to set the modification date
|
||||||
Fs() Fs
|
SetModTime(time.Time)
|
||||||
|
|
||||||
|
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
||||||
|
Open() (io.ReadCloser, error)
|
||||||
|
|
||||||
|
// Update in to the object with the modTime given of the given size
|
||||||
|
Update(in io.Reader, src ObjectInfo) error
|
||||||
|
|
||||||
|
// Removes this object
|
||||||
|
Remove() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// ObjectInfo contains information about an object.
|
||||||
|
type ObjectInfo interface {
|
||||||
|
// Fs returns read only access to the Fs that this object is part of
|
||||||
|
Fs() Info
|
||||||
|
|
||||||
// Remote returns the remote path
|
// Remote returns the remote path
|
||||||
Remote() string
|
Remote() string
|
||||||
|
|
||||||
// Md5sum returns the md5 checksum of the file
|
// Hash returns the selected checksum of the file
|
||||||
// If no Md5sum is available it returns ""
|
// If no checksum is available it returns ""
|
||||||
Hash(HashType) (string, error)
|
Hash(HashType) (string, error)
|
||||||
|
|
||||||
// ModTime returns the modification date of the file
|
// ModTime returns the modification date of the file
|
||||||
// It should return a best guess if one isn't available
|
// It should return a best guess if one isn't available
|
||||||
ModTime() time.Time
|
ModTime() time.Time
|
||||||
|
|
||||||
// SetModTime sets the metadata on the object to set the modification date
|
|
||||||
SetModTime(time.Time)
|
|
||||||
|
|
||||||
// Size returns the size of the file
|
// Size returns the size of the file
|
||||||
Size() int64
|
Size() int64
|
||||||
|
|
||||||
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
|
||||||
Open() (io.ReadCloser, error)
|
|
||||||
|
|
||||||
// Update in to the object with the modTime given of the given size
|
|
||||||
Update(in io.Reader, modTime time.Time, size int64) error
|
|
||||||
|
|
||||||
// Storable says whether this object can be stored
|
// Storable says whether this object can be stored
|
||||||
Storable() bool
|
Storable() bool
|
||||||
|
|
||||||
// Removes this object
|
|
||||||
Remove() error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Purger is an optional interfaces for Fs
|
// Purger is an optional interfaces for Fs
|
||||||
@ -236,7 +246,7 @@ type DirChan chan *Dir
|
|||||||
// Find looks for an Info object for the name passed in
|
// Find looks for an Info object for the name passed in
|
||||||
//
|
//
|
||||||
// Services are looked up in the config file
|
// Services are looked up in the config file
|
||||||
func Find(name string) (*Info, error) {
|
func Find(name string) (*RegInfo, error) {
|
||||||
for _, item := range fsRegistry {
|
for _, item := range fsRegistry {
|
||||||
if item.Name == name {
|
if item.Name == name {
|
||||||
return item, nil
|
return item, nil
|
||||||
@ -316,3 +326,49 @@ func CheckClose(c io.Closer, err *error) {
|
|||||||
*err = cerr
|
*err = cerr
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewStaticObjectInfo returns a static ObjectInfo
|
||||||
|
// If hashes is nil and fs is not nil, the hash map will be replaced with
|
||||||
|
// empty hashes of the types supported by the fs.
|
||||||
|
func NewStaticObjectInfo(remote string, modTime time.Time, size int64, storable bool, hashes map[HashType]string, fs Info) ObjectInfo {
|
||||||
|
info := &staticObjectInfo{
|
||||||
|
remote: remote,
|
||||||
|
modTime: modTime,
|
||||||
|
size: size,
|
||||||
|
storable: storable,
|
||||||
|
hashes: hashes,
|
||||||
|
fs: fs,
|
||||||
|
}
|
||||||
|
if fs != nil && hashes == nil {
|
||||||
|
set := fs.Hashes().Array()
|
||||||
|
info.hashes = make(map[HashType]string)
|
||||||
|
for _, ht := range set {
|
||||||
|
info.hashes[ht] = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return info
|
||||||
|
}
|
||||||
|
|
||||||
|
type staticObjectInfo struct {
|
||||||
|
remote string
|
||||||
|
modTime time.Time
|
||||||
|
size int64
|
||||||
|
storable bool
|
||||||
|
hashes map[HashType]string
|
||||||
|
fs Info
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *staticObjectInfo) Fs() Info { return i.fs }
|
||||||
|
func (i *staticObjectInfo) Remote() string { return i.remote }
|
||||||
|
func (i *staticObjectInfo) ModTime() time.Time { return i.modTime }
|
||||||
|
func (i *staticObjectInfo) Size() int64 { return i.size }
|
||||||
|
func (i *staticObjectInfo) Storable() bool { return i.storable }
|
||||||
|
func (i *staticObjectInfo) Hash(h HashType) (string, error) {
|
||||||
|
if len(i.hashes) == 0 {
|
||||||
|
return "", ErrHashUnsupported
|
||||||
|
}
|
||||||
|
if hash, ok := i.hashes[h]; ok {
|
||||||
|
return hash, nil
|
||||||
|
}
|
||||||
|
return "", ErrHashUnsupported
|
||||||
|
}
|
||||||
|
@ -71,12 +71,13 @@ func (f *Limited) NewFsObject(remote string) Object {
|
|||||||
// May create the object even if it returns an error - if so
|
// May create the object even if it returns an error - if so
|
||||||
// will return the object and the error, otherwise will return
|
// will return the object and the error, otherwise will return
|
||||||
// nil and the error
|
// nil and the error
|
||||||
func (f *Limited) Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error) {
|
func (f *Limited) Put(in io.Reader, src ObjectInfo) (Object, error) {
|
||||||
|
remote := src.Remote()
|
||||||
obj := f.NewFsObject(remote)
|
obj := f.NewFsObject(remote)
|
||||||
if obj == nil {
|
if obj == nil {
|
||||||
return nil, fmt.Errorf("Can't create %q in limited fs", remote)
|
return nil, fmt.Errorf("Can't create %q in limited fs", remote)
|
||||||
}
|
}
|
||||||
return obj, obj.Update(in, modTime, size)
|
return obj, obj.Update(in, src)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mkdir make the directory (container, bucket)
|
// Mkdir make the directory (container, bucket)
|
||||||
|
@ -221,10 +221,10 @@ tryAgain:
|
|||||||
|
|
||||||
if doUpdate {
|
if doUpdate {
|
||||||
actionTaken = "Copied (updated existing)"
|
actionTaken = "Copied (updated existing)"
|
||||||
err = dst.Update(in, src.ModTime(), src.Size())
|
err = dst.Update(in, src)
|
||||||
} else {
|
} else {
|
||||||
actionTaken = "Copied (new)"
|
actionTaken = "Copied (new)"
|
||||||
dst, err = f.Put(in, src.Remote(), src.ModTime(), src.Size())
|
dst, err = f.Put(in, src)
|
||||||
}
|
}
|
||||||
inErr = in.Close()
|
inErr = in.Close()
|
||||||
}
|
}
|
||||||
|
@ -188,7 +188,8 @@ func (r *Run) WriteObjectTo(f fs.Fs, remote, content string, modTime time.Time)
|
|||||||
}
|
}
|
||||||
for tries := 1; ; tries++ {
|
for tries := 1; ; tries++ {
|
||||||
in := bytes.NewBufferString(content)
|
in := bytes.NewBufferString(content)
|
||||||
_, err := f.Put(in, remote, modTime, int64(len(content)))
|
objinfo := fs.NewStaticObjectInfo(remote, modTime, int64(len(content)), true, nil, nil)
|
||||||
|
_, err := f.Put(in, objinfo)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
@ -164,7 +164,8 @@ func testPut(t *testing.T, file *fstest.Item) {
|
|||||||
in := io.TeeReader(buf, hash)
|
in := io.TeeReader(buf, hash)
|
||||||
|
|
||||||
file.Size = int64(buf.Len())
|
file.Size = int64(buf.Len())
|
||||||
obj, err := remote.Put(in, file.Path, file.ModTime, file.Size)
|
obji := fs.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
|
||||||
|
obj, err := remote.Put(in, obji)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal("Put error", err)
|
t.Fatal("Put error", err)
|
||||||
}
|
}
|
||||||
@ -551,7 +552,8 @@ func TestObjectUpdate(t *testing.T) {
|
|||||||
|
|
||||||
file1.Size = int64(buf.Len())
|
file1.Size = int64(buf.Len())
|
||||||
obj := findObject(t, file1.Path)
|
obj := findObject(t, file1.Path)
|
||||||
err := obj.Update(in, file1.ModTime, file1.Size)
|
obji := fs.NewStaticObjectInfo("", file1.ModTime, file1.Size, true, nil, obj.Fs())
|
||||||
|
err := obj.Update(in, obji)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal("Update error", err)
|
t.Fatal("Update error", err)
|
||||||
}
|
}
|
||||||
|
@ -55,7 +55,7 @@ var (
|
|||||||
|
|
||||||
// Register with Fs
|
// Register with Fs
|
||||||
func init() {
|
func init() {
|
||||||
fs.Register(&fs.Info{
|
fs.Register(&fs.RegInfo{
|
||||||
Name: "google cloud storage",
|
Name: "google cloud storage",
|
||||||
NewFs: NewFs,
|
NewFs: NewFs,
|
||||||
Config: func(name string) {
|
Config: func(name string) {
|
||||||
@ -379,13 +379,13 @@ func (f *Fs) ListDir() fs.DirChan {
|
|||||||
// Copy the reader in to the new object which is returned
|
// Copy the reader in to the new object which is returned
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
|
||||||
// Temporary Object under construction
|
// Temporary Object under construction
|
||||||
o := &Object{
|
o := &Object{
|
||||||
fs: f,
|
fs: f,
|
||||||
remote: remote,
|
remote: src.Remote(),
|
||||||
}
|
}
|
||||||
return o, o.Update(in, modTime, size)
|
return o, o.Update(in, src)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mkdir creates the bucket if it doesn't exist
|
// Mkdir creates the bucket if it doesn't exist
|
||||||
@ -466,7 +466,7 @@ func (f *Fs) Hashes() fs.HashSet {
|
|||||||
// ------------------------------------------------------------
|
// ------------------------------------------------------------
|
||||||
|
|
||||||
// Fs returns the parent Fs
|
// Fs returns the parent Fs
|
||||||
func (o *Object) Fs() fs.Fs {
|
func (o *Object) Fs() fs.Info {
|
||||||
return o.fs
|
return o.fs
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -617,7 +617,10 @@ func (o *Object) Open() (in io.ReadCloser, err error) {
|
|||||||
// Update the object with the contents of the io.Reader, modTime and size
|
// Update the object with the contents of the io.Reader, modTime and size
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
|
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
|
||||||
|
size := src.Size()
|
||||||
|
modTime := src.ModTime()
|
||||||
|
|
||||||
object := storage.Object{
|
object := storage.Object{
|
||||||
Bucket: o.fs.bucket,
|
Bucket: o.fs.bucket,
|
||||||
Name: o.fs.root + o.remote,
|
Name: o.fs.root + o.remote,
|
||||||
|
@ -44,7 +44,7 @@ var (
|
|||||||
|
|
||||||
// Register with Fs
|
// Register with Fs
|
||||||
func init() {
|
func init() {
|
||||||
fs.Register(&fs.Info{
|
fs.Register(&fs.RegInfo{
|
||||||
Name: "hubic",
|
Name: "hubic",
|
||||||
NewFs: NewFs,
|
NewFs: NewFs,
|
||||||
Config: func(name string) {
|
Config: func(name string) {
|
||||||
|
@ -19,7 +19,7 @@ import (
|
|||||||
|
|
||||||
// Register with Fs
|
// Register with Fs
|
||||||
func init() {
|
func init() {
|
||||||
fsi := &fs.Info{
|
fsi := &fs.RegInfo{
|
||||||
Name: "local",
|
Name: "local",
|
||||||
NewFs: NewFs,
|
NewFs: NewFs,
|
||||||
Options: []fs.Option{fs.Option{
|
Options: []fs.Option{fs.Option{
|
||||||
@ -230,10 +230,11 @@ func (f *Fs) ListDir() fs.DirChan {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Put the FsObject to the local filesystem
|
// Put the FsObject to the local filesystem
|
||||||
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
|
||||||
|
remote := src.Remote()
|
||||||
// Temporary FsObject under construction - info filled in by Update()
|
// Temporary FsObject under construction - info filled in by Update()
|
||||||
o := f.newFsObject(remote)
|
o := f.newFsObject(remote)
|
||||||
err := o.Update(in, modTime, size)
|
err := o.Update(in, src)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -422,7 +423,7 @@ func (f *Fs) Hashes() fs.HashSet {
|
|||||||
// ------------------------------------------------------------
|
// ------------------------------------------------------------
|
||||||
|
|
||||||
// Fs returns the parent Fs
|
// Fs returns the parent Fs
|
||||||
func (o *Object) Fs() fs.Fs {
|
func (o *Object) Fs() fs.Info {
|
||||||
return o.fs
|
return o.fs
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -568,7 +569,7 @@ func (o *Object) mkdirAll() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Update the object from in with modTime and size
|
// Update the object from in with modTime and size
|
||||||
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
|
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
|
||||||
err := o.mkdirAll()
|
err := o.mkdirAll()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@ -579,7 +580,7 @@ func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Calculate the md5sum of the object we are reading as we go along
|
// Calculate the hash of the object we are reading as we go along
|
||||||
hash := fs.NewMultiHasher()
|
hash := fs.NewMultiHasher()
|
||||||
in = io.TeeReader(in, hash)
|
in = io.TeeReader(in, hash)
|
||||||
|
|
||||||
@ -596,7 +597,7 @@ func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
|
|||||||
o.hashes = hash.Sums()
|
o.hashes = hash.Sums()
|
||||||
|
|
||||||
// Set the mtime
|
// Set the mtime
|
||||||
o.SetModTime(modTime)
|
o.SetModTime(src.ModTime())
|
||||||
|
|
||||||
// ReRead info now that we have finished
|
// ReRead info now that we have finished
|
||||||
return o.lstat()
|
return o.lstat()
|
||||||
|
@ -55,7 +55,7 @@ var (
|
|||||||
|
|
||||||
// Register with Fs
|
// Register with Fs
|
||||||
func init() {
|
func init() {
|
||||||
fs.Register(&fs.Info{
|
fs.Register(&fs.RegInfo{
|
||||||
Name: "onedrive",
|
Name: "onedrive",
|
||||||
NewFs: NewFs,
|
NewFs: NewFs,
|
||||||
Config: func(name string) {
|
Config: func(name string) {
|
||||||
@ -487,12 +487,16 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
|
|||||||
// Copy the reader in to the new object which is returned
|
// Copy the reader in to the new object which is returned
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
|
||||||
|
remote := src.Remote()
|
||||||
|
size := src.Size()
|
||||||
|
modTime := src.ModTime()
|
||||||
|
|
||||||
o, _, _, err := f.createObject(remote, modTime, size)
|
o, _, _, err := f.createObject(remote, modTime, size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return o, o.Update(in, modTime, size)
|
return o, o.Update(in, src)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mkdir creates the container if it doesn't exist
|
// Mkdir creates the container if it doesn't exist
|
||||||
@ -679,7 +683,7 @@ func (f *Fs) Hashes() fs.HashSet {
|
|||||||
// ------------------------------------------------------------
|
// ------------------------------------------------------------
|
||||||
|
|
||||||
// Fs returns the parent Fs
|
// Fs returns the parent Fs
|
||||||
func (o *Object) Fs() fs.Fs {
|
func (o *Object) Fs() fs.Info {
|
||||||
return o.fs
|
return o.fs
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -935,7 +939,10 @@ func (o *Object) uploadMultipart(in io.Reader, size int64) (err error) {
|
|||||||
// Update the object with the contents of the io.Reader, modTime and size
|
// Update the object with the contents of the io.Reader, modTime and size
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) (err error) {
|
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) (err error) {
|
||||||
|
size := src.Size()
|
||||||
|
modTime := src.ModTime()
|
||||||
|
|
||||||
var info *api.Item
|
var info *api.Item
|
||||||
if size <= int64(uploadCutoff) {
|
if size <= int64(uploadCutoff) {
|
||||||
// This is for less than 100 MB of content
|
// This is for less than 100 MB of content
|
||||||
|
14
s3/s3.go
14
s3/s3.go
@ -40,7 +40,7 @@ import (
|
|||||||
|
|
||||||
// Register with Fs
|
// Register with Fs
|
||||||
func init() {
|
func init() {
|
||||||
fs.Register(&fs.Info{
|
fs.Register(&fs.RegInfo{
|
||||||
Name: "s3",
|
Name: "s3",
|
||||||
NewFs: NewFs,
|
NewFs: NewFs,
|
||||||
// AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
|
// AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
|
||||||
@ -495,13 +495,13 @@ func (f *Fs) ListDir() fs.DirChan {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Put the FsObject into the bucket
|
// Put the FsObject into the bucket
|
||||||
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
|
||||||
// Temporary Object under construction
|
// Temporary Object under construction
|
||||||
fs := &Object{
|
fs := &Object{
|
||||||
fs: f,
|
fs: f,
|
||||||
remote: remote,
|
remote: src.Remote(),
|
||||||
}
|
}
|
||||||
return fs, fs.Update(in, modTime, size)
|
return fs, fs.Update(in, src)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mkdir creates the bucket if it doesn't exist
|
// Mkdir creates the bucket if it doesn't exist
|
||||||
@ -582,7 +582,7 @@ func (f *Fs) Hashes() fs.HashSet {
|
|||||||
// ------------------------------------------------------------
|
// ------------------------------------------------------------
|
||||||
|
|
||||||
// Fs returns the parent Fs
|
// Fs returns the parent Fs
|
||||||
func (o *Object) Fs() fs.Fs {
|
func (o *Object) Fs() fs.Info {
|
||||||
return o.fs
|
return o.fs
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -732,7 +732,9 @@ func (o *Object) Open() (in io.ReadCloser, err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Update the Object from in with modTime and size
|
// Update the Object from in with modTime and size
|
||||||
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
|
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
|
||||||
|
modTime := src.ModTime()
|
||||||
|
|
||||||
uploader := s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) {
|
uploader := s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) {
|
||||||
u.Concurrency = 2
|
u.Concurrency = 2
|
||||||
u.LeavePartsOnError = false
|
u.LeavePartsOnError = false
|
||||||
|
@ -29,7 +29,7 @@ var (
|
|||||||
|
|
||||||
// Register with Fs
|
// Register with Fs
|
||||||
func init() {
|
func init() {
|
||||||
fs.Register(&fs.Info{
|
fs.Register(&fs.RegInfo{
|
||||||
Name: "swift",
|
Name: "swift",
|
||||||
NewFs: NewFs,
|
NewFs: NewFs,
|
||||||
Options: []fs.Option{{
|
Options: []fs.Option{{
|
||||||
@ -380,13 +380,13 @@ func (f *Fs) ListDir() fs.DirChan {
|
|||||||
// Copy the reader in to the new object which is returned
|
// Copy the reader in to the new object which is returned
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
|
||||||
// Temporary Object under construction
|
// Temporary Object under construction
|
||||||
fs := &Object{
|
fs := &Object{
|
||||||
fs: f,
|
fs: f,
|
||||||
remote: remote,
|
remote: src.Remote(),
|
||||||
}
|
}
|
||||||
return fs, fs.Update(in, modTime, size)
|
return fs, fs.Update(in, src)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mkdir creates the container if it doesn't exist
|
// Mkdir creates the container if it doesn't exist
|
||||||
@ -457,7 +457,7 @@ func (f *Fs) Hashes() fs.HashSet {
|
|||||||
// ------------------------------------------------------------
|
// ------------------------------------------------------------
|
||||||
|
|
||||||
// Fs returns the parent Fs
|
// Fs returns the parent Fs
|
||||||
func (o *Object) Fs() fs.Fs {
|
func (o *Object) Fs() fs.Info {
|
||||||
return o.fs
|
return o.fs
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -655,7 +655,10 @@ func (o *Object) updateChunks(in io.Reader, headers swift.Headers, size int64) (
|
|||||||
// Update the object with the contents of the io.Reader, modTime and size
|
// Update the object with the contents of the io.Reader, modTime and size
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
|
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
|
||||||
|
size := src.Size()
|
||||||
|
modTime := src.ModTime()
|
||||||
|
|
||||||
// Note whether this has a manifest before starting
|
// Note whether this has a manifest before starting
|
||||||
isManifest, err := o.isManifestFile()
|
isManifest, err := o.isManifestFile()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -41,7 +41,7 @@ var (
|
|||||||
|
|
||||||
// Register with Fs
|
// Register with Fs
|
||||||
func init() {
|
func init() {
|
||||||
fs.Register(&fs.Info{
|
fs.Register(&fs.RegInfo{
|
||||||
Name: "yandex",
|
Name: "yandex",
|
||||||
NewFs: NewFs,
|
NewFs: NewFs,
|
||||||
Config: func(name string) {
|
Config: func(name string) {
|
||||||
@ -326,7 +326,11 @@ func (f *Fs) ListDir() fs.DirChan {
|
|||||||
// Copy the reader in to the new object which is returned
|
// Copy the reader in to the new object which is returned
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
|
||||||
|
remote := src.Remote()
|
||||||
|
size := src.Size()
|
||||||
|
modTime := src.ModTime()
|
||||||
|
|
||||||
o := &Object{
|
o := &Object{
|
||||||
fs: f,
|
fs: f,
|
||||||
remote: remote,
|
remote: remote,
|
||||||
@ -334,7 +338,7 @@ func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs
|
|||||||
modTime: modTime,
|
modTime: modTime,
|
||||||
}
|
}
|
||||||
//TODO maybe read metadata after upload to check if file uploaded successfully
|
//TODO maybe read metadata after upload to check if file uploaded successfully
|
||||||
return o, o.Update(in, modTime, size)
|
return o, o.Update(in, src)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mkdir creates the container if it doesn't exist
|
// Mkdir creates the container if it doesn't exist
|
||||||
@ -390,7 +394,7 @@ func (f *Fs) Hashes() fs.HashSet {
|
|||||||
// ------------------------------------------------------------
|
// ------------------------------------------------------------
|
||||||
|
|
||||||
// Fs returns the parent Fs
|
// Fs returns the parent Fs
|
||||||
func (o *Object) Fs() fs.Fs {
|
func (o *Object) Fs() fs.Info {
|
||||||
return o.fs
|
return o.fs
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -472,7 +476,10 @@ func (o *Object) remotePath() string {
|
|||||||
// Copy the reader into the object updating modTime and size
|
// Copy the reader into the object updating modTime and size
|
||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
|
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
|
||||||
|
size := src.Size()
|
||||||
|
modTime := src.ModTime()
|
||||||
|
|
||||||
remote := o.remotePath()
|
remote := o.remotePath()
|
||||||
//create full path to file before upload.
|
//create full path to file before upload.
|
||||||
err1 := mkDirFullPath(o.fs.yd, remote)
|
err1 := mkDirFullPath(o.fs.yd, remote)
|
||||||
@ -483,7 +490,7 @@ func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
|
|||||||
overwrite := true //overwrite existing file
|
overwrite := true //overwrite existing file
|
||||||
err := o.fs.yd.Upload(in, remote, overwrite)
|
err := o.fs.yd.Upload(in, remote, overwrite)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
//if file uploaded sucessfuly then return metadata
|
//if file uploaded sucessfully then return metadata
|
||||||
o.bytes = uint64(size)
|
o.bytes = uint64(size)
|
||||||
o.modTime = modTime
|
o.modTime = modTime
|
||||||
o.md5sum = "" // according to unit tests after put the md5 is empty.
|
o.md5sum = "" // according to unit tests after put the md5 is empty.
|
||||||
|
Loading…
Reference in New Issue
Block a user