Create separate interface for object information.

Take out read-only information about a Fs in a separate struct to limit access.

See discussion at #282.
This commit is contained in:
klauspost 2016-02-18 12:35:25 +01:00 committed by Nick Craig-Wood
parent 85a0f25b95
commit ef06371c93
16 changed files with 190 additions and 95 deletions

View File

@ -62,7 +62,7 @@ var (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.Info{ fs.Register(&fs.RegInfo{
Name: "amazon cloud drive", Name: "amazon cloud drive",
NewFs: NewFs, NewFs: NewFs,
Config: func(name string) { Config: func(name string) {
@ -521,7 +521,9 @@ func (f *Fs) ListDir() fs.DirChan {
// Copy the reader in to the new object which is returned // Copy the reader in to the new object which is returned
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
remote := src.Remote()
size := src.Size()
// Temporary Object under construction // Temporary Object under construction
o := &Object{ o := &Object{
fs: f, fs: f,
@ -538,7 +540,7 @@ func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs
var info *acd.File var info *acd.File
var resp *http.Response var resp *http.Response
err = f.pacer.CallNoRetry(func() (bool, error) { err = f.pacer.CallNoRetry(func() (bool, error) {
if size != 0 { if src.Size() != 0 {
info, resp, err = folder.Put(in, leaf) info, resp, err = folder.Put(in, leaf)
} else { } else {
info, resp, err = folder.PutSized(in, size, leaf) info, resp, err = folder.PutSized(in, size, leaf)
@ -663,7 +665,7 @@ func (f *Fs) Purge() error {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *Object) Fs() fs.Fs { func (o *Object) Fs() fs.Info {
return o.fs return o.fs
} }
@ -774,7 +776,8 @@ func (o *Object) Open() (in io.ReadCloser, err error) {
// Update the object with the contents of the io.Reader, modTime and size // Update the object with the contents of the io.Reader, modTime and size
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
size := src.Size()
file := acd.File{Node: o.info} file := acd.File{Node: o.info}
var info *acd.File var info *acd.File
var resp *http.Response var resp *http.Response

View File

@ -39,7 +39,7 @@ const (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.Info{ fs.Register(&fs.RegInfo{
Name: "b2", Name: "b2",
NewFs: NewFs, NewFs: NewFs,
Options: []fs.Option{{ Options: []fs.Option{{
@ -462,13 +462,13 @@ func (f *Fs) ListDir() fs.DirChan {
// Copy the reader in to the new object which is returned // Copy the reader in to the new object which is returned
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
// Temporary Object under construction // Temporary Object under construction
fs := &Object{ fs := &Object{
fs: f, fs: f,
remote: remote, remote: src.Remote(),
} }
return fs, fs.Update(in, modTime, size) return fs, fs.Update(in, src)
} }
// Mkdir creates the bucket if it doesn't exist // Mkdir creates the bucket if it doesn't exist
@ -599,7 +599,7 @@ func (f *Fs) Hashes() fs.HashSet {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *Object) Fs() fs.Fs { func (o *Object) Fs() fs.Info {
return o.fs return o.fs
} }
@ -875,7 +875,10 @@ func urlEncode(in string) string {
// Update the object with the contents of the io.Reader, modTime and size // Update the object with the contents of the io.Reader, modTime and size
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) (err error) { func (o *Object) Update(in io.Reader, src fs.ObjectInfo) (err error) {
size := src.Size()
modTime := src.ModTime()
// Open a temp file to copy the input // Open a temp file to copy the input
fd, err := ioutil.TempFile("", "rclone-b2-") fd, err := ioutil.TempFile("", "rclone-b2-")
if err != nil { if err != nil {

View File

@ -84,7 +84,7 @@ var (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.Info{ fs.Register(&fs.RegInfo{
Name: "drive", Name: "drive",
NewFs: NewFs, NewFs: NewFs,
Config: func(name string) { Config: func(name string) {
@ -600,7 +600,11 @@ func (f *Fs) createFileInfo(remote string, modTime time.Time, size int64) (*Obje
// Copy the reader in to the new object which is returned // Copy the reader in to the new object which is returned
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
remote := src.Remote()
size := src.Size()
modTime := src.ModTime()
o, createInfo, err := f.createFileInfo(remote, modTime, size) o, createInfo, err := f.createFileInfo(remote, modTime, size)
if err != nil { if err != nil {
return nil, err return nil, err
@ -818,7 +822,7 @@ func (f *Fs) Hashes() fs.HashSet {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *Object) Fs() fs.Fs { func (o *Object) Fs() fs.Info {
return o.fs return o.fs
} }
@ -1025,7 +1029,9 @@ func (o *Object) Open() (in io.ReadCloser, err error) {
// Copy the reader into the object updating modTime and size // Copy the reader into the object updating modTime and size
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
size := src.Size()
modTime := src.ModTime()
if o.isDocument { if o.isDocument {
return fmt.Errorf("Can't update a google document") return fmt.Errorf("Can't update a google document")
} }

View File

@ -44,7 +44,7 @@ var (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.Info{ fs.Register(&fs.RegInfo{
Name: "dropbox", Name: "dropbox",
NewFs: NewFs, NewFs: NewFs,
Config: configHelper, Config: configHelper,
@ -379,13 +379,13 @@ func (rc *readCloser) Close() error {
// Copy the reader in to the new object which is returned // Copy the reader in to the new object which is returned
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
// Temporary Object under construction // Temporary Object under construction
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: src.Remote(),
} }
return o, o.Update(in, modTime, size) return o, o.Update(in, src)
} }
// Mkdir creates the container if it doesn't exist // Mkdir creates the container if it doesn't exist
@ -531,7 +531,7 @@ func (f *Fs) Hashes() fs.HashSet {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *Object) Fs() fs.Fs { func (o *Object) Fs() fs.Info {
return o.fs return o.fs
} }
@ -656,7 +656,7 @@ func (o *Object) Open() (in io.ReadCloser, err error) {
// Copy the reader into the object updating modTime and size // Copy the reader into the object updating modTime and size
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
remote := o.remotePath() remote := o.remotePath()
if ignoredFiles.MatchString(remote) { if ignoredFiles.MatchString(remote) {
fs.Log(o, "File name disallowed - not uploading") fs.Log(o, "File name disallowed - not uploading")

116
fs/fs.go
View File

@ -22,7 +22,7 @@ const (
// Globals // Globals
var ( var (
// Filesystem registry // Filesystem registry
fsRegistry []*Info fsRegistry []*RegInfo
// ErrorNotFoundInConfigFile is returned by NewFs if not found in config file // ErrorNotFoundInConfigFile is returned by NewFs if not found in config file
ErrorNotFoundInConfigFile = fmt.Errorf("Didn't find section in config file") ErrorNotFoundInConfigFile = fmt.Errorf("Didn't find section in config file")
ErrorCantPurge = fmt.Errorf("Can't purge directory") ErrorCantPurge = fmt.Errorf("Can't purge directory")
@ -32,8 +32,8 @@ var (
ErrorDirExists = fmt.Errorf("Can't copy directory - destination already exists") ErrorDirExists = fmt.Errorf("Can't copy directory - destination already exists")
) )
// Info information about a filesystem // RegInfo provides information about a filesystem
type Info struct { type RegInfo struct {
// Name of this fs // Name of this fs
Name string Name string
// Create a new file system. If root refers to an existing // Create a new file system. If root refers to an existing
@ -63,20 +63,13 @@ type OptionExample struct {
// Register a filesystem // Register a filesystem
// //
// Fs modules should use this in an init() function // Fs modules should use this in an init() function
func Register(info *Info) { func Register(info *RegInfo) {
fsRegistry = append(fsRegistry, info) fsRegistry = append(fsRegistry, info)
} }
// Fs is the interface a cloud storage system must provide // Fs is the interface a cloud storage system must provide
type Fs interface { type Fs interface {
// Name of the remote (as passed into NewFs) Info
Name() string
// Root of the remote (as passed into NewFs)
Root() string
// String returns a description of the FS
String() string
// List the Fs into a channel // List the Fs into a channel
List() ObjectsChan List() ObjectsChan
@ -92,7 +85,7 @@ type Fs interface {
// May create the object even if it returns an error - if so // May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return // will return the object and the error, otherwise will return
// nil and the error // nil and the error
Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error) Put(in io.Reader, src ObjectInfo) (Object, error)
// Mkdir makes the directory (container, bucket) // Mkdir makes the directory (container, bucket)
// //
@ -103,6 +96,18 @@ type Fs interface {
// //
// Return an error if it doesn't exist or isn't empty // Return an error if it doesn't exist or isn't empty
Rmdir() error Rmdir() error
}
// Info provides an interface to reading information about a filesystem.
type Info interface {
// Name of the remote (as passed into NewFs)
Name() string
// Root of the remote (as passed into NewFs)
Root() string
// String returns a description of the FS
String() string
// Precision of the ModTimes in this Fs // Precision of the ModTimes in this Fs
Precision() time.Duration Precision() time.Duration
@ -113,40 +118,45 @@ type Fs interface {
// Object is a filesystem like object provided by an Fs // Object is a filesystem like object provided by an Fs
type Object interface { type Object interface {
ObjectInfo
// String returns a description of the Object // String returns a description of the Object
String() string String() string
// Fs returns the Fs that this object is part of // SetModTime sets the metadata on the object to set the modification date
Fs() Fs SetModTime(time.Time)
// Open opens the file for read. Call Close() on the returned io.ReadCloser
Open() (io.ReadCloser, error)
// Update in to the object with the modTime given of the given size
Update(in io.Reader, src ObjectInfo) error
// Removes this object
Remove() error
}
// ObjectInfo contains information about an object.
type ObjectInfo interface {
// Fs returns read only access to the Fs that this object is part of
Fs() Info
// Remote returns the remote path // Remote returns the remote path
Remote() string Remote() string
// Md5sum returns the md5 checksum of the file // Hash returns the selected checksum of the file
// If no Md5sum is available it returns "" // If no checksum is available it returns ""
Hash(HashType) (string, error) Hash(HashType) (string, error)
// ModTime returns the modification date of the file // ModTime returns the modification date of the file
// It should return a best guess if one isn't available // It should return a best guess if one isn't available
ModTime() time.Time ModTime() time.Time
// SetModTime sets the metadata on the object to set the modification date
SetModTime(time.Time)
// Size returns the size of the file // Size returns the size of the file
Size() int64 Size() int64
// Open opens the file for read. Call Close() on the returned io.ReadCloser
Open() (io.ReadCloser, error)
// Update in to the object with the modTime given of the given size
Update(in io.Reader, modTime time.Time, size int64) error
// Storable says whether this object can be stored // Storable says whether this object can be stored
Storable() bool Storable() bool
// Removes this object
Remove() error
} }
// Purger is an optional interfaces for Fs // Purger is an optional interfaces for Fs
@ -236,7 +246,7 @@ type DirChan chan *Dir
// Find looks for an Info object for the name passed in // Find looks for an Info object for the name passed in
// //
// Services are looked up in the config file // Services are looked up in the config file
func Find(name string) (*Info, error) { func Find(name string) (*RegInfo, error) {
for _, item := range fsRegistry { for _, item := range fsRegistry {
if item.Name == name { if item.Name == name {
return item, nil return item, nil
@ -316,3 +326,49 @@ func CheckClose(c io.Closer, err *error) {
*err = cerr *err = cerr
} }
} }
// NewStaticObjectInfo returns a static ObjectInfo
// If hashes is nil and fs is not nil, the hash map will be replaced with
// empty hashes of the types supported by the fs.
func NewStaticObjectInfo(remote string, modTime time.Time, size int64, storable bool, hashes map[HashType]string, fs Info) ObjectInfo {
info := &staticObjectInfo{
remote: remote,
modTime: modTime,
size: size,
storable: storable,
hashes: hashes,
fs: fs,
}
if fs != nil && hashes == nil {
set := fs.Hashes().Array()
info.hashes = make(map[HashType]string)
for _, ht := range set {
info.hashes[ht] = ""
}
}
return info
}
type staticObjectInfo struct {
remote string
modTime time.Time
size int64
storable bool
hashes map[HashType]string
fs Info
}
func (i *staticObjectInfo) Fs() Info { return i.fs }
func (i *staticObjectInfo) Remote() string { return i.remote }
func (i *staticObjectInfo) ModTime() time.Time { return i.modTime }
func (i *staticObjectInfo) Size() int64 { return i.size }
func (i *staticObjectInfo) Storable() bool { return i.storable }
func (i *staticObjectInfo) Hash(h HashType) (string, error) {
if len(i.hashes) == 0 {
return "", ErrHashUnsupported
}
if hash, ok := i.hashes[h]; ok {
return hash, nil
}
return "", ErrHashUnsupported
}

View File

@ -71,12 +71,13 @@ func (f *Limited) NewFsObject(remote string) Object {
// May create the object even if it returns an error - if so // May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return // will return the object and the error, otherwise will return
// nil and the error // nil and the error
func (f *Limited) Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error) { func (f *Limited) Put(in io.Reader, src ObjectInfo) (Object, error) {
remote := src.Remote()
obj := f.NewFsObject(remote) obj := f.NewFsObject(remote)
if obj == nil { if obj == nil {
return nil, fmt.Errorf("Can't create %q in limited fs", remote) return nil, fmt.Errorf("Can't create %q in limited fs", remote)
} }
return obj, obj.Update(in, modTime, size) return obj, obj.Update(in, src)
} }
// Mkdir make the directory (container, bucket) // Mkdir make the directory (container, bucket)

View File

@ -221,10 +221,10 @@ tryAgain:
if doUpdate { if doUpdate {
actionTaken = "Copied (updated existing)" actionTaken = "Copied (updated existing)"
err = dst.Update(in, src.ModTime(), src.Size()) err = dst.Update(in, src)
} else { } else {
actionTaken = "Copied (new)" actionTaken = "Copied (new)"
dst, err = f.Put(in, src.Remote(), src.ModTime(), src.Size()) dst, err = f.Put(in, src)
} }
inErr = in.Close() inErr = in.Close()
} }

View File

@ -188,7 +188,8 @@ func (r *Run) WriteObjectTo(f fs.Fs, remote, content string, modTime time.Time)
} }
for tries := 1; ; tries++ { for tries := 1; ; tries++ {
in := bytes.NewBufferString(content) in := bytes.NewBufferString(content)
_, err := f.Put(in, remote, modTime, int64(len(content))) objinfo := fs.NewStaticObjectInfo(remote, modTime, int64(len(content)), true, nil, nil)
_, err := f.Put(in, objinfo)
if err == nil { if err == nil {
break break
} }

View File

@ -164,7 +164,8 @@ func testPut(t *testing.T, file *fstest.Item) {
in := io.TeeReader(buf, hash) in := io.TeeReader(buf, hash)
file.Size = int64(buf.Len()) file.Size = int64(buf.Len())
obj, err := remote.Put(in, file.Path, file.ModTime, file.Size) obji := fs.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
obj, err := remote.Put(in, obji)
if err != nil { if err != nil {
t.Fatal("Put error", err) t.Fatal("Put error", err)
} }
@ -551,7 +552,8 @@ func TestObjectUpdate(t *testing.T) {
file1.Size = int64(buf.Len()) file1.Size = int64(buf.Len())
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
err := obj.Update(in, file1.ModTime, file1.Size) obji := fs.NewStaticObjectInfo("", file1.ModTime, file1.Size, true, nil, obj.Fs())
err := obj.Update(in, obji)
if err != nil { if err != nil {
t.Fatal("Update error", err) t.Fatal("Update error", err)
} }

View File

@ -55,7 +55,7 @@ var (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.Info{ fs.Register(&fs.RegInfo{
Name: "google cloud storage", Name: "google cloud storage",
NewFs: NewFs, NewFs: NewFs,
Config: func(name string) { Config: func(name string) {
@ -379,13 +379,13 @@ func (f *Fs) ListDir() fs.DirChan {
// Copy the reader in to the new object which is returned // Copy the reader in to the new object which is returned
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
// Temporary Object under construction // Temporary Object under construction
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: src.Remote(),
} }
return o, o.Update(in, modTime, size) return o, o.Update(in, src)
} }
// Mkdir creates the bucket if it doesn't exist // Mkdir creates the bucket if it doesn't exist
@ -466,7 +466,7 @@ func (f *Fs) Hashes() fs.HashSet {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *Object) Fs() fs.Fs { func (o *Object) Fs() fs.Info {
return o.fs return o.fs
} }
@ -617,7 +617,10 @@ func (o *Object) Open() (in io.ReadCloser, err error) {
// Update the object with the contents of the io.Reader, modTime and size // Update the object with the contents of the io.Reader, modTime and size
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
size := src.Size()
modTime := src.ModTime()
object := storage.Object{ object := storage.Object{
Bucket: o.fs.bucket, Bucket: o.fs.bucket,
Name: o.fs.root + o.remote, Name: o.fs.root + o.remote,

View File

@ -44,7 +44,7 @@ var (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.Info{ fs.Register(&fs.RegInfo{
Name: "hubic", Name: "hubic",
NewFs: NewFs, NewFs: NewFs,
Config: func(name string) { Config: func(name string) {

View File

@ -19,7 +19,7 @@ import (
// Register with Fs // Register with Fs
func init() { func init() {
fsi := &fs.Info{ fsi := &fs.RegInfo{
Name: "local", Name: "local",
NewFs: NewFs, NewFs: NewFs,
Options: []fs.Option{fs.Option{ Options: []fs.Option{fs.Option{
@ -230,10 +230,11 @@ func (f *Fs) ListDir() fs.DirChan {
} }
// Put the FsObject to the local filesystem // Put the FsObject to the local filesystem
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
remote := src.Remote()
// Temporary FsObject under construction - info filled in by Update() // Temporary FsObject under construction - info filled in by Update()
o := f.newFsObject(remote) o := f.newFsObject(remote)
err := o.Update(in, modTime, size) err := o.Update(in, src)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -422,7 +423,7 @@ func (f *Fs) Hashes() fs.HashSet {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *Object) Fs() fs.Fs { func (o *Object) Fs() fs.Info {
return o.fs return o.fs
} }
@ -568,7 +569,7 @@ func (o *Object) mkdirAll() error {
} }
// Update the object from in with modTime and size // Update the object from in with modTime and size
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
err := o.mkdirAll() err := o.mkdirAll()
if err != nil { if err != nil {
return err return err
@ -579,7 +580,7 @@ func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
return err return err
} }
// Calculate the md5sum of the object we are reading as we go along // Calculate the hash of the object we are reading as we go along
hash := fs.NewMultiHasher() hash := fs.NewMultiHasher()
in = io.TeeReader(in, hash) in = io.TeeReader(in, hash)
@ -596,7 +597,7 @@ func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
o.hashes = hash.Sums() o.hashes = hash.Sums()
// Set the mtime // Set the mtime
o.SetModTime(modTime) o.SetModTime(src.ModTime())
// ReRead info now that we have finished // ReRead info now that we have finished
return o.lstat() return o.lstat()

View File

@ -55,7 +55,7 @@ var (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.Info{ fs.Register(&fs.RegInfo{
Name: "onedrive", Name: "onedrive",
NewFs: NewFs, NewFs: NewFs,
Config: func(name string) { Config: func(name string) {
@ -487,12 +487,16 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
// Copy the reader in to the new object which is returned // Copy the reader in to the new object which is returned
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
remote := src.Remote()
size := src.Size()
modTime := src.ModTime()
o, _, _, err := f.createObject(remote, modTime, size) o, _, _, err := f.createObject(remote, modTime, size)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return o, o.Update(in, modTime, size) return o, o.Update(in, src)
} }
// Mkdir creates the container if it doesn't exist // Mkdir creates the container if it doesn't exist
@ -679,7 +683,7 @@ func (f *Fs) Hashes() fs.HashSet {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *Object) Fs() fs.Fs { func (o *Object) Fs() fs.Info {
return o.fs return o.fs
} }
@ -935,7 +939,10 @@ func (o *Object) uploadMultipart(in io.Reader, size int64) (err error) {
// Update the object with the contents of the io.Reader, modTime and size // Update the object with the contents of the io.Reader, modTime and size
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) (err error) { func (o *Object) Update(in io.Reader, src fs.ObjectInfo) (err error) {
size := src.Size()
modTime := src.ModTime()
var info *api.Item var info *api.Item
if size <= int64(uploadCutoff) { if size <= int64(uploadCutoff) {
// This is for less than 100 MB of content // This is for less than 100 MB of content

View File

@ -40,7 +40,7 @@ import (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.Info{ fs.Register(&fs.RegInfo{
Name: "s3", Name: "s3",
NewFs: NewFs, NewFs: NewFs,
// AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region // AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
@ -495,13 +495,13 @@ func (f *Fs) ListDir() fs.DirChan {
} }
// Put the FsObject into the bucket // Put the FsObject into the bucket
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
// Temporary Object under construction // Temporary Object under construction
fs := &Object{ fs := &Object{
fs: f, fs: f,
remote: remote, remote: src.Remote(),
} }
return fs, fs.Update(in, modTime, size) return fs, fs.Update(in, src)
} }
// Mkdir creates the bucket if it doesn't exist // Mkdir creates the bucket if it doesn't exist
@ -582,7 +582,7 @@ func (f *Fs) Hashes() fs.HashSet {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *Object) Fs() fs.Fs { func (o *Object) Fs() fs.Info {
return o.fs return o.fs
} }
@ -732,7 +732,9 @@ func (o *Object) Open() (in io.ReadCloser, err error) {
} }
// Update the Object from in with modTime and size // Update the Object from in with modTime and size
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
modTime := src.ModTime()
uploader := s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) { uploader := s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) {
u.Concurrency = 2 u.Concurrency = 2
u.LeavePartsOnError = false u.LeavePartsOnError = false

View File

@ -29,7 +29,7 @@ var (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.Info{ fs.Register(&fs.RegInfo{
Name: "swift", Name: "swift",
NewFs: NewFs, NewFs: NewFs,
Options: []fs.Option{{ Options: []fs.Option{{
@ -380,13 +380,13 @@ func (f *Fs) ListDir() fs.DirChan {
// Copy the reader in to the new object which is returned // Copy the reader in to the new object which is returned
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
// Temporary Object under construction // Temporary Object under construction
fs := &Object{ fs := &Object{
fs: f, fs: f,
remote: remote, remote: src.Remote(),
} }
return fs, fs.Update(in, modTime, size) return fs, fs.Update(in, src)
} }
// Mkdir creates the container if it doesn't exist // Mkdir creates the container if it doesn't exist
@ -457,7 +457,7 @@ func (f *Fs) Hashes() fs.HashSet {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *Object) Fs() fs.Fs { func (o *Object) Fs() fs.Info {
return o.fs return o.fs
} }
@ -655,7 +655,10 @@ func (o *Object) updateChunks(in io.Reader, headers swift.Headers, size int64) (
// Update the object with the contents of the io.Reader, modTime and size // Update the object with the contents of the io.Reader, modTime and size
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
size := src.Size()
modTime := src.ModTime()
// Note whether this has a manifest before starting // Note whether this has a manifest before starting
isManifest, err := o.isManifestFile() isManifest, err := o.isManifestFile()
if err != nil { if err != nil {

View File

@ -41,7 +41,7 @@ var (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.Info{ fs.Register(&fs.RegInfo{
Name: "yandex", Name: "yandex",
NewFs: NewFs, NewFs: NewFs,
Config: func(name string) { Config: func(name string) {
@ -326,7 +326,11 @@ func (f *Fs) ListDir() fs.DirChan {
// Copy the reader in to the new object which is returned // Copy the reader in to the new object which is returned
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
remote := src.Remote()
size := src.Size()
modTime := src.ModTime()
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
@ -334,7 +338,7 @@ func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs
modTime: modTime, modTime: modTime,
} }
//TODO maybe read metadata after upload to check if file uploaded successfully //TODO maybe read metadata after upload to check if file uploaded successfully
return o, o.Update(in, modTime, size) return o, o.Update(in, src)
} }
// Mkdir creates the container if it doesn't exist // Mkdir creates the container if it doesn't exist
@ -390,7 +394,7 @@ func (f *Fs) Hashes() fs.HashSet {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *Object) Fs() fs.Fs { func (o *Object) Fs() fs.Info {
return o.fs return o.fs
} }
@ -472,7 +476,10 @@ func (o *Object) remotePath() string {
// Copy the reader into the object updating modTime and size // Copy the reader into the object updating modTime and size
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
size := src.Size()
modTime := src.ModTime()
remote := o.remotePath() remote := o.remotePath()
//create full path to file before upload. //create full path to file before upload.
err1 := mkDirFullPath(o.fs.yd, remote) err1 := mkDirFullPath(o.fs.yd, remote)
@ -483,7 +490,7 @@ func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
overwrite := true //overwrite existing file overwrite := true //overwrite existing file
err := o.fs.yd.Upload(in, remote, overwrite) err := o.fs.yd.Upload(in, remote, overwrite)
if err == nil { if err == nil {
//if file uploaded sucessfuly then return metadata //if file uploaded sucessfully then return metadata
o.bytes = uint64(size) o.bytes = uint64(size)
o.modTime = modTime o.modTime = modTime
o.md5sum = "" // according to unit tests after put the md5 is empty. o.md5sum = "" // according to unit tests after put the md5 is empty.