extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-12-22 15:11:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Be more constistent with naming in remotes

Browse Source 
* External objects are called Fs and Object
  * Object.fs always points to the Fs
This commit is contained in:
Nick Craig-Wood 2015-11-07 11:14:46 +00:00
parent 365b4babae
commit b257de4aba
17 changed files with 506 additions and 475 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
amazonclouddrive/amazonclouddrive.go
View File

@ -77,8 +77,8 @@ func init() {
}) })
} }
// FsAcd represents a remote acd server // Fs represents a remote acd server
type FsAcd struct { type Fs struct {
name string // name of this remote name string // name of this remote
c *acd.Client // the connection to the acd server c *acd.Client // the connection to the acd server
root string // the path we are working on root string // the path we are working on
@ -86,11 +86,11 @@ type FsAcd struct {
pacer *pacer.Pacer // pacer for API calls pacer *pacer.Pacer // pacer for API calls
} }
// FsObjectAcd describes a acd object // Object describes a acd object
// //
// Will definitely have info but maybe not meta // Will definitely have info but maybe not meta
type FsObjectAcd struct { type Object struct {
acd *FsAcd // what this object is part of fs *Fs // what this object is part of
remote string // The remote path remote string // The remote path
info *acd.Node // Info from the acd object if known info *acd.Node // Info from the acd object if known
} }
@ -98,17 +98,17 @@ type FsObjectAcd struct {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Name of the remote (as passed into NewFs) // Name of the remote (as passed into NewFs)
func (f *FsAcd) Name() string { func (f *Fs) Name() string {
return f.name return f.name
} }
// Root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
func (f *FsAcd) Root() string { func (f *Fs) Root() string {
return f.root return f.root
} }
// String converts this FsAcd to a string // String converts this Fs to a string
func (f *FsAcd) String() string { func (f *Fs) String() string {
return fmt.Sprintf("Amazon cloud drive root '%s'", f.root) return fmt.Sprintf("Amazon cloud drive root '%s'", f.root)
} }
@ -135,7 +135,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
return fs.ShouldRetry(err) || fs.ShouldRetryHTTP(resp, retryErrorCodes), err return fs.ShouldRetry(err) || fs.ShouldRetryHTTP(resp, retryErrorCodes), err
} }
// NewFs constructs an FsAcd from the path, container:path // NewFs constructs an Fs from the path, container:path
func NewFs(name, root string) (fs.Fs, error) { func NewFs(name, root string) (fs.Fs, error) {
root = parsePath(root) root = parsePath(root)
oAuthClient, err := oauthutil.NewClient(name, acdConfig) oAuthClient, err := oauthutil.NewClient(name, acdConfig)
@ -145,7 +145,7 @@ func NewFs(name, root string) (fs.Fs, error) {
c := acd.NewClient(oAuthClient) c := acd.NewClient(oAuthClient)
c.UserAgent = fs.UserAgent c.UserAgent = fs.UserAgent
f := &FsAcd{ f := &Fs{
name: name, name: name,
root: root, root: root,
c: c, c: c,
@ -202,9 +202,9 @@ func NewFs(name, root string) (fs.Fs, error) {
// Return an FsObject from a path // Return an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsAcd) newFsObjectWithInfo(remote string, info *acd.Node) fs.Object { func (f *Fs) newFsObjectWithInfo(remote string, info *acd.Node) fs.Object {
o := &FsObjectAcd{ o := &Object{
acd: f, fs: f,
remote: remote, remote: remote,
} }
if info != nil { if info != nil {
@ -223,12 +223,12 @@ func (f *FsAcd) newFsObjectWithInfo(remote string, info *acd.Node) fs.Object {
// NewFsObject returns an FsObject from a path // NewFsObject returns an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsAcd) NewFsObject(remote string) fs.Object { func (f *Fs) NewFsObject(remote string) fs.Object {
return f.newFsObjectWithInfo(remote, nil) return f.newFsObjectWithInfo(remote, nil)
} }
// FindLeaf finds a directory of name leaf in the folder with ID pathID // FindLeaf finds a directory of name leaf in the folder with ID pathID
func (f *FsAcd) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) { func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) {
//fs.Debug(f, "FindLeaf(%q, %q)", pathID, leaf) //fs.Debug(f, "FindLeaf(%q, %q)", pathID, leaf)
folder := acd.FolderFromId(pathID, f.c.Nodes) folder := acd.FolderFromId(pathID, f.c.Nodes)
var resp *http.Response var resp *http.Response
@ -255,7 +255,7 @@ func (f *FsAcd) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err
} }
// CreateDir makes a directory with pathID as parent and name leaf // CreateDir makes a directory with pathID as parent and name leaf
func (f *FsAcd) CreateDir(pathID, leaf string) (newID string, err error) { func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) {
//fmt.Printf("CreateDir(%q, %q)\n", pathID, leaf) //fmt.Printf("CreateDir(%q, %q)\n", pathID, leaf)
folder := acd.FolderFromId(pathID, f.c.Nodes) folder := acd.FolderFromId(pathID, f.c.Nodes)
var resp *http.Response var resp *http.Response
@ -283,7 +283,7 @@ type listAllFn func(*acd.Node) bool
// Lists the directory required calling the user function on each item found // Lists the directory required calling the user function on each item found
// //
// If the user fn ever returns true then it early exits with found = true // If the user fn ever returns true then it early exits with found = true
func (f *FsAcd) listAll(dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
query := "parents:" + dirID query := "parents:" + dirID
if directoriesOnly { if directoriesOnly {
query += " AND kind:" + folderKind query += " AND kind:" + folderKind
@ -336,7 +336,7 @@ OUTER:
// //
// This fetches the minimum amount of stuff but does more API calls // This fetches the minimum amount of stuff but does more API calls
// which makes it slow // which makes it slow
func (f *FsAcd) listDirRecursive(dirID string, path string, out fs.ObjectsChan) error { func (f *Fs) listDirRecursive(dirID string, path string, out fs.ObjectsChan) error {
var subError error var subError error
// Make the API request // Make the API request
var wg sync.WaitGroup var wg sync.WaitGroup
@ -377,7 +377,7 @@ func (f *FsAcd) listDirRecursive(dirID string, path string, out fs.ObjectsChan)
} }
// List walks the path returning a channel of FsObjects // List walks the path returning a channel of FsObjects
func (f *FsAcd) List() fs.ObjectsChan { func (f *Fs) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
go func() { go func() {
defer close(out) defer close(out)
@ -397,7 +397,7 @@ func (f *FsAcd) List() fs.ObjectsChan {
} }
// ListDir lists the directories // ListDir lists the directories
func (f *FsAcd) ListDir() fs.DirChan { func (f *Fs) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
go func() { go func() {
defer close(out) defer close(out)
@ -430,17 +430,17 @@ func (f *FsAcd) ListDir() fs.DirChan {
// Copy the reader in to the new object which is returned // Copy the reader in to the new object which is returned
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (f *FsAcd) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction // Temporary Object under construction
o := &FsObjectAcd{ o := &Object{
acd: f, fs: f,
remote: remote, remote: remote,
} }
leaf, directoryID, err := f.dirCache.FindPath(remote, true) leaf, directoryID, err := f.dirCache.FindPath(remote, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
folder := acd.FolderFromId(directoryID, o.acd.c.Nodes) folder := acd.FolderFromId(directoryID, o.fs.c.Nodes)
var info *acd.File var info *acd.File
var resp *http.Response var resp *http.Response
err = f.pacer.CallNoRetry(func() (bool, error) { err = f.pacer.CallNoRetry(func() (bool, error) {
@ -459,13 +459,13 @@ func (f *FsAcd) Put(in io.Reader, remote string, modTime time.Time, size int64)
} }
// Mkdir creates the container if it doesn't exist // Mkdir creates the container if it doesn't exist
func (f *FsAcd) Mkdir() error { func (f *Fs) Mkdir() error {
return f.dirCache.FindRoot(true) return f.dirCache.FindRoot(true)
} }
// purgeCheck remotes the root directory, if check is set then it // purgeCheck remotes the root directory, if check is set then it
// refuses to do so if it has anything in // refuses to do so if it has anything in
func (f *FsAcd) purgeCheck(check bool) error { func (f *Fs) purgeCheck(check bool) error {
if f.root == "" { if f.root == "" {
return fmt.Errorf("Can't purge root directory") return fmt.Errorf("Can't purge root directory")
} }
@ -520,12 +520,12 @@ func (f *FsAcd) purgeCheck(check bool) error {
// Rmdir deletes the root folder // Rmdir deletes the root folder
// //
// Returns an error if it isn't empty // Returns an error if it isn't empty
func (f *FsAcd) Rmdir() error { func (f *Fs) Rmdir() error {
return f.purgeCheck(true) return f.purgeCheck(true)
} }
// Precision return the precision of this Fs // Precision return the precision of this Fs
func (f *FsAcd) Precision() time.Duration { func (f *Fs) Precision() time.Duration {
return fs.ModTimeNotSupported return fs.ModTimeNotSupported
} }
@ -538,13 +538,13 @@ func (f *FsAcd) Precision() time.Duration {
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
// If it isn't possible then return fs.ErrorCantCopy // If it isn't possible then return fs.ErrorCantCopy
//func (f *FsAcd) Copy(src fs.Object, remote string) (fs.Object, error) { //func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
// srcObj, ok := src.(*FsObjectAcd) // srcObj, ok := src.(*Object)
// if !ok { // if !ok {
// fs.Debug(src, "Can't copy - not same remote type") // fs.Debug(src, "Can't copy - not same remote type")
// return nil, fs.ErrorCantCopy // return nil, fs.ErrorCantCopy
// } // }
// srcFs := srcObj.acd // srcFs := srcObj.fs
// _, err := f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil) // _, err := f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil)
// if err != nil { // if err != nil {
// return nil, err // return nil, err
@ -557,19 +557,19 @@ func (f *FsAcd) Precision() time.Duration {
// Optional interface: Only implement this if you have a way of // Optional interface: Only implement this if you have a way of
// deleting all the files quicker than just running Remove() on the // deleting all the files quicker than just running Remove() on the
// result of List() // result of List()
func (f *FsAcd) Purge() error { func (f *Fs) Purge() error {
return f.purgeCheck(false) return f.purgeCheck(false)
} }
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *FsObjectAcd) Fs() fs.Fs { func (o *Object) Fs() fs.Fs {
return o.acd return o.fs
} }
// Return a string version // Return a string version
func (o *FsObjectAcd) String() string { func (o *Object) String() string {
if o == nil { if o == nil {
return "<nil>" return "<nil>"
} }
@ -577,12 +577,12 @@ func (o *FsObjectAcd) String() string {
} }
// Remote returns the remote path // Remote returns the remote path
func (o *FsObjectAcd) Remote() string { func (o *Object) Remote() string {
return o.remote return o.remote
} }
// Md5sum returns the Md5sum of an object returning a lowercase hex string // Md5sum returns the Md5sum of an object returning a lowercase hex string
func (o *FsObjectAcd) Md5sum() (string, error) { func (o *Object) Md5sum() (string, error) {
if o.info.ContentProperties.Md5 != nil { if o.info.ContentProperties.Md5 != nil {
return *o.info.ContentProperties.Md5, nil return *o.info.ContentProperties.Md5, nil
} }
@ -590,25 +590,25 @@ func (o *FsObjectAcd) Md5sum() (string, error) {
} }
// Size returns the size of an object in bytes // Size returns the size of an object in bytes
func (o *FsObjectAcd) Size() int64 { func (o *Object) Size() int64 {
return int64(*o.info.ContentProperties.Size) return int64(*o.info.ContentProperties.Size)
} }
// readMetaData gets the metadata if it hasn't already been fetched // readMetaData gets the metadata if it hasn't already been fetched
// //
// it also sets the info // it also sets the info
func (o *FsObjectAcd) readMetaData() (err error) { func (o *Object) readMetaData() (err error) {
if o.info != nil { if o.info != nil {
return nil return nil
} }
leaf, directoryID, err := o.acd.dirCache.FindPath(o.remote, false) leaf, directoryID, err := o.fs.dirCache.FindPath(o.remote, false)
if err != nil { if err != nil {
return err return err
} }
folder := acd.FolderFromId(directoryID, o.acd.c.Nodes) folder := acd.FolderFromId(directoryID, o.fs.c.Nodes)
var resp *http.Response var resp *http.Response
var info *acd.File var info *acd.File
err = o.acd.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
info, resp, err = folder.GetFile(leaf) info, resp, err = folder.GetFile(leaf)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -625,7 +625,7 @@ func (o *FsObjectAcd) readMetaData() (err error) {
// //
// It attempts to read the objects mtime and if that isn't present the // It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers // LastModified returned in the http headers
func (o *FsObjectAcd) ModTime() time.Time { func (o *Object) ModTime() time.Time {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
fs.Log(o, "Failed to read metadata: %s", err) fs.Log(o, "Failed to read metadata: %s", err)
@ -640,21 +640,21 @@ func (o *FsObjectAcd) ModTime() time.Time {
} }
// SetModTime sets the modification time of the local fs object // SetModTime sets the modification time of the local fs object
func (o *FsObjectAcd) SetModTime(modTime time.Time) { func (o *Object) SetModTime(modTime time.Time) {
// FIXME not implemented // FIXME not implemented
return return
} }
// Storable returns a boolean showing whether this object storable // Storable returns a boolean showing whether this object storable
func (o *FsObjectAcd) Storable() bool { func (o *Object) Storable() bool {
return true return true
} }
// Open an object for read // Open an object for read
func (o *FsObjectAcd) Open() (in io.ReadCloser, err error) { func (o *Object) Open() (in io.ReadCloser, err error) {
file := acd.File{Node: o.info} file := acd.File{Node: o.info}
var resp *http.Response var resp *http.Response
err = o.acd.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
in, resp, err = file.Open() in, resp, err = file.Open()
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -664,12 +664,12 @@ func (o *FsObjectAcd) Open() (in io.ReadCloser, err error) {
// Update the object with the contents of the io.Reader, modTime and size // Update the object with the contents of the io.Reader, modTime and size
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (o *FsObjectAcd) Update(in io.Reader, modTime time.Time, size int64) error { func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
file := acd.File{Node: o.info} file := acd.File{Node: o.info}
var info *acd.File var info *acd.File
var resp *http.Response var resp *http.Response
var err error var err error
err = o.acd.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
if size != 0 { if size != 0 {
info, resp, err = file.OverwriteSized(in, size) info, resp, err = file.OverwriteSized(in, size)
} else { } else {
@ -685,10 +685,10 @@ func (o *FsObjectAcd) Update(in io.Reader, modTime time.Time, size int64) error
} }
// Remove an object // Remove an object
func (o *FsObjectAcd) Remove() error { func (o *Object) Remove() error {
var resp *http.Response var resp *http.Response
var err error var err error
err = o.acd.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.info.Trash() resp, err = o.info.Trash()
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -697,10 +697,10 @@ func (o *FsObjectAcd) Remove() error {
// Check the interfaces are satisfied // Check the interfaces are satisfied
var ( var (
_ fs.Fs = (*FsAcd)(nil) _ fs.Fs = (*Fs)(nil)
_ fs.Purger = (*FsAcd)(nil) _ fs.Purger = (*Fs)(nil)
// _ fs.Copier = (*FsAcd)(nil) // _ fs.Copier = (*Fs)(nil)
// _ fs.Mover = (*FsAcd)(nil) // _ fs.Mover = (*Fs)(nil)
// _ fs.DirMover = (*FsAcd)(nil) // _ fs.DirMover = (*Fs)(nil)
_ fs.Object = (*FsObjectAcd)(nil) _ fs.Object = (*Object)(nil)
) )

2
amazonclouddrive/amazonclouddrive_test.go
View File

@ -13,7 +13,7 @@ import (
) )
func init() { func init() {
fstests.NilObject = fs.Object((*amazonclouddrive.FsObjectAcd)(nil)) fstests.NilObject = fs.Object((*amazonclouddrive.Object)(nil))
fstests.RemoteName = "TestAmazonCloudDrive:" fstests.RemoteName = "TestAmazonCloudDrive:"
} }

158
drive/drive.go
View File

@ -82,8 +82,8 @@ func init() {
pflag.VarP(&chunkSize, "drive-chunk-size", "", "Upload chunk size. Must a power of 2 >= 256k.") pflag.VarP(&chunkSize, "drive-chunk-size", "", "Upload chunk size. Must a power of 2 >= 256k.")
} }
// FsDrive represents a remote drive server // Fs represents a remote drive server
type FsDrive struct { type Fs struct {
name string // name of this remote name string // name of this remote
svc *drive.Service // the connection to the drive server svc *drive.Service // the connection to the drive server
root string // the path we are working on root string // the path we are working on
@ -93,31 +93,31 @@ type FsDrive struct {
pacer *pacer.Pacer // To pace the API calls pacer *pacer.Pacer // To pace the API calls
} }
// FsObjectDrive describes a drive object // Object describes a drive object
type FsObjectDrive struct { type Object struct {
drive *FsDrive // what this object is part of fs *Fs // what this object is part of
remote string // The remote path remote string // The remote path
id string // Drive Id of this object id string // Drive Id of this object
url string // Download URL of this object url string // Download URL of this object
md5sum string // md5sum of the object md5sum string // md5sum of the object
bytes int64 // size of the object bytes int64 // size of the object
modifiedDate string // RFC3339 time it was last modified modifiedDate string // RFC3339 time it was last modified
} }
// ------------------------------------------------------------ // ------------------------------------------------------------
// Name of the remote (as passed into NewFs) // Name of the remote (as passed into NewFs)
func (f *FsDrive) Name() string { func (f *Fs) Name() string {
return f.name return f.name
} }
// Root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
func (f *FsDrive) Root() string { func (f *Fs) Root() string {
return f.root return f.root
} }
// String converts this FsDrive to a string // String converts this Fs to a string
func (f *FsDrive) String() string { func (f *Fs) String() string {
return fmt.Sprintf("Google drive root '%s'", f.root) return fmt.Sprintf("Google drive root '%s'", f.root)
} }
@ -161,7 +161,7 @@ type listAllFn func(*drive.File) bool
// If the user fn ever returns true then it early exits with found = true // If the user fn ever returns true then it early exits with found = true
// //
// Search params: https://developers.google.com/drive/search-parameters // Search params: https://developers.google.com/drive/search-parameters
func (f *FsDrive) listAll(dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
query := fmt.Sprintf("trashed=false") query := fmt.Sprintf("trashed=false")
if dirID != "" { if dirID != "" {
query += fmt.Sprintf(" and '%s' in parents", dirID) query += fmt.Sprintf(" and '%s' in parents", dirID)
@ -216,7 +216,7 @@ func isPowerOfTwo(x int64) bool {
} }
} }
// NewFs contstructs an FsDrive from the path, container:path // NewFs contstructs an Fs from the path, container:path
func NewFs(name, path string) (fs.Fs, error) { func NewFs(name, path string) (fs.Fs, error) {
if !isPowerOfTwo(int64(chunkSize)) { if !isPowerOfTwo(int64(chunkSize)) {
return nil, fmt.Errorf("drive: chunk size %v isn't a power of two", chunkSize) return nil, fmt.Errorf("drive: chunk size %v isn't a power of two", chunkSize)
@ -235,7 +235,7 @@ func NewFs(name, path string) (fs.Fs, error) {
return nil, err return nil, err
} }
f := &FsDrive{ f := &Fs{
name: name, name: name,
root: root, root: root,
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant), pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
@ -286,9 +286,9 @@ func NewFs(name, path string) (fs.Fs, error) {
} }
// Return an FsObject from a path // Return an FsObject from a path
func (f *FsDrive) newFsObjectWithInfoErr(remote string, info *drive.File) (fs.Object, error) { func (f *Fs) newFsObjectWithInfoErr(remote string, info *drive.File) (fs.Object, error) {
fs := &FsObjectDrive{ fs := &Object{
drive: f, fs: f,
remote: remote, remote: remote,
} }
if info != nil { if info != nil {
@ -306,7 +306,7 @@ func (f *FsDrive) newFsObjectWithInfoErr(remote string, info *drive.File) (fs.Ob
// Return an FsObject from a path // Return an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsDrive) newFsObjectWithInfo(remote string, info *drive.File) fs.Object { func (f *Fs) newFsObjectWithInfo(remote string, info *drive.File) fs.Object {
fs, _ := f.newFsObjectWithInfoErr(remote, info) fs, _ := f.newFsObjectWithInfoErr(remote, info)
// Errors have already been logged // Errors have already been logged
return fs return fs
@ -315,12 +315,12 @@ func (f *FsDrive) newFsObjectWithInfo(remote string, info *drive.File) fs.Object
// NewFsObject returns an FsObject from a path // NewFsObject returns an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsDrive) NewFsObject(remote string) fs.Object { func (f *Fs) NewFsObject(remote string) fs.Object {
return f.newFsObjectWithInfo(remote, nil) return f.newFsObjectWithInfo(remote, nil)
} }
// FindLeaf finds a directory of name leaf in the folder with ID pathID // FindLeaf finds a directory of name leaf in the folder with ID pathID
func (f *FsDrive) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) { func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) {
// Find the leaf in pathID // Find the leaf in pathID
found, err = f.listAll(pathID, leaf, true, false, func(item *drive.File) bool { found, err = f.listAll(pathID, leaf, true, false, func(item *drive.File) bool {
if item.Title == leaf { if item.Title == leaf {
@ -333,7 +333,7 @@ func (f *FsDrive) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, e
} }
// CreateDir makes a directory with pathID as parent and name leaf // CreateDir makes a directory with pathID as parent and name leaf
func (f *FsDrive) CreateDir(pathID, leaf string) (newID string, err error) { func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) {
// fmt.Println("Making", path) // fmt.Println("Making", path)
// Define the metadata for the directory we are going to create. // Define the metadata for the directory we are going to create.
createInfo := &drive.File{ createInfo := &drive.File{
@ -359,7 +359,7 @@ func (f *FsDrive) CreateDir(pathID, leaf string) (newID string, err error) {
// //
// This fetches the minimum amount of stuff but does more API calls // This fetches the minimum amount of stuff but does more API calls
// which makes it slow // which makes it slow
func (f *FsDrive) listDirRecursive(dirID string, path string, out fs.ObjectsChan) error { func (f *Fs) listDirRecursive(dirID string, path string, out fs.ObjectsChan) error {
var subError error var subError error
// Make the API request // Make the API request
var wg sync.WaitGroup var wg sync.WaitGroup
@ -407,7 +407,7 @@ func (f *FsDrive) listDirRecursive(dirID string, path string, out fs.ObjectsChan
// //
// This is fast in terms of number of API calls, but slow in terms of // This is fast in terms of number of API calls, but slow in terms of
// fetching more data than it needs // fetching more data than it needs
func (f *FsDrive) listDirFull(dirID string, path string, out fs.ObjectsChan) error { func (f *Fs) listDirFull(dirID string, path string, out fs.ObjectsChan) error {
// Orphans waiting for their parent // Orphans waiting for their parent
orphans := make(map[string][]*drive.File) orphans := make(map[string][]*drive.File)
@ -469,7 +469,7 @@ func (f *FsDrive) listDirFull(dirID string, path string, out fs.ObjectsChan) err
} }
// List walks the path returning a channel of FsObjects // List walks the path returning a channel of FsObjects
func (f *FsDrive) List() fs.ObjectsChan { func (f *Fs) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
go func() { go func() {
defer close(out) defer close(out)
@ -493,7 +493,7 @@ func (f *FsDrive) List() fs.ObjectsChan {
} }
// ListDir walks the path returning a channel of directories // ListDir walks the path returning a channel of directories
func (f *FsDrive) ListDir() fs.DirChan { func (f *Fs) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
go func() { go func() {
defer close(out) defer close(out)
@ -522,13 +522,13 @@ func (f *FsDrive) ListDir() fs.DirChan {
} }
// Creates a drive.File info from the parameters passed in and a half // Creates a drive.File info from the parameters passed in and a half
// finished FsObjectDrive which must have setMetaData called on it // finished Object which must have setMetaData called on it
// //
// Used to create new objects // Used to create new objects
func (f *FsDrive) createFileInfo(remote string, modTime time.Time, size int64) (*FsObjectDrive, *drive.File, error) { func (f *Fs) createFileInfo(remote string, modTime time.Time, size int64) (*Object, *drive.File, error) {
// Temporary FsObject under construction // Temporary Object under construction
o := &FsObjectDrive{ o := &Object{
drive: f, fs: f,
remote: remote, remote: remote,
bytes: size, bytes: size,
} }
@ -558,7 +558,7 @@ func (f *FsDrive) createFileInfo(remote string, modTime time.Time, size int64) (
// Copy the reader in to the new object which is returned // Copy the reader in to the new object which is returned
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
o, createInfo, err := f.createFileInfo(remote, modTime, size) o, createInfo, err := f.createFileInfo(remote, modTime, size)
if err != nil { if err != nil {
return nil, err return nil, err
@ -587,14 +587,14 @@ func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64
} }
// Mkdir creates the container if it doesn't exist // Mkdir creates the container if it doesn't exist
func (f *FsDrive) Mkdir() error { func (f *Fs) Mkdir() error {
return f.dirCache.FindRoot(true) return f.dirCache.FindRoot(true)
} }
// Rmdir deletes the container // Rmdir deletes the container
// //
// Returns an error if it isn't empty // Returns an error if it isn't empty
func (f *FsDrive) Rmdir() error { func (f *Fs) Rmdir() error {
err := f.dirCache.FindRoot(false) err := f.dirCache.FindRoot(false)
if err != nil { if err != nil {
return err return err
@ -629,7 +629,7 @@ func (f *FsDrive) Rmdir() error {
} }
// Precision of the object storage system // Precision of the object storage system
func (f *FsDrive) Precision() time.Duration { func (f *Fs) Precision() time.Duration {
return time.Millisecond return time.Millisecond
} }
@ -642,8 +642,8 @@ func (f *FsDrive) Precision() time.Duration {
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
// If it isn't possible then return fs.ErrorCantCopy // If it isn't possible then return fs.ErrorCantCopy
func (f *FsDrive) Copy(src fs.Object, remote string) (fs.Object, error) { func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectDrive) srcObj, ok := src.(*Object)
if !ok { if !ok {
fs.Debug(src, "Can't copy - not same remote type") fs.Debug(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy return nil, fs.ErrorCantCopy
@ -655,8 +655,8 @@ func (f *FsDrive) Copy(src fs.Object, remote string) (fs.Object, error) {
} }
var info *drive.File var info *drive.File
err = o.drive.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
info, err = o.drive.svc.Files.Copy(srcObj.id, createInfo).Do() info, err = o.fs.svc.Files.Copy(srcObj.id, createInfo).Do()
return shouldRetry(err) return shouldRetry(err)
}) })
if err != nil { if err != nil {
@ -672,7 +672,7 @@ func (f *FsDrive) Copy(src fs.Object, remote string) (fs.Object, error) {
// Optional interface: Only implement this if you have a way of // Optional interface: Only implement this if you have a way of
// deleting all the files quicker than just running Remove() on the // deleting all the files quicker than just running Remove() on the
// result of List() // result of List()
func (f *FsDrive) Purge() error { func (f *Fs) Purge() error {
if f.root == "" { if f.root == "" {
return fmt.Errorf("Can't purge root directory") return fmt.Errorf("Can't purge root directory")
} }
@ -704,8 +704,8 @@ func (f *FsDrive) Purge() error {
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
// If it isn't possible then return fs.ErrorCantMove // If it isn't possible then return fs.ErrorCantMove
func (f *FsDrive) Move(src fs.Object, remote string) (fs.Object, error) { func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectDrive) srcObj, ok := src.(*Object)
if !ok { if !ok {
fs.Debug(src, "Can't move - not same remote type") fs.Debug(src, "Can't move - not same remote type")
return nil, fs.ErrorCantMove return nil, fs.ErrorCantMove
@ -735,8 +735,8 @@ func (f *FsDrive) Move(src fs.Object, remote string) (fs.Object, error) {
// If it isn't possible then return fs.ErrorCantDirMove // If it isn't possible then return fs.ErrorCantDirMove
// //
// If destination exists then return fs.ErrorDirExists // If destination exists then return fs.ErrorDirExists
func (f *FsDrive) DirMove(src fs.Fs) error { func (f *Fs) DirMove(src fs.Fs) error {
srcFs, ok := src.(*FsDrive) srcFs, ok := src.(*Fs)
if !ok { if !ok {
fs.Debug(srcFs, "Can't move directory - not same remote type") fs.Debug(srcFs, "Can't move directory - not same remote type")
return fs.ErrorCantDirMove return fs.ErrorCantDirMove
@ -771,12 +771,12 @@ func (f *FsDrive) DirMove(src fs.Fs) error {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *FsObjectDrive) Fs() fs.Fs { func (o *Object) Fs() fs.Fs {
return o.drive return o.fs
} }
// Return a string version // Return a string version
func (o *FsObjectDrive) String() string { func (o *Object) String() string {
if o == nil { if o == nil {
return "<nil>" return "<nil>"
} }
@ -784,22 +784,22 @@ func (o *FsObjectDrive) String() string {
} }
// Remote returns the remote path // Remote returns the remote path
func (o *FsObjectDrive) Remote() string { func (o *Object) Remote() string {
return o.remote return o.remote
} }
// Md5sum returns the Md5sum of an object returning a lowercase hex string // Md5sum returns the Md5sum of an object returning a lowercase hex string
func (o *FsObjectDrive) Md5sum() (string, error) { func (o *Object) Md5sum() (string, error) {
return o.md5sum, nil return o.md5sum, nil
} }
// Size returns the size of an object in bytes // Size returns the size of an object in bytes
func (o *FsObjectDrive) Size() int64 { func (o *Object) Size() int64 {
return o.bytes return o.bytes
} }
// setMetaData sets the fs data from a drive.File // setMetaData sets the fs data from a drive.File
func (o *FsObjectDrive) setMetaData(info *drive.File) { func (o *Object) setMetaData(info *drive.File) {
o.id = info.Id o.id = info.Id
o.url = info.DownloadUrl o.url = info.DownloadUrl
o.md5sum = strings.ToLower(info.Md5Checksum) o.md5sum = strings.ToLower(info.Md5Checksum)
@ -808,17 +808,17 @@ func (o *FsObjectDrive) setMetaData(info *drive.File) {
} }
// readMetaData gets the info if it hasn't already been fetched // readMetaData gets the info if it hasn't already been fetched
func (o *FsObjectDrive) readMetaData() (err error) { func (o *Object) readMetaData() (err error) {
if o.id != "" { if o.id != "" {
return nil return nil
} }
leaf, directoryID, err := o.drive.dirCache.FindPath(o.remote, false) leaf, directoryID, err := o.fs.dirCache.FindPath(o.remote, false)
if err != nil { if err != nil {
return err return err
} }
found, err := o.drive.listAll(directoryID, leaf, false, true, func(item *drive.File) bool { found, err := o.fs.listAll(directoryID, leaf, false, true, func(item *drive.File) bool {
if item.Title == leaf { if item.Title == leaf {
o.setMetaData(item) o.setMetaData(item)
return true return true
@ -840,7 +840,7 @@ func (o *FsObjectDrive) readMetaData() (err error) {
// //
// It attempts to read the objects mtime and if that isn't present the // It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers // LastModified returned in the http headers
func (o *FsObjectDrive) ModTime() time.Time { func (o *Object) ModTime() time.Time {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
fs.Log(o, "Failed to read metadata: %s", err) fs.Log(o, "Failed to read metadata: %s", err)
@ -855,7 +855,7 @@ func (o *FsObjectDrive) ModTime() time.Time {
} }
// SetModTime sets the modification time of the drive fs object // SetModTime sets the modification time of the drive fs object
func (o *FsObjectDrive) SetModTime(modTime time.Time) { func (o *Object) SetModTime(modTime time.Time) {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
@ -868,8 +868,8 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) {
} }
// Set modified date // Set modified date
var info *drive.File var info *drive.File
err = o.drive.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
info, err = o.drive.svc.Files.Update(o.id, updateInfo).SetModifiedDate(true).Do() info, err = o.fs.svc.Files.Update(o.id, updateInfo).SetModifiedDate(true).Do()
return shouldRetry(err) return shouldRetry(err)
}) })
if err != nil { if err != nil {
@ -882,12 +882,12 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) {
} }
// Storable returns a boolean as to whether this object is storable // Storable returns a boolean as to whether this object is storable
func (o *FsObjectDrive) Storable() bool { func (o *Object) Storable() bool {
return true return true
} }
// Open an object for read // Open an object for read
func (o *FsObjectDrive) Open() (in io.ReadCloser, err error) { func (o *Object) Open() (in io.ReadCloser, err error) {
if o.url == "" { if o.url == "" {
return nil, fmt.Errorf("Forbidden to download - check sharing permission") return nil, fmt.Errorf("Forbidden to download - check sharing permission")
} }
@ -897,8 +897,8 @@ func (o *FsObjectDrive) Open() (in io.ReadCloser, err error) {
} }
req.Header.Set("User-Agent", fs.UserAgent) req.Header.Set("User-Agent", fs.UserAgent)
var res *http.Response var res *http.Response
err = o.drive.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
res, err = o.drive.client.Do(req) res, err = o.fs.client.Do(req)
return shouldRetry(err) return shouldRetry(err)
}) })
if err != nil { if err != nil {
@ -916,7 +916,7 @@ func (o *FsObjectDrive) Open() (in io.ReadCloser, err error) {
// Copy the reader into the object updating modTime and size // Copy the reader into the object updating modTime and size
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (o *FsObjectDrive) Update(in io.Reader, modTime time.Time, size int64) error { func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
updateInfo := &drive.File{ updateInfo := &drive.File{
Id: o.id, Id: o.id,
ModifiedDate: modTime.Format(timeFormatOut), ModifiedDate: modTime.Format(timeFormatOut),
@ -927,8 +927,8 @@ func (o *FsObjectDrive) Update(in io.Reader, modTime time.Time, size int64) erro
var info *drive.File var info *drive.File
if size == 0 || size < int64(driveUploadCutoff) { if size == 0 || size < int64(driveUploadCutoff) {
// Don't retry, return a retry error instead // Don't retry, return a retry error instead
err = o.drive.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
info, err = o.drive.svc.Files.Update(updateInfo.Id, updateInfo).SetModifiedDate(true).Media(in).Do() info, err = o.fs.svc.Files.Update(updateInfo.Id, updateInfo).SetModifiedDate(true).Media(in).Do()
return shouldRetry(err) return shouldRetry(err)
}) })
if err != nil { if err != nil {
@ -936,7 +936,7 @@ func (o *FsObjectDrive) Update(in io.Reader, modTime time.Time, size int64) erro
} }
} else { } else {
// Upload the file in chunks // Upload the file in chunks
info, err = o.drive.Upload(in, size, fs.MimeType(o), updateInfo, o.remote) info, err = o.fs.Upload(in, size, fs.MimeType(o), updateInfo, o.remote)
if err != nil { if err != nil {
return err return err
} }
@ -946,13 +946,13 @@ func (o *FsObjectDrive) Update(in io.Reader, modTime time.Time, size int64) erro
} }
// Remove an object // Remove an object
func (o *FsObjectDrive) Remove() error { func (o *Object) Remove() error {
var err error var err error
err = o.drive.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
if *driveUseTrash { if *driveUseTrash {
_, err = o.drive.svc.Files.Trash(o.id).Do() _, err = o.fs.svc.Files.Trash(o.id).Do()
} else { } else {
err = o.drive.svc.Files.Delete(o.id).Do() err = o.fs.svc.Files.Delete(o.id).Do()
} }
return shouldRetry(err) return shouldRetry(err)
}) })
@ -961,10 +961,10 @@ func (o *FsObjectDrive) Remove() error {
// Check the interfaces are satisfied // Check the interfaces are satisfied
var ( var (
_ fs.Fs = (*FsDrive)(nil) _ fs.Fs = (*Fs)(nil)
_ fs.Purger = (*FsDrive)(nil) _ fs.Purger = (*Fs)(nil)
_ fs.Copier = (*FsDrive)(nil) _ fs.Copier = (*Fs)(nil)
_ fs.Mover = (*FsDrive)(nil) _ fs.Mover = (*Fs)(nil)
_ fs.DirMover = (*FsDrive)(nil) _ fs.DirMover = (*Fs)(nil)
_ fs.Object = (*FsObjectDrive)(nil) _ fs.Object = (*Object)(nil)
) )

2
drive/drive_test.go
View File

@ -13,7 +13,7 @@ import (
) )
func init() { func init() {
fstests.NilObject = fs.Object((*drive.FsObjectDrive)(nil)) fstests.NilObject = fs.Object((*drive.Object)(nil))
fstests.RemoteName = "TestDrive:" fstests.RemoteName = "TestDrive:"
} }

4
drive/upload.go
View File

@ -36,7 +36,7 @@ const (
// resumableUpload is used by the generated APIs to provide resumable uploads. // resumableUpload is used by the generated APIs to provide resumable uploads.
// It is not used by developers directly. // It is not used by developers directly.
type resumableUpload struct { type resumableUpload struct {
f *FsDrive f *Fs
remote string remote string
// URI is the resumable resource destination provided by the server after specifying "&uploadType=resumable". // URI is the resumable resource destination provided by the server after specifying "&uploadType=resumable".
URI string URI string
@ -51,7 +51,7 @@ type resumableUpload struct {
} }
// Upload the io.Reader in of size bytes with contentType and info // Upload the io.Reader in of size bytes with contentType and info
func (f *FsDrive) Upload(in io.Reader, size int64, contentType string, info *drive.File, remote string) (*drive.File, error) { func (f *Fs) Upload(in io.Reader, size int64, contentType string, info *drive.File, remote string) (*drive.File, error) {
fileID := info.Id fileID := info.Id
var body io.Reader var body io.Reader
body, err := googleapi.WithoutDataWrapper.JSONReader(info) body, err := googleapi.WithoutDataWrapper.JSONReader(info)

151
dropbox/dropbox.go
View File

@ -92,8 +92,8 @@ func configHelper(name string) {
} }
} }
// FsDropbox represents a remote dropbox server // Fs represents a remote dropbox server
type FsDropbox struct { type Fs struct {
name string // name of this remote name string // name of this remote
db *dropbox.Dropbox // the connection to the dropbox server db *dropbox.Dropbox // the connection to the dropbox server
root string // the path we are working on root string // the path we are working on
@ -101,29 +101,29 @@ type FsDropbox struct {
slashRootSlash string // root with "/" prefix and postfix, lowercase slashRootSlash string // root with "/" prefix and postfix, lowercase
} }
// FsObjectDropbox describes a dropbox object // Object describes a dropbox object
type FsObjectDropbox struct { type Object struct {
dropbox *FsDropbox // what this object is part of fs *Fs // what this object is part of
remote string // The remote path remote string // The remote path
bytes int64 // size of the object bytes int64 // size of the object
modTime time.Time // time it was last modified modTime time.Time // time it was last modified
hasMetadata bool // metadata is valid hasMetadata bool // metadata is valid
} }
// ------------------------------------------------------------ // ------------------------------------------------------------
// Name of the remote (as passed into NewFs) // Name of the remote (as passed into NewFs)
func (f *FsDropbox) Name() string { func (f *Fs) Name() string {
return f.name return f.name
} }
// Root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
func (f *FsDropbox) Root() string { func (f *Fs) Root() string {
return f.root return f.root
} }
// String converts this FsDropbox to a string // String converts this Fs to a string
func (f *FsDropbox) String() string { func (f *Fs) String() string {
return fmt.Sprintf("Dropbox root '%s'", f.root) return fmt.Sprintf("Dropbox root '%s'", f.root)
} }
@ -144,7 +144,7 @@ func newDropbox(name string) (*dropbox.Dropbox, error) {
return db, err return db, err
} }
// NewFs contstructs an FsDropbox from the path, container:path // NewFs contstructs an Fs from the path, container:path
func NewFs(name, root string) (fs.Fs, error) { func NewFs(name, root string) (fs.Fs, error) {
if uploadChunkSize > maxUploadChunkSize { if uploadChunkSize > maxUploadChunkSize {
return nil, fmt.Errorf("Chunk size too big, must be < %v", maxUploadChunkSize) return nil, fmt.Errorf("Chunk size too big, must be < %v", maxUploadChunkSize)
@ -153,7 +153,7 @@ func NewFs(name, root string) (fs.Fs, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
f := &FsDropbox{ f := &Fs{
name: name, name: name,
db: db, db: db,
} }
@ -186,7 +186,7 @@ func NewFs(name, root string) (fs.Fs, error) {
} }
// Sets root in f // Sets root in f
func (f *FsDropbox) setRoot(root string) { func (f *Fs) setRoot(root string) {
f.root = strings.Trim(root, "/") f.root = strings.Trim(root, "/")
lowerCaseRoot := strings.ToLower(f.root) lowerCaseRoot := strings.ToLower(f.root)
@ -200,10 +200,10 @@ func (f *FsDropbox) setRoot(root string) {
// Return an FsObject from a path // Return an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsDropbox) newFsObjectWithInfo(remote string, info *dropbox.Entry) fs.Object { func (f *Fs) newFsObjectWithInfo(remote string, info *dropbox.Entry) fs.Object {
o := &FsObjectDropbox{ o := &Object{
dropbox: f, fs: f,
remote: remote, remote: remote,
} }
if info != nil { if info != nil {
o.setMetadataFromEntry(info) o.setMetadataFromEntry(info)
@ -220,12 +220,12 @@ func (f *FsDropbox) newFsObjectWithInfo(remote string, info *dropbox.Entry) fs.O
// NewFsObject returns an FsObject from a path // NewFsObject returns an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsDropbox) NewFsObject(remote string) fs.Object { func (f *Fs) NewFsObject(remote string) fs.Object {
return f.newFsObjectWithInfo(remote, nil) return f.newFsObjectWithInfo(remote, nil)
} }
// Strips the root off path and returns it // Strips the root off path and returns it
func (f *FsDropbox) stripRoot(path string) *string { func (f *Fs) stripRoot(path string) *string {
lowercase := strings.ToLower(path) lowercase := strings.ToLower(path)
if !strings.HasPrefix(lowercase, f.slashRootSlash) { if !strings.HasPrefix(lowercase, f.slashRootSlash) {
@ -239,7 +239,7 @@ func (f *FsDropbox) stripRoot(path string) *string {
} }
// Walk the root returning a channel of FsObjects // Walk the root returning a channel of FsObjects
func (f *FsDropbox) list(out fs.ObjectsChan) { func (f *Fs) list(out fs.ObjectsChan) {
// Track path component case, it could be different for entries coming from DropBox API // Track path component case, it could be different for entries coming from DropBox API
// See https://www.dropboxforum.com/hc/communities/public/questions/201665409-Wrong-character-case-of-folder-name-when-calling-listFolder-using-Sync-API?locale=en-us // See https://www.dropboxforum.com/hc/communities/public/questions/201665409-Wrong-character-case-of-folder-name-when-calling-listFolder-using-Sync-API?locale=en-us
// and https://github.com/ncw/rclone/issues/53 // and https://github.com/ncw/rclone/issues/53
@ -318,7 +318,7 @@ func (f *FsDropbox) list(out fs.ObjectsChan) {
} }
// List walks the path returning a channel of FsObjects // List walks the path returning a channel of FsObjects
func (f *FsDropbox) List() fs.ObjectsChan { func (f *Fs) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
go func() { go func() {
defer close(out) defer close(out)
@ -328,7 +328,7 @@ func (f *FsDropbox) List() fs.ObjectsChan {
} }
// ListDir walks the path returning a channel of FsObjects // ListDir walks the path returning a channel of FsObjects
func (f *FsDropbox) ListDir() fs.DirChan { func (f *Fs) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
go func() { go func() {
defer close(out) defer close(out)
@ -379,14 +379,17 @@ func (rc *readCloser) Close() error {
// Copy the reader in to the new object which is returned // Copy the reader in to the new object which is returned
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (f *FsDropbox) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction // Temporary Object under construction
o := &FsObjectDropbox{dropbox: f, remote: remote} o := &Object{
fs: f,
remote: remote,
}
return o, o.Update(in, modTime, size) return o, o.Update(in, modTime, size)
} }
// Mkdir creates the container if it doesn't exist // Mkdir creates the container if it doesn't exist
func (f *FsDropbox) Mkdir() error { func (f *Fs) Mkdir() error {
entry, err := f.db.Metadata(f.slashRoot, false, false, "", "", metadataLimit) entry, err := f.db.Metadata(f.slashRoot, false, false, "", "", metadataLimit)
if err == nil { if err == nil {
if entry.IsDir { if entry.IsDir {
@ -401,7 +404,7 @@ func (f *FsDropbox) Mkdir() error {
// Rmdir deletes the container // Rmdir deletes the container
// //
// Returns an error if it isn't empty // Returns an error if it isn't empty
func (f *FsDropbox) Rmdir() error { func (f *Fs) Rmdir() error {
entry, err := f.db.Metadata(f.slashRoot, true, false, "", "", 16) entry, err := f.db.Metadata(f.slashRoot, true, false, "", "", 16)
if err != nil { if err != nil {
return err return err
@ -413,7 +416,7 @@ func (f *FsDropbox) Rmdir() error {
} }
// Precision returns the precision // Precision returns the precision
func (f *FsDropbox) Precision() time.Duration { func (f *Fs) Precision() time.Duration {
return fs.ModTimeNotSupported return fs.ModTimeNotSupported
} }
@ -426,15 +429,18 @@ func (f *FsDropbox) Precision() time.Duration {
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
// If it isn't possible then return fs.ErrorCantCopy // If it isn't possible then return fs.ErrorCantCopy
func (f *FsDropbox) Copy(src fs.Object, remote string) (fs.Object, error) { func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectDropbox) srcObj, ok := src.(*Object)
if !ok { if !ok {
fs.Debug(src, "Can't copy - not same remote type") fs.Debug(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy return nil, fs.ErrorCantCopy
} }
// Temporary FsObject under construction // Temporary Object under construction
dstObj := &FsObjectDropbox{dropbox: f, remote: remote} dstObj := &Object{
fs: f,
remote: remote,
}
srcPath := srcObj.remotePath() srcPath := srcObj.remotePath()
dstPath := dstObj.remotePath() dstPath := dstObj.remotePath()
@ -451,7 +457,7 @@ func (f *FsDropbox) Copy(src fs.Object, remote string) (fs.Object, error) {
// Optional interface: Only implement this if you have a way of // Optional interface: Only implement this if you have a way of
// deleting all the files quicker than just running Remove() on the // deleting all the files quicker than just running Remove() on the
// result of List() // result of List()
func (f *FsDropbox) Purge() error { func (f *Fs) Purge() error {
// Let dropbox delete the filesystem tree // Let dropbox delete the filesystem tree
_, err := f.db.Delete(f.slashRoot) _, err := f.db.Delete(f.slashRoot)
return err return err
@ -466,15 +472,18 @@ func (f *FsDropbox) Purge() error {
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
// If it isn't possible then return fs.ErrorCantMove // If it isn't possible then return fs.ErrorCantMove
func (f *FsDropbox) Move(src fs.Object, remote string) (fs.Object, error) { func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectDropbox) srcObj, ok := src.(*Object)
if !ok { if !ok {
fs.Debug(src, "Can't move - not same remote type") fs.Debug(src, "Can't move - not same remote type")
return nil, fs.ErrorCantMove return nil, fs.ErrorCantMove
} }
// Temporary FsObject under construction // Temporary Object under construction
dstObj := &FsObjectDropbox{dropbox: f, remote: remote} dstObj := &Object{
fs: f,
remote: remote,
}
srcPath := srcObj.remotePath() srcPath := srcObj.remotePath()
dstPath := dstObj.remotePath() dstPath := dstObj.remotePath()
@ -493,8 +502,8 @@ func (f *FsDropbox) Move(src fs.Object, remote string) (fs.Object, error) {
// If it isn't possible then return fs.ErrorCantDirMove // If it isn't possible then return fs.ErrorCantDirMove
// //
// If destination exists then return fs.ErrorDirExists // If destination exists then return fs.ErrorDirExists
func (f *FsDropbox) DirMove(src fs.Fs) error { func (f *Fs) DirMove(src fs.Fs) error {
srcFs, ok := src.(*FsDropbox) srcFs, ok := src.(*Fs)
if !ok { if !ok {
fs.Debug(srcFs, "Can't move directory - not same remote type") fs.Debug(srcFs, "Can't move directory - not same remote type")
return fs.ErrorCantDirMove return fs.ErrorCantDirMove
@ -517,12 +526,12 @@ func (f *FsDropbox) DirMove(src fs.Fs) error {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *FsObjectDropbox) Fs() fs.Fs { func (o *Object) Fs() fs.Fs {
return o.dropbox return o.fs
} }
// Return a string version // Return a string version
func (o *FsObjectDropbox) String() string { func (o *Object) String() string {
if o == nil { if o == nil {
return "<nil>" return "<nil>"
} }
@ -530,32 +539,32 @@ func (o *FsObjectDropbox) String() string {
} }
// Remote returns the remote path // Remote returns the remote path
func (o *FsObjectDropbox) Remote() string { func (o *Object) Remote() string {
return o.remote return o.remote
} }
// Md5sum returns the Md5sum of an object returning a lowercase hex string // Md5sum returns the Md5sum of an object returning a lowercase hex string
func (o *FsObjectDropbox) Md5sum() (string, error) { func (o *Object) Md5sum() (string, error) {
return "", nil return "", nil
} }
// Size returns the size of an object in bytes // Size returns the size of an object in bytes
func (o *FsObjectDropbox) Size() int64 { func (o *Object) Size() int64 {
return o.bytes return o.bytes
} }
// setMetadataFromEntry sets the fs data from a dropbox.Entry // setMetadataFromEntry sets the fs data from a dropbox.Entry
// //
// This isn't a complete set of metadata and has an inacurate date // This isn't a complete set of metadata and has an inacurate date
func (o *FsObjectDropbox) setMetadataFromEntry(info *dropbox.Entry) { func (o *Object) setMetadataFromEntry(info *dropbox.Entry) {
o.bytes = info.Bytes o.bytes = info.Bytes
o.modTime = time.Time(info.ClientMtime) o.modTime = time.Time(info.ClientMtime)
o.hasMetadata = true o.hasMetadata = true
} }
// Reads the entry from dropbox // Reads the entry from dropbox
func (o *FsObjectDropbox) readEntry() (*dropbox.Entry, error) { func (o *Object) readEntry() (*dropbox.Entry, error) {
entry, err := o.dropbox.db.Metadata(o.remotePath(), false, false, "", "", metadataLimit) entry, err := o.fs.db.Metadata(o.remotePath(), false, false, "", "", metadataLimit)
if err != nil { if err != nil {
fs.Debug(o, "Error reading file: %s", err) fs.Debug(o, "Error reading file: %s", err)
return nil, fmt.Errorf("Error reading file: %s", err) return nil, fmt.Errorf("Error reading file: %s", err)
@ -564,7 +573,7 @@ func (o *FsObjectDropbox) readEntry() (*dropbox.Entry, error) {
} }
// Read entry if not set and set metadata from it // Read entry if not set and set metadata from it
func (o *FsObjectDropbox) readEntryAndSetMetadata() error { func (o *Object) readEntryAndSetMetadata() error {
// Last resort set time from client // Last resort set time from client
if !o.modTime.IsZero() { if !o.modTime.IsZero() {
return nil return nil
@ -578,8 +587,8 @@ func (o *FsObjectDropbox) readEntryAndSetMetadata() error {
} }
// Returns the remote path for the object // Returns the remote path for the object
func (o *FsObjectDropbox) remotePath() string { func (o *Object) remotePath() string {
return o.dropbox.slashRootSlash + o.remote return o.fs.slashRootSlash + o.remote
} }
// Returns the key for the metadata database for a given path // Returns the key for the metadata database for a given path
@ -592,12 +601,12 @@ func metadataKey(path string) string {
} }
// Returns the key for the metadata database // Returns the key for the metadata database
func (o *FsObjectDropbox) metadataKey() string { func (o *Object) metadataKey() string {
return metadataKey(o.remotePath()) return metadataKey(o.remotePath())
} }
// readMetaData gets the info if it hasn't already been fetched // readMetaData gets the info if it hasn't already been fetched
func (o *FsObjectDropbox) readMetaData() (err error) { func (o *Object) readMetaData() (err error) {
if o.hasMetadata { if o.hasMetadata {
return nil return nil
} }
@ -609,7 +618,7 @@ func (o *FsObjectDropbox) readMetaData() (err error) {
// //
// It attempts to read the objects mtime and if that isn't present the // It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers // LastModified returned in the http headers
func (o *FsObjectDropbox) ModTime() time.Time { func (o *Object) ModTime() time.Time {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
fs.Log(o, "Failed to read metadata: %s", err) fs.Log(o, "Failed to read metadata: %s", err)
@ -621,19 +630,19 @@ func (o *FsObjectDropbox) ModTime() time.Time {
// SetModTime sets the modification time of the local fs object // SetModTime sets the modification time of the local fs object
// //
// Commits the datastore // Commits the datastore
func (o *FsObjectDropbox) SetModTime(modTime time.Time) { func (o *Object) SetModTime(modTime time.Time) {
// FIXME not implemented // FIXME not implemented
return return
} }
// Storable returns whether this object is storable // Storable returns whether this object is storable
func (o *FsObjectDropbox) Storable() bool { func (o *Object) Storable() bool {
return true return true
} }
// Open an object for read // Open an object for read
func (o *FsObjectDropbox) Open() (in io.ReadCloser, err error) { func (o *Object) Open() (in io.ReadCloser, err error) {
in, _, err = o.dropbox.db.Download(o.remotePath(), "", 0) in, _, err = o.fs.db.Download(o.remotePath(), "", 0)
return return
} }
@ -642,13 +651,13 @@ func (o *FsObjectDropbox) Open() (in io.ReadCloser, err error) {
// Copy the reader into the object updating modTime and size // Copy the reader into the object updating modTime and size
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (o *FsObjectDropbox) Update(in io.Reader, modTime time.Time, size int64) error { func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
remote := o.remotePath() remote := o.remotePath()
if ignoredFiles.MatchString(remote) { if ignoredFiles.MatchString(remote) {
fs.ErrorLog(o, "File name disallowed - not uploading") fs.ErrorLog(o, "File name disallowed - not uploading")
return nil return nil
} }
entry, err := o.dropbox.db.UploadByChunk(ioutil.NopCloser(in), int(uploadChunkSize), remote, true, "") entry, err := o.fs.db.UploadByChunk(ioutil.NopCloser(in), int(uploadChunkSize), remote, true, "")
if err != nil { if err != nil {
return fmt.Errorf("Upload failed: %s", err) return fmt.Errorf("Upload failed: %s", err)
} }
@ -657,17 +666,17 @@ func (o *FsObjectDropbox) Update(in io.Reader, modTime time.Time, size int64) er
} }
// Remove an object // Remove an object
func (o *FsObjectDropbox) Remove() error { func (o *Object) Remove() error {
_, err := o.dropbox.db.Delete(o.remotePath()) _, err := o.fs.db.Delete(o.remotePath())
return err return err
} }
// Check the interfaces are satisfied // Check the interfaces are satisfied
var ( var (
_ fs.Fs = (*FsDropbox)(nil) _ fs.Fs = (*Fs)(nil)
_ fs.Copier = (*FsDropbox)(nil) _ fs.Copier = (*Fs)(nil)
_ fs.Purger = (*FsDropbox)(nil) _ fs.Purger = (*Fs)(nil)
_ fs.Mover = (*FsDropbox)(nil) _ fs.Mover = (*Fs)(nil)
_ fs.DirMover = (*FsDropbox)(nil) _ fs.DirMover = (*Fs)(nil)
_ fs.Object = (*FsObjectDropbox)(nil) _ fs.Object = (*Object)(nil)
) )

2
dropbox/dropbox_test.go
View File

@ -13,7 +13,7 @@ import (
) )
func init() { func init() {
fstests.NilObject = fs.Object((*dropbox.FsObjectDropbox)(nil)) fstests.NilObject = fs.Object((*dropbox.Object)(nil))
fstests.RemoteName = "TestDropbox:" fstests.RemoteName = "TestDropbox:"
} }

22
fstest/fstests/gen_tests.go
View File

@ -45,7 +45,6 @@ type Data struct {
FsName string FsName string
UpperFsName string UpperFsName string
TestName string TestName string
ObjectName string
Fns []string Fns []string
} }
@ -65,7 +64,7 @@ import (
) )
func init() { func init() {
fstests.NilObject = fs.Object((*{{ .FsName }}.{{ .ObjectName }})(nil)) fstests.NilObject = fs.Object((*{{ .FsName }}.Object)(nil))
fstests.RemoteName = "{{ .TestName }}" fstests.RemoteName = "{{ .TestName }}"
} }
@ -75,7 +74,7 @@ func init() {
` `
// Generate test file piping it through gofmt // Generate test file piping it through gofmt
func generateTestProgram(t *template.Template, fns []string, Fsname, ObjectName string) { func generateTestProgram(t *template.Template, fns []string, Fsname string) {
fsname := strings.ToLower(Fsname) fsname := strings.ToLower(Fsname)
TestName := "Test" + Fsname + ":" TestName := "Test" + Fsname + ":"
outfile := "../../" + fsname + "/" + fsname + "_test.go" outfile := "../../" + fsname + "/" + fsname + "_test.go"
@ -89,7 +88,6 @@ func generateTestProgram(t *template.Template, fns []string, Fsname, ObjectName
FsName: fsname, FsName: fsname,
UpperFsName: Fsname, UpperFsName: Fsname,
TestName: TestName, TestName: TestName,
ObjectName: ObjectName,
Fns: fns, Fns: fns,
} }
@ -126,13 +124,13 @@ func generateTestProgram(t *template.Template, fns []string, Fsname, ObjectName
func main() { func main() {
fns := findTestFunctions() fns := findTestFunctions()
t := template.Must(template.New("main").Parse(testProgram)) t := template.Must(template.New("main").Parse(testProgram))
generateTestProgram(t, fns, "Local", "FsObjectLocal") generateTestProgram(t, fns, "Local")
generateTestProgram(t, fns, "Swift", "FsObjectSwift") generateTestProgram(t, fns, "Swift")
generateTestProgram(t, fns, "S3", "FsObjectS3") generateTestProgram(t, fns, "S3")
generateTestProgram(t, fns, "Drive", "FsObjectDrive") generateTestProgram(t, fns, "Drive")
generateTestProgram(t, fns, "GoogleCloudStorage", "FsObjectStorage") generateTestProgram(t, fns, "GoogleCloudStorage")
generateTestProgram(t, fns, "Dropbox", "FsObjectDropbox") generateTestProgram(t, fns, "Dropbox")
generateTestProgram(t, fns, "AmazonCloudDrive", "FsObjectAcd") generateTestProgram(t, fns, "AmazonCloudDrive")
generateTestProgram(t, fns, "OneDrive", "Object") generateTestProgram(t, fns, "OneDrive")
log.Printf("Done") log.Printf("Done")
} }

132
googlecloudstorage/googlecloudstorage.go
View File

@ -118,8 +118,8 @@ func init() {
}) })
} }
// FsStorage represents a remote storage server // Fs represents a remote storage server
type FsStorage struct { type Fs struct {
name string // name of this remote name string // name of this remote
svc *storage.Service // the connection to the storage server svc *storage.Service // the connection to the storage server
client *http.Client // authorized client client *http.Client // authorized client
@ -130,35 +130,35 @@ type FsStorage struct {
bucketAcl string // used when creating new buckets bucketAcl string // used when creating new buckets
} }
// FsObjectStorage describes a storage object // Object describes a storage object
// //
// Will definitely have info but maybe not meta // Will definitely have info but maybe not meta
type FsObjectStorage struct { type Object struct {
storage *FsStorage // what this object is part of fs *Fs // what this object is part of
remote string // The remote path remote string // The remote path
url string // download path url string // download path
md5sum string // The MD5Sum of the object md5sum string // The MD5Sum of the object
bytes int64 // Bytes in the object bytes int64 // Bytes in the object
modTime time.Time // Modified time of the object modTime time.Time // Modified time of the object
} }
// ------------------------------------------------------------ // ------------------------------------------------------------
// Name of the remote (as passed into NewFs) // Name of the remote (as passed into NewFs)
func (f *FsStorage) Name() string { func (f *Fs) Name() string {
return f.name return f.name
} }
// Root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
func (f *FsStorage) Root() string { func (f *Fs) Root() string {
if f.root == "" { if f.root == "" {
return f.bucket return f.bucket
} }
return f.bucket + "/" + f.root return f.bucket + "/" + f.root
} }
// String converts this FsStorage to a string // String converts this Fs to a string
func (f *FsStorage) String() string { func (f *Fs) String() string {
if f.root == "" { if f.root == "" {
return fmt.Sprintf("Storage bucket %s", f.bucket) return fmt.Sprintf("Storage bucket %s", f.bucket)
} }
@ -180,7 +180,7 @@ func parsePath(path string) (bucket, directory string, err error) {
return return
} }
// NewFs contstructs an FsStorage from the path, bucket:path // NewFs contstructs an Fs from the path, bucket:path
func NewFs(name, root string) (fs.Fs, error) { func NewFs(name, root string) (fs.Fs, error) {
oAuthClient, err := oauthutil.NewClient(name, storageConfig) oAuthClient, err := oauthutil.NewClient(name, storageConfig)
if err != nil { if err != nil {
@ -192,7 +192,7 @@ func NewFs(name, root string) (fs.Fs, error) {
return nil, err return nil, err
} }
f := &FsStorage{ f := &Fs{
name: name, name: name,
bucket: bucket, bucket: bucket,
root: directory, root: directory,
@ -237,10 +237,10 @@ func NewFs(name, root string) (fs.Fs, error) {
// Return an FsObject from a path // Return an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsStorage) newFsObjectWithInfo(remote string, info *storage.Object) fs.Object { func (f *Fs) newFsObjectWithInfo(remote string, info *storage.Object) fs.Object {
o := &FsObjectStorage{ o := &Object{
storage: f, fs: f,
remote: remote, remote: remote,
} }
if info != nil { if info != nil {
o.setMetaData(info) o.setMetaData(info)
@ -257,14 +257,14 @@ func (f *FsStorage) newFsObjectWithInfo(remote string, info *storage.Object) fs.
// NewFsObject returns an FsObject from a path // NewFsObject returns an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsStorage) NewFsObject(remote string) fs.Object { func (f *Fs) NewFsObject(remote string) fs.Object {
return f.newFsObjectWithInfo(remote, nil) return f.newFsObjectWithInfo(remote, nil)
} }
// list the objects into the function supplied // list the objects into the function supplied
// //
// If directories is set it only sends directories // If directories is set it only sends directories
func (f *FsStorage) list(directories bool, fn func(string, *storage.Object)) { func (f *Fs) list(directories bool, fn func(string, *storage.Object)) {
list := f.svc.Objects.List(f.bucket).Prefix(f.root).MaxResults(listChunks) list := f.svc.Objects.List(f.bucket).Prefix(f.root).MaxResults(listChunks)
if directories { if directories {
list = list.Delimiter("/") list = list.Delimiter("/")
@ -303,7 +303,7 @@ func (f *FsStorage) list(directories bool, fn func(string, *storage.Object)) {
} }
// List walks the path returning a channel of FsObjects // List walks the path returning a channel of FsObjects
func (f *FsStorage) List() fs.ObjectsChan { func (f *Fs) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
if f.bucket == "" { if f.bucket == "" {
// Return no objects at top level list // Return no objects at top level list
@ -325,7 +325,7 @@ func (f *FsStorage) List() fs.ObjectsChan {
} }
// ListDir lists the buckets // ListDir lists the buckets
func (f *FsStorage) ListDir() fs.DirChan { func (f *Fs) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
if f.bucket == "" { if f.bucket == "" {
// List the buckets // List the buckets
@ -379,14 +379,17 @@ func (f *FsStorage) ListDir() fs.DirChan {
// Copy the reader in to the new object which is returned // Copy the reader in to the new object which is returned
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (f *FsStorage) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction // Temporary Object under construction
o := &FsObjectStorage{storage: f, remote: remote} o := &Object{
fs: f,
remote: remote,
}
return o, o.Update(in, modTime, size) return o, o.Update(in, modTime, size)
} }
// Mkdir creates the bucket if it doesn't exist // Mkdir creates the bucket if it doesn't exist
func (f *FsStorage) Mkdir() error { func (f *Fs) Mkdir() error {
_, err := f.svc.Buckets.Get(f.bucket).Do() _, err := f.svc.Buckets.Get(f.bucket).Do()
if err == nil { if err == nil {
// Bucket already exists // Bucket already exists
@ -408,12 +411,12 @@ func (f *FsStorage) Mkdir() error {
// //
// Returns an error if it isn't empty: Error 409: The bucket you tried // Returns an error if it isn't empty: Error 409: The bucket you tried
// to delete was not empty. // to delete was not empty.
func (f *FsStorage) Rmdir() error { func (f *Fs) Rmdir() error {
return f.svc.Buckets.Delete(f.bucket).Do() return f.svc.Buckets.Delete(f.bucket).Do()
} }
// Precision returns the precision // Precision returns the precision
func (f *FsStorage) Precision() time.Duration { func (f *Fs) Precision() time.Duration {
return time.Nanosecond return time.Nanosecond
} }
@ -426,18 +429,21 @@ func (f *FsStorage) Precision() time.Duration {
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
// If it isn't possible then return fs.ErrorCantCopy // If it isn't possible then return fs.ErrorCantCopy
func (f *FsStorage) Copy(src fs.Object, remote string) (fs.Object, error) { func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectStorage) srcObj, ok := src.(*Object)
if !ok { if !ok {
fs.Debug(src, "Can't copy - not same remote type") fs.Debug(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy return nil, fs.ErrorCantCopy
} }
// Temporary FsObject under construction // Temporary Object under construction
dstObj := &FsObjectStorage{storage: f, remote: remote} dstObj := &Object{
fs: f,
remote: remote,
}
srcBucket := srcObj.storage.bucket srcBucket := srcObj.fs.bucket
srcObject := srcObj.storage.root + srcObj.remote srcObject := srcObj.fs.root + srcObj.remote
dstBucket := f.bucket dstBucket := f.bucket
dstObject := f.root + remote dstObject := f.root + remote
newObject, err := f.svc.Objects.Copy(srcBucket, srcObject, dstBucket, dstObject, nil).Do() newObject, err := f.svc.Objects.Copy(srcBucket, srcObject, dstBucket, dstObject, nil).Do()
@ -452,12 +458,12 @@ func (f *FsStorage) Copy(src fs.Object, remote string) (fs.Object, error) {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *FsObjectStorage) Fs() fs.Fs { func (o *Object) Fs() fs.Fs {
return o.storage return o.fs
} }
// Return a string version // Return a string version
func (o *FsObjectStorage) String() string { func (o *Object) String() string {
if o == nil { if o == nil {
return "<nil>" return "<nil>"
} }
@ -465,22 +471,22 @@ func (o *FsObjectStorage) String() string {
} }
// Remote returns the remote path // Remote returns the remote path
func (o *FsObjectStorage) Remote() string { func (o *Object) Remote() string {
return o.remote return o.remote
} }
// Md5sum returns the Md5sum of an object returning a lowercase hex string // Md5sum returns the Md5sum of an object returning a lowercase hex string
func (o *FsObjectStorage) Md5sum() (string, error) { func (o *Object) Md5sum() (string, error) {
return o.md5sum, nil return o.md5sum, nil
} }
// Size returns the size of an object in bytes // Size returns the size of an object in bytes
func (o *FsObjectStorage) Size() int64 { func (o *Object) Size() int64 {
return o.bytes return o.bytes
} }
// setMetaData sets the fs data from a storage.Object // setMetaData sets the fs data from a storage.Object
func (o *FsObjectStorage) setMetaData(info *storage.Object) { func (o *Object) setMetaData(info *storage.Object) {
o.url = info.MediaLink o.url = info.MediaLink
o.bytes = int64(info.Size) o.bytes = int64(info.Size)
@ -515,11 +521,11 @@ func (o *FsObjectStorage) setMetaData(info *storage.Object) {
// readMetaData gets the metadata if it hasn't already been fetched // readMetaData gets the metadata if it hasn't already been fetched
// //
// it also sets the info // it also sets the info
func (o *FsObjectStorage) readMetaData() (err error) { func (o *Object) readMetaData() (err error) {
if !o.modTime.IsZero() { if !o.modTime.IsZero() {
return nil return nil
} }
object, err := o.storage.svc.Objects.Get(o.storage.bucket, o.storage.root+o.remote).Do() object, err := o.fs.svc.Objects.Get(o.fs.bucket, o.fs.root+o.remote).Do()
if err != nil { if err != nil {
fs.Debug(o, "Failed to read info: %s", err) fs.Debug(o, "Failed to read info: %s", err)
return err return err
@ -532,7 +538,7 @@ func (o *FsObjectStorage) readMetaData() (err error) {
// //
// It attempts to read the objects mtime and if that isn't present the // It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers // LastModified returned in the http headers
func (o *FsObjectStorage) ModTime() time.Time { func (o *Object) ModTime() time.Time {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
// fs.Log(o, "Failed to read metadata: %s", err) // fs.Log(o, "Failed to read metadata: %s", err)
@ -549,14 +555,14 @@ func metadataFromModTime(modTime time.Time) map[string]string {
} }
// SetModTime sets the modification time of the local fs object // SetModTime sets the modification time of the local fs object
func (o *FsObjectStorage) SetModTime(modTime time.Time) { func (o *Object) SetModTime(modTime time.Time) {
// This only adds metadata so will perserve other metadata // This only adds metadata so will perserve other metadata
object := storage.Object{ object := storage.Object{
Bucket: o.storage.bucket, Bucket: o.fs.bucket,
Name: o.storage.root + o.remote, Name: o.fs.root + o.remote,
Metadata: metadataFromModTime(modTime), Metadata: metadataFromModTime(modTime),
} }
newObject, err := o.storage.svc.Objects.Patch(o.storage.bucket, o.storage.root+o.remote, &object).Do() newObject, err := o.fs.svc.Objects.Patch(o.fs.bucket, o.fs.root+o.remote, &object).Do()
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
fs.ErrorLog(o, "Failed to update remote mtime: %s", err) fs.ErrorLog(o, "Failed to update remote mtime: %s", err)
@ -565,12 +571,12 @@ func (o *FsObjectStorage) SetModTime(modTime time.Time) {
} }
// Storable returns a boolean as to whether this object is storable // Storable returns a boolean as to whether this object is storable
func (o *FsObjectStorage) Storable() bool { func (o *Object) Storable() bool {
return true return true
} }
// Open an object for read // Open an object for read
func (o *FsObjectStorage) Open() (in io.ReadCloser, err error) { func (o *Object) Open() (in io.ReadCloser, err error) {
// This is slightly complicated by Go here insisting on // This is slightly complicated by Go here insisting on
// decoding the %2F in URLs into / which is legal in http, but // decoding the %2F in URLs into / which is legal in http, but
// unfortunately not what the storage server wants. // unfortunately not what the storage server wants.
@ -586,7 +592,7 @@ func (o *FsObjectStorage) Open() (in io.ReadCloser, err error) {
// alter any hex-escaped characters // alter any hex-escaped characters
googleapi.SetOpaque(req.URL) googleapi.SetOpaque(req.URL)
req.Header.Set("User-Agent", fs.UserAgent) req.Header.Set("User-Agent", fs.UserAgent)
res, err := o.storage.client.Do(req) res, err := o.fs.client.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -600,16 +606,16 @@ func (o *FsObjectStorage) Open() (in io.ReadCloser, err error) {
// Update the object with the contents of the io.Reader, modTime and size // Update the object with the contents of the io.Reader, modTime and size
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (o *FsObjectStorage) Update(in io.Reader, modTime time.Time, size int64) error { func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
object := storage.Object{ object := storage.Object{
Bucket: o.storage.bucket, Bucket: o.fs.bucket,
Name: o.storage.root + o.remote, Name: o.fs.root + o.remote,
ContentType: fs.MimeType(o), ContentType: fs.MimeType(o),
Size: uint64(size), Size: uint64(size),
Updated: modTime.Format(timeFormatOut), // Doesn't get set Updated: modTime.Format(timeFormatOut), // Doesn't get set
Metadata: metadataFromModTime(modTime), Metadata: metadataFromModTime(modTime),
} }
newObject, err := o.storage.svc.Objects.Insert(o.storage.bucket, &object).Media(in).Name(object.Name).PredefinedAcl(o.storage.objectAcl).Do() newObject, err := o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in).Name(object.Name).PredefinedAcl(o.fs.objectAcl).Do()
if err != nil { if err != nil {
return err return err
} }
@ -619,11 +625,13 @@ func (o *FsObjectStorage) Update(in io.Reader, modTime time.Time, size int64) er
} }
// Remove an object // Remove an object
func (o *FsObjectStorage) Remove() error { func (o *Object) Remove() error {
return o.storage.svc.Objects.Delete(o.storage.bucket, o.storage.root+o.remote).Do() return o.fs.svc.Objects.Delete(o.fs.bucket, o.fs.root+o.remote).Do()
} }
// Check the interfaces are satisfied // Check the interfaces are satisfied
var _ fs.Fs = &FsStorage{} var (
var _ fs.Copier = &FsStorage{} _ fs.Fs = &Fs{}
var _ fs.Object = &FsObjectStorage{} _ fs.Copier = &Fs{}
_ fs.Object = &Object{}
)

2
googlecloudstorage/googlecloudstorage_test.go
View File

@ -13,7 +13,7 @@ import (
) )
func init() { func init() {
fstests.NilObject = fs.Object((*googlecloudstorage.FsObjectStorage)(nil)) fstests.NilObject = fs.Object((*googlecloudstorage.Object)(nil))
fstests.RemoteName = "TestGoogleCloudStorage:" fstests.RemoteName = "TestGoogleCloudStorage:"
} }

112
local/local.go
View File

@ -28,8 +28,8 @@ func init() {
}) })
} }
// FsLocal represents a local filesystem rooted at root // Fs represents a local filesystem rooted at root
type FsLocal struct { type Fs struct {
name string // the name of the remote name string // the name of the remote
root string // The root directory root string // The root directory
precisionOk sync.Once // Whether we need to read the precision precisionOk sync.Once // Whether we need to read the precision
@ -37,9 +37,9 @@ type FsLocal struct {
warned map[string]struct{} // whether we have warned about this string warned map[string]struct{} // whether we have warned about this string
} }
// FsObjectLocal represents a local filesystem object // Object represents a local filesystem object
type FsObjectLocal struct { type Object struct {
local *FsLocal // The Fs this object is part of fs *Fs // The Fs this object is part of
remote string // The remote path remote string // The remote path
path string // The local path path string // The local path
info os.FileInfo // Interface for file info (always present) info os.FileInfo // Interface for file info (always present)
@ -48,11 +48,11 @@ type FsObjectLocal struct {
// ------------------------------------------------------------ // ------------------------------------------------------------
// NewFs constructs an FsLocal from the path // NewFs constructs an Fs from the path
func NewFs(name, root string) (fs.Fs, error) { func NewFs(name, root string) (fs.Fs, error) {
var err error var err error
f := &FsLocal{ f := &Fs{
name: name, name: name,
warned: make(map[string]struct{}), warned: make(map[string]struct{}),
} }
@ -72,31 +72,35 @@ func NewFs(name, root string) (fs.Fs, error) {
} }
// Name of the remote (as passed into NewFs) // Name of the remote (as passed into NewFs)
func (f *FsLocal) Name() string { func (f *Fs) Name() string {
return f.name return f.name
} }
// Root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
func (f *FsLocal) Root() string { func (f *Fs) Root() string {
return f.root return f.root
} }
// String converts this FsLocal to a string // String converts this Fs to a string
func (f *FsLocal) String() string { func (f *Fs) String() string {
return fmt.Sprintf("Local file system at %s", f.root) return fmt.Sprintf("Local file system at %s", f.root)
} }
// newFsObject makes a half completed FsObjectLocal // newFsObject makes a half completed Object
func (f *FsLocal) newFsObject(remote string) *FsObjectLocal { func (f *Fs) newFsObject(remote string) *Object {
remote = filepath.ToSlash(remote) remote = filepath.ToSlash(remote)
dstPath := filterPath(filepath.Join(f.root, f.cleanUtf8(remote))) dstPath := filterPath(filepath.Join(f.root, f.cleanUtf8(remote)))
return &FsObjectLocal{local: f, remote: remote, path: dstPath} return &Object{
fs: f,
remote: remote,
path: dstPath,
}
} }
// Return an FsObject from a path // Return an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsLocal) newFsObjectWithInfo(remote string, info os.FileInfo) fs.Object { func (f *Fs) newFsObjectWithInfo(remote string, info os.FileInfo) fs.Object {
o := f.newFsObject(remote) o := f.newFsObject(remote)
if info != nil { if info != nil {
o.info = info o.info = info
@ -113,14 +117,14 @@ func (f *FsLocal) newFsObjectWithInfo(remote string, info os.FileInfo) fs.Object
// NewFsObject returns an FsObject from a path // NewFsObject returns an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsLocal) NewFsObject(remote string) fs.Object { func (f *Fs) NewFsObject(remote string) fs.Object {
return f.newFsObjectWithInfo(remote, nil) return f.newFsObjectWithInfo(remote, nil)
} }
// List the path returning a channel of FsObjects // List the path returning a channel of FsObjects
// //
// Ignores everything which isn't Storable, eg links etc // Ignores everything which isn't Storable, eg links etc
func (f *FsLocal) List() fs.ObjectsChan { func (f *Fs) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
go func() { go func() {
err := filepath.Walk(f.root, func(path string, fi os.FileInfo, err error) error { err := filepath.Walk(f.root, func(path string, fi os.FileInfo, err error) error {
@ -158,7 +162,7 @@ func (f *FsLocal) List() fs.ObjectsChan {
// CleanUtf8 makes string a valid UTF-8 string // CleanUtf8 makes string a valid UTF-8 string
// //
// Any invalid UTF-8 characters will be replaced with utf8.RuneError // Any invalid UTF-8 characters will be replaced with utf8.RuneError
func (f *FsLocal) cleanUtf8(name string) string { func (f *Fs) cleanUtf8(name string) string {
if !utf8.ValidString(name) { if !utf8.ValidString(name) {
if _, ok := f.warned[name]; !ok { if _, ok := f.warned[name]; !ok {
fs.Debug(f, "Replacing invalid UTF-8 characters in %q", name) fs.Debug(f, "Replacing invalid UTF-8 characters in %q", name)
@ -173,7 +177,7 @@ func (f *FsLocal) cleanUtf8(name string) string {
} }
// ListDir walks the path returning a channel of FsObjects // ListDir walks the path returning a channel of FsObjects
func (f *FsLocal) ListDir() fs.DirChan { func (f *Fs) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
go func() { go func() {
defer close(out) defer close(out)
@ -216,7 +220,7 @@ func (f *FsLocal) ListDir() fs.DirChan {
} }
// Put the FsObject to the local filesystem // Put the FsObject to the local filesystem
func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction - info filled in by Update() // Temporary FsObject under construction - info filled in by Update()
o := f.newFsObject(remote) o := f.newFsObject(remote)
err := o.Update(in, modTime, size) err := o.Update(in, modTime, size)
@ -227,7 +231,7 @@ func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64
} }
// Mkdir creates the directory if it doesn't exist // Mkdir creates the directory if it doesn't exist
func (f *FsLocal) Mkdir() error { func (f *Fs) Mkdir() error {
// FIXME: https://github.com/syncthing/syncthing/blob/master/lib/osutil/mkdirall_windows.go // FIXME: https://github.com/syncthing/syncthing/blob/master/lib/osutil/mkdirall_windows.go
return os.MkdirAll(f.root, 0777) return os.MkdirAll(f.root, 0777)
} }
@ -235,12 +239,12 @@ func (f *FsLocal) Mkdir() error {
// Rmdir removes the directory // Rmdir removes the directory
// //
// If it isn't empty it will return an error // If it isn't empty it will return an error
func (f *FsLocal) Rmdir() error { func (f *Fs) Rmdir() error {
return os.Remove(f.root) return os.Remove(f.root)
} }
// Precision of the file system // Precision of the file system
func (f *FsLocal) Precision() (precision time.Duration) { func (f *Fs) Precision() (precision time.Duration) {
f.precisionOk.Do(func() { f.precisionOk.Do(func() {
f.precision = f.readPrecision() f.precision = f.readPrecision()
}) })
@ -248,7 +252,7 @@ func (f *FsLocal) Precision() (precision time.Duration) {
} }
// Read the precision // Read the precision
func (f *FsLocal) readPrecision() (precision time.Duration) { func (f *Fs) readPrecision() (precision time.Duration) {
// Default precision of 1s // Default precision of 1s
precision = time.Second precision = time.Second
@ -304,7 +308,7 @@ func (f *FsLocal) readPrecision() (precision time.Duration) {
// Optional interface: Only implement this if you have a way of // Optional interface: Only implement this if you have a way of
// deleting all the files quicker than just running Remove() on the // deleting all the files quicker than just running Remove() on the
// result of List() // result of List()
func (f *FsLocal) Purge() error { func (f *Fs) Purge() error {
fi, err := os.Lstat(f.root) fi, err := os.Lstat(f.root)
if err != nil { if err != nil {
return err return err
@ -324,8 +328,8 @@ func (f *FsLocal) Purge() error {
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
// If it isn't possible then return fs.ErrorCantMove // If it isn't possible then return fs.ErrorCantMove
func (f *FsLocal) Move(src fs.Object, remote string) (fs.Object, error) { func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectLocal) srcObj, ok := src.(*Object)
if !ok { if !ok {
fs.Debug(src, "Can't move - not same remote type") fs.Debug(src, "Can't move - not same remote type")
return nil, fs.ErrorCantMove return nil, fs.ErrorCantMove
@ -374,8 +378,8 @@ func (f *FsLocal) Move(src fs.Object, remote string) (fs.Object, error) {
// If it isn't possible then return fs.ErrorCantDirMove // If it isn't possible then return fs.ErrorCantDirMove
// //
// If destination exists then return fs.ErrorDirExists // If destination exists then return fs.ErrorDirExists
func (f *FsLocal) DirMove(src fs.Fs) error { func (f *Fs) DirMove(src fs.Fs) error {
srcFs, ok := src.(*FsLocal) srcFs, ok := src.(*Fs)
if !ok { if !ok {
fs.Debug(srcFs, "Can't move directory - not same remote type") fs.Debug(srcFs, "Can't move directory - not same remote type")
return fs.ErrorCantDirMove return fs.ErrorCantDirMove
@ -403,12 +407,12 @@ func (f *FsLocal) DirMove(src fs.Fs) error {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *FsObjectLocal) Fs() fs.Fs { func (o *Object) Fs() fs.Fs {
return o.local return o.fs
} }
// Return a string version // Return a string version
func (o *FsObjectLocal) String() string { func (o *Object) String() string {
if o == nil { if o == nil {
return "<nil>" return "<nil>"
} }
@ -416,12 +420,12 @@ func (o *FsObjectLocal) String() string {
} }
// Remote returns the remote path // Remote returns the remote path
func (o *FsObjectLocal) Remote() string { func (o *Object) Remote() string {
return o.local.cleanUtf8(o.remote) return o.fs.cleanUtf8(o.remote)
} }
// Md5sum calculates the Md5sum of a file returning a lowercase hex string // Md5sum calculates the Md5sum of a file returning a lowercase hex string
func (o *FsObjectLocal) Md5sum() (string, error) { func (o *Object) Md5sum() (string, error) {
if o.md5sum != "" { if o.md5sum != "" {
return o.md5sum, nil return o.md5sum, nil
} }
@ -449,17 +453,17 @@ func (o *FsObjectLocal) Md5sum() (string, error) {
} }
// Size returns the size of an object in bytes // Size returns the size of an object in bytes
func (o *FsObjectLocal) Size() int64 { func (o *Object) Size() int64 {
return o.info.Size() return o.info.Size()
} }
// ModTime returns the modification time of the object // ModTime returns the modification time of the object
func (o *FsObjectLocal) ModTime() time.Time { func (o *Object) ModTime() time.Time {
return o.info.ModTime() return o.info.ModTime()
} }
// SetModTime sets the modification time of the local fs object // SetModTime sets the modification time of the local fs object
func (o *FsObjectLocal) SetModTime(modTime time.Time) { func (o *Object) SetModTime(modTime time.Time) {
err := os.Chtimes(o.path, modTime, modTime) err := os.Chtimes(o.path, modTime, modTime)
if err != nil { if err != nil {
fs.Debug(o, "Failed to set mtime on file: %s", err) fs.Debug(o, "Failed to set mtime on file: %s", err)
@ -474,7 +478,7 @@ func (o *FsObjectLocal) SetModTime(modTime time.Time) {
} }
// Storable returns a boolean showing if this object is storable // Storable returns a boolean showing if this object is storable
func (o *FsObjectLocal) Storable() bool { func (o *Object) Storable() bool {
mode := o.info.Mode() mode := o.info.Mode()
if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 { if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
fs.Debug(o, "Can't transfer non file/directory") fs.Debug(o, "Can't transfer non file/directory")
@ -489,9 +493,9 @@ func (o *FsObjectLocal) Storable() bool {
// localOpenFile wraps an io.ReadCloser and updates the md5sum of the // localOpenFile wraps an io.ReadCloser and updates the md5sum of the
// object that is read // object that is read
type localOpenFile struct { type localOpenFile struct {
o *FsObjectLocal // object that is open o *Object // object that is open
in io.ReadCloser // handle we are wrapping in io.ReadCloser // handle we are wrapping
hash hash.Hash // currently accumulating MD5 hash hash.Hash // currently accumulating MD5
} }
// Read bytes from the object - see io.Reader // Read bytes from the object - see io.Reader
@ -516,7 +520,7 @@ func (file *localOpenFile) Close() (err error) {
} }
// Open an object for read // Open an object for read
func (o *FsObjectLocal) Open() (in io.ReadCloser, err error) { func (o *Object) Open() (in io.ReadCloser, err error) {
in, err = os.Open(o.path) in, err = os.Open(o.path)
if err != nil { if err != nil {
return return
@ -531,13 +535,13 @@ func (o *FsObjectLocal) Open() (in io.ReadCloser, err error) {
} }
// mkdirAll makes all the directories needed to store the object // mkdirAll makes all the directories needed to store the object
func (o *FsObjectLocal) mkdirAll() error { func (o *Object) mkdirAll() error {
dir, _ := getDirFile(o.path) dir, _ := getDirFile(o.path)
return os.MkdirAll(dir, 0777) return os.MkdirAll(dir, 0777)
} }
// Update the object from in with modTime and size // Update the object from in with modTime and size
func (o *FsObjectLocal) Update(in io.Reader, modTime time.Time, size int64) error { func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
err := o.mkdirAll() err := o.mkdirAll()
if err != nil { if err != nil {
return err return err
@ -572,14 +576,14 @@ func (o *FsObjectLocal) Update(in io.Reader, modTime time.Time, size int64) erro
} }
// Stat a FsObject into info // Stat a FsObject into info
func (o *FsObjectLocal) lstat() error { func (o *Object) lstat() error {
info, err := os.Lstat(o.path) info, err := os.Lstat(o.path)
o.info = info o.info = info
return err return err
} }
// Remove an object // Remove an object
func (o *FsObjectLocal) Remove() error { func (o *Object) Remove() error {
return os.Remove(o.path) return os.Remove(o.path)
} }
@ -642,7 +646,7 @@ func uncPath(s string) string {
} }
// cleanWindowsName will clean invalid Windows characters // cleanWindowsName will clean invalid Windows characters
func cleanWindowsName(f *FsLocal, name string) string { func cleanWindowsName(f *Fs, name string) string {
original := name original := name
var name2 string var name2 string
if strings.HasPrefix(name, `\\?\`) { if strings.HasPrefix(name, `\\?\`) {
@ -679,8 +683,10 @@ func cleanWindowsName(f *FsLocal, name string) string {
} }
// Check the interfaces are satisfied // Check the interfaces are satisfied
var _ fs.Fs = &FsLocal{} var (
var _ fs.Purger = &FsLocal{} _ fs.Fs = &Fs{}
var _ fs.Mover = &FsLocal{} _ fs.Purger = &Fs{}
var _ fs.DirMover = &FsLocal{} _ fs.Mover = &Fs{}
var _ fs.Object = &FsObjectLocal{} _ fs.DirMover = &Fs{}
_ fs.Object = &Object{}
)

2
local/local_test.go
View File

@ -13,7 +13,7 @@ import (
) )
func init() { func init() {
fstests.NilObject = fs.Object((*local.FsObjectLocal)(nil)) fstests.NilObject = fs.Object((*local.Object)(nil))
fstests.RemoteName = "" fstests.RemoteName = ""
} }

2
local/tests_test.go
View File

@ -58,7 +58,7 @@ var utf8Tests = [][2]string{
} }
func TestCleanUtf8(t *testing.T) { func TestCleanUtf8(t *testing.T) {
f := &FsLocal{} f := &Fs{}
f.warned = make(map[string]struct{}) f.warned = make(map[string]struct{})
for _, test := range utf8Tests { for _, test := range utf8Tests {
got := f.cleanUtf8(test[0]) got := f.cleanUtf8(test[0])

129
s3/s3.go
View File

@ -128,8 +128,8 @@ const (
maxRetries = 10 // number of retries to make of operations maxRetries = 10 // number of retries to make of operations
) )
// FsS3 represents a remote s3 server // Fs represents a remote s3 server
type FsS3 struct { type Fs struct {
name string // the name of the remote name string // the name of the remote
c *s3.S3 // the connection to the s3 server c *s3.S3 // the connection to the s3 server
ses *session.Session // the s3 session ses *session.Session // the s3 session
@ -139,13 +139,13 @@ type FsS3 struct {
locationConstraint string // location constraint of new buckets locationConstraint string // location constraint of new buckets
} }
// FsObjectS3 describes a s3 object // Object describes a s3 object
type FsObjectS3 struct { type Object struct {
// Will definitely have everything but meta which may be nil // Will definitely have everything but meta which may be nil
// //
// List will read everything but meta - to fill that in need to call // List will read everything but meta - to fill that in need to call
// readMetaData // readMetaData
s3 *FsS3 // what this object is part of fs *Fs // what this object is part of
remote string // The remote path remote string // The remote path
etag string // md5sum of the object etag string // md5sum of the object
bytes int64 // size of the object bytes int64 // size of the object
@ -156,20 +156,20 @@ type FsObjectS3 struct {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Name of the remote (as passed into NewFs) // Name of the remote (as passed into NewFs)
func (f *FsS3) Name() string { func (f *Fs) Name() string {
return f.name return f.name
} }
// Root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
func (f *FsS3) Root() string { func (f *Fs) Root() string {
if f.root == "" { if f.root == "" {
return f.bucket return f.bucket
} }
return f.bucket + "/" + f.root return f.bucket + "/" + f.root
} }
// String converts this FsS3 to a string // String converts this Fs to a string
func (f *FsS3) String() string { func (f *Fs) String() string {
if f.root == "" { if f.root == "" {
return fmt.Sprintf("S3 bucket %s", f.bucket) return fmt.Sprintf("S3 bucket %s", f.bucket)
} }
@ -247,7 +247,7 @@ func s3Connection(name string) (*s3.S3, *session.Session, error) {
return c, ses, nil return c, ses, nil
} }
// NewFs contstructs an FsS3 from the path, bucket:path // NewFs contstructs an Fs from the path, bucket:path
func NewFs(name, root string) (fs.Fs, error) { func NewFs(name, root string) (fs.Fs, error) {
bucket, directory, err := s3ParsePath(root) bucket, directory, err := s3ParsePath(root)
if err != nil { if err != nil {
@ -257,7 +257,7 @@ func NewFs(name, root string) (fs.Fs, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
f := &FsS3{ f := &Fs{
name: name, name: name,
c: c, c: c,
bucket: bucket, bucket: bucket,
@ -294,9 +294,9 @@ func NewFs(name, root string) (fs.Fs, error) {
// Return an FsObject from a path // Return an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsS3) newFsObjectWithInfo(remote string, info *s3.Object) fs.Object { func (f *Fs) newFsObjectWithInfo(remote string, info *s3.Object) fs.Object {
o := &FsObjectS3{ o := &Object{
s3: f, fs: f,
remote: remote, remote: remote,
} }
if info != nil { if info != nil {
@ -322,14 +322,14 @@ func (f *FsS3) newFsObjectWithInfo(remote string, info *s3.Object) fs.Object {
// NewFsObject returns an FsObject from a path // NewFsObject returns an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsS3) NewFsObject(remote string) fs.Object { func (f *Fs) NewFsObject(remote string) fs.Object {
return f.newFsObjectWithInfo(remote, nil) return f.newFsObjectWithInfo(remote, nil)
} }
// list the objects into the function supplied // list the objects into the function supplied
// //
// If directories is set it only sends directories // If directories is set it only sends directories
func (f *FsS3) list(directories bool, fn func(string, *s3.Object)) { func (f *Fs) list(directories bool, fn func(string, *s3.Object)) {
maxKeys := int64(listChunkSize) maxKeys := int64(listChunkSize)
delimiter := "" delimiter := ""
if directories { if directories {
@ -394,7 +394,7 @@ func (f *FsS3) list(directories bool, fn func(string, *s3.Object)) {
} }
// List walks the path returning a channel of FsObjects // List walks the path returning a channel of FsObjects
func (f *FsS3) List() fs.ObjectsChan { func (f *Fs) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
if f.bucket == "" { if f.bucket == "" {
// Return no objects at top level list // Return no objects at top level list
@ -415,7 +415,7 @@ func (f *FsS3) List() fs.ObjectsChan {
} }
// ListDir lists the buckets // ListDir lists the buckets
func (f *FsS3) ListDir() fs.DirChan { func (f *Fs) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
if f.bucket == "" { if f.bucket == "" {
// List the buckets // List the buckets
@ -458,14 +458,17 @@ func (f *FsS3) ListDir() fs.DirChan {
} }
// Put the FsObject into the bucket // Put the FsObject into the bucket
func (f *FsS3) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction // Temporary Object under construction
fs := &FsObjectS3{s3: f, remote: remote} fs := &Object{
fs: f,
remote: remote,
}
return fs, fs.Update(in, modTime, size) return fs, fs.Update(in, modTime, size)
} }
// Mkdir creates the bucket if it doesn't exist // Mkdir creates the bucket if it doesn't exist
func (f *FsS3) Mkdir() error { func (f *Fs) Mkdir() error {
req := s3.CreateBucketInput{ req := s3.CreateBucketInput{
Bucket: &f.bucket, Bucket: &f.bucket,
ACL: &f.perm, ACL: &f.perm,
@ -487,7 +490,7 @@ func (f *FsS3) Mkdir() error {
// Rmdir deletes the bucket // Rmdir deletes the bucket
// //
// Returns an error if it isn't empty // Returns an error if it isn't empty
func (f *FsS3) Rmdir() error { func (f *Fs) Rmdir() error {
req := s3.DeleteBucketInput{ req := s3.DeleteBucketInput{
Bucket: &f.bucket, Bucket: &f.bucket,
} }
@ -496,7 +499,7 @@ func (f *FsS3) Rmdir() error {
} }
// Precision of the remote // Precision of the remote
func (f *FsS3) Precision() time.Duration { func (f *Fs) Precision() time.Duration {
return time.Nanosecond return time.Nanosecond
} }
@ -509,13 +512,13 @@ func (f *FsS3) Precision() time.Duration {
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
// If it isn't possible then return fs.ErrorCantCopy // If it isn't possible then return fs.ErrorCantCopy
func (f *FsS3) Copy(src fs.Object, remote string) (fs.Object, error) { func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectS3) srcObj, ok := src.(*Object)
if !ok { if !ok {
fs.Debug(src, "Can't copy - not same remote type") fs.Debug(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy return nil, fs.ErrorCantCopy
} }
srcFs := srcObj.s3 srcFs := srcObj.fs
key := f.root + remote key := f.root + remote
source := srcFs.bucket + "/" + srcFs.root + srcObj.remote source := srcFs.bucket + "/" + srcFs.root + srcObj.remote
req := s3.CopyObjectInput{ req := s3.CopyObjectInput{
@ -534,12 +537,12 @@ func (f *FsS3) Copy(src fs.Object, remote string) (fs.Object, error) {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *FsObjectS3) Fs() fs.Fs { func (o *Object) Fs() fs.Fs {
return o.s3 return o.fs
} }
// Return a string version // Return a string version
func (o *FsObjectS3) String() string { func (o *Object) String() string {
if o == nil { if o == nil {
return "<nil>" return "<nil>"
} }
@ -547,14 +550,14 @@ func (o *FsObjectS3) String() string {
} }
// Remote returns the remote path // Remote returns the remote path
func (o *FsObjectS3) Remote() string { func (o *Object) Remote() string {
return o.remote return o.remote
} }
var matchMd5 = regexp.MustCompile(`^[0-9a-f]{32}$`) var matchMd5 = regexp.MustCompile(`^[0-9a-f]{32}$`)
// Md5sum returns the Md5sum of an object returning a lowercase hex string // Md5sum returns the Md5sum of an object returning a lowercase hex string
func (o *FsObjectS3) Md5sum() (string, error) { func (o *Object) Md5sum() (string, error) {
etag := strings.Trim(strings.ToLower(o.etag), `"`) etag := strings.Trim(strings.ToLower(o.etag), `"`)
// Check the etag is a valid md5sum // Check the etag is a valid md5sum
if !matchMd5.MatchString(etag) { if !matchMd5.MatchString(etag) {
@ -565,23 +568,23 @@ func (o *FsObjectS3) Md5sum() (string, error) {
} }
// Size returns the size of an object in bytes // Size returns the size of an object in bytes
func (o *FsObjectS3) Size() int64 { func (o *Object) Size() int64 {
return o.bytes return o.bytes
} }
// readMetaData gets the metadata if it hasn't already been fetched // readMetaData gets the metadata if it hasn't already been fetched
// //
// it also sets the info // it also sets the info
func (o *FsObjectS3) readMetaData() (err error) { func (o *Object) readMetaData() (err error) {
if o.meta != nil { if o.meta != nil {
return nil return nil
} }
key := o.s3.root + o.remote key := o.fs.root + o.remote
req := s3.HeadObjectInput{ req := s3.HeadObjectInput{
Bucket: &o.s3.bucket, Bucket: &o.fs.bucket,
Key: &key, Key: &key,
} }
resp, err := o.s3.c.HeadObject(&req) resp, err := o.fs.c.HeadObject(&req)
if err != nil { if err != nil {
fs.Debug(o, "Failed to read info: %s", err) fs.Debug(o, "Failed to read info: %s", err)
return err return err
@ -608,7 +611,7 @@ func (o *FsObjectS3) readMetaData() (err error) {
// //
// It attempts to read the objects mtime and if that isn't present the // It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers // LastModified returned in the http headers
func (o *FsObjectS3) ModTime() time.Time { func (o *Object) ModTime() time.Time {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
fs.Log(o, "Failed to read metadata: %s", err) fs.Log(o, "Failed to read metadata: %s", err)
@ -629,7 +632,7 @@ func (o *FsObjectS3) ModTime() time.Time {
} }
// SetModTime sets the modification time of the local fs object // SetModTime sets the modification time of the local fs object
func (o *FsObjectS3) SetModTime(modTime time.Time) { func (o *Object) SetModTime(modTime time.Time) {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
@ -639,18 +642,18 @@ func (o *FsObjectS3) SetModTime(modTime time.Time) {
o.meta[metaMtime] = aws.String(swift.TimeToFloatString(modTime)) o.meta[metaMtime] = aws.String(swift.TimeToFloatString(modTime))
// Copy the object to itself to update the metadata // Copy the object to itself to update the metadata
key := o.s3.root + o.remote key := o.fs.root + o.remote
sourceKey := o.s3.bucket + "/" + key sourceKey := o.fs.bucket + "/" + key
directive := s3.MetadataDirectiveReplace // replace metadata with that passed in directive := s3.MetadataDirectiveReplace // replace metadata with that passed in
req := s3.CopyObjectInput{ req := s3.CopyObjectInput{
Bucket: &o.s3.bucket, Bucket: &o.fs.bucket,
ACL: &o.s3.perm, ACL: &o.fs.perm,
Key: &key, Key: &key,
CopySource: &sourceKey, CopySource: &sourceKey,
Metadata: o.meta, Metadata: o.meta,
MetadataDirective: &directive, MetadataDirective: &directive,
} }
_, err = o.s3.c.CopyObject(&req) _, err = o.fs.c.CopyObject(&req)
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
fs.ErrorLog(o, "Failed to update remote mtime: %s", err) fs.ErrorLog(o, "Failed to update remote mtime: %s", err)
@ -658,18 +661,18 @@ func (o *FsObjectS3) SetModTime(modTime time.Time) {
} }
// Storable raturns a boolean indicating if this object is storable // Storable raturns a boolean indicating if this object is storable
func (o *FsObjectS3) Storable() bool { func (o *Object) Storable() bool {
return true return true
} }
// Open an object for read // Open an object for read
func (o *FsObjectS3) Open() (in io.ReadCloser, err error) { func (o *Object) Open() (in io.ReadCloser, err error) {
key := o.s3.root + o.remote key := o.fs.root + o.remote
req := s3.GetObjectInput{ req := s3.GetObjectInput{
Bucket: &o.s3.bucket, Bucket: &o.fs.bucket,
Key: &key, Key: &key,
} }
resp, err := o.s3.c.GetObject(&req) resp, err := o.fs.c.GetObject(&req)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -677,11 +680,11 @@ func (o *FsObjectS3) Open() (in io.ReadCloser, err error) {
} }
// Update the Object from in with modTime and size // Update the Object from in with modTime and size
func (o *FsObjectS3) Update(in io.Reader, modTime time.Time, size int64) error { func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
uploader := s3manager.NewUploader(o.s3.ses, func(u *s3manager.Uploader) { uploader := s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) {
u.Concurrency = 2 u.Concurrency = 2
u.LeavePartsOnError = false u.LeavePartsOnError = false
u.S3 = o.s3.c u.S3 = o.fs.c
}) })
// Set the mtime in the meta data // Set the mtime in the meta data
@ -692,10 +695,10 @@ func (o *FsObjectS3) Update(in io.Reader, modTime time.Time, size int64) error {
// Guess the content type // Guess the content type
contentType := fs.MimeType(o) contentType := fs.MimeType(o)
key := o.s3.root + o.remote key := o.fs.root + o.remote
req := s3manager.UploadInput{ req := s3manager.UploadInput{
Bucket: &o.s3.bucket, Bucket: &o.fs.bucket,
ACL: &o.s3.perm, ACL: &o.fs.perm,
Key: &key, Key: &key,
Body: in, Body: in,
ContentType: &contentType, ContentType: &contentType,
@ -714,17 +717,19 @@ func (o *FsObjectS3) Update(in io.Reader, modTime time.Time, size int64) error {
} }
// Remove an object // Remove an object
func (o *FsObjectS3) Remove() error { func (o *Object) Remove() error {
key := o.s3.root + o.remote key := o.fs.root + o.remote
req := s3.DeleteObjectInput{ req := s3.DeleteObjectInput{
Bucket: &o.s3.bucket, Bucket: &o.fs.bucket,
Key: &key, Key: &key,
} }
_, err := o.s3.c.DeleteObject(&req) _, err := o.fs.c.DeleteObject(&req)
return err return err
} }
// Check the interfaces are satisfied // Check the interfaces are satisfied
var _ fs.Fs = &FsS3{} var (
var _ fs.Copier = &FsS3{} _ fs.Fs = &Fs{}
var _ fs.Object = &FsObjectS3{} _ fs.Copier = &Fs{}
_ fs.Object = &Object{}
)

2
s3/s3_test.go
View File

@ -13,7 +13,7 @@ import (
) )
func init() { func init() {
fstests.NilObject = fs.Object((*s3.FsObjectS3)(nil)) fstests.NilObject = fs.Object((*s3.Object)(nil))
fstests.RemoteName = "TestS3:" fstests.RemoteName = "TestS3:"
} }

139
swift/swift.go
View File

@ -65,8 +65,8 @@ func init() {
pflag.VarP(&chunkSize, "swift-chunk-size", "", "Above this size files will be chunked into a _segments container.") pflag.VarP(&chunkSize, "swift-chunk-size", "", "Above this size files will be chunked into a _segments container.")
} }
// FsSwift represents a remote swift server // Fs represents a remote swift server
type FsSwift struct { type Fs struct {
name string // name of this remote name string // name of this remote
c swift.Connection // the connection to the swift server c swift.Connection // the connection to the swift server
container string // the container we are working on container string // the container we are working on
@ -74,11 +74,11 @@ type FsSwift struct {
root string // the path we are working on if any root string // the path we are working on if any
} }
// FsObjectSwift describes a swift object // Object describes a swift object
// //
// Will definitely have info but maybe not meta // Will definitely have info but maybe not meta
type FsObjectSwift struct { type Object struct {
swift *FsSwift // what this object is part of fs *Fs // what this object is part of
remote string // The remote path remote string // The remote path
info swift.Object // Info from the swift object if known info swift.Object // Info from the swift object if known
headers *swift.Headers // The object headers if known headers *swift.Headers // The object headers if known
@ -87,20 +87,20 @@ type FsObjectSwift struct {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Name of the remote (as passed into NewFs) // Name of the remote (as passed into NewFs)
func (f *FsSwift) Name() string { func (f *Fs) Name() string {
return f.name return f.name
} }
// Root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
func (f *FsSwift) Root() string { func (f *Fs) Root() string {
if f.root == "" { if f.root == "" {
return f.container return f.container
} }
return f.container + "/" + f.root return f.container + "/" + f.root
} }
// String converts this FsSwift to a string // String converts this Fs to a string
func (f *FsSwift) String() string { func (f *Fs) String() string {
if f.root == "" { if f.root == "" {
return fmt.Sprintf("Swift container %s", f.container) return fmt.Sprintf("Swift container %s", f.container)
} }
@ -154,7 +154,7 @@ func swiftConnection(name string) (*swift.Connection, error) {
return c, nil return c, nil
} }
// NewFs contstructs an FsSwift from the path, container:path // NewFs contstructs an Fs from the path, container:path
func NewFs(name, root string) (fs.Fs, error) { func NewFs(name, root string) (fs.Fs, error) {
container, directory, err := parsePath(root) container, directory, err := parsePath(root)
if err != nil { if err != nil {
@ -164,7 +164,7 @@ func NewFs(name, root string) (fs.Fs, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
f := &FsSwift{ f := &Fs{
name: name, name: name,
c: *c, c: *c,
container: container, container: container,
@ -194,9 +194,9 @@ func NewFs(name, root string) (fs.Fs, error) {
// Return an FsObject from a path // Return an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsSwift) newFsObjectWithInfo(remote string, info *swift.Object) fs.Object { func (f *Fs) newFsObjectWithInfo(remote string, info *swift.Object) fs.Object {
o := &FsObjectSwift{ o := &Object{
swift: f, fs: f,
remote: remote, remote: remote,
} }
if info != nil { if info != nil {
@ -215,7 +215,7 @@ func (f *FsSwift) newFsObjectWithInfo(remote string, info *swift.Object) fs.Obje
// NewFsObject returns an FsObject from a path // NewFsObject returns an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsSwift) NewFsObject(remote string) fs.Object { func (f *Fs) NewFsObject(remote string) fs.Object {
return f.newFsObjectWithInfo(remote, nil) return f.newFsObjectWithInfo(remote, nil)
} }
@ -226,7 +226,7 @@ type listFn func(string, *swift.Object) error
// the container and root supplied // the container and root supplied
// //
// If directories is set it only sends directories // If directories is set it only sends directories
func (f *FsSwift) listContainerRoot(container, root string, directories bool, fn listFn) error { func (f *Fs) listContainerRoot(container, root string, directories bool, fn listFn) error {
// Options for ObjectsWalk // Options for ObjectsWalk
opts := swift.ObjectsOpts{ opts := swift.ObjectsOpts{
Prefix: root, Prefix: root,
@ -266,7 +266,7 @@ func (f *FsSwift) listContainerRoot(container, root string, directories bool, fn
// list the objects into the function supplied // list the objects into the function supplied
// //
// If directories is set it only sends directories // If directories is set it only sends directories
func (f *FsSwift) list(directories bool, fn listFn) { func (f *Fs) list(directories bool, fn listFn) {
err := f.listContainerRoot(f.container, f.root, directories, fn) err := f.listContainerRoot(f.container, f.root, directories, fn)
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
@ -275,7 +275,7 @@ func (f *FsSwift) list(directories bool, fn listFn) {
} }
// List walks the path returning a channel of FsObjects // List walks the path returning a channel of FsObjects
func (f *FsSwift) List() fs.ObjectsChan { func (f *Fs) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
if f.container == "" { if f.container == "" {
// Return no objects at top level list // Return no objects at top level list
@ -290,7 +290,7 @@ func (f *FsSwift) List() fs.ObjectsChan {
if o := f.newFsObjectWithInfo(remote, object); o != nil { if o := f.newFsObjectWithInfo(remote, object); o != nil {
// Do full metadata read on 0 size objects which might be manifest files // Do full metadata read on 0 size objects which might be manifest files
if o.Size() == 0 { if o.Size() == 0 {
err := o.(*FsObjectSwift).readMetaData() err := o.(*Object).readMetaData()
if err != nil { if err != nil {
fs.Debug(o, "Failed to read metadata: %v", err) fs.Debug(o, "Failed to read metadata: %v", err)
} }
@ -305,7 +305,7 @@ func (f *FsSwift) List() fs.ObjectsChan {
} }
// ListDir lists the containers // ListDir lists the containers
func (f *FsSwift) ListDir() fs.DirChan { func (f *Fs) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
if f.container == "" { if f.container == "" {
// List the containers // List the containers
@ -347,26 +347,29 @@ func (f *FsSwift) ListDir() fs.DirChan {
// Copy the reader in to the new object which is returned // Copy the reader in to the new object which is returned
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (f *FsSwift) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction // Temporary Object under construction
fs := &FsObjectSwift{swift: f, remote: remote} fs := &Object{
fs: f,
remote: remote,
}
return fs, fs.Update(in, modTime, size) return fs, fs.Update(in, modTime, size)
} }
// Mkdir creates the container if it doesn't exist // Mkdir creates the container if it doesn't exist
func (f *FsSwift) Mkdir() error { func (f *Fs) Mkdir() error {
return f.c.ContainerCreate(f.container, nil) return f.c.ContainerCreate(f.container, nil)
} }
// Rmdir deletes the container // Rmdir deletes the container
// //
// Returns an error if it isn't empty // Returns an error if it isn't empty
func (f *FsSwift) Rmdir() error { func (f *Fs) Rmdir() error {
return f.c.ContainerDelete(f.container) return f.c.ContainerDelete(f.container)
} }
// Precision of the remote // Precision of the remote
func (f *FsSwift) Precision() time.Duration { func (f *Fs) Precision() time.Duration {
return time.Nanosecond return time.Nanosecond
} }
@ -379,13 +382,13 @@ func (f *FsSwift) Precision() time.Duration {
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
// If it isn't possible then return fs.ErrorCantCopy // If it isn't possible then return fs.ErrorCantCopy
func (f *FsSwift) Copy(src fs.Object, remote string) (fs.Object, error) { func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectSwift) srcObj, ok := src.(*Object)
if !ok { if !ok {
fs.Debug(src, "Can't copy - not same remote type") fs.Debug(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy return nil, fs.ErrorCantCopy
} }
srcFs := srcObj.swift srcFs := srcObj.fs
_, err := f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil) _, err := f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil)
if err != nil { if err != nil {
return nil, err return nil, err
@ -396,12 +399,12 @@ func (f *FsSwift) Copy(src fs.Object, remote string) (fs.Object, error) {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Fs returns the parent Fs // Fs returns the parent Fs
func (o *FsObjectSwift) Fs() fs.Fs { func (o *Object) Fs() fs.Fs {
return o.swift return o.fs
} }
// Return a string version // Return a string version
func (o *FsObjectSwift) String() string { func (o *Object) String() string {
if o == nil { if o == nil {
return "<nil>" return "<nil>"
} }
@ -409,12 +412,12 @@ func (o *FsObjectSwift) String() string {
} }
// Remote returns the remote path // Remote returns the remote path
func (o *FsObjectSwift) Remote() string { func (o *Object) Remote() string {
return o.remote return o.remote
} }
// Md5sum returns the Md5sum of an object returning a lowercase hex string // Md5sum returns the Md5sum of an object returning a lowercase hex string
func (o *FsObjectSwift) Md5sum() (string, error) { func (o *Object) Md5sum() (string, error) {
isManifest, err := o.isManifestFile() isManifest, err := o.isManifestFile()
if err != nil { if err != nil {
return "", err return "", err
@ -427,7 +430,7 @@ func (o *FsObjectSwift) Md5sum() (string, error) {
} }
// isManifestFile checks for manifest header // isManifestFile checks for manifest header
func (o *FsObjectSwift) isManifestFile() (bool, error) { func (o *Object) isManifestFile() (bool, error) {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
if err == swift.ObjectNotFound { if err == swift.ObjectNotFound {
@ -440,18 +443,18 @@ func (o *FsObjectSwift) isManifestFile() (bool, error) {
} }
// Size returns the size of an object in bytes // Size returns the size of an object in bytes
func (o *FsObjectSwift) Size() int64 { func (o *Object) Size() int64 {
return o.info.Bytes return o.info.Bytes
} }
// readMetaData gets the metadata if it hasn't already been fetched // readMetaData gets the metadata if it hasn't already been fetched
// //
// it also sets the info // it also sets the info
func (o *FsObjectSwift) readMetaData() (err error) { func (o *Object) readMetaData() (err error) {
if o.headers != nil { if o.headers != nil {
return nil return nil
} }
info, h, err := o.swift.c.Object(o.swift.container, o.swift.root+o.remote) info, h, err := o.fs.c.Object(o.fs.container, o.fs.root+o.remote)
if err != nil { if err != nil {
return err return err
} }
@ -465,7 +468,7 @@ func (o *FsObjectSwift) readMetaData() (err error) {
// //
// It attempts to read the objects mtime and if that isn't present the // It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers // LastModified returned in the http headers
func (o *FsObjectSwift) ModTime() time.Time { func (o *Object) ModTime() time.Time {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
fs.Debug(o, "Failed to read metadata: %s", err) fs.Debug(o, "Failed to read metadata: %s", err)
@ -480,7 +483,7 @@ func (o *FsObjectSwift) ModTime() time.Time {
} }
// SetModTime sets the modification time of the local fs object // SetModTime sets the modification time of the local fs object
func (o *FsObjectSwift) SetModTime(modTime time.Time) { func (o *Object) SetModTime(modTime time.Time) {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
@ -493,7 +496,7 @@ func (o *FsObjectSwift) SetModTime(modTime time.Time) {
for k, v := range newHeaders { for k, v := range newHeaders {
(*o.headers)[k] = v (*o.headers)[k] = v
} }
err = o.swift.c.ObjectUpdate(o.swift.container, o.swift.root+o.remote, newHeaders) err = o.fs.c.ObjectUpdate(o.fs.container, o.fs.root+o.remote, newHeaders)
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
fs.ErrorLog(o, "Failed to update remote mtime: %s", err) fs.ErrorLog(o, "Failed to update remote mtime: %s", err)
@ -501,13 +504,13 @@ func (o *FsObjectSwift) SetModTime(modTime time.Time) {
} }
// Storable returns if this object is storable // Storable returns if this object is storable
func (o *FsObjectSwift) Storable() bool { func (o *Object) Storable() bool {
return true return true
} }
// Open an object for read // Open an object for read
func (o *FsObjectSwift) Open() (in io.ReadCloser, err error) { func (o *Object) Open() (in io.ReadCloser, err error) {
in, _, err = o.swift.c.ObjectOpen(o.swift.container, o.swift.root+o.remote, true, nil) in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, true, nil)
return return
} }
@ -522,33 +525,33 @@ func min(x, y int64) int64 {
// removeSegments removes any old segments from o // removeSegments removes any old segments from o
// //
// if except is passed in then segments with that prefix won't be deleted // if except is passed in then segments with that prefix won't be deleted
func (o *FsObjectSwift) removeSegments(except string) error { func (o *Object) removeSegments(except string) error {
segmentsRoot := o.swift.root + o.remote + "/" segmentsRoot := o.fs.root + o.remote + "/"
err := o.swift.listContainerRoot(o.swift.segmentsContainer, segmentsRoot, false, func(remote string, object *swift.Object) error { err := o.fs.listContainerRoot(o.fs.segmentsContainer, segmentsRoot, false, func(remote string, object *swift.Object) error {
if except != "" && strings.HasPrefix(remote, except) { if except != "" && strings.HasPrefix(remote, except) {
// fs.Debug(o, "Ignoring current segment file %q in container %q", segmentsRoot+remote, o.swift.segmentsContainer) // fs.Debug(o, "Ignoring current segment file %q in container %q", segmentsRoot+remote, o.fs.segmentsContainer)
return nil return nil
} }
segmentPath := segmentsRoot + remote segmentPath := segmentsRoot + remote
fs.Debug(o, "Removing segment file %q in container %q", segmentPath, o.swift.segmentsContainer) fs.Debug(o, "Removing segment file %q in container %q", segmentPath, o.fs.segmentsContainer)
return o.swift.c.ObjectDelete(o.swift.segmentsContainer, segmentPath) return o.fs.c.ObjectDelete(o.fs.segmentsContainer, segmentPath)
}) })
if err != nil { if err != nil {
return err return err
} }
// remove the segments container if empty, ignore errors // remove the segments container if empty, ignore errors
err = o.swift.c.ContainerDelete(o.swift.segmentsContainer) err = o.fs.c.ContainerDelete(o.fs.segmentsContainer)
if err == nil { if err == nil {
fs.Debug(o, "Removed empty container %q", o.swift.segmentsContainer) fs.Debug(o, "Removed empty container %q", o.fs.segmentsContainer)
} }
return nil return nil
} }
// updateChunks updates the existing object using chunks to a separate // updateChunks updates the existing object using chunks to a separate
// container. It returns a string which prefixes current segments. // container. It returns a string which prefixes current segments.
func (o *FsObjectSwift) updateChunks(in io.Reader, headers swift.Headers, size int64) (string, error) { func (o *Object) updateChunks(in io.Reader, headers swift.Headers, size int64) (string, error) {
// Create the segmentsContainer if it doesn't exist // Create the segmentsContainer if it doesn't exist
err := o.swift.c.ContainerCreate(o.swift.segmentsContainer, nil) err := o.fs.c.ContainerCreate(o.fs.segmentsContainer, nil)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -556,14 +559,14 @@ func (o *FsObjectSwift) updateChunks(in io.Reader, headers swift.Headers, size i
left := size left := size
i := 0 i := 0
uniquePrefix := fmt.Sprintf("%s/%d", swift.TimeToFloatString(time.Now()), size) uniquePrefix := fmt.Sprintf("%s/%d", swift.TimeToFloatString(time.Now()), size)
segmentsPath := fmt.Sprintf("%s%s/%s", o.swift.root, o.remote, uniquePrefix) segmentsPath := fmt.Sprintf("%s%s/%s", o.fs.root, o.remote, uniquePrefix)
for left > 0 { for left > 0 {
n := min(left, int64(chunkSize)) n := min(left, int64(chunkSize))
headers["Content-Length"] = strconv.FormatInt(n, 10) // set Content-Length as we know it headers["Content-Length"] = strconv.FormatInt(n, 10) // set Content-Length as we know it
segmentReader := io.LimitReader(in, n) segmentReader := io.LimitReader(in, n)
segmentPath := fmt.Sprintf("%s/%08d", segmentsPath, i) segmentPath := fmt.Sprintf("%s/%08d", segmentsPath, i)
fs.Debug(o, "Uploading segment file %q into %q", segmentPath, o.swift.segmentsContainer) fs.Debug(o, "Uploading segment file %q into %q", segmentPath, o.fs.segmentsContainer)
_, err := o.swift.c.ObjectPut(o.swift.segmentsContainer, segmentPath, segmentReader, true, "", "", headers) _, err := o.fs.c.ObjectPut(o.fs.segmentsContainer, segmentPath, segmentReader, true, "", "", headers)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -571,18 +574,18 @@ func (o *FsObjectSwift) updateChunks(in io.Reader, headers swift.Headers, size i
i++ i++
} }
// Upload the manifest // Upload the manifest
headers["X-Object-Manifest"] = fmt.Sprintf("%s/%s", o.swift.segmentsContainer, segmentsPath) headers["X-Object-Manifest"] = fmt.Sprintf("%s/%s", o.fs.segmentsContainer, segmentsPath)
headers["Content-Length"] = "0" // set Content-Length as we know it headers["Content-Length"] = "0" // set Content-Length as we know it
emptyReader := bytes.NewReader(nil) emptyReader := bytes.NewReader(nil)
manifestName := o.swift.root + o.remote manifestName := o.fs.root + o.remote
_, err = o.swift.c.ObjectPut(o.swift.container, manifestName, emptyReader, true, "", "", headers) _, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", "", headers)
return uniquePrefix + "/", err return uniquePrefix + "/", err
} }
// Update the object with the contents of the io.Reader, modTime and size // Update the object with the contents of the io.Reader, modTime and size
// //
// The new object may have been created if an error is returned // The new object may have been created if an error is returned
func (o *FsObjectSwift) Update(in io.Reader, modTime time.Time, size int64) error { func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
// Note whether this has a manifest before starting // Note whether this has a manifest before starting
isManifest, err := o.isManifestFile() isManifest, err := o.isManifestFile()
if err != nil { if err != nil {
@ -601,7 +604,7 @@ func (o *FsObjectSwift) Update(in io.Reader, modTime time.Time, size int64) erro
} }
} else { } else {
headers["Content-Length"] = strconv.FormatInt(size, 10) // set Content-Length as we know it headers["Content-Length"] = strconv.FormatInt(size, 10) // set Content-Length as we know it
_, err := o.swift.c.ObjectPut(o.swift.container, o.swift.root+o.remote, in, true, "", "", headers) _, err := o.fs.c.ObjectPut(o.fs.container, o.fs.root+o.remote, in, true, "", "", headers)
if err != nil { if err != nil {
return err return err
} }
@ -621,13 +624,13 @@ func (o *FsObjectSwift) Update(in io.Reader, modTime time.Time, size int64) erro
} }
// Remove an object // Remove an object
func (o *FsObjectSwift) Remove() error { func (o *Object) Remove() error {
isManifestFile, err := o.isManifestFile() isManifestFile, err := o.isManifestFile()
if err != nil { if err != nil {
return err return err
} }
// Remove file/manifest first // Remove file/manifest first
err = o.swift.c.ObjectDelete(o.swift.container, o.swift.root+o.remote) err = o.fs.c.ObjectDelete(o.fs.container, o.fs.root+o.remote)
if err != nil { if err != nil {
return err return err
} }
@ -642,6 +645,8 @@ func (o *FsObjectSwift) Remove() error {
} }
// Check the interfaces are satisfied // Check the interfaces are satisfied
var _ fs.Fs = &FsSwift{} var (
var _ fs.Copier = &FsSwift{} _ fs.Fs = &Fs{}
var _ fs.Object = &FsObjectSwift{} _ fs.Copier = &Fs{}
_ fs.Object = &Object{}
)

2
swift/swift_test.go
View File

@ -13,7 +13,7 @@ import (
) )
func init() { func init() {
fstests.NilObject = fs.Object((*swift.FsObjectSwift)(nil)) fstests.NilObject = fs.Object((*swift.Object)(nil))
fstests.RemoteName = "TestSwift:" fstests.RemoteName = "TestSwift:"
} }