mirror of
https://github.com/rclone/rclone.git
synced 2024-11-22 00:13:49 +01:00
Rename fs.FsX to fs.X
This commit is contained in:
parent
6085dc1b5a
commit
b1f681b912
36
drive/fs.go
36
drive/fs.go
@ -289,7 +289,7 @@ func NewFsDrive(path string) (*FsDrive, error) {
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.FsObject {
|
||||
func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.Object {
|
||||
fs := &FsObjectDrive{
|
||||
drive: f,
|
||||
remote: remote,
|
||||
@ -299,7 +299,7 @@ func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.FsObje
|
||||
} else {
|
||||
err := fs.readMetaData() // reads info and meta, returning an error
|
||||
if err != nil {
|
||||
// logged already fs.FsDebug("Failed to read info: %s", err)
|
||||
// logged already fs.Debug("Failed to read info: %s", err)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@ -309,7 +309,7 @@ func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.FsObje
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func (f *FsDrive) NewFsObject(remote string) fs.FsObject {
|
||||
func (f *FsDrive) NewFsObject(remote string) fs.Object {
|
||||
return f.NewFsObjectWithInfo(remote, nil)
|
||||
}
|
||||
|
||||
@ -319,7 +319,7 @@ func (f *FsDrive) NewFsObject(remote string) fs.FsObject {
|
||||
//
|
||||
// This fetches the minimum amount of stuff but does more API calls
|
||||
// which makes it slow
|
||||
func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.FsObjectsChan) error {
|
||||
func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error {
|
||||
var subError error
|
||||
// Make the API request
|
||||
_, err := f.listAll(dirId, "", false, false, func(item *drive.File) bool {
|
||||
@ -357,7 +357,7 @@ func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.FsObjectsCh
|
||||
//
|
||||
// This is fast in terms of number of API calls, but slow in terms of
|
||||
// fetching more data than it needs
|
||||
func (f *FsDrive) listDirFull(dirId string, path string, out fs.FsObjectsChan) error {
|
||||
func (f *FsDrive) listDirFull(dirId string, path string, out fs.ObjectsChan) error {
|
||||
// Orphans waiting for their parent
|
||||
orphans := make(map[string][]*drive.File)
|
||||
|
||||
@ -547,8 +547,8 @@ func (f *FsDrive) findRoot(create bool) error {
|
||||
}
|
||||
|
||||
// Walk the path returning a channel of FsObjects
|
||||
func (f *FsDrive) List() fs.FsObjectsChan {
|
||||
out := make(fs.FsObjectsChan, fs.Config.Checkers)
|
||||
func (f *FsDrive) List() fs.ObjectsChan {
|
||||
out := make(fs.ObjectsChan, fs.Config.Checkers)
|
||||
go func() {
|
||||
defer close(out)
|
||||
err := f.findRoot(false)
|
||||
@ -571,8 +571,8 @@ func (f *FsDrive) List() fs.FsObjectsChan {
|
||||
}
|
||||
|
||||
// Walk the path returning a channel of FsObjects
|
||||
func (f *FsDrive) ListDir() fs.FsDirChan {
|
||||
out := make(fs.FsDirChan, fs.Config.Checkers)
|
||||
func (f *FsDrive) ListDir() fs.DirChan {
|
||||
out := make(fs.DirChan, fs.Config.Checkers)
|
||||
go func() {
|
||||
defer close(out)
|
||||
err := f.findRoot(false)
|
||||
@ -581,7 +581,7 @@ func (f *FsDrive) ListDir() fs.FsDirChan {
|
||||
log.Printf("Couldn't find root: %s", err)
|
||||
} else {
|
||||
_, err := f.listAll(f.rootId, "", true, false, func(item *drive.File) bool {
|
||||
dir := &fs.FsDir{
|
||||
dir := &fs.Dir{
|
||||
Name: item.Title,
|
||||
Bytes: -1,
|
||||
Count: -1,
|
||||
@ -604,7 +604,7 @@ func (f *FsDrive) ListDir() fs.FsDirChan {
|
||||
// Copy the reader in to the new object which is returned
|
||||
//
|
||||
// The new object may have been created
|
||||
func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) {
|
||||
func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
||||
// Temporary FsObject under construction
|
||||
fs := &FsObjectDrive{drive: f, remote: remote}
|
||||
|
||||
@ -744,7 +744,7 @@ func (o *FsObjectDrive) readMetaData() (err error) {
|
||||
directory, leaf := splitPath(o.remote)
|
||||
directoryId, err := o.drive.findDir(directory, false)
|
||||
if err != nil {
|
||||
fs.FsDebug(o, "Couldn't find directory: %s", err)
|
||||
fs.Debug(o, "Couldn't find directory: %s", err)
|
||||
return fmt.Errorf("Couldn't find directory: %s", err)
|
||||
}
|
||||
|
||||
@ -759,7 +759,7 @@ func (o *FsObjectDrive) readMetaData() (err error) {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
fs.FsDebug(o, "Couldn't find object")
|
||||
fs.Debug(o, "Couldn't find object")
|
||||
return fmt.Errorf("Couldn't find object")
|
||||
}
|
||||
return nil
|
||||
@ -773,12 +773,12 @@ func (o *FsObjectDrive) readMetaData() (err error) {
|
||||
func (o *FsObjectDrive) ModTime() time.Time {
|
||||
err := o.readMetaData()
|
||||
if err != nil {
|
||||
fs.FsLog(o, "Failed to read metadata: %s", err)
|
||||
fs.Log(o, "Failed to read metadata: %s", err)
|
||||
return time.Now()
|
||||
}
|
||||
modTime, err := time.Parse(time.RFC3339, o.modifiedDate)
|
||||
if err != nil {
|
||||
fs.FsLog(o, "Failed to read mtime from object: %s", err)
|
||||
fs.Log(o, "Failed to read mtime from object: %s", err)
|
||||
return time.Now()
|
||||
}
|
||||
return modTime
|
||||
@ -789,7 +789,7 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) {
|
||||
err := o.readMetaData()
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.FsLog(o, "Failed to read metadata: %s", err)
|
||||
fs.Log(o, "Failed to read metadata: %s", err)
|
||||
return
|
||||
}
|
||||
// New metadata
|
||||
@ -800,7 +800,7 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) {
|
||||
_, err = o.drive.svc.Files.Update(o.id, info).SetModifiedDate(true).Do()
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.FsLog(o, "Failed to update remote mtime: %s", err)
|
||||
fs.Log(o, "Failed to update remote mtime: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -832,4 +832,4 @@ func (o *FsObjectDrive) Remove() error {
|
||||
// Check the interfaces are satisfied
|
||||
var _ fs.Fs = &FsDrive{}
|
||||
var _ fs.Purger = &FsDrive{}
|
||||
var _ fs.FsObject = &FsObjectDrive{}
|
||||
var _ fs.Object = &FsObjectDrive{}
|
||||
|
@ -121,32 +121,32 @@ func (s *StatsInfo) Error() {
|
||||
}
|
||||
|
||||
// Checking adds a check into the stats
|
||||
func (s *StatsInfo) Checking(fs FsObject) {
|
||||
func (s *StatsInfo) Checking(o Object) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
s.checking[fs.Remote()] = true
|
||||
s.checking[o.Remote()] = true
|
||||
}
|
||||
|
||||
// DoneChecking removes a check from the stats
|
||||
func (s *StatsInfo) DoneChecking(fs FsObject) {
|
||||
func (s *StatsInfo) DoneChecking(o Object) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
delete(s.checking, fs.Remote())
|
||||
delete(s.checking, o.Remote())
|
||||
s.checks += 1
|
||||
}
|
||||
|
||||
// Transferring adds a transfer into the stats
|
||||
func (s *StatsInfo) Transferring(fs FsObject) {
|
||||
func (s *StatsInfo) Transferring(o Object) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
s.transferring[fs.Remote()] = true
|
||||
s.transferring[o.Remote()] = true
|
||||
}
|
||||
|
||||
// DoneTransferring removes a transfer from the stats
|
||||
func (s *StatsInfo) DoneTransferring(fs FsObject) {
|
||||
func (s *StatsInfo) DoneTransferring(o Object) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
delete(s.transferring, fs.Remote())
|
||||
delete(s.transferring, o.Remote())
|
||||
s.transfers += 1
|
||||
}
|
||||
|
||||
|
86
fs/fs.go
86
fs/fs.go
@ -13,24 +13,24 @@ import (
|
||||
// Globals
|
||||
var (
|
||||
// Global config
|
||||
Config = &FsConfig{}
|
||||
Config = &ConfigInfo{}
|
||||
// Filesystem registry
|
||||
fsRegistry []fsRegistryItem
|
||||
fsRegistry []registryItem
|
||||
)
|
||||
|
||||
// Filesystem config options
|
||||
type FsConfig struct {
|
||||
Verbose bool
|
||||
Quiet bool
|
||||
type ConfigInfo struct {
|
||||
Verbose bool
|
||||
Quiet bool
|
||||
ModifyWindow time.Duration
|
||||
Checkers int
|
||||
Transfers int
|
||||
Checkers int
|
||||
Transfers int
|
||||
}
|
||||
|
||||
// FIXME need local to go last
|
||||
|
||||
// Filesystem registry item
|
||||
type fsRegistryItem struct {
|
||||
type registryItem struct {
|
||||
match *regexp.Regexp // if this matches then can call newFs
|
||||
newFs func(string) (Fs, error) // create a new file system
|
||||
}
|
||||
@ -41,7 +41,7 @@ type fsRegistryItem struct {
|
||||
//
|
||||
// Fs modules should use this in an init() function
|
||||
func Register(match *regexp.Regexp, newFs func(string) (Fs, error)) {
|
||||
fsRegistry = append(fsRegistry, fsRegistryItem{match: match, newFs: newFs})
|
||||
fsRegistry = append(fsRegistry, registryItem{match: match, newFs: newFs})
|
||||
}
|
||||
|
||||
// A Filesystem, describes the local filesystem and the remote object store
|
||||
@ -50,20 +50,20 @@ type Fs interface {
|
||||
String() string
|
||||
|
||||
// List the Fs into a channel
|
||||
List() FsObjectsChan
|
||||
List() ObjectsChan
|
||||
|
||||
// List the Fs directories/buckets/containers into a channel
|
||||
ListDir() FsDirChan
|
||||
ListDir() DirChan
|
||||
|
||||
// Find the FsObject at remote. Returns nil if can't be found
|
||||
NewFsObject(remote string) FsObject
|
||||
// Find the Object at remote. Returns nil if can't be found
|
||||
NewFsObject(remote string) Object
|
||||
|
||||
// Put in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// May create the object even if it returns an error - if so
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
Put(in io.Reader, remote string, modTime time.Time, size int64) (FsObject, error)
|
||||
Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error)
|
||||
|
||||
// Make the directory (container, bucket)
|
||||
Mkdir() error
|
||||
@ -79,7 +79,7 @@ type Fs interface {
|
||||
|
||||
// A filesystem like object which can either be a remote object or a
|
||||
// local file/directory
|
||||
type FsObject interface {
|
||||
type Object interface {
|
||||
// Remote returns the remote path
|
||||
Remote() string
|
||||
|
||||
@ -114,22 +114,22 @@ type Purger interface {
|
||||
Purge() error
|
||||
}
|
||||
|
||||
// A channel of FsObjects
|
||||
type FsObjectsChan chan FsObject
|
||||
// A channel of Objects
|
||||
type ObjectsChan chan Object
|
||||
|
||||
// A slice of FsObjects
|
||||
type FsObjects []FsObject
|
||||
// A slice of Objects
|
||||
type Objects []Object
|
||||
|
||||
// A structure of directory/container/bucket lists
|
||||
type FsDir struct {
|
||||
type Dir struct {
|
||||
Name string // name of the directory
|
||||
When time.Time // modification or creation time - IsZero for unknown
|
||||
Bytes int64 // size of directory and contents -1 for unknown
|
||||
Count int64 // number of objects -1 for unknown
|
||||
}
|
||||
|
||||
// A channel of FsDir objects
|
||||
type FsDirChan chan *FsDir
|
||||
// A channel of Dir objects
|
||||
type DirChan chan *Dir
|
||||
|
||||
// NewFs makes a new Fs object from the path
|
||||
//
|
||||
@ -143,16 +143,16 @@ func NewFs(path string) (Fs, error) {
|
||||
panic("Not found") // FIXME
|
||||
}
|
||||
|
||||
// Write debuging output for this FsObject
|
||||
func FsDebug(fs FsObject, text string, args ...interface{}) {
|
||||
// Write debuging output for this Object
|
||||
func Debug(fs Object, text string, args ...interface{}) {
|
||||
if Config.Verbose {
|
||||
out := fmt.Sprintf(text, args...)
|
||||
log.Printf("%s: %s", fs.Remote(), out)
|
||||
}
|
||||
}
|
||||
|
||||
// Write log output for this FsObject
|
||||
func FsLog(fs FsObject, text string, args ...interface{}) {
|
||||
// Write log output for this Object
|
||||
func Log(fs Object, text string, args ...interface{}) {
|
||||
if !Config.Quiet {
|
||||
out := fmt.Sprintf(text, args...)
|
||||
log.Printf("%s: %s", fs.Remote(), out)
|
||||
@ -173,21 +173,21 @@ func checkClose(c io.Closer, err *error) {
|
||||
// May return an error which will already have been logged
|
||||
//
|
||||
// If an error is returned it will return false
|
||||
func CheckMd5sums(src, dst FsObject) (bool, error) {
|
||||
func CheckMd5sums(src, dst Object) (bool, error) {
|
||||
srcMd5, err := src.Md5sum()
|
||||
if err != nil {
|
||||
Stats.Error()
|
||||
FsLog(src, "Failed to calculate src md5: %s", err)
|
||||
Log(src, "Failed to calculate src md5: %s", err)
|
||||
return false, err
|
||||
}
|
||||
dstMd5, err := dst.Md5sum()
|
||||
if err != nil {
|
||||
Stats.Error()
|
||||
FsLog(dst, "Failed to calculate dst md5: %s", err)
|
||||
Log(dst, "Failed to calculate dst md5: %s", err)
|
||||
return false, err
|
||||
}
|
||||
// FsDebug("Src MD5 %s", srcMd5)
|
||||
// FsDebug("Dst MD5 %s", obj.Hash)
|
||||
// Debug("Src MD5 %s", srcMd5)
|
||||
// Debug("Dst MD5 %s", obj.Hash)
|
||||
return srcMd5 == dstMd5, nil
|
||||
}
|
||||
|
||||
@ -207,9 +207,9 @@ func CheckMd5sums(src, dst FsObject) (bool, error) {
|
||||
//
|
||||
// Otherwise the file is considered to be not equal including if there
|
||||
// were errors reading info.
|
||||
func Equal(src, dst FsObject) bool {
|
||||
func Equal(src, dst Object) bool {
|
||||
if src.Size() != dst.Size() {
|
||||
FsDebug(src, "Sizes differ")
|
||||
Debug(src, "Sizes differ")
|
||||
return false
|
||||
}
|
||||
|
||||
@ -219,9 +219,9 @@ func Equal(src, dst FsObject) bool {
|
||||
dt := dstModTime.Sub(srcModTime)
|
||||
ModifyWindow := Config.ModifyWindow
|
||||
if dt >= ModifyWindow || dt <= -ModifyWindow {
|
||||
FsDebug(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime)
|
||||
Debug(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime)
|
||||
} else {
|
||||
FsDebug(src, "Size and modification time differ by %s (within %s)", dt, ModifyWindow)
|
||||
Debug(src, "Size and modification time differ by %s (within %s)", dt, ModifyWindow)
|
||||
return true
|
||||
}
|
||||
|
||||
@ -229,7 +229,7 @@ func Equal(src, dst FsObject) bool {
|
||||
// check the MD5SUM
|
||||
same, _ := CheckMd5sums(src, dst)
|
||||
if !same {
|
||||
FsDebug(src, "Md5sums differ")
|
||||
Debug(src, "Md5sums differ")
|
||||
return false
|
||||
}
|
||||
|
||||
@ -237,16 +237,16 @@ func Equal(src, dst FsObject) bool {
|
||||
// mtime of the dst object here
|
||||
dst.SetModTime(srcModTime)
|
||||
|
||||
FsDebug(src, "Size and MD5SUM of src and dst objects identical")
|
||||
Debug(src, "Size and MD5SUM of src and dst objects identical")
|
||||
return true
|
||||
}
|
||||
|
||||
// Copy src object to f
|
||||
func Copy(f Fs, src FsObject) {
|
||||
func Copy(f Fs, src Object) {
|
||||
in0, err := src.Open()
|
||||
if err != nil {
|
||||
Stats.Error()
|
||||
FsLog(src, "Failed to open: %s", err)
|
||||
Log(src, "Failed to open: %s", err)
|
||||
return
|
||||
}
|
||||
in := NewAccount(in0) // account the transfer
|
||||
@ -258,16 +258,16 @@ func Copy(f Fs, src FsObject) {
|
||||
}
|
||||
if err != nil {
|
||||
Stats.Error()
|
||||
FsLog(src, "Failed to copy: %s", err)
|
||||
Log(src, "Failed to copy: %s", err)
|
||||
if dst != nil {
|
||||
FsDebug(dst, "Removing failed copy")
|
||||
Debug(dst, "Removing failed copy")
|
||||
removeErr := dst.Remove()
|
||||
if removeErr != nil {
|
||||
Stats.Error()
|
||||
FsLog(dst, "Failed to remove failed copy: %s", removeErr)
|
||||
Log(dst, "Failed to remove failed copy: %s", removeErr)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
FsDebug(src, "Copied")
|
||||
Debug(src, "Copied")
|
||||
}
|
||||
|
30
local/fs.go
30
local/fs.go
@ -55,7 +55,7 @@ func (f *FsLocal) String() string {
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.FsObject {
|
||||
func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.Object {
|
||||
path := filepath.Join(f.root, remote)
|
||||
o := &FsObjectLocal{remote: remote, path: path}
|
||||
if info != nil {
|
||||
@ -63,7 +63,7 @@ func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.FsObje
|
||||
} else {
|
||||
err := o.lstat()
|
||||
if err != nil {
|
||||
fs.FsDebug(o, "Failed to stat %s: %s", path, err)
|
||||
fs.Debug(o, "Failed to stat %s: %s", path, err)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@ -73,15 +73,15 @@ func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.FsObje
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func (f *FsLocal) NewFsObject(remote string) fs.FsObject {
|
||||
func (f *FsLocal) NewFsObject(remote string) fs.Object {
|
||||
return f.NewFsObjectWithInfo(remote, nil)
|
||||
}
|
||||
|
||||
// List the path returning a channel of FsObjects
|
||||
//
|
||||
// Ignores everything which isn't Storable, eg links etc
|
||||
func (f *FsLocal) List() fs.FsObjectsChan {
|
||||
out := make(fs.FsObjectsChan, fs.Config.Checkers)
|
||||
func (f *FsLocal) List() fs.ObjectsChan {
|
||||
out := make(fs.ObjectsChan, fs.Config.Checkers)
|
||||
go func() {
|
||||
err := filepath.Walk(f.root, func(path string, fi os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
@ -116,8 +116,8 @@ func (f *FsLocal) List() fs.FsObjectsChan {
|
||||
}
|
||||
|
||||
// Walk the path returning a channel of FsObjects
|
||||
func (f *FsLocal) ListDir() fs.FsDirChan {
|
||||
out := make(fs.FsDirChan, fs.Config.Checkers)
|
||||
func (f *FsLocal) ListDir() fs.DirChan {
|
||||
out := make(fs.DirChan, fs.Config.Checkers)
|
||||
go func() {
|
||||
defer close(out)
|
||||
items, err := ioutil.ReadDir(f.root)
|
||||
@ -127,7 +127,7 @@ func (f *FsLocal) ListDir() fs.FsDirChan {
|
||||
} else {
|
||||
for _, item := range items {
|
||||
if item.IsDir() {
|
||||
dir := &fs.FsDir{
|
||||
dir := &fs.Dir{
|
||||
Name: item.Name(),
|
||||
When: item.ModTime(),
|
||||
Bytes: 0,
|
||||
@ -159,7 +159,7 @@ func (f *FsLocal) ListDir() fs.FsDirChan {
|
||||
}
|
||||
|
||||
// Puts the FsObject to the local filesystem
|
||||
func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) {
|
||||
func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
||||
dstPath := filepath.Join(f.root, remote)
|
||||
// Temporary FsObject under construction
|
||||
fs := &FsObjectLocal{remote: remote, path: dstPath}
|
||||
@ -270,7 +270,7 @@ func (o *FsObjectLocal) Md5sum() (string, error) {
|
||||
in, err := os.Open(o.path)
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.FsLog(o, "Failed to open: %s", err)
|
||||
fs.Log(o, "Failed to open: %s", err)
|
||||
return "", err
|
||||
}
|
||||
defer in.Close() // FIXME ignoring error
|
||||
@ -278,7 +278,7 @@ func (o *FsObjectLocal) Md5sum() (string, error) {
|
||||
_, err = io.Copy(hash, in)
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.FsLog(o, "Failed to read: %s", err)
|
||||
fs.Log(o, "Failed to read: %s", err)
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%x", hash.Sum(nil)), nil
|
||||
@ -298,7 +298,7 @@ func (o *FsObjectLocal) ModTime() time.Time {
|
||||
func (o *FsObjectLocal) SetModTime(modTime time.Time) {
|
||||
err := os.Chtimes(o.path, modTime, modTime)
|
||||
if err != nil {
|
||||
fs.FsDebug(o, "Failed to set mtime on file: %s", err)
|
||||
fs.Debug(o, "Failed to set mtime on file: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -306,10 +306,10 @@ func (o *FsObjectLocal) SetModTime(modTime time.Time) {
|
||||
func (o *FsObjectLocal) Storable() bool {
|
||||
mode := o.info.Mode()
|
||||
if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
|
||||
fs.FsDebug(o, "Can't transfer non file/directory")
|
||||
fs.Debug(o, "Can't transfer non file/directory")
|
||||
return false
|
||||
} else if mode&os.ModeDir != 0 {
|
||||
fs.FsDebug(o, "FIXME Skipping directory")
|
||||
fs.Debug(o, "FIXME Skipping directory")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
@ -335,4 +335,4 @@ func (o *FsObjectLocal) Remove() error {
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var _ fs.Fs = &FsLocal{}
|
||||
var _ fs.FsObject = &FsObjectLocal{}
|
||||
var _ fs.Object = &FsObjectLocal{}
|
||||
|
48
rclone.go
48
rclone.go
@ -34,17 +34,17 @@ var (
|
||||
modifyWindow = flag.Duration("modify-window", time.Nanosecond, "Max time diff to be considered the same")
|
||||
)
|
||||
|
||||
// A pair of fs.FsObjects
|
||||
// A pair of fs.Objects
|
||||
type PairFsObjects struct {
|
||||
src, dst fs.FsObject
|
||||
src, dst fs.Object
|
||||
}
|
||||
|
||||
type PairFsObjectsChan chan PairFsObjects
|
||||
|
||||
// Check to see if src needs to be copied to dst and if so puts it in out
|
||||
func checkOne(src, dst fs.FsObject, out fs.FsObjectsChan) {
|
||||
func checkOne(src, dst fs.Object, out fs.ObjectsChan) {
|
||||
if dst == nil {
|
||||
fs.FsDebug(src, "Couldn't find local file - download")
|
||||
fs.Debug(src, "Couldn't find local file - download")
|
||||
out <- src
|
||||
return
|
||||
}
|
||||
@ -54,7 +54,7 @@ func checkOne(src, dst fs.FsObject, out fs.FsObjectsChan) {
|
||||
}
|
||||
// Check to see if changed or not
|
||||
if fs.Equal(src, dst) {
|
||||
fs.FsDebug(src, "Unchanged skipping")
|
||||
fs.Debug(src, "Unchanged skipping")
|
||||
return
|
||||
}
|
||||
out <- src
|
||||
@ -63,7 +63,7 @@ func checkOne(src, dst fs.FsObject, out fs.FsObjectsChan) {
|
||||
// Read FsObjects~s on in send to out if they need uploading
|
||||
//
|
||||
// FIXME potentially doing lots of MD5SUMS at once
|
||||
func PairChecker(in PairFsObjectsChan, out fs.FsObjectsChan, wg *sync.WaitGroup) {
|
||||
func PairChecker(in PairFsObjectsChan, out fs.ObjectsChan, wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
for pair := range in {
|
||||
src := pair.src
|
||||
@ -76,7 +76,7 @@ func PairChecker(in PairFsObjectsChan, out fs.FsObjectsChan, wg *sync.WaitGroup)
|
||||
// Read FsObjects~s on in send to out if they need uploading
|
||||
//
|
||||
// FIXME potentially doing lots of MD5SUMS at once
|
||||
func Checker(in, out fs.FsObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) {
|
||||
func Checker(in, out fs.ObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
for src := range in {
|
||||
fs.Stats.Checking(src)
|
||||
@ -87,7 +87,7 @@ func Checker(in, out fs.FsObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) {
|
||||
}
|
||||
|
||||
// Read FsObjects on in and copy them
|
||||
func Copier(in fs.FsObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) {
|
||||
func Copier(in fs.ObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
for src := range in {
|
||||
fs.Stats.Transferring(src)
|
||||
@ -105,7 +105,7 @@ func CopyFs(fdst, fsrc fs.Fs) {
|
||||
}
|
||||
|
||||
to_be_checked := fsrc.List()
|
||||
to_be_uploaded := make(fs.FsObjectsChan, *transfers)
|
||||
to_be_uploaded := make(fs.ObjectsChan, *transfers)
|
||||
|
||||
var checkerWg sync.WaitGroup
|
||||
checkerWg.Add(*checkers)
|
||||
@ -127,7 +127,7 @@ func CopyFs(fdst, fsrc fs.Fs) {
|
||||
}
|
||||
|
||||
// Delete all the files passed in the channel
|
||||
func DeleteFiles(to_be_deleted fs.FsObjectsChan) {
|
||||
func DeleteFiles(to_be_deleted fs.ObjectsChan) {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(*transfers)
|
||||
for i := 0; i < *transfers; i++ {
|
||||
@ -135,16 +135,16 @@ func DeleteFiles(to_be_deleted fs.FsObjectsChan) {
|
||||
defer wg.Done()
|
||||
for dst := range to_be_deleted {
|
||||
if *dry_run {
|
||||
fs.FsDebug(dst, "Not deleting as -dry-run")
|
||||
fs.Debug(dst, "Not deleting as -dry-run")
|
||||
} else {
|
||||
fs.Stats.Checking(dst)
|
||||
err := dst.Remove()
|
||||
fs.Stats.DoneChecking(dst)
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.FsLog(dst, "Couldn't delete: %s", err)
|
||||
fs.Log(dst, "Couldn't delete: %s", err)
|
||||
} else {
|
||||
fs.FsDebug(dst, "Deleted")
|
||||
fs.Debug(dst, "Deleted")
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -167,14 +167,14 @@ func Sync(fdst, fsrc fs.Fs) {
|
||||
|
||||
// Read the destination files first
|
||||
// FIXME could do this in parallel and make it use less memory
|
||||
delFiles := make(map[string]fs.FsObject)
|
||||
delFiles := make(map[string]fs.Object)
|
||||
for dst := range fdst.List() {
|
||||
delFiles[dst.Remote()] = dst
|
||||
}
|
||||
|
||||
// Read source files checking them off against dest files
|
||||
to_be_checked := make(PairFsObjectsChan, *transfers)
|
||||
to_be_uploaded := make(fs.FsObjectsChan, *transfers)
|
||||
to_be_uploaded := make(fs.ObjectsChan, *transfers)
|
||||
|
||||
var checkerWg sync.WaitGroup
|
||||
checkerWg.Add(*checkers)
|
||||
@ -215,7 +215,7 @@ func Sync(fdst, fsrc fs.Fs) {
|
||||
}
|
||||
|
||||
// Delete the spare files
|
||||
toDelete := make(fs.FsObjectsChan, *transfers)
|
||||
toDelete := make(fs.ObjectsChan, *transfers)
|
||||
go func() {
|
||||
for _, fs := range delFiles {
|
||||
toDelete <- fs
|
||||
@ -231,19 +231,19 @@ func Check(fdst, fsrc fs.Fs) {
|
||||
|
||||
// Read the destination files first
|
||||
// FIXME could do this in parallel and make it use less memory
|
||||
dstFiles := make(map[string]fs.FsObject)
|
||||
dstFiles := make(map[string]fs.Object)
|
||||
for dst := range fdst.List() {
|
||||
dstFiles[dst.Remote()] = dst
|
||||
}
|
||||
|
||||
// Read the source files checking them against dstFiles
|
||||
// FIXME could do this in parallel and make it use less memory
|
||||
srcFiles := make(map[string]fs.FsObject)
|
||||
commonFiles := make(map[string][]fs.FsObject)
|
||||
srcFiles := make(map[string]fs.Object)
|
||||
commonFiles := make(map[string][]fs.Object)
|
||||
for src := range fsrc.List() {
|
||||
remote := src.Remote()
|
||||
if dst, ok := dstFiles[remote]; ok {
|
||||
commonFiles[remote] = []fs.FsObject{dst, src}
|
||||
commonFiles[remote] = []fs.Object{dst, src}
|
||||
delete(dstFiles, remote)
|
||||
} else {
|
||||
srcFiles[remote] = src
|
||||
@ -262,7 +262,7 @@ func Check(fdst, fsrc fs.Fs) {
|
||||
log.Printf(remote)
|
||||
}
|
||||
|
||||
checks := make(chan []fs.FsObject, *transfers)
|
||||
checks := make(chan []fs.Object, *transfers)
|
||||
go func() {
|
||||
for _, check := range commonFiles {
|
||||
checks <- check
|
||||
@ -281,7 +281,7 @@ func Check(fdst, fsrc fs.Fs) {
|
||||
if src.Size() != dst.Size() {
|
||||
fs.Stats.DoneChecking(src)
|
||||
fs.Stats.Error()
|
||||
fs.FsLog(src, "Sizes differ")
|
||||
fs.Log(src, "Sizes differ")
|
||||
continue
|
||||
}
|
||||
same, err := fs.CheckMd5sums(src, dst)
|
||||
@ -291,9 +291,9 @@ func Check(fdst, fsrc fs.Fs) {
|
||||
}
|
||||
if !same {
|
||||
fs.Stats.Error()
|
||||
fs.FsLog(src, "Md5sums differ")
|
||||
fs.Log(src, "Md5sums differ")
|
||||
}
|
||||
fs.FsDebug(src, "OK")
|
||||
fs.Debug(src, "OK")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
36
s3/fs.go
36
s3/fs.go
@ -143,7 +143,7 @@ func NewFs(path string) (fs.Fs, error) {
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.FsObject {
|
||||
func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.Object {
|
||||
o := &FsObjectS3{
|
||||
s3: f,
|
||||
remote: remote,
|
||||
@ -153,7 +153,7 @@ func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.FsObject {
|
||||
var err error
|
||||
o.lastModified, err = time.Parse(time.RFC3339, info.LastModified)
|
||||
if err != nil {
|
||||
fs.FsLog(o, "Failed to read last modified: %s", err)
|
||||
fs.Log(o, "Failed to read last modified: %s", err)
|
||||
o.lastModified = time.Now()
|
||||
}
|
||||
o.etag = info.ETag
|
||||
@ -171,13 +171,13 @@ func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.FsObject {
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func (f *FsS3) NewFsObject(remote string) fs.FsObject {
|
||||
func (f *FsS3) NewFsObject(remote string) fs.Object {
|
||||
return f.NewFsObjectWithInfo(remote, nil)
|
||||
}
|
||||
|
||||
// Walk the path returning a channel of FsObjects
|
||||
func (f *FsS3) List() fs.FsObjectsChan {
|
||||
out := make(fs.FsObjectsChan, fs.Config.Checkers)
|
||||
func (f *FsS3) List() fs.ObjectsChan {
|
||||
out := make(fs.ObjectsChan, fs.Config.Checkers)
|
||||
go func() {
|
||||
// FIXME need to implement ALL loop
|
||||
objects, err := f.b.List("", "", "", 10000)
|
||||
@ -198,8 +198,8 @@ func (f *FsS3) List() fs.FsObjectsChan {
|
||||
}
|
||||
|
||||
// Lists the buckets
|
||||
func (f *FsS3) ListDir() fs.FsDirChan {
|
||||
out := make(fs.FsDirChan, fs.Config.Checkers)
|
||||
func (f *FsS3) ListDir() fs.DirChan {
|
||||
out := make(fs.DirChan, fs.Config.Checkers)
|
||||
go func() {
|
||||
defer close(out)
|
||||
buckets, err := f.c.ListBuckets()
|
||||
@ -208,7 +208,7 @@ func (f *FsS3) ListDir() fs.FsDirChan {
|
||||
log.Printf("Couldn't list buckets: %s", err)
|
||||
} else {
|
||||
for _, bucket := range buckets {
|
||||
out <- &fs.FsDir{
|
||||
out <- &fs.Dir{
|
||||
Name: bucket.Name,
|
||||
When: bucket.CreationDate,
|
||||
Bytes: -1,
|
||||
@ -221,7 +221,7 @@ func (f *FsS3) ListDir() fs.FsDirChan {
|
||||
}
|
||||
|
||||
// Put the FsObject into the bucket
|
||||
func (f *FsS3) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) {
|
||||
func (f *FsS3) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
||||
// Temporary FsObject under construction
|
||||
fs := &FsObjectS3{s3: f, remote: remote}
|
||||
|
||||
@ -290,19 +290,19 @@ func (o *FsObjectS3) readMetaData() (err error) {
|
||||
|
||||
headers, err := o.s3.b.Head(o.remote, nil)
|
||||
if err != nil {
|
||||
fs.FsDebug(o, "Failed to read info: %s", err)
|
||||
fs.Debug(o, "Failed to read info: %s", err)
|
||||
return err
|
||||
}
|
||||
size, err := strconv.ParseInt(headers["Content-Length"], 10, 64)
|
||||
if err != nil {
|
||||
fs.FsDebug(o, "Failed to read size from: %q", headers)
|
||||
fs.Debug(o, "Failed to read size from: %q", headers)
|
||||
return err
|
||||
}
|
||||
o.etag = headers["Etag"]
|
||||
o.bytes = size
|
||||
o.meta = headers
|
||||
if o.lastModified, err = time.Parse(http.TimeFormat, headers["Last-Modified"]); err != nil {
|
||||
fs.FsLog(o, "Failed to read last modified from HEAD: %s", err)
|
||||
fs.Log(o, "Failed to read last modified from HEAD: %s", err)
|
||||
o.lastModified = time.Now()
|
||||
}
|
||||
return nil
|
||||
@ -315,18 +315,18 @@ func (o *FsObjectS3) readMetaData() (err error) {
|
||||
func (o *FsObjectS3) ModTime() time.Time {
|
||||
err := o.readMetaData()
|
||||
if err != nil {
|
||||
fs.FsLog(o, "Failed to read metadata: %s", err)
|
||||
fs.Log(o, "Failed to read metadata: %s", err)
|
||||
return time.Now()
|
||||
}
|
||||
// read mtime out of metadata if available
|
||||
d, ok := o.meta[metaMtime]
|
||||
if !ok {
|
||||
// fs.FsDebug(o, "No metadata")
|
||||
// fs.Debug(o, "No metadata")
|
||||
return o.lastModified
|
||||
}
|
||||
modTime, err := swift.FloatStringToTime(d)
|
||||
if err != nil {
|
||||
fs.FsLog(o, "Failed to read mtime from object: %s", err)
|
||||
fs.Log(o, "Failed to read mtime from object: %s", err)
|
||||
return o.lastModified
|
||||
}
|
||||
return modTime
|
||||
@ -337,14 +337,14 @@ func (o *FsObjectS3) SetModTime(modTime time.Time) {
|
||||
err := o.readMetaData()
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.FsLog(o, "Failed to read metadata: %s", err)
|
||||
fs.Log(o, "Failed to read metadata: %s", err)
|
||||
return
|
||||
}
|
||||
o.meta[metaMtime] = swift.TimeToFloatString(modTime)
|
||||
_, err = o.s3.b.Update(o.remote, o.s3.perm, o.meta)
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.FsLog(o, "Failed to update remote mtime: %s", err)
|
||||
fs.Log(o, "Failed to update remote mtime: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -366,4 +366,4 @@ func (o *FsObjectS3) Remove() error {
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var _ fs.Fs = &FsS3{}
|
||||
var _ fs.FsObject = &FsObjectS3{}
|
||||
var _ fs.Object = &FsObjectS3{}
|
||||
|
28
swift/fs.go
28
swift/fs.go
@ -114,7 +114,7 @@ func NewFs(path string) (fs.Fs, error) {
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.FsObject {
|
||||
func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.Object {
|
||||
fs := &FsObjectSwift{
|
||||
swift: f,
|
||||
remote: remote,
|
||||
@ -135,13 +135,13 @@ func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.FsOb
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func (f *FsSwift) NewFsObject(remote string) fs.FsObject {
|
||||
func (f *FsSwift) NewFsObject(remote string) fs.Object {
|
||||
return f.NewFsObjectWithInfo(remote, nil)
|
||||
}
|
||||
|
||||
// Walk the path returning a channel of FsObjects
|
||||
func (f *FsSwift) List() fs.FsObjectsChan {
|
||||
out := make(fs.FsObjectsChan, fs.Config.Checkers)
|
||||
func (f *FsSwift) List() fs.ObjectsChan {
|
||||
out := make(fs.ObjectsChan, fs.Config.Checkers)
|
||||
go func() {
|
||||
// FIXME use a smaller limit?
|
||||
err := f.c.ObjectsWalk(f.container, nil, func(opts *swift.ObjectsOpts) (interface{}, error) {
|
||||
@ -166,8 +166,8 @@ func (f *FsSwift) List() fs.FsObjectsChan {
|
||||
}
|
||||
|
||||
// Lists the containers
|
||||
func (f *FsSwift) ListDir() fs.FsDirChan {
|
||||
out := make(fs.FsDirChan, fs.Config.Checkers)
|
||||
func (f *FsSwift) ListDir() fs.DirChan {
|
||||
out := make(fs.DirChan, fs.Config.Checkers)
|
||||
go func() {
|
||||
defer close(out)
|
||||
containers, err := f.c.ContainersAll(nil)
|
||||
@ -176,7 +176,7 @@ func (f *FsSwift) ListDir() fs.FsDirChan {
|
||||
log.Printf("Couldn't list containers: %s", err)
|
||||
} else {
|
||||
for _, container := range containers {
|
||||
out <- &fs.FsDir{
|
||||
out <- &fs.Dir{
|
||||
Name: container.Name,
|
||||
Bytes: container.Bytes,
|
||||
Count: container.Count,
|
||||
@ -192,7 +192,7 @@ func (f *FsSwift) ListDir() fs.FsDirChan {
|
||||
// Copy the reader in to the new object which is returned
|
||||
//
|
||||
// The new object may have been created
|
||||
func (f *FsSwift) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) {
|
||||
func (f *FsSwift) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
||||
// Temporary FsObject under construction
|
||||
fs := &FsObjectSwift{swift: f, remote: remote}
|
||||
|
||||
@ -246,7 +246,7 @@ func (o *FsObjectSwift) readMetaData() (err error) {
|
||||
}
|
||||
info, h, err := o.swift.c.Object(o.swift.container, o.remote)
|
||||
if err != nil {
|
||||
fs.FsDebug(o, "Failed to read info: %s", err)
|
||||
fs.Debug(o, "Failed to read info: %s", err)
|
||||
return err
|
||||
}
|
||||
meta := h.ObjectMetadata()
|
||||
@ -263,12 +263,12 @@ func (o *FsObjectSwift) readMetaData() (err error) {
|
||||
func (o *FsObjectSwift) ModTime() time.Time {
|
||||
err := o.readMetaData()
|
||||
if err != nil {
|
||||
// fs.FsLog(o, "Failed to read metadata: %s", err)
|
||||
// fs.Log(o, "Failed to read metadata: %s", err)
|
||||
return o.info.LastModified
|
||||
}
|
||||
modTime, err := o.meta.GetModTime()
|
||||
if err != nil {
|
||||
// fs.FsLog(o, "Failed to read mtime from object: %s", err)
|
||||
// fs.Log(o, "Failed to read mtime from object: %s", err)
|
||||
return o.info.LastModified
|
||||
}
|
||||
return modTime
|
||||
@ -279,14 +279,14 @@ func (o *FsObjectSwift) SetModTime(modTime time.Time) {
|
||||
err := o.readMetaData()
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.FsLog(o, "Failed to read metadata: %s", err)
|
||||
fs.Log(o, "Failed to read metadata: %s", err)
|
||||
return
|
||||
}
|
||||
o.meta.SetModTime(modTime)
|
||||
err = o.swift.c.ObjectUpdate(o.swift.container, o.remote, o.meta.ObjectHeaders())
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.FsLog(o, "Failed to update remote mtime: %s", err)
|
||||
fs.Log(o, "Failed to update remote mtime: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -308,4 +308,4 @@ func (o *FsObjectSwift) Remove() error {
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var _ fs.Fs = &FsSwift{}
|
||||
var _ fs.FsObject = &FsObjectSwift{}
|
||||
var _ fs.Object = &FsObjectSwift{}
|
||||
|
Loading…
Reference in New Issue
Block a user