2014-03-28 18:56:04 +01:00
// Generic operations on filesystems and objects
package fs
import (
2017-02-13 11:48:26 +01:00
"bytes"
2014-03-28 18:56:04 +01:00
"fmt"
2014-08-01 18:58:39 +02:00
"io"
2017-08-03 21:42:35 +02:00
"io/ioutil"
2016-03-05 17:10:51 +01:00
"log"
2015-03-01 13:38:31 +01:00
"mime"
"path"
2016-03-05 17:10:51 +01:00
"sort"
2016-01-23 21:16:47 +01:00
"strings"
2014-03-28 18:56:04 +01:00
"sync"
2015-10-02 20:48:48 +02:00
"sync/atomic"
2017-08-03 21:42:35 +02:00
"time"
2016-01-23 21:16:47 +01:00
2016-06-12 16:06:02 +02:00
"github.com/pkg/errors"
2016-10-31 15:45:52 +01:00
"github.com/spf13/pflag"
2017-09-01 17:33:09 +02:00
"golang.org/x/net/context"
2014-03-28 18:56:04 +01:00
)
2015-09-22 19:47:16 +02:00
// CalculateModifyWindow works out modify window for Fses passed in -
// sets Config.ModifyWindow
2014-03-28 18:56:04 +01:00
//
// This is the largest modify window of all the fses in use, and the
// user configured value
func CalculateModifyWindow ( fs ... Fs ) {
for _ , f := range fs {
if f != nil {
precision := f . Precision ( )
if precision > Config . ModifyWindow {
Config . ModifyWindow = precision
}
2015-08-20 21:48:58 +02:00
if precision == ModTimeNotSupported {
2017-02-09 18:08:51 +01:00
Infof ( f , "Modify window not supported" )
2015-08-20 21:48:58 +02:00
return
}
2014-03-28 18:56:04 +01:00
}
}
2017-02-09 18:08:51 +01:00
Infof ( fs [ 0 ] , "Modify window is %s" , Config . ModifyWindow )
2014-03-28 18:56:04 +01:00
}
2016-01-11 13:39:33 +01:00
// HashEquals checks to see if src == dst, but ignores empty strings
// and returns true if either is empty.
func HashEquals ( src , dst string ) bool {
2015-08-17 00:24:34 +02:00
if src == "" || dst == "" {
return true
}
return src == dst
}
2016-01-11 13:39:33 +01:00
// CheckHashes checks the two files to see if they have common
// known hash types and compares them
2014-03-28 18:56:04 +01:00
//
2016-01-24 19:06:57 +01:00
// Returns
2015-08-20 21:48:58 +02:00
//
2016-01-24 19:06:57 +01:00
// equal - which is equality of the hashes
//
// hash - the HashType. This is HashNone if either of the hashes were
// unset or a compatible hash couldn't be found.
//
// err - may return an error which will already have been logged
2014-03-28 18:56:04 +01:00
//
2015-08-20 21:48:58 +02:00
// If an error is returned it will return equal as false
2017-09-11 08:26:53 +02:00
func CheckHashes ( src ObjectInfo , dst Object ) ( equal bool , hash HashType , err error ) {
2016-01-11 13:39:33 +01:00
common := src . Fs ( ) . Hashes ( ) . Overlap ( dst . Fs ( ) . Hashes ( ) )
2017-02-09 12:01:20 +01:00
// Debugf(nil, "Shared hashes: %v", common)
2016-01-11 13:39:33 +01:00
if common . Count ( ) == 0 {
2016-01-24 19:06:57 +01:00
return true , HashNone , nil
2016-01-11 13:39:33 +01:00
}
2016-01-24 19:06:57 +01:00
hash = common . GetOne ( )
srcHash , err := src . Hash ( hash )
2014-03-28 18:56:04 +01:00
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-02-09 12:01:20 +01:00
Errorf ( src , "Failed to calculate src hash: %v" , err )
2016-01-24 19:06:57 +01:00
return false , hash , err
2015-08-20 21:48:58 +02:00
}
2016-01-11 13:39:33 +01:00
if srcHash == "" {
2016-01-24 19:06:57 +01:00
return true , HashNone , nil
2014-03-28 18:56:04 +01:00
}
2016-01-24 19:06:57 +01:00
dstHash , err := dst . Hash ( hash )
2014-03-28 18:56:04 +01:00
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-02-09 12:01:20 +01:00
Errorf ( dst , "Failed to calculate dst hash: %v" , err )
2016-01-24 19:06:57 +01:00
return false , hash , err
2015-08-20 21:48:58 +02:00
}
2016-01-11 13:39:33 +01:00
if dstHash == "" {
2016-01-24 19:06:57 +01:00
return true , HashNone , nil
2014-03-28 18:56:04 +01:00
}
2017-02-23 12:23:19 +01:00
if srcHash != dstHash {
Debugf ( src , "%v = %s (%v)" , hash , srcHash , src . Fs ( ) )
Debugf ( dst , "%v = %s (%v)" , hash , dstHash , dst . Fs ( ) )
}
2016-01-24 19:06:57 +01:00
return srcHash == dstHash , hash , nil
2014-03-28 18:56:04 +01:00
}
2015-09-22 19:47:16 +02:00
// Equal checks to see if the src and dst objects are equal by looking at
2016-01-11 13:39:33 +01:00
// size, mtime and hash
2014-03-28 18:56:04 +01:00
//
// If the src and dst size are different then it is considered to be
2015-06-06 09:38:45 +02:00
// not equal. If --size-only is in effect then this is the only check
2016-06-17 18:20:08 +02:00
// that is done. If --ignore-size is in effect then this check is
// skipped and the files are considered the same size.
2014-03-28 18:56:04 +01:00
//
// If the size is the same and the mtime is the same then it is
2015-06-06 09:38:45 +02:00
// considered to be equal. This check is skipped if using --checksum.
2014-03-28 18:56:04 +01:00
//
2015-06-06 09:38:45 +02:00
// If the size is the same and mtime is different, unreadable or
2016-01-11 13:39:33 +01:00
// --checksum is set and the hash is the same then the file is
2015-06-06 09:38:45 +02:00
// considered to be equal. In this case the mtime on the dst is
// updated if --checksum is not set.
2014-03-28 18:56:04 +01:00
//
// Otherwise the file is considered to be not equal including if there
// were errors reading info.
2017-09-11 08:26:53 +02:00
func Equal ( src ObjectInfo , dst Object ) bool {
2016-12-18 11:03:56 +01:00
return equal ( src , dst , Config . SizeOnly , Config . CheckSum )
}
2017-09-11 08:26:53 +02:00
func equal ( src ObjectInfo , dst Object , sizeOnly , checkSum bool ) bool {
2016-06-17 18:20:08 +02:00
if ! Config . IgnoreSize {
if src . Size ( ) != dst . Size ( ) {
2017-09-16 22:43:48 +02:00
Debugf ( src , "Sizes differ (src %d vs dst %d)" , src . Size ( ) , dst . Size ( ) )
2016-06-17 18:20:08 +02:00
return false
}
2014-03-28 18:56:04 +01:00
}
2016-12-18 11:03:56 +01:00
if sizeOnly {
2017-02-09 12:01:20 +01:00
Debugf ( src , "Sizes identical" )
2015-06-06 09:38:45 +02:00
return true
}
2014-03-28 18:56:04 +01:00
2016-11-28 18:08:15 +01:00
// Assert: Size is equal or being ignored
// If checking checksum and not modtime
2016-12-18 11:03:56 +01:00
if checkSum {
2016-11-28 18:08:15 +01:00
// Check the hash
same , hash , _ := CheckHashes ( src , dst )
if ! same {
2017-02-09 12:01:20 +01:00
Debugf ( src , "%v differ" , hash )
2016-11-28 18:08:15 +01:00
return false
2015-08-20 21:48:58 +02:00
}
2016-11-28 18:08:15 +01:00
if hash == HashNone {
2017-02-09 12:01:20 +01:00
Debugf ( src , "Size of src and dst objects identical" )
2015-06-03 16:08:27 +02:00
} else {
2017-02-09 12:01:20 +01:00
Debugf ( src , "Size and %v of src and dst objects identical" , hash )
2015-06-03 16:08:27 +02:00
}
2016-11-28 18:08:15 +01:00
return true
}
// Sizes the same so check the mtime
if Config . ModifyWindow == ModTimeNotSupported {
2017-02-09 12:01:20 +01:00
Debugf ( src , "Sizes identical" )
2016-11-28 18:08:15 +01:00
return true
}
srcModTime := src . ModTime ( )
dstModTime := dst . ModTime ( )
dt := dstModTime . Sub ( srcModTime )
ModifyWindow := Config . ModifyWindow
if dt < ModifyWindow && dt > - ModifyWindow {
2017-02-09 12:01:20 +01:00
Debugf ( src , "Size and modification time the same (differ by %s, within tolerance %s)" , dt , ModifyWindow )
2016-11-28 18:08:15 +01:00
return true
2014-03-28 18:56:04 +01:00
}
2017-02-09 12:01:20 +01:00
Debugf ( src , "Modification times differ by %s: %v, %v" , dt , srcModTime , dstModTime )
2016-11-28 18:08:15 +01:00
// Check if the hashes are the same
2016-01-24 19:06:57 +01:00
same , hash , _ := CheckHashes ( src , dst )
2014-03-28 18:56:04 +01:00
if ! same {
2017-02-09 12:01:20 +01:00
Debugf ( src , "%v differ" , hash )
2016-11-28 18:08:15 +01:00
return false
}
if hash == HashNone {
// if couldn't check hash, return that they differ
2014-03-28 18:56:04 +01:00
return false
}
2016-11-28 18:08:15 +01:00
// mod time differs but hash is the same to reset mod time if required
if ! Config . NoUpdateModTime {
2017-02-16 00:09:44 +01:00
if Config . DryRun {
Logf ( src , "Not updating modification time as --dry-run" )
2016-11-28 18:08:15 +01:00
} else {
2017-09-02 10:29:01 +02:00
// Size and hash the same but mtime different
// Error if objects are treated as immutable
if Config . Immutable {
Errorf ( dst , "Timestamp mismatch between immutable objects" )
return false
}
// Update the mtime of the dst object here
2017-02-16 00:09:44 +01:00
err := dst . SetModTime ( srcModTime )
if err == ErrorCantSetModTime {
2017-06-13 14:58:39 +02:00
Debugf ( dst , "src and dst identical but can't set mod time without re-uploading" )
return false
} else if err == ErrorCantSetModTimeWithoutDelete {
Debugf ( dst , "src and dst identical but can't set mod time without deleting and re-uploading" )
err = dst . Remove ( )
if err != nil {
Errorf ( dst , "failed to delete before re-upload: %v" , err )
}
2017-02-16 00:09:44 +01:00
return false
} else if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-02-16 00:09:44 +01:00
Errorf ( dst , "Failed to set modification time: %v" , err )
} else {
Infof ( src , "Updated modification time in destination" )
}
2016-03-22 16:07:10 +01:00
}
2015-06-03 16:08:27 +02:00
}
2014-03-28 18:56:04 +01:00
return true
}
2016-09-21 23:13:24 +02:00
// MimeTypeFromName returns a guess at the mime type from the name
func MimeTypeFromName ( remote string ) ( mimeType string ) {
mimeType = mime . TypeByExtension ( path . Ext ( remote ) )
2016-04-07 15:32:01 +02:00
if ! strings . ContainsRune ( mimeType , '/' ) {
2015-03-01 13:38:31 +01:00
mimeType = "application/octet-stream"
}
return mimeType
}
2016-09-21 23:13:24 +02:00
// MimeType returns the MimeType from the object, either by calling
// the MimeTyper interface or using MimeTypeFromName
func MimeType ( o ObjectInfo ) ( mimeType string ) {
// Read the MimeType from the optional interface if available
if do , ok := o . ( MimeTyper ) ; ok {
mimeType = do . MimeType ( )
2017-03-04 11:10:55 +01:00
// Debugf(o, "Read MimeType as %q", mimeType)
2016-09-21 23:13:24 +02:00
if mimeType != "" {
return mimeType
}
}
return MimeTypeFromName ( o . Remote ( ) )
}
2014-07-15 20:27:05 +02:00
// Used to remove a failed copy
2015-03-14 18:54:41 +01:00
//
// Returns whether the file was succesfully removed or not
func removeFailedCopy ( dst Object ) bool {
if dst == nil {
return false
}
2017-02-09 18:08:51 +01:00
Infof ( dst , "Removing failed copy" )
2015-03-14 18:54:41 +01:00
removeErr := dst . Remove ( )
if removeErr != nil {
2017-02-09 18:08:51 +01:00
Infof ( dst , "Failed to remove failed copy: %s" , removeErr )
2015-03-14 18:54:41 +01:00
return false
2014-07-15 20:27:05 +02:00
}
2015-03-14 18:54:41 +01:00
return true
2014-07-15 20:27:05 +02:00
}
2016-10-23 18:34:17 +02:00
// Wrapper to override the remote for an object
type overrideRemoteObject struct {
Object
remote string
}
// Remote returns the overriden remote name
func ( o * overrideRemoteObject ) Remote ( ) string {
return o . remote
}
2017-03-04 11:10:55 +01:00
// MimeType returns the mime type of the underlying object or "" if it
// can't be worked out
func ( o * overrideRemoteObject ) MimeType ( ) string {
if do , ok := o . Object . ( MimeTyper ) ; ok {
return do . MimeType ( )
}
return ""
}
// Check interface is satisfied
var _ MimeTyper = ( * overrideRemoteObject ) ( nil )
2016-10-22 18:53:10 +02:00
// Copy src object to dst or f if nil. If dst is nil then it uses
// remote as the name of the new object.
func Copy ( f Fs , dst Object , remote string , src Object ) ( err error ) {
if Config . DryRun {
2017-02-09 12:01:20 +01:00
Logf ( src , "Not copying as --dry-run" )
2016-10-22 18:53:10 +02:00
return nil
}
2016-01-12 18:38:28 +01:00
maxTries := Config . LowLevelRetries
2015-02-02 18:29:08 +01:00
tries := 0
doUpdate := dst != nil
2017-05-28 13:44:22 +02:00
// work out which hash to use - limit to 1 hash in common
var common HashSet
hashType := HashNone
if ! Config . SizeOnly {
common = src . Fs ( ) . Hashes ( ) . Overlap ( f . Hashes ( ) )
if common . Count ( ) > 0 {
hashType = common . GetOne ( )
common = HashSet ( hashType )
}
}
hashOption := & HashesOption { Hashes : common }
2016-06-18 11:55:58 +02:00
var actionTaken string
for {
// Try server side copy first - if has optional interface and
// is same underlying remote
actionTaken = "Copied (server side copy)"
2017-01-13 18:21:47 +01:00
if doCopy := f . Features ( ) . Copy ; doCopy != nil && SameConfig ( src . Fs ( ) , f ) {
2016-06-18 11:55:58 +02:00
var newDst Object
2017-01-13 18:21:47 +01:00
newDst , err = doCopy ( src , remote )
2016-06-18 11:55:58 +02:00
if err == nil {
dst = newDst
}
} else {
err = ErrorCantCopy
2015-10-06 16:35:22 +02:00
}
2016-06-18 11:55:58 +02:00
// If can't server side copy, do it manually
if err == ErrorCantCopy {
var in0 io . ReadCloser
2017-05-28 13:44:22 +02:00
in0 , err = src . Open ( hashOption )
2016-06-18 11:55:58 +02:00
if err != nil {
err = errors . Wrap ( err , "failed to open source object" )
} else {
2017-02-17 10:15:24 +01:00
in := NewAccount ( in0 , src ) . WithBuffer ( ) // account and buffer the transfer
2017-03-04 11:10:55 +01:00
var wrappedSrc ObjectInfo = src
// We try to pass the original object if possible
if src . Remote ( ) != remote {
wrappedSrc = & overrideRemoteObject { Object : src , remote : remote }
}
2016-06-18 11:55:58 +02:00
if doUpdate {
2016-07-09 11:11:57 +02:00
actionTaken = "Copied (replaced existing)"
2017-05-28 13:44:22 +02:00
err = dst . Update ( in , wrappedSrc , hashOption )
2016-06-18 11:55:58 +02:00
} else {
actionTaken = "Copied (new)"
2017-05-28 13:44:22 +02:00
dst , err = f . Put ( in , wrappedSrc , hashOption )
2016-06-18 11:55:58 +02:00
}
closeErr := in . Close ( )
if err == nil {
err = closeErr
}
}
2015-02-14 19:48:08 +01:00
}
2015-02-02 18:29:08 +01:00
tries ++
2016-06-18 11:55:58 +02:00
if tries >= maxTries {
break
2015-03-14 18:54:41 +01:00
}
2016-06-18 11:55:58 +02:00
// Retry if err returned a retry error
if IsRetryError ( err ) || ShouldRetry ( err ) {
2017-02-09 12:01:20 +01:00
Debugf ( src , "Received error: %v - low level retry %d/%d" , err , tries , maxTries )
2016-06-18 11:55:58 +02:00
continue
}
// otherwise finish
break
2014-03-28 18:56:04 +01:00
}
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-02-09 12:01:20 +01:00
Errorf ( src , "Failed to copy: %v" , err )
2016-07-04 14:12:33 +02:00
return err
2014-03-28 18:56:04 +01:00
}
2014-07-15 20:27:05 +02:00
2014-07-19 13:38:58 +02:00
// Verify sizes are the same after transfer
2016-06-17 18:20:08 +02:00
if ! Config . IgnoreSize && src . Size ( ) != dst . Size ( ) {
2016-06-12 16:06:02 +02:00
err = errors . Errorf ( "corrupted on transfer: sizes differ %d vs %d" , src . Size ( ) , dst . Size ( ) )
2017-02-09 12:01:20 +01:00
Errorf ( dst , "%v" , err )
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2014-07-19 13:38:58 +02:00
removeFailedCopy ( dst )
2016-07-04 14:12:33 +02:00
return err
2014-07-19 13:38:58 +02:00
}
2016-01-11 13:39:33 +01:00
// Verify hashes are the same after transfer - ignoring blank hashes
// TODO(klauspost): This could be extended, so we always create a hash type matching
// the destination, and calculate it while sending.
2017-05-28 13:44:22 +02:00
if hashType != HashNone {
2016-07-04 14:12:33 +02:00
var srcSum string
srcSum , err = src . Hash ( hashType )
2016-01-11 13:39:33 +01:00
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-02-09 12:01:20 +01:00
Errorf ( src , "Failed to read src hash: %v" , err )
2016-01-11 13:39:33 +01:00
} else if srcSum != "" {
2016-07-04 14:12:33 +02:00
var dstSum string
dstSum , err = dst . Hash ( hashType )
2016-01-11 13:39:33 +01:00
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-02-09 12:01:20 +01:00
Errorf ( dst , "Failed to read hash: %v" , err )
2017-02-18 10:13:53 +01:00
} else if ! Config . IgnoreChecksum && ! HashEquals ( srcSum , dstSum ) {
2016-06-12 16:06:02 +02:00
err = errors . Errorf ( "corrupted on transfer: %v hash differ %q vs %q" , hashType , srcSum , dstSum )
2017-02-09 12:01:20 +01:00
Errorf ( dst , "%v" , err )
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2015-06-09 14:18:40 +02:00
removeFailedCopy ( dst )
2016-07-04 14:12:33 +02:00
return err
2015-06-09 14:18:40 +02:00
}
2014-07-15 20:27:05 +02:00
}
}
2017-02-09 18:08:51 +01:00
Infof ( src , actionTaken )
2016-07-04 14:12:33 +02:00
return err
2015-08-24 22:42:23 +02:00
}
2016-10-22 18:53:10 +02:00
// Move src object to dst or fdst if nil. If dst is nil then it uses
// remote as the name of the new object.
func Move ( fdst Fs , dst Object , remote string , src Object ) ( err error ) {
if Config . DryRun {
2017-02-09 12:01:20 +01:00
Logf ( src , "Not moving as --dry-run" )
2016-10-22 18:53:10 +02:00
return nil
}
// See if we have Move available
2017-01-13 18:21:47 +01:00
if doMove := fdst . Features ( ) . Move ; doMove != nil && SameConfig ( src . Fs ( ) , fdst ) {
2016-10-22 18:53:10 +02:00
// Delete destination if it exists
if dst != nil {
err = DeleteFile ( dst )
if err != nil {
return err
}
}
// Move dst <- src
2017-01-13 18:21:47 +01:00
_ , err := doMove ( src , remote )
2016-10-22 18:53:10 +02:00
switch err {
case nil :
2017-02-09 18:08:51 +01:00
Infof ( src , "Moved (server side)" )
2016-10-22 18:53:10 +02:00
return nil
case ErrorCantMove :
2017-02-09 12:01:20 +01:00
Debugf ( src , "Can't move, switching to copy" )
2016-10-22 18:53:10 +02:00
default :
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-03-14 16:35:10 +01:00
Errorf ( src , "Couldn't move: %v" , err )
2016-10-22 18:53:10 +02:00
return err
}
}
// Move not found or didn't work so copy dst <- src
err = Copy ( fdst , dst , remote , src )
if err != nil {
2017-02-09 12:01:20 +01:00
Errorf ( src , "Not deleting source as copy failed: %v" , err )
2016-10-22 18:53:10 +02:00
return err
}
// Delete src if no error on copy
return DeleteFile ( src )
}
2017-01-10 21:03:55 +01:00
// CanServerSideMove returns true if fdst support server side moves or
// server side copies
//
// Some remotes simulate rename by server-side copy and delete, so include
// remotes that implements either Mover or Copier.
func CanServerSideMove ( fdst Fs ) bool {
2017-01-13 18:21:47 +01:00
canMove := fdst . Features ( ) . Move != nil
canCopy := fdst . Features ( ) . Copy != nil
2017-01-10 21:03:55 +01:00
return canMove || canCopy
}
2017-01-10 22:47:03 +01:00
// deleteFileWithBackupDir deletes a single file respecting --dry-run
// and accumulating stats and errors.
//
// If backupDir is set then it moves the file to there instead of
// deleting
func deleteFileWithBackupDir ( dst Object , backupDir Fs ) ( err error ) {
Stats . Checking ( dst . Remote ( ) )
action , actioned , actioning := "delete" , "Deleted" , "deleting"
if backupDir != nil {
action , actioned , actioning = "move into backup dir" , "Moved into backup dir" , "moving into backup dir"
}
2016-03-05 17:10:51 +01:00
if Config . DryRun {
2017-02-09 12:01:20 +01:00
Logf ( dst , "Not %s as --dry-run" , actioning )
2017-01-10 22:47:03 +01:00
} else if backupDir != nil {
if ! SameConfig ( dst . Fs ( ) , backupDir ) {
err = errors . New ( "parameter to --backup-dir has to be on the same remote as destination" )
2016-03-05 17:10:51 +01:00
} else {
2017-01-19 18:26:29 +01:00
remoteWithSuffix := dst . Remote ( ) + Config . Suffix
overwritten , _ := backupDir . NewObject ( remoteWithSuffix )
err = Move ( backupDir , overwritten , remoteWithSuffix , dst )
2016-03-05 17:10:51 +01:00
}
2017-01-10 22:47:03 +01:00
} else {
err = dst . Remove ( )
}
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-02-09 12:01:20 +01:00
Errorf ( dst , "Couldn't %s: %v" , action , err )
2017-04-10 17:07:22 +02:00
} else if ! Config . DryRun {
2017-02-09 18:08:51 +01:00
Infof ( dst , actioned )
2016-03-05 17:10:51 +01:00
}
2017-01-10 22:47:03 +01:00
Stats . DoneChecking ( dst . Remote ( ) )
2016-06-25 15:27:44 +02:00
return err
2016-03-05 17:10:51 +01:00
}
2017-01-10 22:47:03 +01:00
// DeleteFile deletes a single file respecting --dry-run and accumulating stats and errors.
//
// If useBackupDir is set and --backup-dir is in effect then it moves
// the file to there instead of deleting
func DeleteFile ( dst Object ) ( err error ) {
return deleteFileWithBackupDir ( dst , nil )
}
// deleteFilesWithBackupDir removes all the files passed in the
// channel
//
// If backupDir is set the files will be placed into that directory
// instead of being deleted.
func deleteFilesWithBackupDir ( toBeDeleted ObjectsChan , backupDir Fs ) error {
2014-03-28 18:56:04 +01:00
var wg sync . WaitGroup
wg . Add ( Config . Transfers )
2016-06-25 15:27:44 +02:00
var errorCount int32
2014-03-28 18:56:04 +01:00
for i := 0 ; i < Config . Transfers ; i ++ {
go func ( ) {
defer wg . Done ( )
2015-09-22 19:47:16 +02:00
for dst := range toBeDeleted {
2017-01-10 22:47:03 +01:00
err := deleteFileWithBackupDir ( dst , backupDir )
2016-06-25 15:27:44 +02:00
if err != nil {
atomic . AddInt32 ( & errorCount , 1 )
}
2014-03-28 18:56:04 +01:00
}
} ( )
}
2017-02-09 18:08:51 +01:00
Infof ( nil , "Waiting for deletions to finish" )
2014-03-28 18:56:04 +01:00
wg . Wait ( )
2016-06-25 15:27:44 +02:00
if errorCount > 0 {
return errors . Errorf ( "failed to delete %d files" , errorCount )
}
return nil
2014-03-28 18:56:04 +01:00
}
2017-01-10 22:47:03 +01:00
// DeleteFiles removes all the files passed in the channel
func DeleteFiles ( toBeDeleted ObjectsChan ) error {
return deleteFilesWithBackupDir ( toBeDeleted , nil )
}
2016-06-25 15:28:26 +02:00
// Read a Objects into add() for the given Fs.
2016-04-23 22:46:52 +02:00
// dir is the start directory, "" for root
2016-01-12 14:33:03 +01:00
// If includeAll is specified all files will be added,
// otherwise only files passing the filter will be added.
2016-07-04 14:12:33 +02:00
//
// Each object is passed ito the function provided. If that returns
// an error then the listing will be aborted and that error returned.
func readFilesFn ( fs Fs , includeAll bool , dir string , add func ( Object ) error ) ( err error ) {
2017-02-24 23:51:01 +01:00
return Walk ( fs , "" , includeAll , Config . MaxDepth , func ( dirPath string , entries DirEntries , err error ) error {
2016-04-21 21:06:21 +02:00
if err != nil {
2016-06-25 15:28:26 +02:00
return err
2016-04-21 21:06:21 +02:00
}
2017-02-24 23:51:01 +01:00
return entries . ForObjectError ( add )
} )
2017-01-24 12:00:05 +01:00
}
// DirEntries is a slice of Object or *Dir
2017-06-30 11:54:14 +02:00
type DirEntries [ ] DirEntry
2017-01-24 12:00:05 +01:00
// Len is part of sort.Interface.
func ( ds DirEntries ) Len ( ) int {
return len ( ds )
}
// Swap is part of sort.Interface.
func ( ds DirEntries ) Swap ( i , j int ) {
ds [ i ] , ds [ j ] = ds [ j ] , ds [ i ]
}
// Less is part of sort.Interface.
func ( ds DirEntries ) Less ( i , j int ) bool {
return ds [ i ] . Remote ( ) < ds [ j ] . Remote ( )
}
2017-02-24 23:51:01 +01:00
// ForObject runs the function supplied on every object in the entries
func ( ds DirEntries ) ForObject ( fn func ( o Object ) ) {
for _ , entry := range ds {
o , ok := entry . ( Object )
if ok {
fn ( o )
}
}
}
// ForObjectError runs the function supplied on every object in the entries
func ( ds DirEntries ) ForObjectError ( fn func ( o Object ) error ) error {
for _ , entry := range ds {
o , ok := entry . ( Object )
if ok {
err := fn ( o )
if err != nil {
return err
}
}
}
return nil
}
2017-06-30 14:37:29 +02:00
// ForDir runs the function supplied on every Directory in the entries
func ( ds DirEntries ) ForDir ( fn func ( dir Directory ) ) {
2017-02-24 23:51:01 +01:00
for _ , entry := range ds {
2017-06-30 14:37:29 +02:00
dir , ok := entry . ( Directory )
2017-02-24 23:51:01 +01:00
if ok {
fn ( dir )
}
}
}
2017-06-30 14:37:29 +02:00
// ForDirError runs the function supplied on every Directory in the entries
func ( ds DirEntries ) ForDirError ( fn func ( dir Directory ) error ) error {
2017-02-24 23:51:01 +01:00
for _ , entry := range ds {
2017-06-30 14:37:29 +02:00
dir , ok := entry . ( Directory )
2017-02-24 23:51:01 +01:00
if ok {
err := fn ( dir )
if err != nil {
return err
}
}
}
return nil
}
2017-07-08 16:42:18 +02:00
// DirEntryType returns a string description of the DirEntry, either
// "object", "directory" or "unknown type XXX"
func DirEntryType ( d DirEntry ) string {
switch d . ( type ) {
case Object :
return "object"
case Directory :
return "directory"
}
return fmt . Sprintf ( "unknown type %T" , d )
}
2017-01-24 12:00:05 +01:00
// ListDirSorted reads Object and *Dir into entries for the given Fs.
//
// dir is the start directory, "" for root
//
// If includeAll is specified all files will be added, otherwise only
2017-06-11 23:43:31 +02:00
// files and directories passing the filter will be added.
2017-01-24 12:00:05 +01:00
//
// Files will be returned in sorted order
func ListDirSorted ( fs Fs , includeAll bool , dir string ) ( entries DirEntries , err error ) {
2017-06-11 23:43:31 +02:00
// Get unfiltered entries from the fs
entries , err = fs . List ( dir )
if err != nil {
return nil , err
2017-01-24 12:00:05 +01:00
}
2017-11-09 10:28:36 +01:00
// This should happen only if exclude files lives in the
// starting directory, otherwise ListDirSorted should not be
// called.
2017-11-16 15:38:25 +01:00
if ! includeAll && Config . Filter . ListContainsExcludeFile ( entries ) {
2017-11-09 10:28:36 +01:00
Debugf ( dir , "Excluded from sync (and deletion)" )
2017-11-16 15:38:25 +01:00
return nil , nil
}
2017-11-09 10:28:36 +01:00
return filterAndSortDir ( entries , includeAll , dir , Config . Filter . IncludeObject , Config . Filter . IncludeDirectory ( fs ) )
2017-07-19 10:36:27 +02:00
}
2017-06-11 23:43:31 +02:00
2017-07-19 10:36:27 +02:00
// filter (if required) and check the entries, then sort them
func filterAndSortDir ( entries DirEntries , includeAll bool , dir string ,
IncludeObject func ( o Object ) bool ,
2017-11-09 10:28:36 +01:00
IncludeDirectory func ( remote string ) ( bool , error ) ) ( newEntries DirEntries , err error ) {
2017-07-19 10:36:27 +02:00
newEntries = entries [ : 0 ] // in place filter
2017-07-19 00:39:42 +02:00
prefix := ""
if dir != "" {
prefix = dir + "/"
}
for _ , entry := range entries {
ok := true
2017-07-19 10:36:27 +02:00
// check includes and types
switch x := entry . ( type ) {
case Object :
// Make sure we don't delete excluded files if not required
if ! includeAll && ! IncludeObject ( x ) {
ok = false
Debugf ( x , "Excluded from sync (and deletion)" )
}
case Directory :
2017-11-09 10:28:36 +01:00
if ! includeAll {
include , err := IncludeDirectory ( x . Remote ( ) )
if err != nil {
return nil , err
}
if ! include {
ok = false
Debugf ( x , "Excluded from sync (and deletion)" )
}
2017-01-24 12:00:05 +01:00
}
2017-07-19 10:36:27 +02:00
default :
return nil , errors . Errorf ( "unknown object type %T" , entry )
2017-01-24 12:00:05 +01:00
}
2017-07-19 10:36:27 +02:00
// check remote name belongs in this directry
2017-07-19 00:39:42 +02:00
remote := entry . Remote ( )
2017-07-19 10:36:27 +02:00
switch {
case ! ok :
// ignore
case ! strings . HasPrefix ( remote , prefix ) :
ok = false
Errorf ( entry , "Entry doesn't belong in directory %q (too short) - ignoring" , dir )
case remote == prefix :
2017-07-19 00:39:42 +02:00
ok = false
2017-07-19 10:36:27 +02:00
Errorf ( entry , "Entry doesn't belong in directory %q (same as directory) - ignoring" , dir )
case strings . ContainsRune ( remote [ len ( prefix ) : ] , '/' ) :
ok = false
Errorf ( entry , "Entry doesn't belong in directory %q (contains subdir) - ignoring" , dir )
default :
// ok
2017-07-19 00:39:42 +02:00
}
if ok {
newEntries = append ( newEntries , entry )
}
2017-01-24 12:00:05 +01:00
}
2017-07-19 00:39:42 +02:00
entries = newEntries
2017-06-11 23:43:31 +02:00
2017-07-06 15:07:26 +02:00
// Sort the directory entries by Remote
//
// We use a stable sort here just in case there are
// duplicates. Assuming the remote delivers the entries in a
// consistent order, this will give the best user experience
// in syncing as it will use the first entry for the sync
// comparison.
sort . Stable ( entries )
2017-01-24 12:00:05 +01:00
return entries , nil
2016-06-25 15:28:26 +02:00
}
2017-01-11 15:59:53 +01:00
// SameConfig returns true if fdst and fsrc are using the same config
// file entry
func SameConfig ( fdst , fsrc Info ) bool {
return fdst . Name ( ) == fsrc . Name ( )
}
2015-09-22 19:47:16 +02:00
// Same returns true if fdst and fsrc point to the same underlying Fs
2017-01-11 15:59:53 +01:00
func Same ( fdst , fsrc Info ) bool {
return SameConfig ( fdst , fsrc ) && fdst . Root ( ) == fsrc . Root ( )
2015-09-01 21:50:28 +02:00
}
2016-07-11 12:36:46 +02:00
// Overlapping returns true if fdst and fsrc point to the same
2017-01-11 15:59:53 +01:00
// underlying Fs and they overlap.
func Overlapping ( fdst , fsrc Info ) bool {
if ! SameConfig ( fdst , fsrc ) {
return false
}
// Return the Root with a trailing / if not empty
fixedRoot := func ( f Info ) string {
s := strings . Trim ( f . Root ( ) , "/" )
if s != "" {
s += "/"
}
return s
}
fdstRoot := fixedRoot ( fdst )
fsrcRoot := fixedRoot ( fsrc )
return strings . HasPrefix ( fdstRoot , fsrcRoot ) || strings . HasPrefix ( fsrcRoot , fdstRoot )
2016-07-11 12:36:46 +02:00
}
2016-04-07 15:56:27 +02:00
// checkIdentical checks to see if dst and src are identical
//
// it returns true if differences were found
2016-10-12 11:59:55 +02:00
// it also returns whether it couldn't be hashed
2016-11-05 19:17:21 +01:00
func checkIdentical ( dst , src Object ) ( differ bool , noHash bool ) {
2017-02-13 11:48:26 +01:00
same , hash , err := CheckHashes ( src , dst )
if err != nil {
// CheckHashes will log and count errors
2016-10-12 11:59:55 +02:00
return true , false
2016-04-07 15:56:27 +02:00
}
2017-02-13 11:48:26 +01:00
if hash == HashNone {
return false , true
}
if ! same {
2017-11-15 06:32:00 +01:00
err = errors . Errorf ( "%v differ" , hash )
Errorf ( src , "%v" , err )
Stats . Error ( err )
2017-02-13 11:48:26 +01:00
return true , false
2016-04-07 15:56:27 +02:00
}
2016-10-12 11:59:55 +02:00
return false , false
2016-04-07 15:56:27 +02:00
}
2017-09-01 17:33:09 +02:00
// checkFn is the the type of the checking function used in CheckFn()
type checkFn func ( a , b Object ) ( differ bool , noHash bool )
2015-11-24 17:54:12 +01:00
2017-09-01 17:33:09 +02:00
// checkMarch is used to march over two Fses in the same way as
// sync/copy
type checkMarch struct {
fdst , fsrc Fs
check checkFn
differences int32
noHashes int32
srcFilesMissing int32
dstFilesMissing int32
}
2015-03-14 18:11:24 +01:00
2017-09-01 17:33:09 +02:00
// DstOnly have an object which is in the destination only
func ( c * checkMarch ) DstOnly ( dst DirEntry ) ( recurse bool ) {
switch dst . ( type ) {
case Object :
2017-11-15 06:32:00 +01:00
err := errors . Errorf ( "File not in %v" , c . fsrc )
Errorf ( dst , "%v" , err )
Stats . Error ( err )
2017-09-01 17:33:09 +02:00
atomic . AddInt32 ( & c . differences , 1 )
atomic . AddInt32 ( & c . srcFilesMissing , 1 )
case Directory :
// Do the same thing to the entire contents of the directory
return true
default :
panic ( "Bad object in DirEntries" )
2014-03-28 18:56:04 +01:00
}
2017-09-01 17:33:09 +02:00
return false
}
2014-03-28 18:56:04 +01:00
2017-09-01 17:33:09 +02:00
// SrcOnly have an object which is in the source only
func ( c * checkMarch ) SrcOnly ( src DirEntry ) ( recurse bool ) {
switch src . ( type ) {
case Object :
2017-11-15 06:32:00 +01:00
err := errors . Errorf ( "File not in %v" , c . fdst )
Errorf ( src , "%v" , err )
Stats . Error ( err )
2017-09-01 17:33:09 +02:00
atomic . AddInt32 ( & c . differences , 1 )
atomic . AddInt32 ( & c . dstFilesMissing , 1 )
case Directory :
// Do the same thing to the entire contents of the directory
return true
default :
panic ( "Bad object in DirEntries" )
2014-03-28 18:56:04 +01:00
}
2017-09-01 17:33:09 +02:00
return false
}
2014-03-28 18:56:04 +01:00
2017-09-01 17:33:09 +02:00
// check to see if two objects are identical using the check function
func ( c * checkMarch ) checkIdentical ( dst , src Object ) ( differ bool , noHash bool ) {
Stats . Checking ( src . Remote ( ) )
defer Stats . DoneChecking ( src . Remote ( ) )
if ! Config . IgnoreSize && src . Size ( ) != dst . Size ( ) {
2017-11-15 06:32:00 +01:00
err := errors . Errorf ( "Sizes differ" )
Errorf ( src , "%v" , err )
Stats . Error ( err )
2017-09-01 17:33:09 +02:00
return true , false
2014-03-28 18:56:04 +01:00
}
2017-09-01 17:33:09 +02:00
if Config . SizeOnly {
return false , false
}
return c . check ( dst , src )
}
2014-03-28 18:56:04 +01:00
2017-09-01 17:33:09 +02:00
// Match is called when src and dst are present, so sync src to dst
func ( c * checkMarch ) Match ( dst , src DirEntry ) ( recurse bool ) {
switch srcX := src . ( type ) {
case Object :
dstX , ok := dst . ( Object )
if ok {
differ , noHash := c . checkIdentical ( dstX , srcX )
if differ {
atomic . AddInt32 ( & c . differences , 1 )
} else {
Debugf ( dstX , "OK" )
}
if noHash {
atomic . AddInt32 ( & c . noHashes , 1 )
}
} else {
2017-11-15 06:32:00 +01:00
err := errors . Errorf ( "is file on %v but directory on %v" , c . fsrc , c . fdst )
Errorf ( src , "%v" , err )
Stats . Error ( err )
2017-09-01 17:33:09 +02:00
atomic . AddInt32 ( & c . differences , 1 )
atomic . AddInt32 ( & c . dstFilesMissing , 1 )
2017-02-13 11:48:26 +01:00
}
2017-09-01 17:33:09 +02:00
case Directory :
// Do the same thing to the entire contents of the directory
_ , ok := dst . ( Directory )
if ok {
return true
2017-02-13 11:48:26 +01:00
}
2017-11-15 06:32:00 +01:00
err := errors . Errorf ( "is file on %v but directory on %v" , c . fdst , c . fsrc )
Errorf ( dst , "%v" , err )
Stats . Error ( err )
2017-09-01 17:33:09 +02:00
atomic . AddInt32 ( & c . differences , 1 )
atomic . AddInt32 ( & c . srcFilesMissing , 1 )
default :
panic ( "Bad object in DirEntries" )
2017-02-13 11:48:26 +01:00
}
2017-09-01 17:33:09 +02:00
return false
}
2017-02-13 11:48:26 +01:00
2017-09-01 17:33:09 +02:00
// CheckFn checks the files in fsrc and fdst according to Size and
// hash using checkFunction on each file to check the hashes.
//
// checkFunction sees if dst and src are identical
//
// it returns true if differences were found
// it also returns whether it couldn't be hashed
func CheckFn ( fdst , fsrc Fs , check checkFn ) error {
c := & checkMarch {
fdst : fdst ,
fsrc : fsrc ,
check : check ,
2014-03-28 18:56:04 +01:00
}
2017-09-01 17:33:09 +02:00
// set up a march over fdst and fsrc
m := newMarch ( context . Background ( ) , fdst , fsrc , "" , c )
2017-02-09 18:08:51 +01:00
Infof ( fdst , "Waiting for checks to finish" )
2017-09-01 17:33:09 +02:00
m . run ( )
if c . dstFilesMissing > 0 {
Logf ( fdst , "%d files missing" , c . dstFilesMissing )
}
if c . srcFilesMissing > 0 {
Logf ( fsrc , "%d files missing" , c . srcFilesMissing )
}
2017-02-09 12:01:20 +01:00
Logf ( fdst , "%d differences found" , Stats . GetErrors ( ) )
2017-09-01 17:33:09 +02:00
if c . noHashes > 0 {
Logf ( fdst , "%d hashes could not be checked" , c . noHashes )
2016-10-12 11:59:55 +02:00
}
2017-09-01 17:33:09 +02:00
if c . differences > 0 {
return errors . Errorf ( "%d differences found" , c . differences )
2014-03-28 18:56:04 +01:00
}
return nil
}
2017-02-12 17:30:18 +01:00
// Check the files in fsrc and fdst according to Size and hash
func Check ( fdst , fsrc Fs ) error {
return CheckFn ( fdst , fsrc , checkIdentical )
}
2017-02-13 11:48:26 +01:00
// ReadFill reads as much data from r into buf as it can
//
// It reads until the buffer is full or r.Read returned an error.
//
// This is io.ReadFull but when you just want as much data as
// possible, not an exact size of block.
func ReadFill ( r io . Reader , buf [ ] byte ) ( n int , err error ) {
var nn int
for n < len ( buf ) && err == nil {
nn , err = r . Read ( buf [ n : ] )
n += nn
}
return n , err
}
// CheckEqualReaders checks to see if in1 and in2 have the same
// content when read.
//
// it returns true if differences were found
func CheckEqualReaders ( in1 , in2 io . Reader ) ( differ bool , err error ) {
const bufSize = 64 * 1024
buf1 := make ( [ ] byte , bufSize )
buf2 := make ( [ ] byte , bufSize )
for {
n1 , err1 := ReadFill ( in1 , buf1 )
n2 , err2 := ReadFill ( in2 , buf2 )
// check errors
if err1 != nil && err1 != io . EOF {
return true , err1
} else if err2 != nil && err2 != io . EOF {
return true , err2
}
// err1 && err2 are nil or io.EOF here
// process the data
if n1 != n2 || ! bytes . Equal ( buf1 [ : n1 ] , buf2 [ : n2 ] ) {
return true , nil
}
// if both streams finished the we have finished
if err1 == io . EOF && err2 == io . EOF {
break
}
}
return false , nil
}
// CheckIdentical checks to see if dst and src are identical by
// reading all their bytes if necessary.
//
// it returns true if differences were found
func CheckIdentical ( dst , src Object ) ( differ bool , err error ) {
in1 , err := dst . Open ( )
if err != nil {
return true , errors . Wrapf ( err , "failed to open %q" , dst )
}
2017-02-17 10:15:24 +01:00
in1 = NewAccount ( in1 , dst ) . WithBuffer ( ) // account and buffer the transfer
2017-02-13 11:48:26 +01:00
defer CheckClose ( in1 , & err )
in2 , err := src . Open ( )
if err != nil {
return true , errors . Wrapf ( err , "failed to open %q" , src )
}
2017-02-17 10:15:24 +01:00
in2 = NewAccount ( in2 , src ) . WithBuffer ( ) // account and buffer the transfer
2017-02-13 11:48:26 +01:00
defer CheckClose ( in2 , & err )
return CheckEqualReaders ( in1 , in2 )
}
// CheckDownload checks the files in fsrc and fdst according to Size
// and the actual contents of the files.
func CheckDownload ( fdst , fsrc Fs ) error {
check := func ( a , b Object ) ( differ bool , noHash bool ) {
differ , err := CheckIdentical ( a , b )
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-02-13 11:48:26 +01:00
Errorf ( a , "Failed to download: %v" , err )
return true , true
}
return differ , false
}
return CheckFn ( fdst , fsrc , check )
}
2015-09-22 19:47:16 +02:00
// ListFn lists the Fs to the supplied function
2014-03-28 18:56:04 +01:00
//
// Lists in parallel which may get them out of order
2014-07-12 13:09:20 +02:00
func ListFn ( f Fs , fn func ( Object ) ) error {
2017-02-24 23:51:01 +01:00
return Walk ( f , "" , false , Config . MaxDepth , func ( dirPath string , entries DirEntries , err error ) error {
if err != nil {
// FIXME count errors and carry on for listing
return err
}
entries . ForObject ( fn )
return nil
} )
2014-03-28 18:56:04 +01:00
}
2015-02-28 16:30:40 +01:00
// mutex for synchronized output
var outMutex sync . Mutex
// Synchronized fmt.Fprintf
2015-09-22 08:31:12 +02:00
//
// Ignores errors from Fprintf
func syncFprintf ( w io . Writer , format string , a ... interface { } ) {
2015-02-28 16:30:40 +01:00
outMutex . Lock ( )
defer outMutex . Unlock ( )
2015-09-22 08:31:12 +02:00
_ , _ = fmt . Fprintf ( w , format , a ... )
2015-02-28 16:30:40 +01:00
}
2015-09-15 16:46:06 +02:00
// List the Fs to the supplied writer
2014-07-12 13:09:20 +02:00
//
2015-11-24 17:54:12 +01:00
// Shows size and path - obeys includes and excludes
2014-07-12 13:09:20 +02:00
//
// Lists in parallel which may get them out of order
2014-08-01 18:58:39 +02:00
func List ( f Fs , w io . Writer ) error {
2014-07-12 13:09:20 +02:00
return ListFn ( f , func ( o Object ) {
2015-02-28 16:30:40 +01:00
syncFprintf ( w , "%9d %s\n" , o . Size ( ) , o . Remote ( ) )
2014-07-12 13:09:20 +02:00
} )
}
2015-09-22 19:47:16 +02:00
// ListLong lists the Fs to the supplied writer
2014-07-12 13:09:20 +02:00
//
2015-11-24 17:54:12 +01:00
// Shows size, mod time and path - obeys includes and excludes
2014-07-12 13:09:20 +02:00
//
// Lists in parallel which may get them out of order
2014-08-01 18:58:39 +02:00
func ListLong ( f Fs , w io . Writer ) error {
2014-07-12 13:09:20 +02:00
return ListFn ( f , func ( o Object ) {
2016-07-02 17:58:50 +02:00
Stats . Checking ( o . Remote ( ) )
2014-07-12 13:09:20 +02:00
modTime := o . ModTime ( )
2016-07-02 17:58:50 +02:00
Stats . DoneChecking ( o . Remote ( ) )
2015-09-22 20:04:12 +02:00
syncFprintf ( w , "%9d %s %s\n" , o . Size ( ) , modTime . Local ( ) . Format ( "2006-01-02 15:04:05.000000000" ) , o . Remote ( ) )
2014-07-12 13:09:20 +02:00
} )
}
2015-09-22 19:47:16 +02:00
// Md5sum list the Fs to the supplied writer
2014-07-12 13:09:20 +02:00
//
2015-11-24 17:54:12 +01:00
// Produces the same output as the md5sum command - obeys includes and
// excludes
2014-07-12 13:09:20 +02:00
//
// Lists in parallel which may get them out of order
2014-08-01 18:58:39 +02:00
func Md5sum ( f Fs , w io . Writer ) error {
2016-01-11 13:39:33 +01:00
return hashLister ( HashMD5 , f , w )
}
// Sha1sum list the Fs to the supplied writer
//
// Obeys includes and excludes
//
// Lists in parallel which may get them out of order
func Sha1sum ( f Fs , w io . Writer ) error {
return hashLister ( HashSHA1 , f , w )
}
2017-05-26 16:09:31 +02:00
// DropboxHashSum list the Fs to the supplied writer
//
// Obeys includes and excludes
//
// Lists in parallel which may get them out of order
func DropboxHashSum ( f Fs , w io . Writer ) error {
return hashLister ( HashDropbox , f , w )
}
2016-01-11 13:39:33 +01:00
func hashLister ( ht HashType , f Fs , w io . Writer ) error {
2014-07-12 13:09:20 +02:00
return ListFn ( f , func ( o Object ) {
2016-07-02 17:58:50 +02:00
Stats . Checking ( o . Remote ( ) )
2016-01-11 13:39:33 +01:00
sum , err := o . Hash ( ht )
2016-07-02 17:58:50 +02:00
Stats . DoneChecking ( o . Remote ( ) )
2016-01-11 13:39:33 +01:00
if err == ErrHashUnsupported {
sum = "UNSUPPORTED"
} else if err != nil {
2017-02-09 12:01:20 +01:00
Debugf ( o , "Failed to read %v: %v" , ht , err )
2016-01-11 13:39:33 +01:00
sum = "ERROR"
2014-07-12 13:09:20 +02:00
}
2016-01-17 14:56:00 +01:00
syncFprintf ( w , "%*s %s\n" , HashWidth [ ht ] , sum , o . Remote ( ) )
2014-07-12 13:09:20 +02:00
} )
}
2015-10-02 20:48:48 +02:00
// Count counts the objects and their sizes in the Fs
2015-11-24 17:54:12 +01:00
//
// Obeys includes and excludes
2015-10-02 20:48:48 +02:00
func Count ( f Fs ) ( objects int64 , size int64 , err error ) {
err = ListFn ( f , func ( o Object ) {
atomic . AddInt64 ( & objects , 1 )
atomic . AddInt64 ( & size , o . Size ( ) )
} )
return
}
2017-02-24 23:51:01 +01:00
// ConfigMaxDepth returns the depth to use for a recursive or non recursive listing.
func ConfigMaxDepth ( recursive bool ) int {
depth := Config . MaxDepth
if ! recursive && depth < 0 {
depth = 1
}
return depth
}
2015-09-22 19:47:16 +02:00
// ListDir lists the directories/buckets/containers in the Fs to the supplied writer
2014-08-01 18:58:39 +02:00
func ListDir ( f Fs , w io . Writer ) error {
2017-02-24 23:51:01 +01:00
return Walk ( f , "" , false , ConfigMaxDepth ( false ) , func ( dirPath string , entries DirEntries , err error ) error {
2016-04-21 21:06:21 +02:00
if err != nil {
2017-02-24 23:51:01 +01:00
// FIXME count errors and carry on for listing
return err
2016-04-21 21:06:21 +02:00
}
2017-06-30 14:37:29 +02:00
entries . ForDir ( func ( dir Directory ) {
2017-02-24 23:51:01 +01:00
if dir != nil {
2017-06-30 14:37:29 +02:00
syncFprintf ( w , "%12d %13s %9d %s\n" , dir . Size ( ) , dir . ModTime ( ) . Format ( "2006-01-02 15:04:05" ) , dir . Items ( ) , dir . Remote ( ) )
2017-02-24 23:51:01 +01:00
}
} )
return nil
} )
2014-03-28 18:56:04 +01:00
}
2017-06-27 23:18:54 +02:00
// logDirName returns an object for the logger
func logDirName ( f Fs , dir string ) interface { } {
if dir != "" {
return dir
}
return f
}
2015-09-22 19:47:16 +02:00
// Mkdir makes a destination directory or container
2016-11-25 22:52:43 +01:00
func Mkdir ( f Fs , dir string ) error {
2016-02-28 20:47:22 +01:00
if Config . DryRun {
2017-06-27 23:18:54 +02:00
Logf ( logDirName ( f , dir ) , "Not making directory as dry run is set" )
2016-02-28 20:47:22 +01:00
return nil
}
2017-06-27 23:18:54 +02:00
Debugf ( logDirName ( f , dir ) , "Making directory" )
2016-11-25 22:52:43 +01:00
err := f . Mkdir ( dir )
2014-03-28 18:56:04 +01:00
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2014-03-28 18:56:04 +01:00
return err
}
return nil
}
2016-02-25 21:05:34 +01:00
// TryRmdir removes a container but not if not empty. It doesn't
// count errors but may return one.
2016-11-25 22:52:43 +01:00
func TryRmdir ( f Fs , dir string ) error {
2014-03-28 18:56:04 +01:00
if Config . DryRun {
2017-06-27 23:18:54 +02:00
Logf ( logDirName ( f , dir ) , "Not deleting as dry run is set" )
2016-02-25 21:05:34 +01:00
return nil
2014-03-28 18:56:04 +01:00
}
2017-06-27 23:18:54 +02:00
Debugf ( logDirName ( f , dir ) , "Removing directory" )
2016-11-25 22:52:43 +01:00
return f . Rmdir ( dir )
2016-02-25 21:05:34 +01:00
}
// Rmdir removes a container but not if not empty
2016-11-25 22:52:43 +01:00
func Rmdir ( f Fs , dir string ) error {
err := TryRmdir ( f , dir )
2016-02-25 21:05:34 +01:00
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2016-02-25 21:05:34 +01:00
return err
}
return err
2014-03-28 18:56:04 +01:00
}
2015-09-22 19:47:16 +02:00
// Purge removes a container and all of its contents
2014-03-28 18:56:04 +01:00
func Purge ( f Fs ) error {
2015-11-08 15:16:00 +01:00
doFallbackPurge := true
2014-07-25 19:19:49 +02:00
var err error
2017-01-13 18:21:47 +01:00
if doPurge := f . Features ( ) . Purge ; doPurge != nil {
2015-11-08 15:16:00 +01:00
doFallbackPurge = false
2014-07-13 11:45:13 +02:00
if Config . DryRun {
2017-02-09 12:01:20 +01:00
Logf ( f , "Not purging as --dry-run set" )
2014-07-13 11:45:13 +02:00
} else {
2017-01-13 18:21:47 +01:00
err = doPurge ( )
2015-11-08 15:16:00 +01:00
if err == ErrorCantPurge {
doFallbackPurge = true
}
2014-03-28 18:56:04 +01:00
}
2015-11-08 15:16:00 +01:00
}
if doFallbackPurge {
2014-07-25 19:19:49 +02:00
// DeleteFiles and Rmdir observe --dry-run
2017-02-24 23:51:01 +01:00
err = DeleteFiles ( listToChan ( f ) )
2016-06-25 15:27:44 +02:00
if err != nil {
return err
}
2017-02-01 14:23:31 +01:00
err = Rmdirs ( f , "" )
2014-07-25 19:19:49 +02:00
}
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2014-07-25 19:19:49 +02:00
return err
2014-03-28 18:56:04 +01:00
}
return nil
}
2015-12-02 23:25:32 +01:00
// Delete removes all the contents of a container. Unlike Purge, it
// obeys includes and excludes.
func Delete ( f Fs ) error {
delete := make ( ObjectsChan , Config . Transfers )
2016-06-25 15:27:44 +02:00
delErr := make ( chan error , 1 )
2015-12-02 23:25:32 +01:00
go func ( ) {
2016-06-25 15:27:44 +02:00
delErr <- DeleteFiles ( delete )
2015-12-02 23:25:32 +01:00
} ( )
err := ListFn ( f , func ( o Object ) {
delete <- o
} )
close ( delete )
2016-06-25 15:27:44 +02:00
delError := <- delErr
if err == nil {
err = delError
}
2015-12-02 23:25:32 +01:00
return err
}
2016-01-31 13:58:41 +01:00
2016-03-05 17:10:51 +01:00
// dedupeRename renames the objs slice to different names
func dedupeRename ( remote string , objs [ ] Object ) {
f := objs [ 0 ] . Fs ( )
2017-01-13 18:21:47 +01:00
doMove := f . Features ( ) . Move
if doMove == nil {
2016-03-05 17:10:51 +01:00
log . Fatalf ( "Fs %v doesn't support Move" , f )
}
ext := path . Ext ( remote )
base := remote [ : len ( remote ) - len ( ext ) ]
for i , o := range objs {
newName := fmt . Sprintf ( "%s-%d%s" , base , i + 1 , ext )
if ! Config . DryRun {
2017-01-13 18:21:47 +01:00
newObj , err := doMove ( o , newName )
2016-03-05 17:10:51 +01:00
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-02-09 12:01:20 +01:00
Errorf ( o , "Failed to rename: %v" , err )
2016-03-05 17:10:51 +01:00
continue
}
2017-02-09 18:08:51 +01:00
Infof ( newObj , "renamed from: %v" , o )
2016-03-05 17:10:51 +01:00
} else {
2017-02-09 12:01:20 +01:00
Logf ( remote , "Not renaming to %q as --dry-run" , newName )
2016-03-05 17:10:51 +01:00
}
}
}
// dedupeDeleteAllButOne deletes all but the one in keep
func dedupeDeleteAllButOne ( keep int , remote string , objs [ ] Object ) {
for i , o := range objs {
if i == keep {
continue
}
2016-06-25 15:27:44 +02:00
_ = DeleteFile ( o )
2016-03-05 17:10:51 +01:00
}
2017-02-09 12:01:20 +01:00
Logf ( remote , "Deleted %d extra copies" , len ( objs ) - 1 )
2016-03-05 17:10:51 +01:00
}
// dedupeDeleteIdentical deletes all but one of identical (by hash) copies
func dedupeDeleteIdentical ( remote string , objs [ ] Object ) [ ] Object {
// See how many of these duplicates are identical
byHash := make ( map [ string ] [ ] Object , len ( objs ) )
for _ , o := range objs {
md5sum , err := o . Hash ( HashMD5 )
if err == nil {
byHash [ md5sum ] = append ( byHash [ md5sum ] , o )
}
2016-01-31 13:58:41 +01:00
}
2016-03-05 17:10:51 +01:00
// Delete identical duplicates, refilling obj with the ones remaining
objs = nil
for md5sum , hashObjs := range byHash {
if len ( hashObjs ) > 1 {
2017-02-09 12:01:20 +01:00
Logf ( remote , "Deleting %d/%d identical duplicates (md5sum %q)" , len ( hashObjs ) - 1 , len ( hashObjs ) , md5sum )
2016-03-05 17:10:51 +01:00
for _ , o := range hashObjs [ 1 : ] {
2016-06-25 15:27:44 +02:00
_ = DeleteFile ( o )
2016-03-05 17:10:51 +01:00
}
}
objs = append ( objs , hashObjs [ 0 ] )
}
return objs
}
// dedupeInteractive interactively dedupes the slice of objects
func dedupeInteractive ( remote string , objs [ ] Object ) {
fmt . Printf ( "%s: %d duplicates remain\n" , remote , len ( objs ) )
for i , o := range objs {
md5sum , err := o . Hash ( HashMD5 )
if err != nil {
md5sum = err . Error ( )
}
fmt . Printf ( " %d: %12d bytes, %s, md5sum %32s\n" , i + 1 , o . Size ( ) , o . ModTime ( ) . Format ( "2006-01-02 15:04:05.000000000" ) , md5sum )
}
switch Command ( [ ] string { "sSkip and do nothing" , "kKeep just one (choose which in next step)" , "rRename all to be different (by changing file.jpg to file-1.jpg)" } ) {
case 's' :
case 'k' :
keep := ChooseNumber ( "Enter the number of the file to keep" , 1 , len ( objs ) )
dedupeDeleteAllButOne ( keep - 1 , remote , objs )
case 'r' :
dedupeRename ( remote , objs )
}
}
type objectsSortedByModTime [ ] Object
func ( objs objectsSortedByModTime ) Len ( ) int { return len ( objs ) }
func ( objs objectsSortedByModTime ) Swap ( i , j int ) { objs [ i ] , objs [ j ] = objs [ j ] , objs [ i ] }
func ( objs objectsSortedByModTime ) Less ( i , j int ) bool {
return objs [ i ] . ModTime ( ) . Before ( objs [ j ] . ModTime ( ) )
}
// DeduplicateMode is how the dedupe command chooses what to do
type DeduplicateMode int
// Deduplicate modes
const (
DeduplicateInteractive DeduplicateMode = iota // interactively ask the user
DeduplicateSkip // skip all conflicts
DeduplicateFirst // choose the first object
DeduplicateNewest // choose the newest object
DeduplicateOldest // choose the oldest object
DeduplicateRename // rename the objects
)
2016-08-03 18:35:29 +02:00
func ( x DeduplicateMode ) String ( ) string {
switch x {
2016-03-05 17:10:51 +01:00
case DeduplicateInteractive :
return "interactive"
case DeduplicateSkip :
return "skip"
case DeduplicateFirst :
return "first"
case DeduplicateNewest :
return "newest"
case DeduplicateOldest :
return "oldest"
case DeduplicateRename :
return "rename"
}
return "unknown"
}
2016-08-03 18:35:29 +02:00
// Set a DeduplicateMode from a string
func ( x * DeduplicateMode ) Set ( s string ) error {
switch strings . ToLower ( s ) {
case "interactive" :
* x = DeduplicateInteractive
case "skip" :
* x = DeduplicateSkip
case "first" :
* x = DeduplicateFirst
case "newest" :
* x = DeduplicateNewest
case "oldest" :
* x = DeduplicateOldest
case "rename" :
* x = DeduplicateRename
default :
return errors . Errorf ( "Unknown mode for dedupe %q." , s )
}
return nil
}
// Type of the value
func ( x * DeduplicateMode ) Type ( ) string {
return "string"
}
// Check it satisfies the interface
var _ pflag . Value = ( * DeduplicateMode ) ( nil )
2017-08-02 22:34:22 +02:00
// dedupeFindDuplicateDirs scans f for duplicate directories
func dedupeFindDuplicateDirs ( f Fs ) ( [ ] [ ] Directory , error ) {
duplicateDirs := [ ] [ ] Directory { }
err := Walk ( f , "" , true , Config . MaxDepth , func ( dirPath string , entries DirEntries , err error ) error {
if err != nil {
return err
}
dirs := map [ string ] [ ] Directory { }
entries . ForDir ( func ( d Directory ) {
dirs [ d . Remote ( ) ] = append ( dirs [ d . Remote ( ) ] , d )
} )
for _ , ds := range dirs {
if len ( ds ) > 1 {
duplicateDirs = append ( duplicateDirs , ds )
}
}
return nil
} )
if err != nil {
return nil , errors . Wrap ( err , "find duplicate dirs" )
}
return duplicateDirs , nil
}
// dedupeMergeDuplicateDirs merges all the duplicate directories found
func dedupeMergeDuplicateDirs ( f Fs , duplicateDirs [ ] [ ] Directory ) error {
mergeDirs := f . Features ( ) . MergeDirs
if mergeDirs == nil {
return errors . Errorf ( "%v: can't merge directories" , f )
}
dirCacheFlush := f . Features ( ) . DirCacheFlush
if dirCacheFlush == nil {
return errors . Errorf ( "%v: can't flush dir cache" , f )
}
for _ , dirs := range duplicateDirs {
if ! Config . DryRun {
Infof ( dirs [ 0 ] , "Merging contents of duplicate directories" )
err := mergeDirs ( dirs )
if err != nil {
return errors . Wrap ( err , "merge duplicate dirs" )
}
} else {
Infof ( dirs [ 0 ] , "NOT Merging contents of duplicate directories as --dry-run" )
}
}
dirCacheFlush ( )
return nil
}
2016-03-05 17:10:51 +01:00
// Deduplicate interactively finds duplicate files and offers to
// delete all but one or rename them to be different. Only useful with
// Google Drive which can have duplicate file names.
func Deduplicate ( f Fs , mode DeduplicateMode ) error {
2017-02-09 18:08:51 +01:00
Infof ( f , "Looking for duplicates using %v mode." , mode )
2017-08-02 22:34:22 +02:00
// Find duplicate directories first and fix them - repeat
// until all fixed
for {
duplicateDirs , err := dedupeFindDuplicateDirs ( f )
if err != nil {
return err
}
if len ( duplicateDirs ) == 0 {
break
}
err = dedupeMergeDuplicateDirs ( f , duplicateDirs )
if err != nil {
return err
}
if Config . DryRun {
break
}
}
// Now find duplicate files
2016-01-31 13:58:41 +01:00
files := map [ string ] [ ] Object { }
2017-02-24 23:51:01 +01:00
err := Walk ( f , "" , true , Config . MaxDepth , func ( dirPath string , entries DirEntries , err error ) error {
2016-04-21 21:06:21 +02:00
if err != nil {
return err
}
2017-02-24 23:51:01 +01:00
entries . ForObject ( func ( o Object ) {
remote := o . Remote ( )
files [ remote ] = append ( files [ remote ] , o )
} )
return nil
} )
if err != nil {
return err
2016-01-31 13:58:41 +01:00
}
for remote , objs := range files {
if len ( objs ) > 1 {
2017-02-09 12:01:20 +01:00
Logf ( remote , "Found %d duplicates - deleting identical copies" , len ( objs ) )
2016-03-05 17:10:51 +01:00
objs = dedupeDeleteIdentical ( remote , objs )
if len ( objs ) <= 1 {
2017-02-09 12:01:20 +01:00
Logf ( remote , "All duplicates removed" )
2016-03-05 17:10:51 +01:00
continue
2016-01-31 13:58:41 +01:00
}
2016-03-05 17:10:51 +01:00
switch mode {
case DeduplicateInteractive :
dedupeInteractive ( remote , objs )
case DeduplicateFirst :
dedupeDeleteAllButOne ( 0 , remote , objs )
case DeduplicateNewest :
sort . Sort ( objectsSortedByModTime ( objs ) ) // sort oldest first
dedupeDeleteAllButOne ( len ( objs ) - 1 , remote , objs )
case DeduplicateOldest :
sort . Sort ( objectsSortedByModTime ( objs ) ) // sort oldest first
dedupeDeleteAllButOne ( 0 , remote , objs )
case DeduplicateRename :
dedupeRename ( remote , objs )
case DeduplicateSkip :
// skip
default :
//skip
2016-01-31 13:58:41 +01:00
}
}
}
return nil
}
2016-04-21 21:06:21 +02:00
2017-02-24 23:51:01 +01:00
// listToChan will transfer all objects in the listing to the output
2016-04-21 21:06:21 +02:00
//
// If an error occurs, the error will be logged, and it will close the
// channel.
//
// If the error was ErrorDirNotFound then it will be ignored
2017-02-24 23:51:01 +01:00
func listToChan ( f Fs ) ObjectsChan {
2016-04-21 21:06:21 +02:00
o := make ( ObjectsChan , Config . Checkers )
go func ( ) {
defer close ( o )
2017-02-24 23:51:01 +01:00
_ = Walk ( f , "" , true , Config . MaxDepth , func ( dirPath string , entries DirEntries , err error ) error {
2016-04-21 21:06:21 +02:00
if err != nil {
2017-02-24 23:51:01 +01:00
if err == ErrorDirNotFound {
return nil
2016-04-21 21:06:21 +02:00
}
2017-11-15 06:32:00 +01:00
err = errors . Errorf ( "Failed to list: %v" , err )
Stats . Error ( err )
Errorf ( nil , "%v" , err )
2017-02-24 23:51:01 +01:00
return nil
2016-04-21 21:06:21 +02:00
}
2017-02-24 23:51:01 +01:00
entries . ForObject ( func ( obj Object ) {
o <- obj
} )
return nil
} )
2016-04-21 21:06:21 +02:00
} ( )
return o
}
2016-07-01 17:35:36 +02:00
// CleanUp removes the trash for the Fs
func CleanUp ( f Fs ) error {
2017-01-13 18:21:47 +01:00
doCleanUp := f . Features ( ) . CleanUp
if doCleanUp == nil {
2016-07-01 17:35:36 +02:00
return errors . Errorf ( "%v doesn't support cleanup" , f )
}
2016-07-02 17:58:50 +02:00
if Config . DryRun {
2017-02-09 12:01:20 +01:00
Logf ( f , "Not running cleanup as --dry-run set" )
2016-07-02 17:58:50 +02:00
return nil
}
2017-01-13 18:21:47 +01:00
return doCleanUp ( )
2016-07-01 17:35:36 +02:00
}
2016-08-18 23:43:02 +02:00
2017-02-09 12:25:36 +01:00
// wrap a Reader and a Closer together into a ReadCloser
type readCloser struct {
io . Reader
2017-11-11 19:43:00 +01:00
io . Closer
2017-02-09 12:25:36 +01:00
}
2016-08-18 23:43:02 +02:00
// Cat any files to the io.Writer
2017-02-08 09:09:41 +01:00
//
// if offset == 0 it will be ignored
// if offset > 0 then the file will be seeked to that offset
// if offset < 0 then the file will be seeked that far from the end
//
// if count < 0 then it will be ignored
// if count >= 0 then only that many characters will be output
func Cat ( f Fs , w io . Writer , offset , count int64 ) error {
2016-08-18 23:43:02 +02:00
var mu sync . Mutex
return ListFn ( f , func ( o Object ) {
2016-09-12 19:15:58 +02:00
var err error
2016-08-18 23:43:02 +02:00
Stats . Transferring ( o . Remote ( ) )
2016-09-12 19:15:58 +02:00
defer func ( ) {
Stats . DoneTransferring ( o . Remote ( ) , err == nil )
} ( )
2017-02-09 12:46:53 +01:00
size := o . Size ( )
2017-02-08 09:09:41 +01:00
thisOffset := offset
if thisOffset < 0 {
2017-02-09 12:46:53 +01:00
thisOffset += size
2017-02-08 09:09:41 +01:00
}
2017-02-09 12:46:53 +01:00
// size remaining is now reduced by thisOffset
size -= thisOffset
2017-02-08 09:09:41 +01:00
var options [ ] OpenOption
if thisOffset > 0 {
options = append ( options , & SeekOption { Offset : thisOffset } )
}
in , err := o . Open ( options ... )
2016-08-18 23:43:02 +02:00
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-02-09 12:01:20 +01:00
Errorf ( o , "Failed to open: %v" , err )
2016-08-18 23:43:02 +02:00
return
}
2017-02-08 09:09:41 +01:00
if count >= 0 {
2017-02-09 12:25:36 +01:00
in = & readCloser { Reader : & io . LimitedReader { R : in , N : count } , Closer : in }
2017-02-09 12:46:53 +01:00
// reduce remaining size to count
if size > count {
size = count
}
2017-02-08 09:09:41 +01:00
}
2017-02-17 10:15:24 +01:00
in = NewAccountSizeName ( in , size , o . Remote ( ) ) . WithBuffer ( ) // account and buffer the transfer
2016-08-18 23:43:02 +02:00
defer func ( ) {
err = in . Close ( )
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-02-09 12:01:20 +01:00
Errorf ( o , "Failed to close: %v" , err )
2016-08-18 23:43:02 +02:00
}
} ( )
2017-02-08 09:09:41 +01:00
// take the lock just before we output stuff, so at the last possible moment
mu . Lock ( )
defer mu . Unlock ( )
2017-02-09 12:25:36 +01:00
_ , err = io . Copy ( w , in )
2016-08-18 23:43:02 +02:00
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-02-09 12:01:20 +01:00
Errorf ( o , "Failed to send to output: %v" , err )
2016-08-18 23:43:02 +02:00
}
} )
}
2016-11-27 12:49:31 +01:00
2017-08-03 21:42:35 +02:00
// Rcat reads data from the Reader until EOF and uploads it to a file on remote
2017-11-11 19:43:00 +01:00
func Rcat ( fdst Fs , dstFileName string , in io . ReadCloser , modTime time . Time ) ( dst Object , err error ) {
2017-08-03 21:42:35 +02:00
Stats . Transferring ( dstFileName )
2017-11-11 19:43:00 +01:00
in = NewAccountSizeName ( in , - 1 , dstFileName ) . WithBuffer ( )
2017-08-03 21:42:35 +02:00
defer func ( ) {
Stats . DoneTransferring ( dstFileName , err == nil )
2017-11-11 19:43:00 +01:00
if otherErr := in . Close ( ) ; otherErr != nil {
2017-09-11 08:25:34 +02:00
Debugf ( fdst , "Rcat: failed to close source: %v" , err )
}
2017-08-03 21:42:35 +02:00
} ( )
2017-09-11 08:26:53 +02:00
hashOption := & HashesOption { Hashes : fdst . Hashes ( ) }
hash , err := NewMultiHasherTypes ( fdst . Hashes ( ) )
if err != nil {
2017-09-16 22:49:08 +02:00
return nil , err
2017-09-11 08:26:53 +02:00
}
2017-11-11 19:43:00 +01:00
readCounter := NewCountingReader ( in )
2017-09-11 08:26:53 +02:00
trackingIn := io . TeeReader ( readCounter , hash )
2017-09-11 08:25:34 +02:00
2017-09-11 08:26:53 +02:00
compare := func ( dst Object ) error {
src := NewStaticObjectInfo ( dstFileName , modTime , int64 ( readCounter . BytesRead ( ) ) , false , hash . Sums ( ) , fdst )
if ! Equal ( src , dst ) {
err = errors . Errorf ( "corrupted on transfer" )
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-09-11 08:26:53 +02:00
Errorf ( dst , "%v" , err )
return err
2017-09-11 08:25:34 +02:00
}
2017-09-11 08:26:53 +02:00
return nil
}
// check if file small enough for direct upload
buf := make ( [ ] byte , Config . StreamingUploadCutoff )
if n , err := io . ReadFull ( trackingIn , buf ) ; err == io . EOF || err == io . ErrUnexpectedEOF {
Debugf ( fdst , "File to upload is small (%d bytes), uploading instead of streaming" , n )
in := ioutil . NopCloser ( bytes . NewReader ( buf [ : n ] ) )
2017-09-11 08:25:34 +02:00
objInfo := NewStaticObjectInfo ( dstFileName , modTime , int64 ( n ) , false , nil , nil )
2017-09-11 22:28:16 +02:00
if Config . DryRun {
Logf ( "stdin" , "Not uploading as --dry-run" )
2017-09-16 22:49:08 +02:00
return nil , nil
2017-09-11 22:28:16 +02:00
}
2017-09-11 08:26:53 +02:00
dst , err := fdst . Put ( in , objInfo , hashOption )
if err != nil {
2017-09-16 22:49:08 +02:00
return dst , err
2017-09-11 08:26:53 +02:00
}
2017-09-16 22:49:08 +02:00
return dst , compare ( dst )
2017-09-11 08:25:34 +02:00
}
2017-11-11 19:43:00 +01:00
// Make a new ReadCloser with the bits we've already read
in = & readCloser {
Reader : io . MultiReader ( bytes . NewReader ( buf ) , trackingIn ) ,
Closer : in ,
}
2017-09-11 08:25:34 +02:00
2017-08-03 21:42:35 +02:00
fStreamTo := fdst
canStream := fdst . Features ( ) . PutStream != nil
if ! canStream {
Debugf ( fdst , "Target remote doesn't support streaming uploads, creating temporary local FS to spool file" )
tmpLocalFs , err := temporaryLocalFs ( )
if err != nil {
2017-09-16 22:49:08 +02:00
return nil , errors . Wrap ( err , "Failed to create temporary local FS to spool file" )
2017-08-03 21:42:35 +02:00
}
defer func ( ) {
err := Purge ( tmpLocalFs )
if err != nil {
Infof ( tmpLocalFs , "Failed to cleanup temporary FS: %v" , err )
}
} ( )
fStreamTo = tmpLocalFs
}
if Config . DryRun {
2017-09-11 22:28:16 +02:00
Logf ( "stdin" , "Not uploading as --dry-run" )
2017-08-03 21:42:35 +02:00
// prevents "broken pipe" errors
_ , err = io . Copy ( ioutil . Discard , in )
2017-09-16 22:49:08 +02:00
return nil , err
2017-08-03 21:42:35 +02:00
}
2017-09-11 08:25:34 +02:00
objInfo := NewStaticObjectInfo ( dstFileName , modTime , - 1 , false , nil , nil )
2017-09-16 22:49:08 +02:00
if dst , err = fStreamTo . Features ( ) . PutStream ( in , objInfo , hashOption ) ; err != nil {
return dst , err
2017-08-03 21:42:35 +02:00
}
2017-09-16 22:49:08 +02:00
if err = compare ( dst ) ; err != nil {
return dst , err
2017-09-11 08:26:53 +02:00
}
if ! canStream {
2017-09-16 22:49:08 +02:00
return dst , Copy ( fdst , nil , dstFileName , dst )
2017-09-11 08:26:53 +02:00
}
2017-09-16 22:49:08 +02:00
return dst , nil
2017-08-03 21:42:35 +02:00
}
2016-11-27 12:49:31 +01:00
// Rmdirs removes any empty directories (or directories only
// containing empty directories) under f, including f.
2017-02-01 14:23:31 +01:00
func Rmdirs ( f Fs , dir string ) error {
2016-11-27 12:49:31 +01:00
dirEmpty := make ( map [ string ] bool )
dirEmpty [ "" ] = true
2017-02-24 23:51:01 +01:00
err := Walk ( f , dir , true , Config . MaxDepth , func ( dirPath string , entries DirEntries , err error ) error {
2016-11-27 12:49:31 +01:00
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-02-24 23:51:01 +01:00
Errorf ( f , "Failed to list %q: %v" , dirPath , err )
return nil
}
for _ , entry := range entries {
switch x := entry . ( type ) {
2017-06-30 14:37:29 +02:00
case Directory :
2017-02-24 23:51:01 +01:00
// add a new directory as empty
2017-06-30 14:37:29 +02:00
dir := x . Remote ( )
2017-02-24 23:51:01 +01:00
_ , found := dirEmpty [ dir ]
if ! found {
dirEmpty [ dir ] = true
2016-11-27 12:49:31 +01:00
}
2017-02-24 23:51:01 +01:00
case Object :
// mark the parents of the file as being non-empty
dir := x . Remote ( )
for dir != "" {
dir = path . Dir ( dir )
if dir == "." || dir == "/" {
dir = ""
}
empty , found := dirEmpty [ dir ]
// End if we reach a directory which is non-empty
if found && ! empty {
break
}
dirEmpty [ dir ] = false
2016-11-27 12:49:31 +01:00
}
}
}
2017-02-24 23:51:01 +01:00
return nil
} )
if err != nil {
return errors . Wrap ( err , "failed to rmdirs" )
2016-11-27 12:49:31 +01:00
}
// Now delete the empty directories, starting from the longest path
var toDelete [ ] string
for dir , empty := range dirEmpty {
if empty {
toDelete = append ( toDelete , dir )
}
}
sort . Strings ( toDelete )
for i := len ( toDelete ) - 1 ; i >= 0 ; i -- {
dir := toDelete [ i ]
err := TryRmdir ( f , dir )
if err != nil {
2017-11-15 06:32:00 +01:00
Stats . Error ( err )
2017-02-09 12:01:20 +01:00
Errorf ( dir , "Failed to rmdir: %v" , err )
2016-11-27 12:49:31 +01:00
return err
}
}
return nil
}
2016-10-23 18:34:17 +02:00
// moveOrCopyFile moves or copies a single file possibly to a new name
func moveOrCopyFile ( fdst Fs , fsrc Fs , dstFileName string , srcFileName string , cp bool ) ( err error ) {
2017-10-12 21:45:36 +02:00
dstFilePath := path . Join ( fdst . Root ( ) , dstFileName )
srcFilePath := path . Join ( fsrc . Root ( ) , srcFileName )
if fdst . Name ( ) == fsrc . Name ( ) && dstFilePath == srcFilePath {
2017-05-27 17:30:26 +02:00
Debugf ( fdst , "don't need to copy/move %s, it is already at target location" , dstFileName )
return nil
}
2016-10-23 18:34:17 +02:00
// Choose operations
Op := Move
if cp {
Op = Copy
}
// Find src object
srcObj , err := fsrc . NewObject ( srcFileName )
if err != nil {
return err
}
// Find dst object if it exists
dstObj , err := fdst . NewObject ( dstFileName )
if err == ErrorObjectNotFound {
dstObj = nil
} else if err != nil {
return err
}
if NeedTransfer ( dstObj , srcObj ) {
2017-06-07 14:02:21 +02:00
Stats . Transferring ( srcFileName )
err = Op ( fdst , dstObj , dstFileName , srcObj )
Stats . DoneTransferring ( srcFileName , err == nil )
} else {
Stats . Checking ( srcFileName )
if ! cp {
err = DeleteFile ( srcObj )
}
defer Stats . DoneChecking ( srcFileName )
2016-10-23 18:34:17 +02:00
}
2017-06-07 14:02:21 +02:00
return err
2016-10-23 18:34:17 +02:00
}
// MoveFile moves a single file possibly to a new name
func MoveFile ( fdst Fs , fsrc Fs , dstFileName string , srcFileName string ) ( err error ) {
return moveOrCopyFile ( fdst , fsrc , dstFileName , srcFileName , false )
}
// CopyFile moves a single file possibly to a new name
func CopyFile ( fdst Fs , fsrc Fs , dstFileName string , srcFileName string ) ( err error ) {
return moveOrCopyFile ( fdst , fsrc , dstFileName , srcFileName , true )
}