diff --git a/.gitignore b/.gitignore index 5b90117b7..5ec9012d1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ test-env* _junk/ rclone +upload diff --git a/drive/fs.go b/drive/fs.go index 20bdd0555..68f5bb3dc 100644 --- a/drive/fs.go +++ b/drive/fs.go @@ -22,30 +22,52 @@ package drive // * files with / in name import ( - "code.google.com/p/goauth2/oauth" - "code.google.com/p/google-api-go-client/drive/v2" "errors" "flag" "fmt" - "github.com/ncw/rclone/fs" "io" "log" "mime" "net/http" "os" "path" - "regexp" "strings" "sync" "time" -) -// Pattern to match a drive url -var Match = regexp.MustCompile(`^drive://(.*)$`) + "code.google.com/p/goauth2/oauth" + "code.google.com/p/google-api-go-client/drive/v2" + "github.com/ncw/rclone/fs" +) // Register with Fs func init() { - fs.Register(Match, NewFs) + fs.Register(&fs.FsInfo{ + Name: "drive", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "client_id", + Help: "Google Application Client Id.", + Examples: []fs.OptionExample{{ + Value: "202264815644.apps.googleusercontent.com", + Help: "rclone's client id - use this or your own if you want", + }}, + }, { + Name: "client_secret", + Help: "Google Application Client Secret.", + Examples: []fs.OptionExample{{ + Value: "X4Z3ca8xfWDb1Voo-F9a7ZxJ", + Help: "rclone's client secret - use this or your own if you want", + }}, + }, { + Name: "token_file", + Help: "Path to store token file.", + Examples: []fs.OptionExample{{ + Value: path.Join(fs.HomeDir, ".gdrive-token-file"), + Help: "Suggested path for token file", + }}, + }}, + }) } // FsDrive represents a remote drive server @@ -128,11 +150,8 @@ const ( // Globals var ( // Flags - driveClientId = flag.String("drive-client-id", os.Getenv("GDRIVE_CLIENT_ID"), "Auth URL for server. Defaults to environment var GDRIVE_CLIENT_ID.") - driveClientSecret = flag.String("drive-client-secret", os.Getenv("GDRIVE_CLIENT_SECRET"), "User name. Defaults to environment var GDRIVE_CLIENT_SECRET.") - driveTokenFile = flag.String("drive-token-file", os.Getenv("GDRIVE_TOKEN_FILE"), "API key (password). Defaults to environment var GDRIVE_TOKEN_FILE.") - driveAuthCode = flag.String("drive-auth-code", "", "Pass in when requested to make the drive token file.") - driveFullList = flag.Bool("drive-full-list", true, "Use a full listing for directory list. More data but usually quicker.") + driveAuthCode = flag.String("drive-auth-code", "", "Pass in when requested to make the drive token file.") + driveFullList = flag.Bool("drive-full-list", true, "Use a full listing for directory list. More data but usually quicker.") ) // String converts this FsDrive to a string @@ -142,13 +161,7 @@ func (f *FsDrive) String() string { // parseParse parses a drive 'url' func parseDrivePath(path string) (root string, err error) { - parts := Match.FindAllStringSubmatch(path, -1) - if len(parts) != 1 || len(parts[0]) != 2 { - err = fmt.Errorf("Couldn't parse drive url %q", path) - } else { - root = parts[0][1] - root = strings.Trim(root, "/") - } + root = strings.Trim(root, "/") return } @@ -222,26 +235,29 @@ func MakeNewToken(t *oauth.Transport) error { } // NewFs contstructs an FsDrive from the path, container:path -func NewFs(path string) (fs.Fs, error) { - if *driveClientId == "" { - return nil, errors.New("Need -drive-client-id or environmental variable GDRIVE_CLIENT_ID") +func NewFs(name, path string) (fs.Fs, error) { + clientId := fs.ConfigFile.MustValue(name, "client_id") + if clientId == "" { + return nil, errors.New("client_id not found") } - if *driveClientSecret == "" { - return nil, errors.New("Need -drive-client-secret or environmental variable GDRIVE_CLIENT_SECRET") + clientSecret := fs.ConfigFile.MustValue(name, "client_secret") + if clientSecret == "" { + return nil, errors.New("client_secret not found") } - if *driveTokenFile == "" { - return nil, errors.New("Need -drive-token-file or environmental variable GDRIVE_TOKEN_FILE") + tokenFile := fs.ConfigFile.MustValue(name, "token_file") + if tokenFile == "" { + return nil, errors.New("token-file not found") } // Settings for authorization. var driveConfig = &oauth.Config{ - ClientId: *driveClientId, - ClientSecret: *driveClientSecret, + ClientId: clientId, + ClientSecret: clientSecret, Scope: "https://www.googleapis.com/auth/drive", RedirectURL: "urn:ietf:wg:oauth:2.0:oob", AuthURL: "https://accounts.google.com/o/oauth2/auth", TokenURL: "https://accounts.google.com/o/oauth2/token", - TokenCache: oauth.CacheFile(*driveTokenFile), + TokenCache: oauth.CacheFile(tokenFile), } root, err := parseDrivePath(path) diff --git a/fs/config.go b/fs/config.go new file mode 100644 index 000000000..fc299f597 --- /dev/null +++ b/fs/config.go @@ -0,0 +1,260 @@ +// Read and write the config file +package fs + +import ( + "bufio" + "flag" + "fmt" + "log" + "os" + "os/user" + "path" + "sort" + "strconv" + "strings" + "time" + + "github.com/Unknwon/goconfig" +) + +const ( + configFileName = ".rclone.conf" +) + +// Global +var ( + // Config file + ConfigFile *goconfig.ConfigFile + // Config file path + ConfigPath string + // Global config + Config = &ConfigInfo{} + // Home directory + HomeDir string + // Flags + verbose = flag.Bool("verbose", false, "Print lots more stuff") + quiet = flag.Bool("quiet", false, "Print as little stuff as possible") + modifyWindow = flag.Duration("modify-window", time.Nanosecond, "Max time diff to be considered the same") + checkers = flag.Int("checkers", 8, "Number of checkers to run in parallel.") + transfers = flag.Int("transfers", 4, "Number of file transfers to run in parallel.") +) + +// Filesystem config options +type ConfigInfo struct { + Verbose bool + Quiet bool + ModifyWindow time.Duration + Checkers int + Transfers int +} + +// Loads the config file +func LoadConfig() { + // Read some flags if set + // + // FIXME read these from the config file too + Config.Verbose = *verbose + Config.Quiet = *quiet + Config.ModifyWindow = *modifyWindow + Config.Checkers = *checkers + Config.Transfers = *transfers + + // Find users home directory + usr, err := user.Current() + if err != nil { + log.Printf("Couldn't find home directory: %v", err) + return + } + HomeDir = usr.HomeDir + ConfigPath = path.Join(HomeDir, configFileName) + + // Load configuration file. + ConfigFile, err = goconfig.LoadConfigFile(ConfigPath) + if err != nil { + log.Printf("Failed to load config file %v - using defaults", ConfigPath) + } +} + +// Save configuration file. +func SaveConfig() { + err := goconfig.SaveConfigFile(ConfigFile, ConfigPath) + if err != nil { + log.Fatalf("Failed to save config file: %v", err) + } +} + +// Show an overview of the config file +func ShowConfig() { + remotes := ConfigFile.GetSectionList() + sort.Strings(remotes) + fmt.Printf("%-20s %s\n", "Name", "Type") + fmt.Printf("%-20s %s\n", "====", "====") + for _, remote := range remotes { + fmt.Printf("%-20s %s\n", remote, ConfigFile.MustValue(remote, "type")) + } +} + +// ChooseRemote chooses a remote name +func ChooseRemote() string { + remotes := ConfigFile.GetSectionList() + sort.Strings(remotes) + return Choose("remote", remotes, nil, false) +} + +// Read some input +func ReadLine() string { + buf := bufio.NewReader(os.Stdin) + line, err := buf.ReadString('\n') + if err != nil { + log.Fatalf("Failed to read line: %v", err) + } + return strings.TrimSpace(line) +} + +// Command - choose one +func Command(commands []string) int { + opts := []string{} + for _, text := range commands { + fmt.Printf("%c) %s\n", text[0], text[1:]) + opts = append(opts, text[:1]) + } + optString := strings.Join(opts, "") + optHelp := strings.Join(opts, "/") + for { + fmt.Printf("%s> ", optHelp) + result := strings.ToLower(ReadLine()) + if len(result) != 1 { + continue + } + i := strings.IndexByte(optString, result[0]) + if i >= 0 { + return i + } + } +} + +// Choose one of the defaults or type a new string if newOk is set +func Choose(what string, defaults, help []string, newOk bool) string { + fmt.Printf("Choose a number from below") + if newOk { + fmt.Printf(", or type in your own value") + } + fmt.Println() + for i, text := range defaults { + if help != nil { + parts := strings.Split(help[i], "\n") + for _, part := range parts { + fmt.Printf(" * %s\n", part) + } + } + fmt.Printf("%2d) %s\n", i+1, text) + } + for { + fmt.Printf("%s> ", what) + result := ReadLine() + i, err := strconv.Atoi(result) + if err != nil { + if newOk { + return result + } + continue + } + if i >= 1 && i <= len(defaults) { + return defaults[i-1] + } + } +} + +// Show the contents of the remote +func ShowRemote(name string) { + fmt.Printf("--------------------\n") + fmt.Printf("[%s]\n", name) + for _, key := range ConfigFile.GetKeyList(name) { + fmt.Printf("%s = %s\n", key, ConfigFile.MustValue(name, key)) + } + fmt.Printf("--------------------\n") +} + +// Print the contents of the remote and ask if it is OK +func OkRemote(name string) bool { + ShowRemote(name) + switch i := Command([]string{"yYes this is OK", "eEdit this remote", "dDelete this remote"}); i { + case 0: + return true + case 1: + return false + case 2: + ConfigFile.DeleteSection(name) + return true + default: + log.Printf("Bad choice %d", i) + } + return false +} + +// Make a new remote +func NewRemote(name string) { + fmt.Printf("What type of source is it?\n") + types := []string{} + for _, item := range fsRegistry { + types = append(types, item.Name) + } + newType := Choose("type", types, nil, false) + ConfigFile.SetValue(name, "type", newType) + fs, err := Find(newType) + if err != nil { + log.Fatalf("Failed to find fs: %v", err) + } + for _, option := range fs.Options { + ConfigFile.SetValue(name, option.Name, option.Choose()) + } + if OkRemote(name) { + SaveConfig() + return + } + EditRemote(name) +} + +// Edit a remote +func EditRemote(name string) { + ShowRemote(name) + fmt.Printf("Edit remote\n") + for { + for _, key := range ConfigFile.GetKeyList(name) { + value := ConfigFile.MustValue(name, key) + fmt.Printf("Press enter to accept current value, or type in a new one\n") + fmt.Printf("%s = %s>", key, value) + newValue := ReadLine() + if newValue != "" { + ConfigFile.SetValue(name, key, newValue) + } + } + if OkRemote(name) { + break + } + } + SaveConfig() +} + +// Edit the config file interactively +func EditConfig() { + for { + fmt.Printf("Current remotes:\n\n") + ShowConfig() + fmt.Printf("\n") + switch i := Command([]string{"eEdit existing remote", "nNew remote", "dDelete remote", "qQuit config"}); i { + case 0: + name := ChooseRemote() + EditRemote(name) + case 1: + fmt.Printf("name> ") + name := ReadLine() + NewRemote(name) + case 2: + name := ChooseRemote() + ConfigFile.DeleteSection(name) + case 3: + return + } + } +} diff --git a/fs/fs.go b/fs/fs.go index d87d7f75a..e20e4b5eb 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -12,41 +12,52 @@ import ( // Globals var ( - // Global config - Config = &ConfigInfo{} // Filesystem registry - fsRegistry []registryItem + fsRegistry []*FsInfo ) -// Filesystem config options -type ConfigInfo struct { - Verbose bool - Quiet bool - ModifyWindow time.Duration - Checkers int - Transfers int +// Filesystem info +type FsInfo struct { + Name string // name of this fs + NewFs func(string, string) (Fs, error) // create a new file system + Options []Option } -// Filesystem registry item -type registryItem struct { - match *regexp.Regexp // if this matches then can call newFs - newFs func(string) (Fs, error) // create a new file system +// An options for a Fs +type Option struct { + Name string + Help string + Optional bool + Examples []OptionExample +} + +// An example for an option +type OptionExample struct { + Value string + Help string +} + +// Choose an option +func (o *Option) Choose() string { + fmt.Println(o.Help) + if len(o.Examples) > 0 { + var values []string + var help []string + for _, example := range o.Examples { + values = append(values, example.Value) + help = append(help, example.Help) + } + return Choose(o.Name, values, help, true) + } + fmt.Printf("%s> ", o.Name) + return ReadLine() } // Register a filesystem // -// If a path matches with match then can call newFs on it -// -// Pass with match nil goes last and matches everything (used by local fs) -// // Fs modules should use this in an init() function -func Register(match *regexp.Regexp, newFs func(string) (Fs, error)) { - fsRegistry = append(fsRegistry, registryItem{match: match, newFs: newFs}) - // Keep one nil match at the end - last := len(fsRegistry) - 1 - if last >= 1 && fsRegistry[last-1].match == nil { - fsRegistry[last], fsRegistry[last-1] = fsRegistry[last-1], fsRegistry[last] - } +func Register(info *FsInfo) { + fsRegistry = append(fsRegistry, info) } // A Filesystem, describes the local filesystem and the remote object store @@ -136,16 +147,42 @@ type Dir struct { // A channel of Dir objects type DirChan chan *Dir -// NewFs makes a new Fs object from the path +// Pattern to match a url +var matcher = regexp.MustCompile(`^([\w_-]+)://(.*)$`) + +// Finds a FsInfo object for the name passed in // -// FIXME make more generic -func NewFs(path string) (Fs, error) { +// Services are looked up in the config file +func Find(name string) (*FsInfo, error) { for _, item := range fsRegistry { - if item.match == nil || item.match.MatchString(path) { - return item.newFs(path) + if item.Name == name { + return item, nil } } - panic("Not found") // FIXME + return nil, fmt.Errorf("Didn't find filing system for %q", name) +} + +// NewFs makes a new Fs object from the path +// +// The path is of the form service://path +// +// Services are looked up in the config file +func NewFs(path string) (Fs, error) { + parts := matcher.FindStringSubmatch(path) + fsName, configName, fsPath := "local", "local", path + if parts != nil { + configName, fsPath = parts[1], parts[2] + var err error + fsName, err = ConfigFile.GetValue(configName, "type") + if err != nil { + return nil, fmt.Errorf("Didn't find section in config file for %q", configName) + } + } + fs, err := Find(fsName) + if err != nil { + return nil, err + } + return fs.NewFs(configName, fsPath) } // Write debuging output for this Object diff --git a/local/fs.go b/local/fs.go index 48788e1d8..b0b3a9af9 100644 --- a/local/fs.go +++ b/local/fs.go @@ -4,7 +4,6 @@ package local import ( "crypto/md5" "fmt" - "github.com/ncw/rclone/fs" "io" "io/ioutil" "log" @@ -13,11 +12,16 @@ import ( "path/filepath" "sync" "time" + + "github.com/ncw/rclone/fs" ) // Register with Fs func init() { - fs.Register(nil, NewFs) + fs.Register(&fs.FsInfo{ + Name: "local", + NewFs: NewFs, + }) } // FsLocal represents a local filesystem rooted at root @@ -37,7 +41,7 @@ type FsObjectLocal struct { // ------------------------------------------------------------ // NewFs contstructs an FsLocal from the path -func NewFs(root string) (fs.Fs, error) { +func NewFs(name, root string) (fs.Fs, error) { root = path.Clean(root) f := &FsLocal{root: root} return f, nil diff --git a/rclone.go b/rclone.go index 361468bdf..090d5a611 100644 --- a/rclone.go +++ b/rclone.go @@ -6,7 +6,6 @@ package main import ( "flag" "fmt" - "github.com/ncw/rclone/fs" "log" "os" "runtime" @@ -14,6 +13,8 @@ import ( "strings" "sync" "time" + + "github.com/ncw/rclone/fs" // Active file systems _ "github.com/ncw/rclone/drive" _ "github.com/ncw/rclone/local" @@ -25,13 +26,8 @@ import ( var ( // Flags cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file") - verbose = flag.Bool("verbose", false, "Print lots more stuff") - quiet = flag.Bool("quiet", false, "Print as little stuff as possible") dry_run = flag.Bool("dry-run", false, "Do a trial run with no permanent changes") - checkers = flag.Int("checkers", 8, "Number of checkers to run in parallel.") - transfers = flag.Int("transfers", 4, "Number of file transfers to run in parallel.") statsInterval = flag.Duration("stats", time.Minute*1, "Interval to print stats") - modifyWindow = flag.Duration("modify-window", time.Nanosecond, "Max time diff to be considered the same") ) // A pair of fs.Objects @@ -105,17 +101,17 @@ func CopyFs(fdst, fsrc fs.Fs) { } to_be_checked := fsrc.List() - to_be_uploaded := make(fs.ObjectsChan, *transfers) + to_be_uploaded := make(fs.ObjectsChan, fs.Config.Transfers) var checkerWg sync.WaitGroup - checkerWg.Add(*checkers) - for i := 0; i < *checkers; i++ { + checkerWg.Add(fs.Config.Checkers) + for i := 0; i < fs.Config.Checkers; i++ { go Checker(to_be_checked, to_be_uploaded, fdst, &checkerWg) } var copierWg sync.WaitGroup - copierWg.Add(*transfers) - for i := 0; i < *transfers; i++ { + copierWg.Add(fs.Config.Transfers) + for i := 0; i < fs.Config.Transfers; i++ { go Copier(to_be_uploaded, fdst, &copierWg) } @@ -129,8 +125,8 @@ func CopyFs(fdst, fsrc fs.Fs) { // Delete all the files passed in the channel func DeleteFiles(to_be_deleted fs.ObjectsChan) { var wg sync.WaitGroup - wg.Add(*transfers) - for i := 0; i < *transfers; i++ { + wg.Add(fs.Config.Transfers) + for i := 0; i < fs.Config.Transfers; i++ { go func() { defer wg.Done() for dst := range to_be_deleted { @@ -173,18 +169,18 @@ func Sync(fdst, fsrc fs.Fs) { } // Read source files checking them off against dest files - to_be_checked := make(PairFsObjectsChan, *transfers) - to_be_uploaded := make(fs.ObjectsChan, *transfers) + to_be_checked := make(PairFsObjectsChan, fs.Config.Transfers) + to_be_uploaded := make(fs.ObjectsChan, fs.Config.Transfers) var checkerWg sync.WaitGroup - checkerWg.Add(*checkers) - for i := 0; i < *checkers; i++ { + checkerWg.Add(fs.Config.Checkers) + for i := 0; i < fs.Config.Checkers; i++ { go PairChecker(to_be_checked, to_be_uploaded, &checkerWg) } var copierWg sync.WaitGroup - copierWg.Add(*transfers) - for i := 0; i < *transfers; i++ { + copierWg.Add(fs.Config.Transfers) + for i := 0; i < fs.Config.Transfers; i++ { go Copier(to_be_uploaded, fdst, &copierWg) } @@ -215,7 +211,7 @@ func Sync(fdst, fsrc fs.Fs) { } // Delete the spare files - toDelete := make(fs.ObjectsChan, *transfers) + toDelete := make(fs.ObjectsChan, fs.Config.Transfers) go func() { for _, fs := range delFiles { toDelete <- fs @@ -262,7 +258,7 @@ func Check(fdst, fsrc fs.Fs) { log.Printf(remote) } - checks := make(chan []fs.Object, *transfers) + checks := make(chan []fs.Object, fs.Config.Transfers) go func() { for _, check := range commonFiles { checks <- check @@ -271,8 +267,8 @@ func Check(fdst, fsrc fs.Fs) { }() var checkerWg sync.WaitGroup - checkerWg.Add(*checkers) - for i := 0; i < *checkers; i++ { + checkerWg.Add(fs.Config.Checkers) + for i := 0; i < fs.Config.Checkers; i++ { go func() { defer checkerWg.Done() for check := range checks { @@ -309,8 +305,8 @@ func Check(fdst, fsrc fs.Fs) { func List(f, _ fs.Fs) { in := f.List() var wg sync.WaitGroup - wg.Add(*checkers) - for i := 0; i < *checkers; i++ { + wg.Add(fs.Config.Checkers) + for i := 0; i < fs.Config.Checkers; i++ { go func() { defer wg.Done() for o := range in { @@ -370,117 +366,128 @@ func purge(fdst, fsrc fs.Fs) { } } +// Edits the config file +func EditConfig(fdst, fsrc fs.Fs) { + fs.EditConfig() +} + type Command struct { - name string - help string - run func(fdst, fsrc fs.Fs) - minArgs, maxArgs int + Name string + Help string + ArgsHelp string + Run func(fdst, fsrc fs.Fs) + MinArgs int + MaxArgs int + NoStats bool } // checkArgs checks there are enough arguments and prints a message if not func (cmd *Command) checkArgs(args []string) { - if len(args) < cmd.minArgs { + if len(args) < cmd.MinArgs { syntaxError() - fmt.Fprintf(os.Stderr, "Command %s needs %d arguments mininum\n", cmd.name, cmd.minArgs) + fmt.Fprintf(os.Stderr, "Command %s needs %d arguments mininum\n", cmd.Name, cmd.MinArgs) os.Exit(1) - } else if len(args) > cmd.maxArgs { + } else if len(args) > cmd.MaxArgs { syntaxError() - fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.name, cmd.maxArgs) + fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.Name, cmd.MaxArgs) os.Exit(1) } } var Commands = []Command{ { - "copy", - ` - + Name: "copy", + ArgsHelp: "source://path dest://path", + Help: ` Copy the source to the destination. Doesn't transfer unchanged files, testing first by modification time then by - MD5SUM. Doesn't delete files from the destination. - -`, - CopyFs, - 2, 2, + MD5SUM. Doesn't delete files from the destination.`, + Run: CopyFs, + MinArgs: 2, + MaxArgs: 2, }, { - "sync", - ` - + Name: "sync", + ArgsHelp: "source://path dest://path", + Help: ` Sync the source to the destination. Doesn't transfer unchanged files, testing first by modification time then by MD5SUM. Deletes any files that exist in source that don't exist in destination. Since this can cause data loss, test first with the -dry-run flag.`, - - Sync, - 2, 2, + Run: Sync, + MinArgs: 2, + MaxArgs: 2, }, { - "ls", - `[] - + Name: "ls", + ArgsHelp: "[remote://path]", + Help: ` List all the objects in the the path.`, - - List, - 1, 1, + Run: List, + MinArgs: 1, + MaxArgs: 1, }, { - "lsd", - `[] - + Name: "lsd", + ArgsHelp: "[remote://path]", + Help: ` List all directoryes/objects/buckets in the the path.`, - - ListDir, - 1, 1, + Run: ListDir, + MinArgs: 1, + MaxArgs: 1, }, { - "mkdir", - ` - + Name: "mkdir", + ArgsHelp: "remote://path", + Help: ` Make the path if it doesn't already exist`, - - mkdir, - 1, 1, + Run: mkdir, + MinArgs: 1, + MaxArgs: 1, }, { - "rmdir", - ` - + Name: "rmdir", + ArgsHelp: "remote://path", + Help: ` Remove the path. Note that you can't remove a path with objects in it, use purge for that.`, - - rmdir, - 1, 1, + Run: rmdir, + MinArgs: 1, + MaxArgs: 1, }, { - "purge", - ` - + Name: "purge", + ArgsHelp: "remote://path", + Help: ` Remove the path and all of its contents.`, - - purge, - 1, 1, + Run: purge, + MinArgs: 1, + MaxArgs: 1, }, { - "check", - ` - + Name: "check", + ArgsHelp: "source://path dest://path", + Help: ` Checks the files in the source and destination match. It compares sizes and MD5SUMs and prints a report of files which don't match. It doesn't alter the source or destination.`, - - Check, - 2, 2, + Run: Check, + MinArgs: 2, + MaxArgs: 2, }, { - "help", - ` - + Name: "config", + Help: ` + Enter an interactive configuration session.`, + Run: EditConfig, + NoStats: true, + }, + { + Name: "help", + Help: ` This help.`, - - nil, - 0, 0, + NoStats: true, }, } @@ -495,7 +502,8 @@ Subcommands: `) for i := range Commands { cmd := &Commands[i] - fmt.Fprintf(os.Stderr, " %s: %s\n\n", cmd.name, cmd.help) + fmt.Fprintf(os.Stderr, " %s %s\n", cmd.Name, cmd.ArgsHelp) + fmt.Fprintf(os.Stderr, "%s\n\n", cmd.Help) } fmt.Fprintf(os.Stderr, "Options:\n") @@ -517,13 +525,7 @@ func main() { flag.Parse() args := flag.Args() runtime.GOMAXPROCS(runtime.NumCPU()) - - // Pass on some flags to fs.Config - fs.Config.Verbose = *verbose - fs.Config.Quiet = *quiet - fs.Config.ModifyWindow = *modifyWindow - fs.Config.Checkers = *checkers - fs.Config.Transfers = *transfers + fs.LoadConfig() // Setup profiling if desired if *cpuprofile != "" { @@ -548,10 +550,10 @@ func main() { for i := range Commands { command := &Commands[i] // exact command name found - use that - if command.name == cmd { + if command.Name == cmd { found = command break - } else if strings.HasPrefix(command.name, cmd) { + } else if strings.HasPrefix(command.Name, cmd) { if found != nil { fs.Stats.Error() log.Fatalf("Not unique - matches multiple commands %q", cmd) @@ -572,14 +574,14 @@ func main() { fdst, err = fs.NewFs(args[0]) if err != nil { fs.Stats.Error() - log.Fatal("Failed to create file system: ", err) + log.Fatalf("Failed to create file system for %q: %v", args[0], err) } } if len(args) >= 2 { fsrc, err = fs.NewFs(args[1]) if err != nil { fs.Stats.Error() - log.Fatal("Failed to create destination file system: ", err) + log.Fatalf("Failed to create destination file system for %q: %v", args[1], err) } fsrc, fdst = fdst, fsrc } @@ -599,22 +601,30 @@ func main() { fs.Config.ModifyWindow = precision } } - log.Printf("Modify window is %s\n", fs.Config.ModifyWindow) + if fs.Config.Verbose { + log.Printf("Modify window is %s\n", fs.Config.ModifyWindow) + } // Print the stats every statsInterval - go func() { - ch := time.Tick(*statsInterval) - for { - <-ch - fs.Stats.Log() - } - }() + if !found.NoStats { + go func() { + ch := time.Tick(*statsInterval) + for { + <-ch + fs.Stats.Log() + } + }() + } // Run the actual command - if found.run != nil { - found.run(fdst, fsrc) - fmt.Println(fs.Stats) - log.Printf("*** Go routines at exit %d\n", runtime.NumGoroutine()) + if found.Run != nil { + found.Run(fdst, fsrc) + if !found.NoStats { + fmt.Println(fs.Stats) + } + if fs.Config.Verbose { + log.Printf("*** Go routines at exit %d\n", runtime.NumGoroutine()) + } if fs.Stats.Errored() { os.Exit(1) } diff --git a/s3/fs.go b/s3/fs.go index 8c83596da..36689dff4 100644 --- a/s3/fs.go +++ b/s3/fs.go @@ -5,30 +5,99 @@ package s3 import ( "errors" - "flag" "fmt" - "github.com/ncw/goamz/aws" - "github.com/ncw/goamz/s3" - "github.com/ncw/rclone/fs" - "github.com/ncw/swift" "io" "log" "mime" "net/http" - "os" "path" "regexp" "strconv" "strings" "time" -) -// Pattern to match a s3 url -var Match = regexp.MustCompile(`^s3://([^/]*)(.*)$`) + "github.com/ncw/goamz/aws" + "github.com/ncw/goamz/s3" + "github.com/ncw/rclone/fs" + "github.com/ncw/swift" +) // Register with Fs func init() { - fs.Register(Match, NewFs) + fs.Register(&fs.FsInfo{ + Name: "s3", + NewFs: NewFs, + // AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region + Options: []fs.Option{{ + Name: "access_key_id", + Help: "AWS Access Key ID.", + }, { + Name: "secret_access_key", + Help: "AWS Secret Access Key (password). ", + }, { + Name: "endpoint", + Help: "Endpoint for S3 API.", + Examples: []fs.OptionExample{{ + Value: "https://s3.amazonaws.com/", + Help: "The default endpoint - a good choice if you are unsure.\nUS Region, Northern Virginia or Pacific Northwest.\nLeave location constraint empty.", + }, { + Value: "https://s3-external-1.amazonaws.com", + Help: "US Region, Northern Virginia only.\nLeave location constraint empty.", + }, { + Value: "https://s3-us-west-2.amazonaws.com", + Help: "US West (Oregon) Region\nNeeds location constraint us-west-2.", + }, { + Value: "https://s3-us-west-1.amazonaws.com", + Help: "US West (Northern California) Region\nNeeds location constraint us-west-1.", + }, { + Value: "https://s3-eu-west-1.amazonaws.com", + Help: "EU (Ireland) Region Region\nNeeds location constraint EU or eu-west-1.", + }, { + Value: "https://s3-ap-southeast-1.amazonaws.com", + Help: "Asia Pacific (Singapore) Region\nNeeds location constraint ap-southeast-1.", + }, { + Value: "https://s3-ap-southeast-2.amazonaws.com", + Help: "Asia Pacific (Sydney) Region\nNeeds location constraint .", + }, { + Value: "https://s3-ap-northeast-1.amazonaws.com", + Help: "Asia Pacific (Tokyo) Region\nNeeds location constraint ap-northeast-1.", + }, { + Value: "https://s3-sa-east-1.amazonaws.com", + Help: "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.", + }}, + }, { + Name: "location_constraint", + Help: "Location constraint - must be set to match the Endpoint.", + Examples: []fs.OptionExample{{ + Value: "", + Help: "Empty for US Region, Northern Virginia or Pacific Northwest.", + }, { + Value: "us-west-2", + Help: "US West (Oregon) Region.", + }, { + Value: "us-west-1", + Help: "US West (Northern California) Region.", + }, { + Value: "eu-west-1", + Help: "EU (Ireland) Region.", + }, { + Value: "EU", + Help: "EU Region.", + }, { + Value: "ap-southeast-1", + Help: "Asia Pacific (Singapore) Region.", + }, { + Value: "ap-southeast-2", + Help: "Asia Pacific (Sydney) Region.", + }, { + Value: "ap-northeast-1", + Help: "Asia Pacific (Tokyo) Region.", + }, { + Value: "sa-east-1", + Help: "South America (Sao Paulo) Region.", + }}, + }}, + }) } // Constants @@ -60,57 +129,54 @@ type FsObjectS3 struct { // ------------------------------------------------------------ -// Globals -var ( - // Flags - awsAccessKeyId = flag.String("aws-access-key-id", os.Getenv("AWS_ACCESS_KEY_ID"), "AWS Access Key ID. Defaults to environment var AWS_ACCESS_KEY_ID.") - awsSecretAccessKey = flag.String("aws-secret-access-key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "AWS Secret Access Key (password). Defaults to environment var AWS_SECRET_ACCESS_KEY.") - // AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region - s3Endpoint = flag.String("s3-endpoint", os.Getenv("S3_ENDPOINT"), "S3 Endpoint. Defaults to environment var S3_ENDPOINT then https://s3.amazonaws.com/.") - s3LocationConstraint = flag.String("s3-location-constraint", os.Getenv("S3_LOCATION_CONSTRAINT"), "Location constraint for creating buckets only. Defaults to environment var S3_LOCATION_CONSTRAINT.") -) - // String converts this FsS3 to a string func (f *FsS3) String() string { return fmt.Sprintf("S3 bucket %s", f.bucket) } +// Pattern to match a s3 path +var matcher = regexp.MustCompile(`^([^/]*)(.*)$`) + // parseParse parses a s3 'url' func s3ParsePath(path string) (bucket, directory string, err error) { - parts := Match.FindAllStringSubmatch(path, -1) - if len(parts) != 1 || len(parts[0]) != 3 { - err = fmt.Errorf("Couldn't parse s3 url %q", path) + parts := matcher.FindStringSubmatch(path) + if parts == nil { + err = fmt.Errorf("Couldn't parse bucket out of s3 path %q", path) } else { - bucket, directory = parts[0][1], parts[0][2] + bucket, directory = parts[1], parts[2] directory = strings.Trim(directory, "/") } return } // s3Connection makes a connection to s3 -func s3Connection() (*s3.S3, error) { +func s3Connection(name string) (*s3.S3, error) { // Make the auth - if *awsAccessKeyId == "" { - return nil, errors.New("Need -aws-access-key-id or environmental variable AWS_ACCESS_KEY_ID") + accessKeyId := fs.ConfigFile.MustValue(name, "access_key_id") + if accessKeyId == "" { + return nil, errors.New("access_key_id not found") } - if *awsSecretAccessKey == "" { - return nil, errors.New("Need -aws-secret-access-key or environmental variable AWS_SECRET_ACCESS_KEY") + secretAccessKey := fs.ConfigFile.MustValue(name, "secret_access_key") + if secretAccessKey == "" { + return nil, errors.New("secret_access_key not found") } - auth := aws.Auth{AccessKey: *awsAccessKeyId, SecretKey: *awsSecretAccessKey} + auth := aws.Auth{AccessKey: accessKeyId, SecretKey: secretAccessKey} // FIXME look through all the regions by name and use one of them if found // Synthesize the region - if *s3Endpoint == "" { - *s3Endpoint = "https://s3.amazonaws.com/" + s3Endpoint := fs.ConfigFile.MustValue(name, "endpoint") + if s3Endpoint == "" { + s3Endpoint = "https://s3.amazonaws.com/" } region := aws.Region{ Name: "s3", - S3Endpoint: *s3Endpoint, + S3Endpoint: s3Endpoint, S3LocationConstraint: false, } - if *s3LocationConstraint != "" { - region.Name = *s3LocationConstraint + s3LocationConstraint := fs.ConfigFile.MustValue(name, "location_constraint") + if s3LocationConstraint != "" { + region.Name = s3LocationConstraint region.S3LocationConstraint = true } @@ -119,7 +185,7 @@ func s3Connection() (*s3.S3, error) { } // NewFsS3 contstructs an FsS3 from the path, bucket:path -func NewFs(path string) (fs.Fs, error) { +func NewFs(name, path string) (fs.Fs, error) { bucket, directory, err := s3ParsePath(path) if err != nil { return nil, err @@ -127,7 +193,7 @@ func NewFs(path string) (fs.Fs, error) { if directory != "" { return nil, fmt.Errorf("Directories not supported yet in %q: %q", path, directory) } - c, err := s3Connection() + c, err := s3Connection(name) if err != nil { return nil, err } diff --git a/swift/fs.go b/swift/fs.go index d1cb70fb2..8a92421cd 100644 --- a/swift/fs.go +++ b/swift/fs.go @@ -5,24 +5,51 @@ package swift import ( "errors" - "flag" "fmt" - "github.com/ncw/rclone/fs" - "github.com/ncw/swift" "io" "log" - "os" "regexp" "strings" "time" -) -// Pattern to match a swift url -var Match = regexp.MustCompile(`^swift://([^/]*)(.*)$`) + "github.com/ncw/rclone/fs" + "github.com/ncw/swift" +) // Register with Fs func init() { - fs.Register(Match, NewFs) + fs.Register(&fs.FsInfo{ + Name: "swift", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "user", + Help: "User name to log in.", + }, { + Name: "key", + Help: "API key or password.", + }, { + Name: "auth", + Help: "Authentication URL for server.", + Examples: []fs.OptionExample{{ + Help: "Rackspace US", + Value: "https://auth.api.rackspacecloud.com/v1.0", + }, { + Help: "Rackspace UK", + Value: "https://lon.auth.api.rackspacecloud.com/v1.0", + }, { + Help: "Rackspace v2", + Value: "https://identity.api.rackspacecloud.com/v2.0", + }, { + Help: "Memset Memstore UK", + Value: "https://auth.storage.memset.com/v1.0", + }, { + Help: "Memset Memstore UK v2", + Value: "https://auth.storage.memset.com/v2.0", + }}, + }, + // snet = flag.Bool("swift-snet", false, "Use internal service network") // FIXME not implemented + }, + }) } // FsSwift represents a remote swift server @@ -44,48 +71,44 @@ type FsObjectSwift struct { // ------------------------------------------------------------ -// Globals -var ( - // Flags - // FIXME make these part of swift so we get a standard set of flags? - authUrl = flag.String("swift-auth", os.Getenv("ST_AUTH"), "Auth URL for server. Defaults to environment var ST_AUTH.") - userName = flag.String("swift-user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.") - apiKey = flag.String("swift-key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.") - snet = flag.Bool("swift-snet", false, "Use internal service network") // FIXME not implemented -) - // String converts this FsSwift to a string func (f *FsSwift) String() string { return fmt.Sprintf("Swift container %s", f.container) } +// Pattern to match a swift path +var matcher = regexp.MustCompile(`^([^/]*)(.*)$`) + // parseParse parses a swift 'url' func parsePath(path string) (container, directory string, err error) { - parts := Match.FindAllStringSubmatch(path, -1) - if len(parts) != 1 || len(parts[0]) != 3 { - err = fmt.Errorf("Couldn't parse swift url %q", path) + parts := matcher.FindStringSubmatch(path) + if parts == nil { + err = fmt.Errorf("Couldn't find container in swift path %q", path) } else { - container, directory = parts[0][1], parts[0][2] + container, directory = parts[1], parts[2] directory = strings.Trim(directory, "/") } return } // swiftConnection makes a connection to swift -func swiftConnection() (*swift.Connection, error) { - if *userName == "" { - return nil, errors.New("Need -user or environmental variable ST_USER") +func swiftConnection(name string) (*swift.Connection, error) { + userName := fs.ConfigFile.MustValue(name, "user") + if userName == "" { + return nil, errors.New("user not found") } - if *apiKey == "" { - return nil, errors.New("Need -key or environmental variable ST_KEY") + apiKey := fs.ConfigFile.MustValue(name, "key") + if apiKey == "" { + return nil, errors.New("key not found") } - if *authUrl == "" { - return nil, errors.New("Need -auth or environmental variable ST_AUTH") + authUrl := fs.ConfigFile.MustValue(name, "auth") + if authUrl == "" { + return nil, errors.New("auth not found") } c := &swift.Connection{ - UserName: *userName, - ApiKey: *apiKey, - AuthUrl: *authUrl, + UserName: userName, + ApiKey: apiKey, + AuthUrl: authUrl, } err := c.Authenticate() if err != nil { @@ -95,7 +118,7 @@ func swiftConnection() (*swift.Connection, error) { } // NewFs contstructs an FsSwift from the path, container:path -func NewFs(path string) (fs.Fs, error) { +func NewFs(name, path string) (fs.Fs, error) { container, directory, err := parsePath(path) if err != nil { return nil, err @@ -103,7 +126,7 @@ func NewFs(path string) (fs.Fs, error) { if directory != "" { return nil, fmt.Errorf("Directories not supported yet in %q", path) } - c, err := swiftConnection() + c, err := swiftConnection(name) if err != nil { return nil, err }