Implement Hubic storage system - fixes #200

docs/content/about.md

@@ -20,6 +20,7 @@ Rclone is a command line program to sync files and directories to and from
   * Google Cloud Storage
   * Amazon Cloud Drive
   * Microsoft One Drive
+  * Hubic
   * The local filesystem
 
 Features

docs/content/hubic.md

@@ -0,0 +1,98 @@
+---
+title: "Hubic"
+description: "Rclone docs for Hubic"
+date: "2015-11-08"
+---
+
+<i class="fa fa-space-shuttle"></i> Hubic
+-----------------------------------------
+
+Paths are specified as `remote:path`
+
+Paths are specified as `remote:container` (or `remote:` for the `lsd`
+command.)  You may put subdirectories in too, eg `remote:container/path/to/dir`.
+
+The initial setup for Hubic involves getting a token from Hubic which
+you need to do in your browser.  `rclone config` walks you through it.
+
+Here is an example of how to make a remote called `remote`.  First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+```
+n) New remote
+d) Delete remote
+q) Quit config
+e/n/d/q> n
+name> remote
+What type of source is it?
+Choose a number from below
+ 1) amazon cloud drive
+ 2) drive
+ 3) dropbox
+ 4) google cloud storage
+ 5) local
+ 6) onedrive
+ 7) hubic
+ 8) s3
+ 9) swift
+type> 7
+Hubic App Client Id - leave blank normally.
+client_id> 
+Hubic App Client Secret - leave blank normally.
+client_secret> 
+Remote config
+If your browser doesn't open automatically go to the following link: http://localhost:53682/auth
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+--------------------
+[remote]
+client_id = 
+client_secret = 
+token = {"access_token":"XXXXXX"}
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+Note that rclone runs a webserver on your local machine to collect the
+token as returned from Hubic. This only runs from the moment it opens
+your browser to the moment you get back the verification code.  This
+is on `http://127.0.0.1:53682/` and this it may require you to unblock
+it temporarily if you are running a host firewall.
+
+Once configured you can then use `rclone` like this,
+
+List containers in the top level of your Hubic
+
+    rclone lsd remote:
+
+List all the files in your Hubic
+
+    rclone ls remote:
+
+To copy a local directory to an Hubic directory called backup
+
+    rclone copy /home/source remote:backup
+
+### Modified time ###
+
+The modified time is stored as metadata on the object as
+`X-Object-Meta-Mtime` as floating point since the epoch accurate to 1
+ns.
+
+This is a defacto standard (used in the official python-swiftclient
+amongst others) for storing the modification time for an object.
+
+Note that Hubic wraps the Swift backend, so most of the properties of
+are the same.
+
+### Limitations ###
+
+Code to refresh the OpenStack token isn't done yet which may cause
+problems with very long transfers.

docs/content/overview.md

@@ -24,6 +24,7 @@ Here is an overview of the major features of each cloud storage system.
 | Google Cloud Storage   | Yes     | Yes     | No             | No              |
 | Amazon Cloud Drive     | Yes     | No      | Yes            | No              |
 | Microsoft One Drive    | No      | Yes     | Yes            | No              |
+| Hubic                  | Yes     | Yes     | No             | No              |
 | The local filesystem   | Yes     | Yes     | Depends        | No              |
 
 ### MD5SUM ###

docs/layouts/chrome/navbar.html

@@ -38,6 +38,7 @@
                     <li><a href="/googlecloudstorage/"><i class="fa fa-google"></i> Google Cloud Storage</a></li>
                     <li><a href="/amazonclouddrive/"><i class="fa fa-amazon"></i> Amazon Cloud Drive</a></li>
                     <li><a href="/onedrive/"><i class="fa fa-windows"></i> Microsoft One Drive</a></li>
+                    <li><a href="/hubic/"><i class="fa fa-space-shuttle"></i> Hubic</a></li>
                     <li><a href="/local/"><i class="fa fa-file"></i> Local</a></li>
                   </ul>
                 </li>

fs/operations_test.go

@@ -24,6 +24,7 @@ import (
 	_ "github.com/ncw/rclone/drive"
 	_ "github.com/ncw/rclone/dropbox"
 	_ "github.com/ncw/rclone/googlecloudstorage"
+	_ "github.com/ncw/rclone/hubic"
 	_ "github.com/ncw/rclone/local"
 	_ "github.com/ncw/rclone/onedrive"
 	_ "github.com/ncw/rclone/s3"

fs/test_all.sh

@@ -10,6 +10,7 @@ TestGoogleCloudStorage:
 TestDropbox:
 TestAmazonCloudDrive:
 TestOneDrive:
+TestHubic:
 "
 
 function test_remote {

fstest/fstests/fstests.go

@@ -435,7 +435,14 @@ func TestObjectString(t *testing.T) {
 func TestObjectFs(t *testing.T) {
 	skipIfNotOk(t)
 	obj := findObject(t, file1.Path)
-	if obj.Fs() != remote {
+	equal := obj.Fs() == remote
+	if !equal {
+		// Check to see if this wraps something else
+		if unwrap, ok := remote.(fs.UnWrapper); ok {
+			equal = obj.Fs() == unwrap.UnWrap()
+		}
+	}
+	if !equal {
 		t.Errorf("Fs is wrong %v != %v", obj.Fs(), remote)
 	}
 }
@@ -558,7 +565,13 @@ func TestLimitedFs(t *testing.T) {
 	fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy})
 	_, ok := fileRemote.(*fs.Limited)
 	if !ok {
-		t.Errorf("%v is not a fs.Limited", fileRemote)
+		// Check to see if this wraps a Limited FS
+		if unwrap, hasUnWrap := fileRemote.(fs.UnWrapper); hasUnWrap {
+			_, ok = unwrap.UnWrap().(*fs.Limited)
+		}
+		if !ok {
+			t.Errorf("%v is not a fs.Limited", fileRemote)
+		}
 	}
 }
 

fstest/fstests/gen_tests.go

@@ -132,5 +132,6 @@ func main() {
 	generateTestProgram(t, fns, "Dropbox")
 	generateTestProgram(t, fns, "AmazonCloudDrive")
 	generateTestProgram(t, fns, "OneDrive")
+	generateTestProgram(t, fns, "Hubic")
 	log.Printf("Done")
 }

hubic/auth.go

@@ -0,0 +1,54 @@
+package hubic
+
+import (
+	"net/http"
+
+	"github.com/ncw/swift"
+)
+
+// auth is an authenticator for swift
+type auth struct {
+	f *Fs
+}
+
+// newAuth creates a swift authenticator
+func newAuth(f *Fs) *auth {
+	return &auth{
+		f: f,
+	}
+}
+
+// Request constructs a http.Request for authentication
+//
+// returns nil for not needed
+func (a *auth) Request(*swift.Connection) (*http.Request, error) {
+	err := a.f.getCredentials()
+	if err != nil {
+		return nil, err
+	}
+	return nil, nil
+}
+
+// Response parses the result of an http request
+func (a *auth) Response(resp *http.Response) error {
+	return nil
+}
+
+// The public storage URL - set Internal to true to read
+// internal/service net URL
+func (a *auth) StorageUrl(Internal bool) string {
+	return a.f.credentials.Endpoint
+}
+
+// The access token
+func (a *auth) Token() string {
+	return a.f.credentials.Token
+}
+
+// The CDN url if available
+func (a *auth) CdnUrl() string {
+	return ""
+}
+
+// Check the interfaces are satisfied
+var _ swift.Authenticator = (*auth)(nil)

hubic/hubic.go

@@ -0,0 +1,226 @@
+// Package hubic provides an interface to the Hubic object storage
+// system.
+package hubic
+
+// This uses the normal swift mechanism to update the credentials and
+// ignores the expires field returned by the Hubic API.  This may need
+// to be revisted after some actual experience.
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/oauthutil"
+	"github.com/ncw/rclone/swift"
+	swiftLib "github.com/ncw/swift"
+	"golang.org/x/oauth2"
+)
+
+const (
+	rcloneClientID     = "api_hubic_svWP970PvSWbw5G3PzrAqZ6X2uHeZBPI"
+	rcloneClientSecret = "8MrG3pjWyJya4OnO9ZTS4emI+9fa1ouPgvfD2MbTzfDYvO/H5czFxsTXtcji4/Hz3snz8/CrzMzlxvP9//Ty/Q=="
+)
+
+// Globals
+var (
+	// Description of how to auth for this app
+	oauthConfig = &oauth2.Config{
+		Scopes: []string{
+			"credentials.r", // Read Openstack credentials
+		},
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  "https://api.hubic.com/oauth/auth/",
+			TokenURL: "https://api.hubic.com/oauth/token/",
+		},
+		ClientID:     rcloneClientID,
+		ClientSecret: fs.Reveal(rcloneClientSecret),
+		RedirectURL:  oauthutil.RedirectLocalhostURL,
+	}
+)
+
+// Register with Fs
+func init() {
+	fs.Register(&fs.Info{
+		Name:  "hubic",
+		NewFs: NewFs,
+		Config: func(name string) {
+			err := oauthutil.Config(name, oauthConfig)
+			if err != nil {
+				log.Fatalf("Failed to configure token: %v", err)
+			}
+		},
+		Options: []fs.Option{{
+			Name: oauthutil.ConfigClientID,
+			Help: "Hubic Client Id - leave blank normally.",
+		}, {
+			Name: oauthutil.ConfigClientSecret,
+			Help: "Hubic Client Secret - leave blank normally.",
+		}},
+	})
+}
+
+// credentials is the JSON returned from the Hubic API to read the
+// OpenStack credentials
+type credentials struct {
+	Token    string `json:"token"`    // Openstack token
+	Endpoint string `json:"endpoint"` // Openstack endpoint
+	Expires  string `json:"expires"`  // Expires date - eg "2015-11-09T14:24:56+01:00"
+}
+
+// Fs represents a remote hubic
+type Fs struct {
+	fs.Fs                    // wrapped Fs
+	client      *http.Client // client for oauth api
+	credentials credentials  // returned from the Hubic API
+	expires     time.Time    // time credentials expire
+}
+
+// Object describes a swift object
+type Object struct {
+	*swift.Object
+}
+
+// Return a string version
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.Object.String()
+}
+
+// ------------------------------------------------------------
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+	if f.Fs == nil {
+		return "Hubic"
+	}
+	return fmt.Sprintf("Hubic %s", f.Fs.String())
+}
+
+// checkClose is a utility function used to check the return from
+// Close in a defer statement.
+func checkClose(c io.Closer, err *error) {
+	cerr := c.Close()
+	if *err == nil {
+		*err = cerr
+	}
+}
+
+// getCredentials reads the OpenStack Credentials using the Hubic API
+//
+// The credentials are read into the Fs
+func (f *Fs) getCredentials() (err error) {
+	req, err := http.NewRequest("GET", "https://api.hubic.com/1.0/account/credentials", nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Add("User-Agent", fs.UserAgent)
+	resp, err := f.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer checkClose(resp.Body, &err)
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return fmt.Errorf("Failed to get credentials: %s", resp.Status)
+	}
+	decoder := json.NewDecoder(resp.Body)
+	var result credentials
+	err = decoder.Decode(&result)
+	if err != nil {
+		return err
+	}
+	// fs.Debug(f, "Got credentials %+v", result)
+	if result.Token == "" || result.Endpoint == "" || result.Expires == "" {
+		return fmt.Errorf("Couldn't read token, result and expired from credentials")
+	}
+	f.credentials = result
+	expires, err := time.Parse(time.RFC3339, result.Expires)
+	if err != nil {
+		return err
+	}
+	f.expires = expires
+	fs.Debug(f, "Got swift credentials (expiry %v in %v)", f.expires, f.expires.Sub(time.Now()))
+	return nil
+}
+
+// NewFs constructs an Fs from the path, container:path
+func NewFs(name, root string) (fs.Fs, error) {
+	client, err := oauthutil.NewClient(name, oauthConfig)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to configure Hubic: %v", err)
+	}
+
+	f := &Fs{
+		client: client,
+	}
+
+	// Make the swift Connection
+	c := &swiftLib.Connection{
+		Auth:           newAuth(f),
+		UserAgent:      fs.UserAgent,
+		ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
+		Timeout:        10 * fs.Config.Timeout,        // Use the timeouts in the transport
+		Transport:      fs.Config.Transport(),
+	}
+	err = c.Authenticate()
+	if err != nil {
+		return nil, fmt.Errorf("Error authenticating swift connection: %v", err)
+	}
+
+	// Make inner swift Fs from the connection
+	swiftFs, err := swift.NewFsWithConnection(name, root, c)
+	if err != nil {
+		return nil, err
+	}
+	f.Fs = swiftFs
+	return f, nil
+}
+
+// Purge deletes all the files and the container
+//
+// Optional interface: Only implement this if you have a way of
+// deleting all the files quicker than just running Remove() on the
+// result of List()
+func (f *Fs) Purge() error {
+	fPurge, ok := f.Fs.(fs.Purger)
+	if !ok {
+		return fs.ErrorCantPurge
+	}
+	return fPurge.Purge()
+}
+
+// Copy src to this remote using server side copy operations.
+//
+// This is stored with the remote path given
+//
+// It returns the destination Object and a possible error
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantCopy
+func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
+	fCopy, ok := f.Fs.(fs.Copier)
+	if !ok {
+		return nil, fs.ErrorCantCopy
+	}
+	return fCopy.Copy(src, remote)
+}
+
+// UnWrap returns the Fs that this Fs is wrapping
+func (f *Fs) UnWrap() fs.Fs {
+	return f.Fs
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs        = (*Fs)(nil)
+	_ fs.Purger    = (*Fs)(nil)
+	_ fs.Copier    = (*Fs)(nil)
+	_ fs.UnWrapper = (*Fs)(nil)
+)

hubic/hubic_test.go

@@ -0,0 +1,56 @@
+// Test Hubic filesystem interface
+//
+// Automatically generated - DO NOT EDIT
+// Regenerate with: make gen_tests
+package hubic_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+	"github.com/ncw/rclone/hubic"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*hubic.Object)(nil))
+	fstests.RemoteName = "TestHubic:"
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                  { fstests.TestInit(t) }
+func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
+func TestFsListDirRoot(t *testing.T)         { fstests.TestFsListDirRoot(t) }
+func TestFsListRoot(t *testing.T)            { fstests.TestFsListRoot(t) }
+func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
+func TestFsCopy(t *testing.T)                { fstests.TestFsCopy(t) }
+func TestFsMove(t *testing.T)                { fstests.TestFsMove(t) }
+func TestFsDirMove(t *testing.T)             { fstests.TestFsDirMove(t) }
+func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
+func TestLimitedFs(t *testing.T)             { fstests.TestLimitedFs(t) }
+func TestLimitedFsNotFound(t *testing.T)     { fstests.TestLimitedFsNotFound(t) }
+func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

make_manual.py

@@ -25,6 +25,7 @@ docs = [
     "googlecloudstorage.md",
     "amazonclouddrive.md",
     "onedrive.md",
+    "hubic.md",
     "local.md",
     "changelog.md",
     "bugs.md",

rclone.go

@@ -20,6 +20,7 @@ import (
 	_ "github.com/ncw/rclone/drive"
 	_ "github.com/ncw/rclone/dropbox"
 	_ "github.com/ncw/rclone/googlecloudstorage"
+	_ "github.com/ncw/rclone/hubic"
 	_ "github.com/ncw/rclone/local"
 	_ "github.com/ncw/rclone/onedrive"
 	_ "github.com/ncw/rclone/s3"

swift/swift.go

@@ -159,16 +159,13 @@ func swiftConnection(name string) (*swift.Connection, error) {
 	return c, nil
 }
 
-// NewFs contstructs an Fs from the path, container:path
-func NewFs(name, root string) (fs.Fs, error) {
+// NewFsWithConnection contstructs an Fs from the path, container:path
+// and authenticated connection
+func NewFsWithConnection(name, root string, c *swift.Connection) (fs.Fs, error) {
 	container, directory, err := parsePath(root)
 	if err != nil {
 		return nil, err
 	}
-	c, err := swiftConnection(name)
-	if err != nil {
-		return nil, err
-	}
 	f := &Fs{
 		name:              name,
 		c:                 *c,
@@ -196,6 +193,15 @@ func NewFs(name, root string) (fs.Fs, error) {
 	return f, nil
 }
 
+// NewFs contstructs an Fs from the path, container:path
+func NewFs(name, root string) (fs.Fs, error) {
+	c, err := swiftConnection(name)
+	if err != nil {
+		return nil, err
+	}
+	return NewFsWithConnection(name, root, c)
+}
+
 // Return an FsObject from a path
 //
 // May return nil if an error occurred