Implement Gofile backend - fixes #4632

README.md

@@ -56,6 +56,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
   * Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
+  * GoFile [:page_facing_up:](https://rclone.org/gofile/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)
   * Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)

backend/all/all.go

@@ -18,6 +18,7 @@ import (
 	_ "github.com/rclone/rclone/backend/fichier"
 	_ "github.com/rclone/rclone/backend/filefabric"
 	_ "github.com/rclone/rclone/backend/ftp"
+	_ "github.com/rclone/rclone/backend/gofile"
 	_ "github.com/rclone/rclone/backend/googlecloudstorage"
 	_ "github.com/rclone/rclone/backend/googlephotos"
 	_ "github.com/rclone/rclone/backend/hasher"

backend/gofile/api/types.go

@@ -0,0 +1,311 @@
+// Package api has type definitions for gofile
+//
+// Converted from the API docs with help from https://mholt.github.io/json-to-go/
+package api
+
+import (
+	"fmt"
+	"time"
+)
+
+const (
+	// 2017-05-03T07:26:10-07:00
+	timeFormat = `"` + time.RFC3339 + `"`
+)
+
+// Time represents date and time information for the
+// gofile API, by using RFC3339
+type Time time.Time
+
+// MarshalJSON turns a Time into JSON (in UTC)
+func (t *Time) MarshalJSON() (out []byte, err error) {
+	timeString := (*time.Time)(t).Format(timeFormat)
+	return []byte(timeString), nil
+}
+
+// UnmarshalJSON turns JSON into a Time
+func (t *Time) UnmarshalJSON(data []byte) error {
+	newT, err := time.Parse(timeFormat, string(data))
+	if err != nil {
+		return err
+	}
+	*t = Time(newT)
+	return nil
+}
+
+// Error is returned from gofile when things go wrong
+type Error struct {
+	Status string `json:"status"`
+}
+
+// Error returns a string for the error and satisfies the error interface
+func (e Error) Error() string {
+	out := fmt.Sprintf("Error %q", e.Status)
+	return out
+}
+
+// IsError returns true if there is an error
+func (e Error) IsError() bool {
+	return e.Status != "ok"
+}
+
+// Err returns err if not nil, or e if IsError or nil
+func (e Error) Err(err error) error {
+	if err != nil {
+		return err
+	}
+	if e.IsError() {
+		return e
+	}
+	return nil
+}
+
+// Check Error satisfies the error interface
+var _ error = (*Error)(nil)
+
+// Types of things in Item
+const (
+	ItemTypeFolder = "folder"
+	ItemTypeFile   = "file"
+)
+
+// Item describes a folder or a file as returned by /contents
+type Item struct {
+	ID            string                 `json:"id"`
+	ParentFolder  string                 `json:"parentFolder"`
+	Type          string                 `json:"type"`
+	Name          string                 `json:"name"`
+	Size          int64                  `json:"size"`
+	Code          string                 `json:"code"`
+	CreateTime    int64                  `json:"createTime"`
+	ModTime       int64                  `json:"modTime"`
+	Link          string                 `json:"link"`
+	MD5           string                 `json:"md5"`
+	MimeType      string                 `json:"mimetype"`
+	ChildrenCount int                    `json:"childrenCount"`
+	DirectLinks   map[string]*DirectLink `json:"directLinks"`
+	//Public         bool     `json:"public"`
+	//ServerSelected string   `json:"serverSelected"`
+	//Thumbnail      string   `json:"thumbnail"`
+	//DownloadCount int      `json:"downloadCount"`
+	//TotalDownloadCount int64            `json:"totalDownloadCount"`
+	//TotalSize int64            `json:"totalSize"`
+	//ChildrenIDs   []string               `json:"childrenIds"`
+	Children map[string]*Item `json:"children"`
+}
+
+// ToNativeTime converts a go time to a native time
+func ToNativeTime(t time.Time) int64 {
+	return t.Unix()
+}
+
+// FromNativeTime converts native time to a go time
+func FromNativeTime(t int64) time.Time {
+	return time.Unix(t, 0)
+}
+
+// DirectLink describes a direct link to a file so it can be
+// downloaded by third parties.
+type DirectLink struct {
+	ExpireTime       int64  `json:"expireTime"`
+	SourceIpsAllowed []any  `json:"sourceIpsAllowed"`
+	DomainsAllowed   []any  `json:"domainsAllowed"`
+	Auth             []any  `json:"auth"`
+	IsReqLink        bool   `json:"isReqLink"`
+	DirectLink       string `json:"directLink"`
+}
+
+// Contents is returned from the /contents call
+type Contents struct {
+	Error
+	Data struct {
+		Item
+	} `json:"data"`
+	Metadata Metadata `json:"metadata"`
+}
+
+// Metadata is returned when paging is in use
+type Metadata struct {
+	TotalCount  int  `json:"totalCount"`
+	TotalPages  int  `json:"totalPages"`
+	Page        int  `json:"page"`
+	PageSize    int  `json:"pageSize"`
+	HasNextPage bool `json:"hasNextPage"`
+}
+
+// AccountsGetID is the result of /accounts/getid
+type AccountsGetID struct {
+	Error
+	Data struct {
+		ID string `json:"id"`
+	} `json:"data"`
+}
+
+// Stats of storage and traffic
+type Stats struct {
+	FolderCount            int64 `json:"folderCount"`
+	FileCount              int64 `json:"fileCount"`
+	Storage                int64 `json:"storage"`
+	TrafficDirectGenerated int64 `json:"trafficDirectGenerated"`
+	TrafficReqDownloaded   int64 `json:"trafficReqDownloaded"`
+	TrafficWebDownloaded   int64 `json:"trafficWebDownloaded"`
+}
+
+// AccountsGet is the result of /accounts/{id}
+type AccountsGet struct {
+	Error
+	Data struct {
+		ID                             string `json:"id"`
+		Email                          string `json:"email"`
+		Tier                           string `json:"tier"`
+		PremiumType                    string `json:"premiumType"`
+		Token                          string `json:"token"`
+		RootFolder                     string `json:"rootFolder"`
+		SubscriptionProvider           string `json:"subscriptionProvider"`
+		SubscriptionEndDate            int    `json:"subscriptionEndDate"`
+		SubscriptionLimitDirectTraffic int64  `json:"subscriptionLimitDirectTraffic"`
+		SubscriptionLimitStorage       int64  `json:"subscriptionLimitStorage"`
+		StatsCurrent                   Stats  `json:"statsCurrent"`
+		// StatsHistory                   map[int]map[int]map[int]Stats `json:"statsHistory"`
+	} `json:"data"`
+}
+
+// CreateFolderRequest is the input to /contents/createFolder
+type CreateFolderRequest struct {
+	ParentFolderID string `json:"parentFolderId"`
+	FolderName     string `json:"folderName"`
+	ModTime        int64  `json:"modTime,omitempty"`
+}
+
+// CreateFolderResponse is the output from /contents/createFolder
+type CreateFolderResponse struct {
+	Error
+	Data Item `json:"data"`
+}
+
+// DeleteRequest is the input to DELETE /contents
+type DeleteRequest struct {
+	ContentsID string `json:"contentsId"` // comma separated list of IDs
+}
+
+// DeleteResponse is the input to DELETE /contents
+type DeleteResponse struct {
+	Error
+	Data map[string]Error
+}
+
+// Server is an upload server
+type Server struct {
+	Name string `json:"name"`
+	Zone string `json:"zone"`
+}
+
+// String returns a string representation of the Server
+func (s *Server) String() string {
+	return fmt.Sprintf("%s (%s)", s.Name, s.Zone)
+}
+
+// Root returns the root URL for the server
+func (s *Server) Root() string {
+	return fmt.Sprintf("https://%s.gofile.io/", s.Name)
+}
+
+// URL returns the upload URL for the server
+func (s *Server) URL() string {
+	return fmt.Sprintf("https://%s.gofile.io/contents/uploadfile", s.Name)
+}
+
+// ServersResponse is the output from /servers
+type ServersResponse struct {
+	Error
+	Data struct {
+		Servers []Server `json:"servers"`
+	} `json:"data"`
+}
+
+// UploadResponse is returned by POST /contents/uploadfile
+type UploadResponse struct {
+	Error
+	Data Item `json:"data"`
+}
+
+// DirectLinksRequest specifies the parameters for the direct link
+type DirectLinksRequest struct {
+	ExpireTime       int64 `json:"expireTime,omitempty"`
+	SourceIpsAllowed []any `json:"sourceIpsAllowed,omitempty"`
+	DomainsAllowed   []any `json:"domainsAllowed,omitempty"`
+	Auth             []any `json:"auth,omitempty"`
+}
+
+// DirectLinksResult is returned from POST /contents/{id}/directlinks
+type DirectLinksResult struct {
+	Error
+	Data struct {
+		ExpireTime       int64  `json:"expireTime"`
+		SourceIpsAllowed []any  `json:"sourceIpsAllowed"`
+		DomainsAllowed   []any  `json:"domainsAllowed"`
+		Auth             []any  `json:"auth"`
+		IsReqLink        bool   `json:"isReqLink"`
+		ID               string `json:"id"`
+		DirectLink       string `json:"directLink"`
+	} `json:"data"`
+}
+
+// UpdateItemRequest describes the updates to be done to an item for PUT /contents/{id}/update
+//
+// The Value of the attribute to define :
+// For Attribute "name" : The name of the content (file or folder)
+// For Attribute "description" : The description displayed on the download page (folder only)
+// For Attribute "tags" : A comma-separated list of tags (folder only)
+// For Attribute "public" : either true or false (folder only)
+// For Attribute "expiry" : A unix timestamp of the expiration date (folder only)
+// For Attribute "password" : The password to set (folder only)
+type UpdateItemRequest struct {
+	Attribute string `json:"attribute"`
+	Value     any    `json:"attributeValue"`
+}
+
+// UpdateItemResponse is returned by PUT /contents/{id}/update
+type UpdateItemResponse struct {
+	Error
+	Data Item `json:"data"`
+}
+
+// MoveRequest is the input to /contents/move
+type MoveRequest struct {
+	FolderID   string `json:"folderId"`
+	ContentsID string `json:"contentsId"` // comma separated list of IDs
+}
+
+// MoveResponse is returned by POST /contents/move
+type MoveResponse struct {
+	Error
+	Data map[string]struct {
+		Error
+		Item `json:"data"`
+	} `json:"data"`
+}
+
+// CopyRequest is the input to /contents/copy
+type CopyRequest struct {
+	FolderID   string `json:"folderId"`
+	ContentsID string `json:"contentsId"` // comma separated list of IDs
+}
+
+// CopyResponse is returned by POST /contents/copy
+type CopyResponse struct {
+	Error
+	Data map[string]struct {
+		Error
+		Item `json:"data"`
+	} `json:"data"`
+}
+
+// UploadServerStatus is returned when fetching the root of an upload server
+type UploadServerStatus struct {
+	Error
+	Data struct {
+		Server string `json:"server"`
+		Test   string `json:"test"`
+	} `json:"data"`
+}

backend/gofile/gofile_test.go

@@ -0,0 +1,17 @@
+// Test Gofile filesystem interface
+package gofile_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/gofile"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestGoFile:",
+		NilObject:  (*gofile.Object)(nil),
+	})
+}

bin/make_manual.py

@@ -42,6 +42,7 @@ docs = [
     "dropbox.md",
     "filefabric.md",
     "ftp.md",
+    "gofile.md",
     "googlecloudstorage.md",
     "drive.md",
     "googlephotos.md",

docs/content/_index.md

@@ -123,6 +123,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="Enterprise File Fabric" home="https://storagemadeeasy.com/about/" config="/filefabric/" >}}
 {{< provider name="Fastmail Files" home="https://www.fastmail.com/" config="/webdav/#fastmail-files" >}}
 {{< provider name="FTP" home="https://en.wikipedia.org/wiki/File_Transfer_Protocol" config="/ftp/" >}}
+{{< provider name="Gofile" home="https://gofile.io/" config="/gofile/" >}}
 {{< provider name="Google Cloud Storage" home="https://cloud.google.com/storage/" config="/googlecloudstorage/" >}}
 {{< provider name="Google Drive" home="https://www.google.com/drive/" config="/drive/" >}}
 {{< provider name="Google Photos" home="https://www.google.com/photos/about/" config="/googlephotos/" >}}

docs/content/docs.md

@@ -43,6 +43,7 @@ See the following for detailed instructions for
   * [Dropbox](/dropbox/)
   * [Enterprise File Fabric](/filefabric/)
   * [FTP](/ftp/)
+  * [Gofile](/gofile/)
   * [Google Cloud Storage](/googlecloudstorage/)
   * [Google Drive](/drive/)
   * [Google Photos](/googlephotos/)

docs/content/gofile.md

@@ -0,0 +1,256 @@
+---
+title: "Gofile"
+description: "Rclone docs for Gofile"
+versionIntroduced: "v1.68"
+---
+
+# {{< icon "fa fa-folder" >}} Gofile
+
+[Gofile](https://gofile.io) is a content storage and distribution
+platform. Its aim is to provide as much service as possible for free
+or at a very low price.
+
+The initial setup for Gofile involves logging in to the web interface
+and going to the "My Profile" section. Copy the "Account API token"
+for use in the config file.
+
+Note that if you wish to connect rclone to Gofile you will need a
+premium account.
+
+## Configuration
+
+Here is an example of how to make a remote called `remote`.  First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+```
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+
+Enter name for new remote.
+name> remote
+
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+XX / Gofile
+   \ (gofile)
+Storage> gofile
+
+Option access_token.
+API Access token
+You can get this from the web control panel.
+Enter a value. Press Enter to leave empty.
+access_token> YOURACCESSTOKEN
+
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+
+Configuration complete.
+Options:
+- type: gofile
+- access_token: YOURACCESSTOKEN
+Keep this "remote" remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+Once configured you can then use `rclone` like this,
+
+List directories and files in the top level of your Gofile
+
+    rclone lsf remote:
+
+To copy a local directory to an Gofile directory called backup
+
+    rclone copy /home/source remote:backup
+
+### Modification times and hashes
+
+Gofile supports modification times with a resolution of 1 second.
+
+Gofile supports MD5 hashes, so you can use the `--checksum` flag.
+
+### Restricted filename characters
+
+In addition to the [default restricted characters set](/overview/#restricted-characters)
+the following characters are also replaced:
+
+| Character | Value | Replacement |
+| --------- |:-----:|:-----------:|
+| !         | 0x21  | ！           |
+| "         | 0x22  | ＂           |
+| *         | 0x2A  | ＊           |
+| :         | 0x3A  | ：           |
+| <         | 0x3C  | ＜           |
+| >         | 0x3E  | ＞           |
+| ?         | 0x3F  | ？           |
+| \         | 0x5C  | ＼           |
+| \|        | 0x7C  | ｜           |
+
+
+File names can also not start or end with the following characters.
+These only get replaced if they are the first or last character in the
+name:
+
+| Character | Value | Replacement |
+| --------- |:-----:|:-----------:|
+| .         | 0x2E  | ．           |
+
+Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),
+as they can't be used in JSON strings.
+
+### Public Links
+
+Gofile supports `rclone link` to make public links to files or
+directories. If you specify a directory it will download as a `zip`
+file. You can use the `--expire` flag to specify the time the link
+should be valid. Note that `rclone link --unlink` removes all the
+public links for a file.
+
+### Root folder ID
+
+You can set the `root_folder_id` for rclone.  This is the directory
+(identified by its `Folder ID`) that rclone considers to be the root
+of your Gofile drive.
+
+Normally you will leave this blank and rclone will determine the
+correct root to use itself and fill in the value in the config file.
+
+However you can set this to restrict rclone to a specific folder
+hierarchy.
+
+In order to do this you will have to find the `Folder ID` of the
+directory you wish rclone to display.
+
+You can do this with rclone
+
+```
+$ rclone lsf -Fip --dirs-only remote:
+d6341f53-ee65-4f29-9f59-d11e8070b2a0;Files/
+f4f5c9b8-6ece-478b-b03e-4538edfe5a1c;Photos/
+d50e356c-29ca-4b27-a3a7-494d91026e04;Videos/
+```
+
+The ID to use is the part before the `;` so you could set
+
+```
+root_folder_id = d6341f53-ee65-4f29-9f59-d11e8070b2a0
+```
+
+To restrict rclone to the `Files` directory.
+
+{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/gofile/gofile.go then run make backenddocs" >}}
+### Standard options
+
+Here are the Standard options specific to gofile (Gofile).
+
+#### --gofile-access-token
+
+API Access token
+
+You can get this from the web control panel.
+
+Properties:
+
+- Config:      access_token
+- Env Var:     RCLONE_GOFILE_ACCESS_TOKEN
+- Type:        string
+- Required:    false
+
+### Advanced options
+
+Here are the Advanced options specific to gofile (Gofile).
+
+#### --gofile-root-folder-id
+
+ID of the root folder
+
+Leave this blank normally, rclone will fill it in automatically.
+
+If you want rclone to be restricted to a particular folder you can
+fill it in - see the docs for more info.
+
+
+Properties:
+
+- Config:      root_folder_id
+- Env Var:     RCLONE_GOFILE_ROOT_FOLDER_ID
+- Type:        string
+- Required:    false
+
+#### --gofile-account-id
+
+Account ID
+
+Leave this blank normally, rclone will fill it in automatically.
+
+
+Properties:
+
+- Config:      account_id
+- Env Var:     RCLONE_GOFILE_ACCOUNT_ID
+- Type:        string
+- Required:    false
+
+#### --gofile-encoding
+
+The encoding for the backend.
+
+See the [encoding section in the overview](/overview/#encoding) for more info.
+
+Properties:
+
+- Config:      encoding
+- Env Var:     RCLONE_GOFILE_ENCODING
+- Type:        Encoding
+- Default:     Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftPeriod,RightPeriod,InvalidUtf8,Dot,Exclamation
+
+#### --gofile-description
+
+Description of the remote.
+
+Properties:
+
+- Config:      description
+- Env Var:     RCLONE_GOFILE_DESCRIPTION
+- Type:        string
+- Required:    false
+
+{{< rem autogenerated options stop >}}
+
+## Limitations
+
+Gofile only supports filenames up to 255 characters in length, where a
+character is a unicode character.
+
+Directories should not be cached for more than 24h otherwise files in
+the directory may not be downloadable. In practice this means when
+using a VFS based rclone command such as `rclone mount` you should
+make sure `--dir-cache-time` is less than `24h`.
+
+Note that Gofile is currently limited to a total of 100,000 items. If
+you attempt to upload more than that you will get
+`error-limit-100000`. This limit may be lifted in the future.
+
+### Duplicated files
+
+Gofile is capable of having files with duplicated file names. For
+instance two files called `hello.txt` in the same directory.
+
+Rclone cannot sync that to a normal file system but it can be fixed
+with the `rclone dedupe` command.
+
+Duplicated files cause problems with the syncing and you will see
+messages in the log about duplicates.
+
+Use `rclone dedupe` to fix duplicated files.

docs/content/overview.md

@@ -25,6 +25,7 @@ Here is an overview of the major features of each cloud storage system.
 | Dropbox                      | DBHASH ¹          | R       | Yes              | No              | -         | -        |
 | Enterprise File Fabric       | -                 | R/W     | Yes              | No              | R/W       | -        |
 | FTP                          | -                 | R/W ¹⁰  | No               | No              | -         | -        |
+| Gofile                       | MD5               | DR/W    | No               | Yes             | R         | -        |
 | Google Cloud Storage         | MD5               | R/W     | No               | No              | R/W       | -        |
 | Google Drive                 | MD5, SHA1, SHA256 | DR/W    | No               | Yes             | R/W       | DRWU     |
 | Google Photos                | -                 | -       | No               | Yes             | R         | -        |
@@ -501,6 +502,7 @@ upon backend-specific capabilities.
 | Dropbox                      | Yes   | Yes  | Yes  | Yes     | No      | No    | Yes          | No                | Yes          | Yes   | Yes      |
 | Enterprise File Fabric       | Yes   | Yes  | Yes  | Yes     | Yes     | No    | No           | No                | No           | No    | Yes      |
 | FTP                          | No    | No   | Yes  | Yes     | No      | No    | Yes          | No                | No           | No    | Yes      |
+| Gofile                       | Yes   | Yes  | Yes  | Yes     | No      | No    | Yes          | No                | Yes          | Yes   | Yes      |
 | Google Cloud Storage         | Yes   | Yes  | No   | No      | No      | Yes   | Yes          | No                | No           | No    | No       |
 | Google Drive                 | Yes   | Yes  | Yes  | Yes     | Yes     | Yes   | Yes          | No                | Yes          | Yes   | Yes      |
 | Google Photos                | No    | No   | No   | No      | No      | No    | No           | No                | No           | No    | No       |

docs/layouts/chrome/navbar.html

@@ -66,6 +66,7 @@
           <a class="dropdown-item" href="/dropbox/"><i class="fab fa-dropbox fa-fw"></i> Dropbox</a>
           <a class="dropdown-item" href="/filefabric/"><i class="fa fa-cloud fa-fw"></i> Enterprise File Fabric</a>
           <a class="dropdown-item" href="/ftp/"><i class="fa fa-file fa-fw"></i> FTP</a>
+          <a class="dropdown-item" href="/gofile/"><i class="fa fa-folder fa-fw"></i> Gofile</a>
           <a class="dropdown-item" href="/googlecloudstorage/"><i class="fab fa-google fa-fw"></i> Google Cloud Storage</a>
           <a class="dropdown-item" href="/drive/"><i class="fab fa-google fa-fw"></i> Google Drive</a>
           <a class="dropdown-item" href="/googlephotos/"><i class="fas fa-images fa-fw"></i> Google Photos</a>

fstest/test_all/config.yaml

@@ -137,6 +137,9 @@ backends:
  #   remote:   "TestFileFabric:"
  #   fastlist: false
  #   extratime: 2.0
+ - backend:  "gofile"
+   remote:   "TestGoFile:"
+   fastlist: true
  - backend:  "googlecloudstorage"
    remote:   "TestGoogleCloudStorage:"
    fastlist: true