From 09b79679cd9cfdb0843a109d8b7419d490262572 Mon Sep 17 00:00:00 2001 From: Chaitanya Bankanhal Date: Thu, 20 Aug 2020 23:31:38 +0530 Subject: [PATCH] plugins: restructure and add tests for pluginsctl/* calls --- fs/rc/rcserver/rcserver.go | 20 +- fs/rc/webgui/plugins.go | 466 +++++++------------------------------ fs/rc/webgui/rc.go | 310 ++++++++++++++++++++++++ fs/rc/webgui/rc_test.go | 148 ++++++++++++ fs/rc/webgui/webgui.go | 7 +- 5 files changed, 547 insertions(+), 404 deletions(-) create mode 100644 fs/rc/webgui/rc.go create mode 100644 fs/rc/webgui/rc_test.go diff --git a/fs/rc/rcserver/rcserver.go b/fs/rc/rcserver/rcserver.go index 622e08597..c46501db2 100644 --- a/fs/rc/rcserver/rcserver.go +++ b/fs/rc/rcserver/rcserver.go @@ -6,7 +6,6 @@ import ( "encoding/json" "flag" "fmt" - "github.com/rclone/rclone/fs/rc/webgui" "log" "mime" "net/http" @@ -18,6 +17,8 @@ import ( "sync" "time" + "github.com/rclone/rclone/fs/rc/webgui" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -368,7 +369,6 @@ func (s *Server) serveRemote(w http.ResponseWriter, r *http.Request, path string // Match URLS of the form [fs]/remote var fsMatch = regexp.MustCompile(`^\[(.*?)\](.*)$`) -var referrerPathReg = regexp.MustCompile("^(https?)://(.+):([0-9]+)?/(.*)$") func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string) { // Look to see if this has an fs in the path @@ -397,20 +397,8 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string) return } return - } else if s.opt.WebUI { - referrer := r.Referer() - referrerPathMatch := referrerPathReg.FindStringSubmatch(referrer) - - if referrerPathMatch != nil { - referrerPluginMatch := webgui.PluginsMatch.FindStringSubmatch(referrerPathMatch[4]) - if referrerPluginMatch != nil { - path = fmt.Sprintf("/plugins/%s/%s/%s", referrerPluginMatch[1], referrerPluginMatch[2], path) - - http.Redirect(w, r, path, http.StatusMovedPermanently) - //s.pluginsHandler.ServeHTTP(w, r) - return - } - } + } else if s.opt.WebUI && webgui.ServePluginWithReferrerOK(w, r, path) { + return } // Serve the files r.URL.Path = "/" + path diff --git a/fs/rc/webgui/plugins.go b/fs/rc/webgui/plugins.go index 2db0febc7..645a02830 100644 --- a/fs/rc/webgui/plugins.go +++ b/fs/rc/webgui/plugins.go @@ -1,24 +1,22 @@ package webgui import ( - "context" "encoding/json" - "errors" "fmt" - "github.com/rclone/rclone/fs" - "github.com/rclone/rclone/fs/config" - "github.com/rclone/rclone/fs/rc" "io/ioutil" "net/http" "net/http/httputil" "net/url" - "os" "path/filepath" "regexp" "strings" "sync" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config" ) +// PackageJSON is the structure of package.json of a plugin type PackageJSON struct { Name string `json:"name"` Version string `json:"version"` @@ -36,23 +34,33 @@ type PackageJSON struct { Bugs struct { URL string `json:"url"` } `json:"bugs"` - - //RcloneHandlesType []string `json:"rcloneHandlesType"` Rclone RcloneConfig `json:"rclone"` } +// RcloneConfig represents the rclone specific config type RcloneConfig struct { - HandlesType []string `json:"handlesType"` - PluginType string `json:"pluginType"` + HandlesType []string `json:"handlesType"` + PluginType string `json:"pluginType"` + RedirectReferrer bool `json:"redirectReferrer"` + Test bool `json:"-"` +} + +func (r *PackageJSON) isTesting() bool { + return r.Rclone.Test } var ( - loadedTestPlugins *Plugins - cachePath string - PluginsPath string - pluginsConfigPath string - loadedPlugins *Plugins - pluginsProxy = &httputil.ReverseProxy{} + //loadedTestPlugins *Plugins + cachePath string + + loadedPlugins *Plugins + pluginsProxy = &httputil.ReverseProxy{} + // PluginsMatch is used for matching author and plugin name in the url path + PluginsMatch = regexp.MustCompile(`^plugins\/([^\/]*)\/([^\/\?]+)[\/]?(.*)$`) + // PluginsPath is the base path where webgui plugins are stored + PluginsPath string + pluginsConfigPath string + availablePluginsJSONPath = "availablePlugins.json" ) func init() { @@ -60,19 +68,14 @@ func init() { PluginsPath = filepath.Join(cachePath, "plugins") pluginsConfigPath = filepath.Join(PluginsPath, "config") - loadedPlugins = newPlugins("availablePlugins.json") + loadedPlugins = newPlugins(availablePluginsJSONPath) err := loadedPlugins.readFromFile() if err != nil { - fs.Errorf(nil, "error reading available plugins", err) - } - loadedTestPlugins = newPlugins("testPlugins.json") - err = loadedTestPlugins.readFromFile() - - if err != nil { - fs.Errorf(nil, "error reading test plugins", err) + fs.Errorf(nil, "error reading available plugins: %v", err) } } +// Plugins represents the structure how plugins are saved onto disk type Plugins struct { mutex sync.Mutex LoadedPlugins map[string]PackageJSON `json:"loadedPlugins"` @@ -93,8 +96,8 @@ func (p *Plugins) readFromFile() (err error) { if err != nil { return err } - availablePluginsJson := filepath.Join(pluginsConfigPath, p.fileName) - data, err := ioutil.ReadFile(availablePluginsJson) + availablePluginsJSON := filepath.Join(pluginsConfigPath, p.fileName) + data, err := ioutil.ReadFile(availablePluginsJSON) if err != nil { // create a file ? } @@ -105,19 +108,19 @@ func (p *Plugins) readFromFile() (err error) { return nil } -func (p *Plugins) addPlugin(pluginName string, packageJsonPath string) (err error) { +func (p *Plugins) addPlugin(pluginName string, packageJSONPath string) (err error) { p.mutex.Lock() defer p.mutex.Unlock() - data, err := ioutil.ReadFile(packageJsonPath) + data, err := ioutil.ReadFile(packageJSONPath) if err != nil { return err } - var pkgJson = PackageJSON{} - err = json.Unmarshal(data, &pkgJson) + var pkgJSON = PackageJSON{} + err = json.Unmarshal(data, &pkgJSON) if err != nil { return err } - p.LoadedPlugins[pluginName] = pkgJson + p.LoadedPlugins[pluginName] = pkgJSON err = p.writeToFile() if err != nil { @@ -135,15 +138,16 @@ func (p *Plugins) addTestPlugin(pluginName string, testURL string, handlesType [ return err } - var pkgJson = PackageJSON{ + var pkgJSON = PackageJSON{ Name: pluginName, TestURL: testURL, Rclone: RcloneConfig{ HandlesType: handlesType, + Test: true, }, } - p.LoadedPlugins[pluginName] = pkgJson + p.LoadedPlugins[pluginName] = pkgJSON err = p.writeToFile() if err != nil { @@ -156,11 +160,11 @@ func (p *Plugins) addTestPlugin(pluginName string, testURL string, handlesType [ func (p *Plugins) writeToFile() (err error) { //p.mutex.Lock() //defer p.mutex.Unlock() - availablePluginsJson := filepath.Join(pluginsConfigPath, p.fileName) + availablePluginsJSON := filepath.Join(pluginsConfigPath, p.fileName) file, err := json.MarshalIndent(p, "", " ") - err = ioutil.WriteFile(availablePluginsJson, file, 0755) + err = ioutil.WriteFile(availablePluginsJSON, file, 0755) if err != nil { fs.Logf(nil, "%s", err) } @@ -177,7 +181,7 @@ func (p *Plugins) removePlugin(name string) (err error) { _, ok := p.LoadedPlugins[name] if !ok { - return errors.New(fmt.Sprintf("plugin %s not loaded", name)) + return fmt.Errorf("plugin %s not loaded", name) } delete(p.LoadedPlugins, name) @@ -188,219 +192,18 @@ func (p *Plugins) removePlugin(name string) (err error) { return nil } +// GetPluginByName returns the plugin object for the key (author/plugin-name) func (p *Plugins) GetPluginByName(name string) (out *PackageJSON, err error) { p.mutex.Lock() defer p.mutex.Unlock() po, ok := p.LoadedPlugins[name] if !ok { - return nil, errors.New(fmt.Sprintf("plugin %s not loaded", name)) + return nil, fmt.Errorf("plugin %s not loaded", name) } return &po, nil } -func init() { - rc.Add(rc.Call{ - Path: "pluginsctl/addTestPlugin", - AuthRequired: true, - Fn: rcAddTestPlugin, - Title: "Show current mount points", - Help: `This shows currently mounted points, which can be used for performing an unmount - -This takes no parameters and returns - -- mountPoints: list of current mount points - -Eg - - rclone rc pluginsctl/addTestPlugin -`, - }) -} - -func rcAddTestPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) { - name, err := in.GetString("name") - if err != nil { - return nil, err - } - - loadUrl, err := in.GetString("loadUrl") - if err != nil { - return nil, err - } - var handlesTypes []string - err = in.GetStructMissingOK("handlesTypes", &handlesTypes) - if err != nil { - return nil, err - } - - err = loadedTestPlugins.addTestPlugin(name, loadUrl, handlesTypes) - if err != nil { - return nil, err - } - - return nil, nil -} - -func init() { - rc.Add(rc.Call{ - Path: "pluginsctl/listTestPlugins", - AuthRequired: true, - Fn: rcGetLoadedPlugins, - Title: "Show current mount points", - Help: `This shows currently mounted points, which can be used for performing an unmount - -This takes no parameters and returns - -- mountPoints: list of current mount points - -Eg - - rclone rc pluginsctl/listTestPlugins -`, - }) -} - -func rcGetLoadedPlugins(_ context.Context, in rc.Params) (out rc.Params, err error) { - return rc.Params{ - "loadedTestPlugins": loadedTestPlugins.LoadedPlugins, - }, nil -} - -func init() { - rc.Add(rc.Call{ - Path: "pluginsctl/removeTestPlugin", - AuthRequired: true, - Fn: rcRemoveTestPlugin, - Title: "Show current mount points", - Help: `This shows currently mounted points, which can be used for performing an unmount - -This takes no parameters and returns - -- mountPoints: list of current mount points - -Eg - - rclone rc pluginsctl/removeTestPlugin -`, - }) -} - -func rcRemoveTestPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) { - name, err := in.GetString("name") - if err != nil { - return nil, err - } - err = loadedTestPlugins.removePlugin(name) - if err != nil { - return nil, err - } - return nil, nil -} - -func init() { - rc.Add(rc.Call{ - Path: "pluginsctl/addPlugin", - AuthRequired: true, - Fn: rcAddPlugin, - Title: "Show current mount points", - Help: `This shows currently mounted points, which can be used for performing an unmount - -This takes no parameters and returns - -- mountPoints: list of current mount points - -Eg - - rclone rc pluginsctl/addPlugin -`, - }) -} - -func rcAddPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) { - pluginUrl, err := in.GetString("url") - if err != nil { - return nil, err - } - - author, repoName, repoBranch, err := getAuthorRepoBranchGithub(pluginUrl) - if err != nil { - return nil, err - } - - branch, err := in.GetString("branch") - if err != nil || branch == "" { - branch = repoBranch - } - - version, err := in.GetString("version") - if err != nil || version == "" { - version = "latest" - } - - err = CreatePathIfNotExist(PluginsPath) - if err != nil { - return nil, err - } - - // fetch and package.json - // https://raw.githubusercontent.com/rclone/rclone-webui-react/master/package.json - - pluginID := fmt.Sprintf("%s/%s", author, repoName) - - currentPluginPath := filepath.Join(PluginsPath, pluginID) - - err = CreatePathIfNotExist(currentPluginPath) - if err != nil { - return nil, err - } - - packageJsonUrl := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/package.json", author, repoName, branch) - packageJsonFilePath := filepath.Join(currentPluginPath, "package.json") - err = DownloadFile(packageJsonFilePath, packageJsonUrl) - if err != nil { - return nil, err - } - // register in plugins - - // download release and save in plugins//repo-name/app - // https://api.github.com/repos/rclone/rclone-webui-react/releases/latest - releaseUrl, tag, _, err := GetLatestReleaseURL(fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/%s", author, repoName, version)) - zipName := tag + ".zip" - zipPath := filepath.Join(currentPluginPath, zipName) - - err = DownloadFile(zipPath, releaseUrl) - if err != nil { - return nil, err - } - - extractPath := filepath.Join(currentPluginPath, "app") - - err = CreatePathIfNotExist(extractPath) - if err != nil { - return nil, err - } - err = os.RemoveAll(extractPath) - if err != nil { - fs.Logf(nil, "No previous downloads to remove") - } - - fs.Logf(nil, "Unzipping plugin binary") - - err = Unzip(zipPath, extractPath) - if err != nil { - return nil, err - } - - err = loadedPlugins.addPlugin(pluginID, packageJsonFilePath) - if err != nil { - return nil, err - } - - return nil, nil - -} - // getAuthorRepoBranchGithub gives author, repoName and branch from a github.com url // url examples: // https://github.com/rclone/rclone-webui-react/ @@ -409,14 +212,14 @@ func rcAddPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) { // github.com/rclone/rclone-webui-react // func getAuthorRepoBranchGithub(url string) (author string, repoName string, branch string, err error) { - repoUrl := url - repoUrl = strings.Replace(repoUrl, "https://", "", 1) - repoUrl = strings.Replace(repoUrl, "http://", "", 1) + repoURL := url + repoURL = strings.Replace(repoURL, "https://", "", 1) + repoURL = strings.Replace(repoURL, "http://", "", 1) - urlSplits := strings.Split(repoUrl, "/") + urlSplits := strings.Split(repoURL, "/") if len(urlSplits) < 3 || len(urlSplits) > 5 || urlSplits[0] != "github.com" { - return "", "", "", errors.New(fmt.Sprintf("Invalid github url: %s", url)) + return "", "", "", fmt.Errorf("invalid github url: %s", url) } // get branch name @@ -427,94 +230,6 @@ func getAuthorRepoBranchGithub(url string) (author string, repoName string, bran return urlSplits[1], urlSplits[2], "master", nil } -func init() { - rc.Add(rc.Call{ - Path: "pluginsctl/listPlugins", - AuthRequired: true, - Fn: rcGetPlugins, - Title: "Get the list of currently loaded plugins", - Help: `This allows you to get the currently enabled plugins and their details. - -This takes no parameters and returns - -- loadedPlugins: list of current production plugins -- testPlugins: list of temporarily loaded development plugins, usually running on a different server. - -Eg - - rclone rc pluginsctl/listPlugins -`, - }) -} - -func rcGetPlugins(_ context.Context, in rc.Params) (out rc.Params, err error) { - err = loadedPlugins.readFromFile() - if err != nil { - return nil, err - } - err = loadedTestPlugins.readFromFile() - if err != nil { - return nil, err - } - - return rc.Params{ - "loadedPlugins": loadedPlugins.LoadedPlugins, - "loadedTestPlugins": loadedTestPlugins.LoadedPlugins, - }, nil -} - -func init() { - rc.Add(rc.Call{ - Path: "pluginsctl/removePlugin", - AuthRequired: true, - Fn: rcRemovePlugin, - Title: "Get the list of currently loaded plugins", - Help: `This allows you to get the currently enabled plugins and their details. - -This takes parameters - -- name: name of the plugin in the format / - -Eg - - rclone rc pluginsctl/removePlugin name=rclone/video-plugin -`, - }) -} - -func rcRemovePlugin(_ context.Context, in rc.Params) (out rc.Params, err error) { - name, err := in.GetString("name") - if err != nil { - return nil, err - } - - err = loadedPlugins.removePlugin(name) - if err != nil { - return nil, err - } - return nil, nil -} - -func init() { - rc.Add(rc.Call{ - Path: "pluginsctl/getPluginsForType", - AuthRequired: true, - Fn: rcGetPluginsForType, - Title: "Get the list of currently loaded plugins", - Help: `This allows you to get the currently enabled plugins and their details. - -This takes no parameters and returns - -- loadedPlugins: list of current production plugins -- testPlugins: list of temporarily loaded development plugins, usually running on a different server. - -Eg - - rclone rc pluginsctl/getPlugins -`, - }) -} - func filterPlugins(plugins *Plugins, compare func(packageJSON *PackageJSON) bool) map[string]PackageJSON { output := map[string]PackageJSON{} @@ -527,58 +242,7 @@ func filterPlugins(plugins *Plugins, compare func(packageJSON *PackageJSON) bool return output } -func rcGetPluginsForType(_ context.Context, in rc.Params) (out rc.Params, err error) { - handlesType, err := in.GetString("type") - if err != nil { - handlesType = "" - } - - pluginType, err := in.GetString("pluginType") - if err != nil { - pluginType = "" - } - var loadedPluginsResult map[string]PackageJSON - - var loadedTestPluginsResult map[string]PackageJSON - - if pluginType == "" || pluginType == "FileHandler" { - - loadedPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool { - for i := range packageJSON.Rclone.HandlesType { - if packageJSON.Rclone.HandlesType[i] == handlesType { - return true - } - } - return false - }) - - loadedTestPluginsResult = filterPlugins(loadedTestPlugins, func(packageJSON *PackageJSON) bool { - for i := range packageJSON.Rclone.HandlesType { - if packageJSON.Rclone.HandlesType[i] == handlesType { - return true - } - } - return false - }) - } else { - loadedPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool { - return packageJSON.Rclone.PluginType == pluginType - }) - - loadedTestPluginsResult = filterPlugins(loadedTestPlugins, func(packageJSON *PackageJSON) bool { - return packageJSON.Rclone.PluginType == pluginType - }) - } - - return rc.Params{ - "loadedPlugins": loadedPluginsResult, - "testPlugins": loadedTestPluginsResult, - }, nil - -} - -var PluginsMatch = regexp.MustCompile(`^plugins\/([^\/]*)\/([^\/\?]+)[\/]?(.*)$`) - +// getDirectorForProxy is a helper function for reverse proxy of test plugins func getDirectorForProxy(origin *url.URL) func(req *http.Request) { return func(req *http.Request) { req.Header.Add("X-Forwarded-Host", req.Host) @@ -589,12 +253,15 @@ func getDirectorForProxy(origin *url.URL) func(req *http.Request) { } } +// ServePluginOK checks the plugin url and uses reverse proxy to allow redirection for content not being served by rclone func ServePluginOK(w http.ResponseWriter, r *http.Request, pluginsMatchResult []string) (ok bool) { - testPlugin, err := loadedTestPlugins.GetPluginByName(fmt.Sprintf("%s/%s", pluginsMatchResult[1], pluginsMatchResult[2])) + testPlugin, err := loadedPlugins.GetPluginByName(fmt.Sprintf("%s/%s", pluginsMatchResult[1], pluginsMatchResult[2])) if err != nil { return false } - + if !testPlugin.Rclone.Test { + return false + } origin, _ := url.Parse(fmt.Sprintf("%s/%s", testPlugin.TestURL, pluginsMatchResult[3])) director := getDirectorForProxy(origin) @@ -603,3 +270,30 @@ func ServePluginOK(w http.ResponseWriter, r *http.Request, pluginsMatchResult [] pluginsProxy.ServeHTTP(w, r) return true } + +var referrerPathReg = regexp.MustCompile("^(https?)://(.+):([0-9]+)?/(.*)$") + +// ServePluginWithReferrerOK check if redirectReferrer is set for the referred a plugin, if yes, +// sends a redirect to actual url. This function is useful for plugins to refer to absolute paths when +// the referrer in http.Request is set +func ServePluginWithReferrerOK(w http.ResponseWriter, r *http.Request, path string) (ok bool) { + referrer := r.Referer() + referrerPathMatch := referrerPathReg.FindStringSubmatch(referrer) + + if referrerPathMatch != nil { + referrerPluginMatch := PluginsMatch.FindStringSubmatch(referrerPathMatch[4]) + pluginKey := fmt.Sprintf("%s/%s", referrerPluginMatch[1], referrerPluginMatch[2]) + currentPlugin, err := loadedPlugins.GetPluginByName(pluginKey) + if err != nil { + return false + } + if referrerPluginMatch != nil && currentPlugin.Rclone.RedirectReferrer { + path = fmt.Sprintf("/plugins/%s/%s/%s", referrerPluginMatch[1], referrerPluginMatch[2], path) + + http.Redirect(w, r, path, http.StatusMovedPermanently) + //s.pluginsHandler.ServeHTTP(w, r) + return true + } + } + return false +} diff --git a/fs/rc/webgui/rc.go b/fs/rc/webgui/rc.go new file mode 100644 index 000000000..8006cce48 --- /dev/null +++ b/fs/rc/webgui/rc.go @@ -0,0 +1,310 @@ +package webgui + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/rc" +) + +func init() { + rc.Add(rc.Call{ + Path: "pluginsctl/listTestPlugins", + AuthRequired: true, + Fn: rcListTestPlugins, + Title: "Show currently loaded test plugins", + Help: `allows listing of test plugins with the rclone.test set to true in package.json of the plugin + +This takes no parameters and returns + +- loadedTestPlugins: list of currently available test plugins + +Eg + + rclone rc pluginsctl/listTestPlugins +`, + }) +} + +func rcListTestPlugins(_ context.Context, _ rc.Params) (out rc.Params, err error) { + return rc.Params{ + "loadedTestPlugins": filterPlugins(loadedPlugins, func(json *PackageJSON) bool { return json.isTesting() }), + }, nil +} + +func init() { + rc.Add(rc.Call{ + Path: "pluginsctl/removeTestPlugin", + AuthRequired: true, + Fn: rcRemoveTestPlugin, + Title: "Remove a test plugin", + Help: `This allows you to remove a plugin using it's name + +This takes the following parameters + +- name: name of the plugin in the format / + +Eg + + rclone rc pluginsctl/removeTestPlugin name=rclone/rclone-webui-react +`, + }) +} +func rcRemoveTestPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) { + name, err := in.GetString("name") + if err != nil { + return nil, err + } + err = loadedPlugins.removePlugin(name) + if err != nil { + return nil, err + } + return nil, nil +} + +func init() { + rc.Add(rc.Call{ + Path: "pluginsctl/addPlugin", + AuthRequired: true, + Fn: rcAddPlugin, + Title: "Add a plugin using url", + Help: `used for adding a plugin to the webgui + +This takes the following parameters + +- url: http url of the github repo where the plugin is hosted (http://github.com/rclone/rclone-webui-react) + +Eg + + rclone rc pluginsctl/addPlugin +`, + }) +} + +func rcAddPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) { + pluginURL, err := in.GetString("url") + if err != nil { + return nil, err + } + + author, repoName, repoBranch, err := getAuthorRepoBranchGithub(pluginURL) + if err != nil { + return nil, err + } + + branch, err := in.GetString("branch") + if err != nil || branch == "" { + branch = repoBranch + } + + version, err := in.GetString("version") + if err != nil || version == "" { + version = "latest" + } + + err = CreatePathIfNotExist(PluginsPath) + if err != nil { + return nil, err + } + + // fetch and package.json + // https://raw.githubusercontent.com/rclone/rclone-webui-react/master/package.json + + pluginID := fmt.Sprintf("%s/%s", author, repoName) + + currentPluginPath := filepath.Join(PluginsPath, pluginID) + + err = CreatePathIfNotExist(currentPluginPath) + if err != nil { + return nil, err + } + + packageJSONUrl := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/package.json", author, repoName, branch) + packageJSONFilePath := filepath.Join(currentPluginPath, "package.json") + err = DownloadFile(packageJSONFilePath, packageJSONUrl) + if err != nil { + return nil, err + } + // register in plugins + + // download release and save in plugins//repo-name/app + // https://api.github.com/repos/rclone/rclone-webui-react/releases/latest + releaseURL, tag, _, err := GetLatestReleaseURL(fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/%s", author, repoName, version)) + if err != nil { + return nil, err + } + zipName := tag + ".zip" + zipPath := filepath.Join(currentPluginPath, zipName) + + err = DownloadFile(zipPath, releaseURL) + if err != nil { + return nil, err + } + + extractPath := filepath.Join(currentPluginPath, "app") + + err = CreatePathIfNotExist(extractPath) + if err != nil { + return nil, err + } + err = os.RemoveAll(extractPath) + if err != nil { + fs.Logf(nil, "No previous downloads to remove") + } + + fs.Logf(nil, "Unzipping plugin binary") + + err = Unzip(zipPath, extractPath) + if err != nil { + return nil, err + } + + err = loadedPlugins.addPlugin(pluginID, packageJSONFilePath) + if err != nil { + return nil, err + } + + return nil, nil + +} + +func init() { + rc.Add(rc.Call{ + Path: "pluginsctl/listPlugins", + AuthRequired: true, + Fn: rcGetPlugins, + Title: "Get the list of currently loaded plugins", + Help: `This allows you to get the currently enabled plugins and their details. + +This takes no parameters and returns + +- loadedPlugins: list of current production plugins +- testPlugins: list of temporarily loaded development plugins, usually running on a different server. + +Eg + + rclone rc pluginsctl/listPlugins +`, + }) +} + +func rcGetPlugins(_ context.Context, _ rc.Params) (out rc.Params, err error) { + err = loadedPlugins.readFromFile() + if err != nil { + return nil, err + } + return rc.Params{ + "loadedPlugins": filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool { return !packageJSON.isTesting() }), + "loadedTestPlugins": filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool { return packageJSON.isTesting() }), + }, nil +} + +func init() { + rc.Add(rc.Call{ + Path: "pluginsctl/removePlugin", + AuthRequired: true, + Fn: rcRemovePlugin, + Title: "Remove a loaded plugin", + Help: `This allows you to remove a plugin using it's name + +This takes parameters + +- name: name of the plugin in the format / + +Eg + + rclone rc pluginsctl/removePlugin name=rclone/video-plugin +`, + }) +} + +func rcRemovePlugin(_ context.Context, in rc.Params) (out rc.Params, err error) { + name, err := in.GetString("name") + if err != nil { + return nil, err + } + + err = loadedPlugins.removePlugin(name) + if err != nil { + return nil, err + } + return nil, nil +} + +func init() { + rc.Add(rc.Call{ + Path: "pluginsctl/getPluginsForType", + AuthRequired: true, + Fn: rcGetPluginsForType, + Title: "Get plugins with type criteria", + Help: `This shows all possible plugins by a mime type + +This takes the following parameters + +- type: supported mime type by a loaded plugin eg (video/mp4, audio/mp3) +- pluginType: filter plugins based on their type eg (DASHBOARD, FILE_HANDLER, TERMINAL) + +and returns + +- loadedPlugins: list of current production plugins +- testPlugins: list of temporarily loaded development plugins, usually running on a different server. + +Eg + + rclone rc pluginsctl/getPluginsForType type=video/mp4 +`, + }) +} + +func rcGetPluginsForType(_ context.Context, in rc.Params) (out rc.Params, err error) { + handlesType, err := in.GetString("type") + if err != nil { + handlesType = "" + } + + pluginType, err := in.GetString("pluginType") + if err != nil { + pluginType = "" + } + var loadedPluginsResult map[string]PackageJSON + + var loadedTestPluginsResult map[string]PackageJSON + + if pluginType == "" || pluginType == "FileHandler" { + + loadedPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool { + for i := range packageJSON.Rclone.HandlesType { + if packageJSON.Rclone.HandlesType[i] == handlesType && !packageJSON.Rclone.Test { + return true + } + } + return false + }) + + loadedTestPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool { + for i := range packageJSON.Rclone.HandlesType { + if packageJSON.Rclone.HandlesType[i] == handlesType && packageJSON.Rclone.Test { + return true + } + } + return false + }) + } else { + loadedPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool { + return packageJSON.Rclone.PluginType == pluginType && !packageJSON.isTesting() + }) + + loadedTestPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool { + return packageJSON.Rclone.PluginType == pluginType && packageJSON.isTesting() + }) + } + + return rc.Params{ + "loadedPlugins": loadedPluginsResult, + "loadedTestPlugins": loadedTestPluginsResult, + }, nil + +} diff --git a/fs/rc/webgui/rc_test.go b/fs/rc/webgui/rc_test.go new file mode 100644 index 000000000..efe02aa2e --- /dev/null +++ b/fs/rc/webgui/rc_test.go @@ -0,0 +1,148 @@ +package webgui + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/rclone/rclone/fs/rc" + "github.com/stretchr/testify/assert" +) + +const testPluginName = "rclone-webui-react" +const testPluginAuthor = "rclone" +const testPluginKey = testPluginAuthor + "/" + testPluginName +const testPluginURL = "https://github.com/" + testPluginAuthor + "/" + testPluginName + "/" + +func setCacheDir(t *testing.T) string { + cacheDir, err := ioutil.TempDir("", "rclone-cache-dir") + assert.Nil(t, err) + PluginsPath = filepath.Join(cacheDir, "plugins") + pluginsConfigPath = filepath.Join(cacheDir, "config") + + loadedPlugins = newPlugins(availablePluginsJSONPath) + err = loadedPlugins.readFromFile() + assert.Nil(t, err) + return cacheDir +} + +func cleanCacheDir(t *testing.T, cacheDir string) { + _ = os.RemoveAll(cacheDir) +} + +func addPlugin(t *testing.T) { + addPlugin := rc.Calls.Get("pluginsctl/addPlugin") + assert.NotNil(t, addPlugin) + in := rc.Params{ + "url": testPluginURL, + } + out, err := addPlugin.Fn(context.Background(), in) + assert.Nil(t, err) + assert.Nil(t, out) + +} + +func removePlugin(t *testing.T) { + addPlugin := rc.Calls.Get("pluginsctl/removePlugin") + assert.NotNil(t, addPlugin) + + in := rc.Params{ + "name": testPluginKey, + } + out, err := addPlugin.Fn(context.Background(), in) + assert.NotNil(t, err) + assert.Nil(t, out) +} + +//func TestListTestPlugins(t *testing.T) { +// addPlugin := rc.Calls.Get("pluginsctl/listTestPlugins") +// assert.NotNil(t, addPlugin) +// in := rc.Params{} +// out, err := addPlugin.Fn(context.Background(), in) +// assert.Nil(t, err) +// expected := rc.Params{ +// "loadedTestPlugins": map[string]PackageJSON{}, +// } +// assert.Equal(t, expected, out) +//} + +//func TestRemoveTestPlugin(t *testing.T) { +// addPlugin := rc.Calls.Get("pluginsctl/removeTestPlugin") +// assert.NotNil(t, addPlugin) +// in := rc.Params{ +// "name": "", +// } +// out, err := addPlugin.Fn(context.Background(), in) +// assert.NotNil(t, err) +// assert.Nil(t, out) +//} + +func TestAddPlugin(t *testing.T) { + cacheDir := setCacheDir(t) + defer cleanCacheDir(t, cacheDir) + + addPlugin(t) + _, ok := loadedPlugins.LoadedPlugins[testPluginKey] + assert.True(t, ok) + + //removePlugin(t) + //_, ok = loadedPlugins.LoadedPlugins[testPluginKey] + //assert.False(t, ok) +} + +func TestListPlugins(t *testing.T) { + cacheDir := setCacheDir(t) + defer cleanCacheDir(t, cacheDir) + + addPlugin := rc.Calls.Get("pluginsctl/listPlugins") + assert.NotNil(t, addPlugin) + in := rc.Params{} + out, err := addPlugin.Fn(context.Background(), in) + assert.Nil(t, err) + expected := rc.Params{ + "loadedPlugins": map[string]PackageJSON{}, + "loadedTestPlugins": map[string]PackageJSON{}, + } + assert.Equal(t, expected, out) +} + +func TestRemovePlugin(t *testing.T) { + cacheDir := setCacheDir(t) + defer cleanCacheDir(t, cacheDir) + + addPlugin(t) + removePluginCall := rc.Calls.Get("pluginsctl/removePlugin") + assert.NotNil(t, removePlugin) + + in := rc.Params{ + "name": testPluginKey, + } + out, err := removePluginCall.Fn(context.Background(), in) + assert.Nil(t, err) + assert.Nil(t, out) + removePlugin(t) + assert.Equal(t, len(loadedPlugins.LoadedPlugins), 0) + +} + +func TestPluginsForType(t *testing.T) { + addPlugin := rc.Calls.Get("pluginsctl/getPluginsForType") + assert.NotNil(t, addPlugin) + in := rc.Params{ + "type": "", + "pluginType": "FileHandler", + } + out, err := addPlugin.Fn(context.Background(), in) + assert.Nil(t, err) + assert.NotNil(t, out) + + in = rc.Params{ + "type": "video/mp4", + "pluginType": "", + } + _, err = addPlugin.Fn(context.Background(), in) + assert.Nil(t, err) + assert.NotNil(t, out) +} diff --git a/fs/rc/webgui/webgui.go b/fs/rc/webgui/webgui.go index a8479d026..b794af468 100644 --- a/fs/rc/webgui/webgui.go +++ b/fs/rc/webgui/webgui.go @@ -27,9 +27,12 @@ func GetLatestReleaseURL(fetchURL string) (string, string, int, error) { } results := gitHubRequest{} if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { - return "", "", 0, errors.New("Could not decode results from http request") + return "", "", 0, errors.New("could not decode results from http request") + } + if len(results.Assets) < 1 { + return "", "", 0, errors.New("could not find an asset in the release. " + + "check if asset was successfully added in github release assets") } - res := results.Assets[0].BrowserDownloadURL tag := results.TagName size := results.Assets[0].Size