2018-10-27 19:29:20 +02:00
// Package rcserver implements the HTTP endpoint to serve the remote control
package rcserver
import (
2020-11-05 16:18:51 +01:00
"context"
2019-08-20 20:25:04 +02:00
"encoding/base64"
2018-10-27 19:29:20 +02:00
"encoding/json"
2019-07-18 12:13:54 +02:00
"flag"
"fmt"
2020-01-12 10:12:04 +01:00
"log"
2018-10-27 19:29:20 +02:00
"mime"
"net/http"
2018-10-28 15:31:24 +01:00
"net/url"
2019-08-04 13:32:37 +02:00
"path/filepath"
2018-10-28 15:31:24 +01:00
"regexp"
"sort"
2018-10-27 19:29:20 +02:00
"strings"
2020-05-08 17:15:21 +02:00
"time"
2018-10-27 19:29:20 +02:00
2022-12-11 15:47:47 +01:00
"github.com/go-chi/chi/v5/middleware"
2020-02-26 09:34:32 +01:00
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
2019-07-28 19:47:38 +02:00
"github.com/rclone/rclone/fs"
2020-02-26 09:34:32 +01:00
"github.com/rclone/rclone/fs/accounting"
2019-07-28 19:47:38 +02:00
"github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fs/config"
2021-09-06 16:42:23 +02:00
"github.com/rclone/rclone/fs/fshttp"
2019-07-28 19:47:38 +02:00
"github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/fs/rc"
2019-08-08 06:56:58 +02:00
"github.com/rclone/rclone/fs/rc/jobs"
"github.com/rclone/rclone/fs/rc/rcflags"
2021-09-06 16:42:23 +02:00
"github.com/rclone/rclone/fs/rc/webgui"
2022-12-11 15:47:47 +01:00
libhttp "github.com/rclone/rclone/lib/http"
2021-04-20 06:50:08 +02:00
"github.com/rclone/rclone/lib/http/serve"
2020-01-12 10:12:04 +01:00
"github.com/rclone/rclone/lib/random"
2021-09-06 16:42:23 +02:00
"github.com/skratchdot/open-golang/open"
2018-10-27 19:29:20 +02:00
)
2020-02-26 09:34:32 +01:00
var promHandler http . Handler
func init ( ) {
2020-11-05 17:59:59 +01:00
rcloneCollector := accounting . NewRcloneCollector ( context . Background ( ) )
2020-02-26 09:34:32 +01:00
prometheus . MustRegister ( rcloneCollector )
2021-09-06 16:42:23 +02:00
m := fshttp . NewMetrics ( "rclone" )
for _ , c := range m . Collectors ( ) {
prometheus . MustRegister ( c )
}
fshttp . DefaultMetrics = m
2020-02-26 09:34:32 +01:00
promHandler = promhttp . Handler ( )
}
2018-10-27 19:29:20 +02:00
// Start the remote control server if configured
2018-11-01 18:20:04 +01:00
//
// If the server wasn't configured the *Server returned may be nil
2020-11-05 16:18:51 +01:00
func Start ( ctx context . Context , opt * rc . Options ) ( * Server , error ) {
2019-08-10 18:12:22 +02:00
jobs . SetOpt ( opt ) // set the defaults for jobs
2018-10-27 19:29:20 +02:00
if opt . Enabled {
2018-10-28 15:31:24 +01:00
// Serve on the DefaultServeMux so can have global registrations appear
2022-12-11 15:47:47 +01:00
s , err := newServer ( ctx , opt , http . DefaultServeMux )
if err != nil {
return nil , err
}
2018-11-01 18:20:04 +01:00
return s , s . Serve ( )
2018-10-27 19:29:20 +02:00
}
2018-11-01 18:20:04 +01:00
return nil , nil
2018-10-27 19:29:20 +02:00
}
2018-11-01 18:20:04 +01:00
// Server contains everything to run the rc server
type Server struct {
2020-11-05 16:18:51 +01:00
ctx context . Context // for global config
2022-12-11 15:47:47 +01:00
server * libhttp . Server
2020-07-27 20:32:45 +02:00
files http . Handler
pluginsHandler http . Handler
opt * rc . Options
2018-10-27 19:29:20 +02:00
}
2022-12-11 15:47:47 +01:00
func newServer ( ctx context . Context , opt * rc . Options , mux * http . ServeMux ) ( * Server , error ) {
2020-04-23 21:22:47 +02:00
fileHandler := http . Handler ( nil )
2020-07-27 20:32:45 +02:00
pluginsHandler := http . Handler ( nil )
2018-10-27 19:29:20 +02:00
// Add some more mime types which are often missing
_ = mime . AddExtensionType ( ".wasm" , "application/wasm" )
_ = mime . AddExtensionType ( ".js" , "application/javascript" )
2021-09-10 15:35:53 +02:00
cachePath := filepath . Join ( config . GetCacheDir ( ) , "webgui" )
2019-08-04 13:32:37 +02:00
extractPath := filepath . Join ( cachePath , "current/build" )
2018-10-27 19:29:20 +02:00
// File handling
if opt . Files != "" {
2019-08-04 13:32:37 +02:00
if opt . WebUI {
fs . Logf ( nil , "--rc-files overrides --rc-web-gui command\n" )
}
2018-10-27 19:29:20 +02:00
fs . Logf ( nil , "Serving files from %q" , opt . Files )
2020-04-23 21:22:47 +02:00
fileHandler = http . FileServer ( http . Dir ( opt . Files ) )
2019-08-04 13:32:37 +02:00
} else if opt . WebUI {
2021-09-10 15:35:53 +02:00
if err := webgui . CheckAndDownloadWebGUIRelease ( opt . WebGUIUpdate , opt . WebGUIForceUpdate , opt . WebGUIFetchURL , config . GetCacheDir ( ) ) ; err != nil {
2021-10-05 09:12:10 +02:00
fs . Errorf ( nil , "Error while fetching the latest release of Web GUI: %v" , err )
2020-01-12 10:12:04 +01:00
}
if opt . NoAuth {
2022-01-23 14:12:42 +01:00
fs . Logf ( nil , "It is recommended to use web gui with auth." )
} else {
2022-12-11 15:47:47 +01:00
if opt . Auth . BasicUser == "" && opt . Auth . HtPasswd == "" {
opt . Auth . BasicUser = "gui"
fs . Infof ( nil , "No username specified. Using default username: %s \n" , rcflags . Opt . Auth . BasicUser )
2022-01-23 14:12:42 +01:00
}
2022-12-11 15:47:47 +01:00
if opt . Auth . BasicPass == "" && opt . Auth . HtPasswd == "" {
2022-01-23 14:12:42 +01:00
randomPass , err := random . Password ( 128 )
if err != nil {
log . Fatalf ( "Failed to make password: %v" , err )
}
2022-12-11 15:47:47 +01:00
opt . Auth . BasicPass = randomPass
2022-01-23 14:12:42 +01:00
fs . Infof ( nil , "No password specified. Using random password: %s \n" , randomPass )
2020-01-12 10:12:04 +01:00
}
}
opt . Serve = true
fs . Logf ( nil , "Serving Web GUI" )
2020-04-23 21:22:47 +02:00
fileHandler = http . FileServer ( http . Dir ( extractPath ) )
2020-07-27 20:32:45 +02:00
pluginsHandler = http . FileServer ( http . Dir ( webgui . PluginsPath ) )
2018-10-27 19:29:20 +02:00
}
2020-04-23 21:22:47 +02:00
s := & Server {
2020-11-05 16:18:51 +01:00
ctx : ctx ,
2020-07-27 20:32:45 +02:00
opt : opt ,
files : fileHandler ,
pluginsHandler : pluginsHandler ,
2020-04-23 21:22:47 +02:00
}
2022-12-11 15:47:47 +01:00
var err error
s . server , err = libhttp . NewServer ( ctx ,
libhttp . WithConfig ( opt . HTTP ) ,
libhttp . WithAuth ( opt . Auth ) ,
libhttp . WithTemplate ( opt . Template ) ,
)
if err != nil {
return nil , fmt . Errorf ( "failed to init server: %w" , err )
}
router := s . server . Router ( )
router . Use (
middleware . SetHeader ( "Accept-Ranges" , "bytes" ) ,
middleware . SetHeader ( "Server" , "rclone/" + fs . Version ) ,
)
// Add the debug handler which is installed in the default mux
router . Handle ( "/debug/*" , mux )
// FIXME split these up into individual functions
router . Get ( "/*" , s . handler )
router . Head ( "/*" , s . handler )
router . Post ( "/*" , s . handler )
router . Options ( "/*" , s . handler )
return s , nil
2018-10-27 19:29:20 +02:00
}
2018-11-01 18:20:04 +01:00
// Serve runs the http server in the background.
//
// Use s.Close() and s.Wait() to shutdown server
func ( s * Server ) Serve ( ) error {
2022-12-11 15:47:47 +01:00
s . server . Serve ( )
for _ , URL := range s . server . URLs ( ) {
fs . Logf ( nil , "Serving remote control on %s" , URL )
// Open the files in the browser if set
if s . files != nil {
openURL , err := url . Parse ( URL )
if err != nil {
return fmt . Errorf ( "invalid serving URL: %w" , err )
}
// Add username, password into the URL if they are set
user , pass := s . opt . Auth . BasicUser , s . opt . Auth . BasicPass
if user != "" && pass != "" {
openURL . User = url . UserPassword ( user , pass )
// Base64 encode username and password to be sent through url
loginToken := user + ":" + pass
parameters := url . Values { }
encodedToken := base64 . URLEncoding . EncodeToString ( [ ] byte ( loginToken ) )
fs . Debugf ( nil , "login_token %q" , encodedToken )
parameters . Add ( "login_token" , encodedToken )
openURL . RawQuery = parameters . Encode ( )
openURL . RawPath = "/#/login"
}
// Don't open browser if serving in testing environment or required not to do so.
if flag . Lookup ( "test.v" ) == nil && ! s . opt . WebGUINoOpenBrowser {
if err := open . Start ( openURL . String ( ) ) ; err != nil {
fs . Errorf ( nil , "Failed to open Web GUI in browser: %v. Manually access it at: %s" , err , openURL . String ( ) )
}
} else {
fs . Logf ( nil , "Web GUI is not automatically opening browser. Navigate to %s to use." , openURL . String ( ) )
2020-01-12 10:12:04 +01:00
}
2019-07-18 12:13:54 +02:00
}
2018-10-27 19:29:20 +02:00
}
2018-11-01 18:20:04 +01:00
return nil
2018-10-27 19:29:20 +02:00
}
// writeError writes a formatted error to the output
func writeError ( path string , in rc . Params , w http . ResponseWriter , err error , status int ) {
fs . Errorf ( nil , "rc: %q: error: %v" , path , err )
2021-03-30 10:42:15 +02:00
params , status := rc . Error ( path , in , err , status )
2018-10-27 19:29:20 +02:00
w . WriteHeader ( status )
2021-03-30 10:42:15 +02:00
err = rc . WriteJSON ( w , params )
2018-10-27 19:29:20 +02:00
if err != nil {
// can't return the error at this point
2021-01-10 13:24:22 +01:00
fs . Errorf ( nil , "rc: writeError: failed to write JSON output from %#v: %v" , in , err )
2018-10-27 19:29:20 +02:00
}
}
// handler reads incoming requests and dispatches them
2018-11-01 18:20:04 +01:00
func ( s * Server ) handler ( w http . ResponseWriter , r * http . Request ) {
2022-12-11 15:47:47 +01:00
path := strings . TrimLeft ( r . URL . Path , "/" )
2018-10-27 19:29:20 +02:00
switch r . Method {
case "POST" :
s . handlePost ( w , r , path )
case "OPTIONS" :
s . handleOptions ( w , r , path )
2018-10-28 15:31:24 +01:00
case "GET" , "HEAD" :
2018-10-27 19:29:20 +02:00
s . handleGet ( w , r , path )
default :
2021-11-04 11:12:57 +01:00
writeError ( path , nil , w , fmt . Errorf ( "method %q not allowed" , r . Method ) , http . StatusMethodNotAllowed )
2018-10-27 19:29:20 +02:00
return
}
}
2018-11-01 18:20:04 +01:00
func ( s * Server ) handlePost ( w http . ResponseWriter , r * http . Request , path string ) {
2020-12-12 16:35:30 +01:00
ctx := r . Context ( )
2018-10-28 15:31:24 +01:00
contentType := r . Header . Get ( "Content-Type" )
values := r . URL . Query ( )
if contentType == "application/x-www-form-urlencoded" {
// Parse the POST and URL parameters into r.Form, for others r.Form will be empty value
err := r . ParseForm ( )
if err != nil {
2021-11-04 11:12:57 +01:00
writeError ( path , nil , w , fmt . Errorf ( "failed to parse form/URL parameters: %w" , err ) , http . StatusBadRequest )
2018-10-28 15:31:24 +01:00
return
}
values = r . Form
2018-10-27 19:29:20 +02:00
}
// Read the POST and URL parameters into in
in := make ( rc . Params )
2018-10-28 15:31:24 +01:00
for k , vs := range values {
2018-10-27 19:29:20 +02:00
if len ( vs ) > 0 {
in [ k ] = vs [ len ( vs ) - 1 ]
}
}
// Parse a JSON blob from the input
2018-10-28 15:31:24 +01:00
if contentType == "application/json" {
2018-10-27 19:29:20 +02:00
err := json . NewDecoder ( r . Body ) . Decode ( & in )
if err != nil {
2021-11-04 11:12:57 +01:00
writeError ( path , in , w , fmt . Errorf ( "failed to read input JSON: %w" , err ) , http . StatusBadRequest )
2018-10-27 19:29:20 +02:00
return
}
}
// Find the call
call := rc . Calls . Get ( path )
if call == nil {
2021-11-04 11:12:57 +01:00
writeError ( path , in , w , fmt . Errorf ( "couldn't find method %q" , path ) , http . StatusNotFound )
2018-10-27 19:29:20 +02:00
return
}
2018-11-03 17:37:09 +01:00
// Check to see if it requires authorisation
2022-12-11 15:47:47 +01:00
if ! s . opt . NoAuth && call . AuthRequired && ! s . server . UsingAuth ( ) {
2021-11-04 11:12:57 +01:00
writeError ( path , in , w , fmt . Errorf ( "authentication must be set up on the rc server to use %q or the --rc-no-auth flag must be in use" , path ) , http . StatusForbidden )
2018-11-03 17:37:09 +01:00
return
}
2021-01-10 13:24:22 +01:00
inOrig := in . Copy ( )
2020-06-05 16:26:46 +02:00
if call . NeedsRequest {
// Add the request to RC
in [ "_request" ] = r
}
2018-11-03 17:37:09 +01:00
2020-07-27 20:01:35 +02:00
if call . NeedsResponse {
2020-08-15 20:03:07 +02:00
in [ "_response" ] = w
2020-07-27 20:01:35 +02:00
}
2018-10-27 19:29:20 +02:00
fs . Debugf ( nil , "rc: %q: with parameters %+v" , path , in )
2020-12-12 16:35:30 +01:00
job , out , err := jobs . NewJob ( ctx , call . Fn , in )
if job != nil {
w . Header ( ) . Add ( "x-rclone-jobid" , fmt . Sprintf ( "%d" , job . ID ) )
2018-10-27 19:29:20 +02:00
}
if err != nil {
2021-01-10 13:24:22 +01:00
writeError ( path , inOrig , w , err , http . StatusInternalServerError )
2018-10-27 19:29:20 +02:00
return
}
if out == nil {
out = make ( rc . Params )
}
fs . Debugf ( nil , "rc: %q: reply %+v: %v" , path , out , err )
err = rc . WriteJSON ( w , out )
if err != nil {
2019-08-10 17:22:17 +02:00
// can't return the error at this point - but have a go anyway
2021-01-10 13:24:22 +01:00
writeError ( path , inOrig , w , err , http . StatusInternalServerError )
fs . Errorf ( nil , "rc: handlePost: failed to write JSON output: %v" , err )
2018-10-27 19:29:20 +02:00
}
}
2018-11-01 18:20:04 +01:00
func ( s * Server ) handleOptions ( w http . ResponseWriter , r * http . Request , path string ) {
2018-10-27 19:29:20 +02:00
w . WriteHeader ( http . StatusOK )
}
2018-11-01 18:20:04 +01:00
func ( s * Server ) serveRoot ( w http . ResponseWriter , r * http . Request ) {
2018-10-28 15:31:24 +01:00
remotes := config . FileSections ( )
sort . Strings ( remotes )
2022-12-11 15:47:47 +01:00
directory := serve . NewDirectory ( "" , s . server . HTMLTemplate ( ) )
2020-05-08 17:15:21 +02:00
directory . Name = "List of all rclone remotes."
2018-10-28 15:31:24 +01:00
q := url . Values { }
for _ , remote := range remotes {
q . Set ( "fs" , remote )
2020-05-08 17:15:21 +02:00
directory . AddHTMLEntry ( "[" + remote + ":]" , true , - 1 , time . Time { } )
2018-10-28 15:31:24 +01:00
}
2020-05-08 17:15:21 +02:00
sortParm := r . URL . Query ( ) . Get ( "sort" )
orderParm := r . URL . Query ( ) . Get ( "order" )
directory . ProcessQueryParams ( sortParm , orderParm )
2018-10-28 15:31:24 +01:00
directory . Serve ( w , r )
}
2018-11-01 18:20:04 +01:00
func ( s * Server ) serveRemote ( w http . ResponseWriter , r * http . Request , path string , fsName string ) {
2020-11-05 16:18:51 +01:00
f , err := cache . Get ( s . ctx , fsName )
2018-10-28 15:31:24 +01:00
if err != nil {
2021-11-04 11:12:57 +01:00
writeError ( path , nil , w , fmt . Errorf ( "failed to make Fs: %w" , err ) , http . StatusInternalServerError )
2018-10-28 15:31:24 +01:00
return
}
if path == "" || strings . HasSuffix ( path , "/" ) {
path = strings . Trim ( path , "/" )
2019-06-17 10:34:30 +02:00
entries , err := list . DirSorted ( r . Context ( ) , f , false , path )
2018-10-27 19:29:20 +02:00
if err != nil {
2021-11-04 11:12:57 +01:00
writeError ( path , nil , w , fmt . Errorf ( "failed to list directory: %w" , err ) , http . StatusInternalServerError )
2018-10-27 19:29:20 +02:00
return
}
2018-10-28 15:31:24 +01:00
// Make the entries for display
2022-12-11 15:47:47 +01:00
directory := serve . NewDirectory ( path , s . server . HTMLTemplate ( ) )
2018-10-28 15:31:24 +01:00
for _ , entry := range entries {
_ , isDir := entry . ( fs . Directory )
2020-05-08 17:15:21 +02:00
//directory.AddHTMLEntry(entry.Remote(), isDir, entry.Size(), entry.ModTime(r.Context()))
directory . AddHTMLEntry ( entry . Remote ( ) , isDir , entry . Size ( ) , time . Time { } )
2018-10-28 15:31:24 +01:00
}
2020-05-08 17:15:21 +02:00
sortParm := r . URL . Query ( ) . Get ( "sort" )
orderParm := r . URL . Query ( ) . Get ( "order" )
directory . ProcessQueryParams ( sortParm , orderParm )
2018-10-28 15:31:24 +01:00
directory . Serve ( w , r )
} else {
2019-06-10 12:59:06 +02:00
path = strings . Trim ( path , "/" )
2019-06-17 10:34:30 +02:00
o , err := f . NewObject ( r . Context ( ) , path )
2018-10-27 19:29:20 +02:00
if err != nil {
2021-11-04 11:12:57 +01:00
writeError ( path , nil , w , fmt . Errorf ( "failed to find object: %w" , err ) , http . StatusInternalServerError )
2018-10-27 19:29:20 +02:00
return
}
serve . Object ( w , r , o )
2018-10-28 15:31:24 +01:00
}
}
// Match URLS of the form [fs]/remote
var fsMatch = regexp . MustCompile ( ` ^\[(.*?)\](.*)$ ` )
2018-11-01 18:20:04 +01:00
func ( s * Server ) handleGet ( w http . ResponseWriter , r * http . Request , path string ) {
2018-10-28 15:31:24 +01:00
// Look to see if this has an fs in the path
2020-07-27 20:32:45 +02:00
fsMatchResult := fsMatch . FindStringSubmatch ( path )
2018-10-28 15:31:24 +01:00
switch {
2020-07-27 20:32:45 +02:00
case fsMatchResult != nil && s . opt . Serve :
2018-10-28 15:31:24 +01:00
// Serve /[fs]/remote files
2020-07-27 20:32:45 +02:00
s . serveRemote ( w , r , fsMatchResult [ 2 ] , fsMatchResult [ 1 ] )
2018-10-28 15:31:24 +01:00
return
2020-02-26 09:34:32 +01:00
case path == "metrics" && s . opt . EnableMetrics :
promHandler . ServeHTTP ( w , r )
return
2018-10-28 15:31:24 +01:00
case path == "*" && s . opt . Serve :
// Serve /* as the remote listing
s . serveRoot ( w , r )
return
case s . files != nil :
2020-09-17 21:06:51 +02:00
if s . opt . WebUI {
pluginsMatchResult := webgui . PluginsMatch . FindStringSubmatch ( path )
2022-06-08 22:25:17 +02:00
if len ( pluginsMatchResult ) > 2 {
2020-09-17 21:06:51 +02:00
ok := webgui . ServePluginOK ( w , r , pluginsMatchResult )
if ! ok {
r . URL . Path = fmt . Sprintf ( "/%s/%s/app/build/%s" , pluginsMatchResult [ 1 ] , pluginsMatchResult [ 2 ] , pluginsMatchResult [ 3 ] )
s . pluginsHandler . ServeHTTP ( w , r )
return
}
return
} else if webgui . ServePluginWithReferrerOK ( w , r , path ) {
2020-07-27 20:32:45 +02:00
return
}
}
2018-10-28 15:31:24 +01:00
// Serve the files
2019-08-20 20:47:57 +02:00
r . URL . Path = "/" + path
2018-10-27 19:29:20 +02:00
s . files . ServeHTTP ( w , r )
2018-10-28 15:31:24 +01:00
return
case path == "" && s . opt . Serve :
// Serve the root as a remote listing
s . serveRoot ( w , r )
return
2018-10-27 19:29:20 +02:00
}
2018-10-28 15:31:24 +01:00
http . Error ( w , http . StatusText ( http . StatusNotFound ) , http . StatusNotFound )
2018-10-27 19:29:20 +02:00
}
2022-12-11 15:47:47 +01:00
// Wait blocks while the server is serving requests
func ( s * Server ) Wait ( ) {
s . server . Wait ( )
}
// Shutdown gracefully shuts down the server
func ( s * Server ) Shutdown ( ) error {
return s . server . Shutdown ( )
}