// Package httplib provides common functionality for http servers package httplib import ( "context" "crypto/tls" "crypto/x509" "encoding/base64" "fmt" "html/template" "io/ioutil" "log" "net" "net/http" "strings" "time" auth "github.com/abbot/go-http-auth" "github.com/pkg/errors" "github.com/rclone/rclone/cmd/serve/httplib/serve/data" "github.com/rclone/rclone/fs" ) // Globals var () // Help contains text describing the http server to add to the command // help. var Help = ` ### Server options Use --addr to specify which IP address and port the server should listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. --server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer. --max-header-bytes controls the maximum number of bytes the server will accept in the HTTP header. --baseurl controls the URL prefix that rclone serves from. By default rclone will serve from the root. If you used --baseurl "/rclone" then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl "/rclone" and --baseurl "/rclone/" are all treated identically. --template allows a user to specify a custom markup template for http and webdav serve functions. The server exports the following markup to be used within the template to server pages: | Parameter | Description | | :---------- | :---------- | | .Name | The full path of a file/directory. | | .Title | Directory listing of .Name | | .Sort | The current sort used. This is changeable via ?sort= parameter | | | Sort Options: namedirfist,name,size,time (default namedirfirst) | | .Order | The current ordering used. This is changeable via ?order= parameter | | | Order Options: asc,desc (default asc) | | .Query | Currently unused. | | .Breadcrumb | Allows for creating a relative navigation | |-- .Link | The relative to the root link of the Text. | |-- .Text | The Name of the directory. | | .Entries | Information about a specific file/directory. | |-- .URL | The 'url' of an entry. | |-- .Leaf | Currently same as 'URL' but intended to be 'just' the name. | |-- .IsDir | Boolean for if an entry is a directory or not. | |-- .Size | Size in Bytes of the entry. | |-- .ModTime | The UTC timestamp of an entry. | #### Authentication By default this will serve files without needing a login. You can either use an htpasswd file which can take lots of users, or set a single username and password with the --user and --pass flags. Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended. To create an htpasswd file: touch htpasswd htpasswd -B htpasswd user htpasswd -B htpasswd anotherUser The password file can be updated while rclone is running. Use --realm to set the authentication realm. #### SSL/TLS By default this will serve over http. If you want you can serve over https. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also. --cert should be either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate. ` // Options contains options for the http Server type Options struct { ListenAddr string // Port to listen on BaseURL string // prefix to strip from URLs ServerReadTimeout time.Duration // Timeout for server reading data ServerWriteTimeout time.Duration // Timeout for server writing data MaxHeaderBytes int // Maximum size of request header SslCert string // SSL PEM key (concatenation of certificate and CA certificate) SslKey string // SSL PEM Private key ClientCA string // Client certificate authority to verify clients with HtPasswd string // htpasswd file - if not provided no authentication is done Realm string // realm for authentication BasicUser string // single username for basic auth if not using Htpasswd BasicPass string // password for BasicUser Auth AuthFn `json:"-"` // custom Auth (not set by command line flags) Template string // User specified template } // AuthFn if used will be used to authenticate user, pass. If an error // is returned then the user is not authenticated. // // If a non nil value is returned then it is added to the context under the key type AuthFn func(user, pass string) (value interface{}, err error) // DefaultOpt is the default values used for Options var DefaultOpt = Options{ ListenAddr: "localhost:8080", Realm: "rclone", ServerReadTimeout: 1 * time.Hour, ServerWriteTimeout: 1 * time.Hour, MaxHeaderBytes: 4096, } // Server contains info about the running http server type Server struct { Opt Options handler http.Handler // original handler listener net.Listener waitChan chan struct{} // for waiting on the listener to close httpServer *http.Server basicPassHashed string useSSL bool // if server is configured for SSL/TLS usingAuth bool // set if authentication is configured HTMLTemplate *template.Template // HTML template for web interface } type contextUserType struct{} // ContextUserKey is a simple context key for storing the username of the request var ContextUserKey = &contextUserType{} type contextAuthType struct{} // ContextAuthKey is a simple context key for storing info returned by AuthFn var ContextAuthKey = &contextAuthType{} // singleUserProvider provides the encrypted password for a single user func (s *Server) singleUserProvider(user, realm string) string { if user == s.Opt.BasicUser { return s.basicPassHashed } return "" } // parseAuthorization parses the Authorization header into user, pass // it returns a boolean as to whether the parse was successful func parseAuthorization(r *http.Request) (user, pass string, ok bool) { authHeader := r.Header.Get("Authorization") if authHeader != "" { s := strings.SplitN(authHeader, " ", 2) if len(s) == 2 && s[0] == "Basic" { b, err := base64.StdEncoding.DecodeString(s[1]) if err == nil { parts := strings.SplitN(string(b), ":", 2) user = parts[0] if len(parts) > 1 { pass = parts[1] ok = true } } } } return } // NewServer creates an http server. The opt can be nil in which case // the default options will be used. func NewServer(handler http.Handler, opt *Options) *Server { s := &Server{ handler: handler, } // Make a copy of the options if opt != nil { s.Opt = *opt } else { s.Opt = DefaultOpt } // Use htpasswd if required on everything if s.Opt.HtPasswd != "" || s.Opt.BasicUser != "" || s.Opt.Auth != nil { var authenticator *auth.BasicAuth if s.Opt.Auth == nil { var secretProvider auth.SecretProvider if s.Opt.HtPasswd != "" { fs.Infof(nil, "Using %q as htpasswd storage", s.Opt.HtPasswd) secretProvider = auth.HtpasswdFileProvider(s.Opt.HtPasswd) } else { fs.Infof(nil, "Using --user %s --pass XXXX as authenticated user", s.Opt.BasicUser) s.basicPassHashed = string(auth.MD5Crypt([]byte(s.Opt.BasicPass), []byte("dlPL2MqE"), []byte("$1$"))) secretProvider = s.singleUserProvider } authenticator = auth.NewBasicAuthenticator(s.Opt.Realm, secretProvider) } oldHandler := handler handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // No auth wanted for OPTIONS method if r.Method == "OPTIONS" { oldHandler.ServeHTTP(w, r) return } unauthorized := func() { w.Header().Set("Content-Type", "text/plain") w.Header().Set("WWW-Authenticate", `Basic realm="`+s.Opt.Realm+`"`) http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) } user, pass, authValid := parseAuthorization(r) if !authValid { unauthorized() return } if s.Opt.Auth == nil { if username := authenticator.CheckAuth(r); username == "" { fs.Infof(r.URL.Path, "%s: Unauthorized request from %s", r.RemoteAddr, user) unauthorized() return } } else { // Custom Auth value, err := s.Opt.Auth(user, pass) if err != nil { fs.Infof(r.URL.Path, "%s: Auth failed from %s: %v", r.RemoteAddr, user, err) unauthorized() return } if value != nil { r = r.WithContext(context.WithValue(r.Context(), ContextAuthKey, value)) } } r = r.WithContext(context.WithValue(r.Context(), ContextUserKey, user)) oldHandler.ServeHTTP(w, r) }) s.usingAuth = true } s.useSSL = s.Opt.SslKey != "" if (s.Opt.SslCert != "") != s.useSSL { log.Fatalf("Need both -cert and -key to use SSL") } // If a Base URL is set then serve from there s.Opt.BaseURL = strings.Trim(s.Opt.BaseURL, "/") if s.Opt.BaseURL != "" { s.Opt.BaseURL = "/" + s.Opt.BaseURL } // FIXME make a transport? s.httpServer = &http.Server{ Addr: s.Opt.ListenAddr, Handler: handler, ReadTimeout: s.Opt.ServerReadTimeout, WriteTimeout: s.Opt.ServerWriteTimeout, MaxHeaderBytes: s.Opt.MaxHeaderBytes, ReadHeaderTimeout: 10 * time.Second, // time to send the headers IdleTimeout: 60 * time.Second, // time to keep idle connections open TLSConfig: &tls.Config{ MinVersion: tls.VersionTLS10, // disable SSL v3.0 and earlier }, } if s.Opt.ClientCA != "" { if !s.useSSL { log.Fatalf("Can't use --client-ca without --cert and --key") } certpool := x509.NewCertPool() pem, err := ioutil.ReadFile(s.Opt.ClientCA) if err != nil { log.Fatalf("Failed to read client certificate authority: %v", err) } if !certpool.AppendCertsFromPEM(pem) { log.Fatalf("Can't parse client certificate authority") } s.httpServer.TLSConfig.ClientCAs = certpool s.httpServer.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert } htmlTemplate, templateErr := data.GetTemplate(s.Opt.Template) if templateErr != nil { log.Fatalf(templateErr.Error()) } s.HTMLTemplate = htmlTemplate return s } // Serve runs the server - returns an error only if // the listener was not started; does not block, so // use s.Wait() to block on the listener indefinitely. func (s *Server) Serve() error { ln, err := net.Listen("tcp", s.httpServer.Addr) if err != nil { return errors.Wrapf(err, "start server failed") } s.listener = ln s.waitChan = make(chan struct{}) go func() { var err error if s.useSSL { // hacky hack to get this to work with old Go versions, which // don't have ServeTLS on http.Server; see PR #2194. type tlsServer interface { ServeTLS(ln net.Listener, cert, key string) error } srvIface := interface{}(s.httpServer) if tlsSrv, ok := srvIface.(tlsServer); ok { // yay -- we get easy TLS support with HTTP/2 err = tlsSrv.ServeTLS(s.listener, s.Opt.SslCert, s.Opt.SslKey) } else { // oh well -- we can still do TLS but might not have HTTP/2 tlsConfig := new(tls.Config) tlsConfig.Certificates = make([]tls.Certificate, 1) tlsConfig.Certificates[0], err = tls.LoadX509KeyPair(s.Opt.SslCert, s.Opt.SslKey) if err != nil { log.Printf("Error loading key pair: %v", err) } tlsLn := tls.NewListener(s.listener, tlsConfig) err = s.httpServer.Serve(tlsLn) } } else { err = s.httpServer.Serve(s.listener) } if err != nil { log.Printf("Error on serving HTTP server: %v", err) } }() return nil } // Wait blocks while the listener is open. func (s *Server) Wait() { <-s.waitChan } // Close shuts the running server down func (s *Server) Close() { err := s.httpServer.Close() if err != nil { log.Printf("Error on closing HTTP server: %v", err) return } close(s.waitChan) } // URL returns the serving address of this server func (s *Server) URL() string { proto := "http" if s.useSSL { proto = "https" } addr := s.Opt.ListenAddr if s.listener != nil { // prefer actual listener address; required if using 0-port // (i.e. port assigned by operating system) addr = s.listener.Addr().String() } return fmt.Sprintf("%s://%s%s/", proto, addr, s.Opt.BaseURL) } // UsingAuth returns true if authentication is required func (s *Server) UsingAuth() bool { return s.usingAuth } // Path returns the current path with the Prefix stripped // // If it returns false, then the path was invalid and the handler // should exit as the error response has already been sent func (s *Server) Path(w http.ResponseWriter, r *http.Request) (Path string, ok bool) { Path = r.URL.Path if s.Opt.BaseURL == "" { return Path, true } if !strings.HasPrefix(Path, s.Opt.BaseURL+"/") { http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) return Path, false } Path = Path[len(s.Opt.BaseURL):] return Path, true }