serve http/webdav: support SSL/TLS

cmd/serve/http/http.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/ncw/rclone/cmd"
 	"github.com/ncw/rclone/cmd/serve/httplib"
+	"github.com/ncw/rclone/cmd/serve/httplib/httpflags"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/accounting"
 	"github.com/ncw/rclone/lib/rest"
@@ -20,7 +21,7 @@ import (
 )
 
 func init() {
-	httplib.AddFlags(Command.Flags())
+	httpflags.AddFlags(Command.Flags())
 	vfsflags.AddFlags(Command.Flags())
 }
 
@@ -44,7 +45,7 @@ control the stats printing.
 		cmd.CheckArgs(1, 1, command, args)
 		f := cmd.NewFsSrc(args)
 		cmd.Run(false, true, command, func() error {
-			s := newServer(f)
+			s := newServer(f, &httpflags.Opt)
 			s.serve()
 			return nil
 		})
@@ -58,12 +59,12 @@ type server struct {
 	srv *httplib.Server
 }
 
-func newServer(f fs.Fs) *server {
+func newServer(f fs.Fs, opt *httplib.Options) *server {
 	mux := http.NewServeMux()
 	s := &server{
 		f:   f,
 		vfs: vfs.New(f, &vfsflags.Opt),
-		srv: httplib.NewServer(mux),
+		srv: httplib.NewServer(mux, opt),
 	}
 	mux.HandleFunc("/", s.handler)
 	return s

cmd/serve/http/http_test.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	_ "github.com/ncw/rclone/backend/local"
+	"github.com/ncw/rclone/cmd/serve/httplib"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config"
 	"github.com/ncw/rclone/fs/filter"
@@ -28,8 +29,9 @@ const (
 )
 
 func startServer(t *testing.T, f fs.Fs) {
-	s := newServer(f)
-	s.srv.SetBindAddress(testBindAddress)
+	opt := httplib.DefaultOpt
+	opt.ListenAddr = testBindAddress
+	s := newServer(f, &opt)
 	go s.serve()
 
 	// try to connect to the test server

cmd/serve/httplib/httpflags/httpflags.go

@@ -0,0 +1,27 @@
+package httpflags
+
+import (
+	"github.com/ncw/rclone/cmd/serve/httplib"
+	"github.com/ncw/rclone/fs/config/flags"
+	"github.com/spf13/pflag"
+)
+
+// Options set by command line flags
+var (
+	Opt = httplib.DefaultOpt
+)
+
+// AddFlags adds flags for the httplib
+func AddFlags(flagSet *pflag.FlagSet) {
+	flags.StringVarP(flagSet, &Opt.ListenAddr, "addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to.")
+	flags.DurationVarP(flagSet, &Opt.ServerReadTimeout, "server-read-timeout", "", Opt.ServerReadTimeout, "Timeout for server reading data")
+	flags.DurationVarP(flagSet, &Opt.ServerWriteTimeout, "server-write-timeout", "", Opt.ServerWriteTimeout, "Timeout for server writing data")
+	flags.IntVarP(flagSet, &Opt.MaxHeaderBytes, "max-header-bytes", "", Opt.MaxHeaderBytes, "Maximum size of request header")
+	flags.StringVarP(flagSet, &Opt.SslCert, "cert", "", Opt.SslCert, "SSL PEM key (concatenation of certificate and CA certificate)")
+	flags.StringVarP(flagSet, &Opt.SslKey, "key", "", Opt.SslKey, "SSL PEM Private key")
+	flags.StringVarP(flagSet, &Opt.ClientCA, "client-ca", "", Opt.ClientCA, "Client certificate authority to verify clients with")
+	flags.StringVarP(flagSet, &Opt.HtPasswd, "htpasswd", "", Opt.HtPasswd, "htpasswd file - if not provided no authentication is done")
+	flags.StringVarP(flagSet, &Opt.Realm, "realm", "", Opt.Realm, "realm for authentication")
+	flags.StringVarP(flagSet, &Opt.BasicUser, "user", "", Opt.BasicUser, "User name for authentication.")
+	flags.StringVarP(flagSet, &Opt.BasicPass, "pass", "", Opt.BasicPass, "Password for authentication.")
+}

cmd/serve/httplib/httplib.go

@@ -2,32 +2,20 @@
 package httplib
 
 import (
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net/http"
+	"time"
 
 	auth "github.com/abbot/go-http-auth"
 	"github.com/ncw/rclone/fs"
-	"github.com/spf13/pflag"
 )
 
 // Globals
-var (
-	bindAddress  = "localhost:8080"
-	htPasswdFile = ""
-	realm        = "rclone"
-	basicUser    = ""
-	basicPass    = ""
-)
-
-// AddFlags adds the http server specific flags
-func AddFlags(flagSet *pflag.FlagSet) {
-	flagSet.StringVarP(&bindAddress, "addr", "", bindAddress, "IPaddress:Port to bind server to.")
-	flagSet.StringVarP(&htPasswdFile, "htpasswd", "", htPasswdFile, "File to use for htpasswd authentication.")
-	flagSet.StringVarP(&realm, "realm", "", realm, "Realm name for authentication.")
-	flagSet.StringVarP(&basicUser, "user", "", basicUser, "User name for authentication.")
-	flagSet.StringVarP(&basicPass, "pass", "", basicPass, "Password for authentication.")
-}
+var ()
 
 // Help contains text describing the http server to add to the command
 // help.
@@ -38,6 +26,13 @@ Use --addr to specify which IP address and port the server should
 listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all
 IPs.  By default it only listens on localhost.
 
+--server-read-timeout and --server-write-timeout can be used to
+control the timeouts on the server.  Note that this is the total time
+for a transfer.
+
+--max-header-bytes controls the maximum number of bytes the server will
+accept in the HTTP header.
+
 #### Authentication
 
 By default this will serve files without needing a login.
@@ -55,69 +50,142 @@ To create an htpasswd file:
     htpasswd -B htpasswd user
     htpasswd -B htpasswd anotherUser
 
-Use --realm to set the authentication realm.`
+The password file can be updated while rclone is running.
+
+Use --realm to set the authentication realm.
+
+#### SSL/TLS
+
+By default this will serve over http.  If you want you can serve over
+https.  You will need to supply the --ssl along with --cert and --key.
+If you wish to do client side certificate validation then you will
+need to supply --client-ca also.
+`
+
+// Options contains options for the http Server
+type Options struct {
+	ListenAddr         string        // Port to listen on
+	ServerReadTimeout  time.Duration // Timeout for server reading data
+	ServerWriteTimeout time.Duration // Timeout for server writing data
+	MaxHeaderBytes     int           // Maximum size of request header
+	SslCert            string        // SSL PEM key (concatenation of certificate and CA certificate)
+	SslKey             string        // SSL PEM Private key
+	ClientCA           string        // Client certificate authority to verify clients with
+	HtPasswd           string        // htpasswd file - if not provided no authentication is done
+	Realm              string        // realm for authentication
+	BasicUser          string        // single username for basic auth if not using Htpasswd
+	BasicPass          string        // password for BasicUser
+}
+
+// DefaultOpt is the default values used for Options
+var DefaultOpt = Options{
+	ListenAddr:         "localhost:8080",
+	Realm:              "rclone",
+	ServerReadTimeout:  1 * time.Hour,
+	ServerWriteTimeout: 1 * time.Hour,
+	MaxHeaderBytes:     4096,
+}
 
 // Server contains info about the running http server
 type Server struct {
-	bindAddress     string
+	Opt             Options
+	handler         http.Handler // original handler
 	httpServer      *http.Server
-	basicUser       string
 	basicPassHashed string
+	useSSL          bool // if server is configured for SSL/TLS
 }
 
 // singleUserProvider provides the encrypted password for a single user
 func (s *Server) singleUserProvider(user, realm string) string {
-	if user == s.basicUser {
+	if user == s.Opt.BasicUser {
 		return s.basicPassHashed
 	}
 	return ""
 }
 
-// NewServer creates an http server
-func NewServer(handler http.Handler) *Server {
+// NewServer creates an http server.  The opt can be nil in which case
+// the default options will be used.
+func NewServer(handler http.Handler, opt *Options) *Server {
 	s := &Server{
-		bindAddress: bindAddress,
+		handler: handler,
+	}
+
+	// Make a copy of the options
+	if opt != nil {
+		s.Opt = *opt
+	} else {
+		s.Opt = DefaultOpt
 	}
 
 	// Use htpasswd if required on everything
-	if htPasswdFile != "" || basicUser != "" {
+	if s.Opt.HtPasswd != "" || s.Opt.BasicUser != "" {
 		var secretProvider auth.SecretProvider
-		if htPasswdFile != "" {
-			fs.Infof(nil, "Using %q as htpasswd storage", htPasswdFile)
-			secretProvider = auth.HtpasswdFileProvider(htPasswdFile)
+		if s.Opt.HtPasswd != "" {
+			fs.Infof(nil, "Using %q as htpasswd storage", s.Opt.HtPasswd)
+			secretProvider = auth.HtpasswdFileProvider(s.Opt.HtPasswd)
 		} else {
-			fs.Infof(nil, "Using --user %s --pass XXXX as authenticated user", basicUser)
-			s.basicUser = basicUser
-			s.basicPassHashed = string(auth.MD5Crypt([]byte(basicPass), []byte("dlPL2MqE"), []byte("$1$")))
+			fs.Infof(nil, "Using --user %s --pass XXXX as authenticated user", s.Opt.BasicUser)
+			s.basicPassHashed = string(auth.MD5Crypt([]byte(s.Opt.BasicPass), []byte("dlPL2MqE"), []byte("$1$")))
 			secretProvider = s.singleUserProvider
 		}
-		authenticator := auth.NewBasicAuthenticator(realm, secretProvider)
+		authenticator := auth.NewBasicAuthenticator(s.Opt.Realm, secretProvider)
 		handler = auth.JustCheck(authenticator, handler.ServeHTTP)
 	}
 
+	s.useSSL = s.Opt.SslKey != ""
+	if (s.Opt.SslCert != "") != s.useSSL {
+		log.Fatalf("Need both -cert and -key to use SSL")
+	}
+
 	// FIXME make a transport?
 	s.httpServer = &http.Server{
-		Addr:           s.bindAddress,
+		Addr:           s.Opt.ListenAddr,
 		Handler:        handler,
-		MaxHeaderBytes: 1 << 20,
+		ReadTimeout:    s.Opt.ServerReadTimeout,
+		WriteTimeout:   s.Opt.ServerWriteTimeout,
+		MaxHeaderBytes: s.Opt.MaxHeaderBytes,
+		TLSConfig: &tls.Config{
+			MinVersion: tls.VersionTLS10, // disable SSL v3.0 and earlier
+		},
 	}
 	// go version specific initialisation
 	initServer(s.httpServer)
-	return s
-}
 
-// SetBindAddress overrides the config flag
-func (s *Server) SetBindAddress(addr string) {
-	s.bindAddress = addr
-	s.httpServer.Addr = addr
+	if s.Opt.ClientCA != "" {
+		if !s.useSSL {
+			log.Fatalf("Can't use --client-ca without --cert and --key")
+		}
+		certpool := x509.NewCertPool()
+		pem, err := ioutil.ReadFile(s.Opt.ClientCA)
+		if err != nil {
+			log.Fatalf("Failed to read client certificate authority: %v", err)
+		}
+		if !certpool.AppendCertsFromPEM(pem) {
+			log.Fatalf("Can't parse client certificate authority")
+		}
+		s.httpServer.TLSConfig.ClientCAs = certpool
+		s.httpServer.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert
+	}
+
+	return s
 }
 
 // Serve runs the server - doesn't return
 func (s *Server) Serve() {
-	log.Fatal(s.httpServer.ListenAndServe())
+	var err error
+	if s.useSSL {
+		err = s.httpServer.ListenAndServeTLS(s.Opt.SslCert, s.Opt.SslKey)
+	} else {
+		err = s.httpServer.ListenAndServe()
+	}
+	log.Fatalf("Fatal error while serving HTTP: %v", err)
 }
 
 // URL returns the serving address of this server
 func (s *Server) URL() string {
-	return fmt.Sprintf("http://%s/", s.bindAddress)
+	proto := "http"
+	if s.useSSL {
+		proto = "https"
+	}
+	return fmt.Sprintf("%s://%s/", proto, s.Opt.ListenAddr)
 }