mirror of
https://github.com/rclone/rclone.git
synced 2024-11-07 09:04:52 +01:00
rcserver: implement prometheus metrics on a dedicated port - fixes #7940
This commit is contained in:
parent
26bc9826e5
commit
d15704ef9f
@ -421,11 +421,18 @@ func initConfig() {
|
||||
}
|
||||
|
||||
// Start the remote control server if configured
|
||||
_, err = rcserver.Start(context.Background(), &rc.Opt)
|
||||
_, err = rcserver.Start(ctx, &rc.Opt)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to start remote control: %v", err)
|
||||
}
|
||||
|
||||
// Start the metrics server if configured
|
||||
_, err = rcserver.MetricsStart(ctx, &rc.Opt)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to start metrics server: %v", err)
|
||||
|
||||
}
|
||||
|
||||
// Setup CPU profiling if desired
|
||||
if *cpuProfile != "" {
|
||||
fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile)
|
||||
|
@ -2836,6 +2836,17 @@ Rclone prefixes all log messages with their level in capitals, e.g. INFO
|
||||
which makes it easy to grep the log file for different kinds of
|
||||
information.
|
||||
|
||||
Metrics
|
||||
-------
|
||||
|
||||
Rclone can publish metrics in the OpenMetrics/Prometheus format.
|
||||
|
||||
To enable the metrics endpoint, use the `--metrics-addr` flag. Metrics can also be published on the `--rc-addr` port if the `--rc` flag and `--rc-enable-metrics` flags are supplied or if using rclone rcd `--rc-enable-metrics`
|
||||
|
||||
Rclone provides extensive configuration options for the metrics HTTP endpoint. These settings are grouped under the Metrics section and have a prefix `--metrics-*`.
|
||||
|
||||
When metrics are enabled with `--rc-enable-metrics`, they will be published on the same port as the rc API. In this case, the `--metrics-*` flags will be ignored, and the HTTP endpoint configuration will be managed by the `--rc-*` parameters.
|
||||
|
||||
Exit Code
|
||||
---------
|
||||
|
||||
|
@ -100,6 +100,7 @@ Default Off.
|
||||
### --rc-enable-metrics
|
||||
|
||||
Enable OpenMetrics/Prometheus compatible endpoint at `/metrics`.
|
||||
If more control over the metrics is desired (for example running it on a different port or with different auth) then endpoint can be enabled with the `--metrics-*` flags instead.
|
||||
|
||||
Default Off.
|
||||
|
||||
|
@ -124,6 +124,7 @@ func init() {
|
||||
All.NewGroup("Logging", "Flags for logging and statistics")
|
||||
All.NewGroup("Metadata", "Flags to control metadata")
|
||||
All.NewGroup("RC", "Flags to control the Remote Control API")
|
||||
All.NewGroup("Metrics", "Flags to control the Metrics HTTP endpoint.")
|
||||
}
|
||||
|
||||
// installFlag constructs a name from the flag passed in and
|
||||
|
15
fs/rc/rc.go
15
fs/rc/rc.go
@ -71,8 +71,8 @@ var OptionsInfo = fs.Options{{
|
||||
}, {
|
||||
Name: "rc_enable_metrics",
|
||||
Default: false,
|
||||
Help: "Enable prometheus metrics on /metrics",
|
||||
Groups: "RC",
|
||||
Help: "Enable the Prometheus metrics path at the remote control server",
|
||||
Groups: "RC,Metrics",
|
||||
}, {
|
||||
Name: "rc_job_expire_duration",
|
||||
Default: 60 * time.Second,
|
||||
@ -83,10 +83,18 @@ var OptionsInfo = fs.Options{{
|
||||
Default: 10 * time.Second,
|
||||
Help: "Interval to check for expired async jobs",
|
||||
Groups: "RC",
|
||||
}, {
|
||||
Name: "metrics_addr",
|
||||
Default: []string{""},
|
||||
Help: "IPaddress:Port or :Port to bind metrics server to",
|
||||
Groups: "Metrics",
|
||||
}}.
|
||||
AddPrefix(libhttp.ConfigInfo, "rc", "RC").
|
||||
AddPrefix(libhttp.AuthConfigInfo, "rc", "RC").
|
||||
AddPrefix(libhttp.TemplateConfigInfo, "rc", "RC").
|
||||
AddPrefix(libhttp.ConfigInfo, "metrics", "Metrics").
|
||||
AddPrefix(libhttp.AuthConfigInfo, "metrics", "Metrics").
|
||||
AddPrefix(libhttp.TemplateConfigInfo, "metrics", "Metrics").
|
||||
SetDefault("rc_addr", []string{"localhost:5572"})
|
||||
|
||||
func init() {
|
||||
@ -109,6 +117,9 @@ type Options struct {
|
||||
WebGUINoOpenBrowser bool `config:"rc_web_gui_no_open_browser"` // set to disable auto opening browser
|
||||
WebGUIFetchURL string `config:"rc_web_fetch_url"` // set the default url for fetching webgui
|
||||
EnableMetrics bool `config:"rc_enable_metrics"` // set to disable prometheus metrics on /metrics
|
||||
MetricsHTTP libhttp.Config `config:"metrics"`
|
||||
MetricsAuth libhttp.AuthConfig `config:"metrics"`
|
||||
MetricsTemplate libhttp.TemplateConfig `config:"metrics"`
|
||||
JobExpireDuration time.Duration `config:"rc_job_expire_duration"`
|
||||
JobExpireInterval time.Duration `config:"rc_job_expire_interval"`
|
||||
}
|
||||
|
97
fs/rc/rcserver/metrics.go
Normal file
97
fs/rc/rcserver/metrics.go
Normal file
@ -0,0 +1,97 @@
|
||||
// Package rcserver implements the HTTP endpoint to serve the remote control
|
||||
package rcserver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/rc"
|
||||
"github.com/rclone/rclone/fs/rc/jobs"
|
||||
libhttp "github.com/rclone/rclone/lib/http"
|
||||
)
|
||||
|
||||
const path = "/metrics"
|
||||
|
||||
var promHandlerFunc http.HandlerFunc
|
||||
|
||||
func init() {
|
||||
rcloneCollector := accounting.NewRcloneCollector(context.Background())
|
||||
prometheus.MustRegister(rcloneCollector)
|
||||
|
||||
m := fshttp.NewMetrics("rclone")
|
||||
for _, c := range m.Collectors() {
|
||||
prometheus.MustRegister(c)
|
||||
}
|
||||
fshttp.DefaultMetrics = m
|
||||
|
||||
promHandlerFunc = promhttp.Handler().ServeHTTP
|
||||
}
|
||||
|
||||
// MetricsStart the remote control server if configured
|
||||
//
|
||||
// If the server wasn't configured the *Server returned may be nil
|
||||
func MetricsStart(ctx context.Context, opt *rc.Options) (*MetricsServer, error) {
|
||||
jobs.SetOpt(opt) // set the defaults for jobs
|
||||
if opt.MetricsHTTP.ListenAddr[0] != "" {
|
||||
// Serve on the DefaultServeMux so can have global registrations appear
|
||||
s, err := newMetricsServer(ctx, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s, s.Serve()
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// MetricsServer contains everything to run the rc server
|
||||
type MetricsServer struct {
|
||||
ctx context.Context // for global config
|
||||
server *libhttp.Server
|
||||
promHandlerFunc http.Handler
|
||||
opt *rc.Options
|
||||
}
|
||||
|
||||
func newMetricsServer(ctx context.Context, opt *rc.Options) (*MetricsServer, error) {
|
||||
s := &MetricsServer{
|
||||
ctx: ctx,
|
||||
opt: opt,
|
||||
promHandlerFunc: promHandlerFunc,
|
||||
}
|
||||
|
||||
var err error
|
||||
s.server, err = libhttp.NewServer(ctx,
|
||||
libhttp.WithConfig(opt.MetricsHTTP),
|
||||
libhttp.WithAuth(opt.MetricsAuth),
|
||||
libhttp.WithTemplate(opt.MetricsTemplate),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to init server: %w", err)
|
||||
}
|
||||
|
||||
router := s.server.Router()
|
||||
router.Get(path, promHandlerFunc)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Serve runs the http server in the background.
|
||||
//
|
||||
// Use s.Close() and s.Wait() to shutdown server
|
||||
func (s *MetricsServer) Serve() error {
|
||||
s.server.Serve()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Wait blocks while the server is serving requests
|
||||
func (s *MetricsServer) Wait() {
|
||||
s.server.Wait()
|
||||
}
|
||||
|
||||
// Shutdown gracefully shuts down the server
|
||||
func (s *MetricsServer) Shutdown() error {
|
||||
return s.server.Shutdown()
|
||||
}
|
88
fs/rc/rcserver/metrics_test.go
Normal file
88
fs/rc/rcserver/metrics_test.go
Normal file
@ -0,0 +1,88 @@
|
||||
package rcserver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/config/configfile"
|
||||
"github.com/rclone/rclone/fs/rc"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Run a suite of tests
|
||||
func testMetricsServer(t *testing.T, tests []testRun, opt *rc.Options) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
configfile.Install()
|
||||
rcServer, err := newMetricsServer(ctx, opt)
|
||||
require.NoError(t, err)
|
||||
testURL := rcServer.server.URLs()[0]
|
||||
mux := rcServer.server.Router()
|
||||
emulateCalls(t, tests, mux, testURL)
|
||||
}
|
||||
|
||||
// return an enabled rc
|
||||
func newMetricsTestOpt() rc.Options {
|
||||
opt := rc.Opt
|
||||
opt.MetricsHTTP.ListenAddr = []string{testBindAddress}
|
||||
return opt
|
||||
}
|
||||
|
||||
func TestMetrics(t *testing.T) {
|
||||
stats := accounting.GlobalStats()
|
||||
tests := makeMetricsTestCases(stats)
|
||||
opt := newMetricsTestOpt()
|
||||
testMetricsServer(t, tests, &opt)
|
||||
|
||||
// Test changing a couple options
|
||||
stats.Bytes(500)
|
||||
for i := 0; i < 30; i++ {
|
||||
require.NoError(t, stats.DeleteFile(context.Background(), 0))
|
||||
}
|
||||
stats.Errors(2)
|
||||
stats.Bytes(324)
|
||||
|
||||
tests = makeMetricsTestCases(stats)
|
||||
testMetricsServer(t, tests, &opt)
|
||||
}
|
||||
|
||||
func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) {
|
||||
tests = []testRun{{
|
||||
Name: "Bytes Transferred Metric",
|
||||
URL: "metrics",
|
||||
Method: "GET",
|
||||
Status: http.StatusOK,
|
||||
Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())),
|
||||
}, {
|
||||
Name: "Checked Files Metric",
|
||||
URL: "metrics",
|
||||
Method: "GET",
|
||||
Status: http.StatusOK,
|
||||
Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())),
|
||||
}, {
|
||||
Name: "Errors Metric",
|
||||
URL: "metrics",
|
||||
Method: "GET",
|
||||
Status: http.StatusOK,
|
||||
Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())),
|
||||
}, {
|
||||
Name: "Deleted Files Metric",
|
||||
URL: "metrics",
|
||||
Method: "GET",
|
||||
Status: http.StatusOK,
|
||||
Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.GetDeletes())),
|
||||
}, {
|
||||
Name: "Files Transferred Metric",
|
||||
URL: "metrics",
|
||||
Method: "GET",
|
||||
Status: http.StatusOK,
|
||||
Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())),
|
||||
},
|
||||
}
|
||||
return
|
||||
}
|
@ -18,13 +18,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/list"
|
||||
"github.com/rclone/rclone/fs/rc"
|
||||
"github.com/rclone/rclone/fs/rc/jobs"
|
||||
@ -35,21 +31,6 @@ import (
|
||||
"github.com/skratchdot/open-golang/open"
|
||||
)
|
||||
|
||||
var promHandler http.Handler
|
||||
|
||||
func init() {
|
||||
rcloneCollector := accounting.NewRcloneCollector(context.Background())
|
||||
prometheus.MustRegister(rcloneCollector)
|
||||
|
||||
m := fshttp.NewMetrics("rclone")
|
||||
for _, c := range m.Collectors() {
|
||||
prometheus.MustRegister(c)
|
||||
}
|
||||
fshttp.DefaultMetrics = m
|
||||
|
||||
promHandler = promhttp.Handler()
|
||||
}
|
||||
|
||||
// Start the remote control server if configured
|
||||
//
|
||||
// If the server wasn't configured the *Server returned may be nil
|
||||
@ -376,7 +357,7 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string)
|
||||
s.serveRemote(w, r, fsMatchResult[2], fsMatchResult[1])
|
||||
return
|
||||
case path == "metrics" && s.opt.EnableMetrics:
|
||||
promHandler.ServeHTTP(w, r)
|
||||
promHandlerFunc(w, r)
|
||||
return
|
||||
case path == "*" && s.opt.Serve:
|
||||
// Serve /* as the remote listing
|
||||
|
@ -15,9 +15,10 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/config/configfile"
|
||||
"github.com/rclone/rclone/fs/rc"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@ -115,6 +116,10 @@ func testServer(t *testing.T, tests []testRun, opt *rc.Options) {
|
||||
require.NoError(t, err)
|
||||
testURL := rcServer.server.URLs()[0]
|
||||
mux := rcServer.server.Router()
|
||||
emulateCalls(t, tests, mux, testURL)
|
||||
}
|
||||
|
||||
func emulateCalls(t *testing.T, tests []testRun, mux chi.Router, testURL string) {
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
t.Helper()
|
||||
@ -568,61 +573,6 @@ Unknown command
|
||||
testServer(t, tests, &opt)
|
||||
}
|
||||
|
||||
func TestMetrics(t *testing.T) {
|
||||
stats := accounting.GlobalStats()
|
||||
tests := makeMetricsTestCases(stats)
|
||||
opt := newTestOpt()
|
||||
opt.EnableMetrics = true
|
||||
testServer(t, tests, &opt)
|
||||
|
||||
// Test changing a couple options
|
||||
stats.Bytes(500)
|
||||
for i := 0; i < 30; i++ {
|
||||
require.NoError(t, stats.DeleteFile(context.Background(), 0))
|
||||
}
|
||||
stats.Errors(2)
|
||||
stats.Bytes(324)
|
||||
|
||||
tests = makeMetricsTestCases(stats)
|
||||
testServer(t, tests, &opt)
|
||||
}
|
||||
|
||||
func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) {
|
||||
tests = []testRun{{
|
||||
Name: "Bytes Transferred Metric",
|
||||
URL: "/metrics",
|
||||
Method: "GET",
|
||||
Status: http.StatusOK,
|
||||
Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())),
|
||||
}, {
|
||||
Name: "Checked Files Metric",
|
||||
URL: "/metrics",
|
||||
Method: "GET",
|
||||
Status: http.StatusOK,
|
||||
Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())),
|
||||
}, {
|
||||
Name: "Errors Metric",
|
||||
URL: "/metrics",
|
||||
Method: "GET",
|
||||
Status: http.StatusOK,
|
||||
Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())),
|
||||
}, {
|
||||
Name: "Deleted Files Metric",
|
||||
URL: "/metrics",
|
||||
Method: "GET",
|
||||
Status: http.StatusOK,
|
||||
Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.GetDeletes())),
|
||||
}, {
|
||||
Name: "Files Transferred Metric",
|
||||
URL: "/metrics",
|
||||
Method: "GET",
|
||||
Status: http.StatusOK,
|
||||
Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())),
|
||||
},
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
var matchRemoteDirListing = regexp.MustCompile(`<title>Directory listing of /</title>`)
|
||||
|
||||
func TestServingRoot(t *testing.T) {
|
||||
|
Loading…
Reference in New Issue
Block a user