rcserver: implement prometheus metrics on a dedicated port - fixes #7940

This commit is contained in:
Oleg Kunitsyn 2024-09-06 16:00:36 +02:00 committed by GitHub
parent 26bc9826e5
commit d15704ef9f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 226 additions and 79 deletions

View File

@ -421,11 +421,18 @@ func initConfig() {
}
// Start the remote control server if configured
_, err = rcserver.Start(context.Background(), &rc.Opt)
_, err = rcserver.Start(ctx, &rc.Opt)
if err != nil {
log.Fatalf("Failed to start remote control: %v", err)
}
// Start the metrics server if configured
_, err = rcserver.MetricsStart(ctx, &rc.Opt)
if err != nil {
log.Fatalf("Failed to start metrics server: %v", err)
}
// Setup CPU profiling if desired
if *cpuProfile != "" {
fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile)

View File

@ -2836,6 +2836,17 @@ Rclone prefixes all log messages with their level in capitals, e.g. INFO
which makes it easy to grep the log file for different kinds of
information.
Metrics
-------
Rclone can publish metrics in the OpenMetrics/Prometheus format.
To enable the metrics endpoint, use the `--metrics-addr` flag. Metrics can also be published on the `--rc-addr` port if the `--rc` flag and `--rc-enable-metrics` flags are supplied or if using rclone rcd `--rc-enable-metrics`
Rclone provides extensive configuration options for the metrics HTTP endpoint. These settings are grouped under the Metrics section and have a prefix `--metrics-*`.
When metrics are enabled with `--rc-enable-metrics`, they will be published on the same port as the rc API. In this case, the `--metrics-*` flags will be ignored, and the HTTP endpoint configuration will be managed by the `--rc-*` parameters.
Exit Code
---------

View File

@ -100,6 +100,7 @@ Default Off.
### --rc-enable-metrics
Enable OpenMetrics/Prometheus compatible endpoint at `/metrics`.
If more control over the metrics is desired (for example running it on a different port or with different auth) then endpoint can be enabled with the `--metrics-*` flags instead.
Default Off.

View File

@ -124,6 +124,7 @@ func init() {
All.NewGroup("Logging", "Flags for logging and statistics")
All.NewGroup("Metadata", "Flags to control metadata")
All.NewGroup("RC", "Flags to control the Remote Control API")
All.NewGroup("Metrics", "Flags to control the Metrics HTTP endpoint.")
}
// installFlag constructs a name from the flag passed in and

View File

@ -71,8 +71,8 @@ var OptionsInfo = fs.Options{{
}, {
Name: "rc_enable_metrics",
Default: false,
Help: "Enable prometheus metrics on /metrics",
Groups: "RC",
Help: "Enable the Prometheus metrics path at the remote control server",
Groups: "RC,Metrics",
}, {
Name: "rc_job_expire_duration",
Default: 60 * time.Second,
@ -83,10 +83,18 @@ var OptionsInfo = fs.Options{{
Default: 10 * time.Second,
Help: "Interval to check for expired async jobs",
Groups: "RC",
}, {
Name: "metrics_addr",
Default: []string{""},
Help: "IPaddress:Port or :Port to bind metrics server to",
Groups: "Metrics",
}}.
AddPrefix(libhttp.ConfigInfo, "rc", "RC").
AddPrefix(libhttp.AuthConfigInfo, "rc", "RC").
AddPrefix(libhttp.TemplateConfigInfo, "rc", "RC").
AddPrefix(libhttp.ConfigInfo, "metrics", "Metrics").
AddPrefix(libhttp.AuthConfigInfo, "metrics", "Metrics").
AddPrefix(libhttp.TemplateConfigInfo, "metrics", "Metrics").
SetDefault("rc_addr", []string{"localhost:5572"})
func init() {
@ -109,6 +117,9 @@ type Options struct {
WebGUINoOpenBrowser bool `config:"rc_web_gui_no_open_browser"` // set to disable auto opening browser
WebGUIFetchURL string `config:"rc_web_fetch_url"` // set the default url for fetching webgui
EnableMetrics bool `config:"rc_enable_metrics"` // set to disable prometheus metrics on /metrics
MetricsHTTP libhttp.Config `config:"metrics"`
MetricsAuth libhttp.AuthConfig `config:"metrics"`
MetricsTemplate libhttp.TemplateConfig `config:"metrics"`
JobExpireDuration time.Duration `config:"rc_job_expire_duration"`
JobExpireInterval time.Duration `config:"rc_job_expire_interval"`
}

97
fs/rc/rcserver/metrics.go Normal file
View File

@ -0,0 +1,97 @@
// Package rcserver implements the HTTP endpoint to serve the remote control
package rcserver
import (
"context"
"fmt"
"net/http"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/fs/rc/jobs"
libhttp "github.com/rclone/rclone/lib/http"
)
const path = "/metrics"
var promHandlerFunc http.HandlerFunc
func init() {
rcloneCollector := accounting.NewRcloneCollector(context.Background())
prometheus.MustRegister(rcloneCollector)
m := fshttp.NewMetrics("rclone")
for _, c := range m.Collectors() {
prometheus.MustRegister(c)
}
fshttp.DefaultMetrics = m
promHandlerFunc = promhttp.Handler().ServeHTTP
}
// MetricsStart the remote control server if configured
//
// If the server wasn't configured the *Server returned may be nil
func MetricsStart(ctx context.Context, opt *rc.Options) (*MetricsServer, error) {
jobs.SetOpt(opt) // set the defaults for jobs
if opt.MetricsHTTP.ListenAddr[0] != "" {
// Serve on the DefaultServeMux so can have global registrations appear
s, err := newMetricsServer(ctx, opt)
if err != nil {
return nil, err
}
return s, s.Serve()
}
return nil, nil
}
// MetricsServer contains everything to run the rc server
type MetricsServer struct {
ctx context.Context // for global config
server *libhttp.Server
promHandlerFunc http.Handler
opt *rc.Options
}
func newMetricsServer(ctx context.Context, opt *rc.Options) (*MetricsServer, error) {
s := &MetricsServer{
ctx: ctx,
opt: opt,
promHandlerFunc: promHandlerFunc,
}
var err error
s.server, err = libhttp.NewServer(ctx,
libhttp.WithConfig(opt.MetricsHTTP),
libhttp.WithAuth(opt.MetricsAuth),
libhttp.WithTemplate(opt.MetricsTemplate),
)
if err != nil {
return nil, fmt.Errorf("failed to init server: %w", err)
}
router := s.server.Router()
router.Get(path, promHandlerFunc)
return s, nil
}
// Serve runs the http server in the background.
//
// Use s.Close() and s.Wait() to shutdown server
func (s *MetricsServer) Serve() error {
s.server.Serve()
return nil
}
// Wait blocks while the server is serving requests
func (s *MetricsServer) Wait() {
s.server.Wait()
}
// Shutdown gracefully shuts down the server
func (s *MetricsServer) Shutdown() error {
return s.server.Shutdown()
}

View File

@ -0,0 +1,88 @@
package rcserver
import (
"context"
"fmt"
"net/http"
"regexp"
"testing"
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/config/configfile"
"github.com/rclone/rclone/fs/rc"
"github.com/stretchr/testify/require"
)
// Run a suite of tests
func testMetricsServer(t *testing.T, tests []testRun, opt *rc.Options) {
t.Helper()
ctx := context.Background()
configfile.Install()
rcServer, err := newMetricsServer(ctx, opt)
require.NoError(t, err)
testURL := rcServer.server.URLs()[0]
mux := rcServer.server.Router()
emulateCalls(t, tests, mux, testURL)
}
// return an enabled rc
func newMetricsTestOpt() rc.Options {
opt := rc.Opt
opt.MetricsHTTP.ListenAddr = []string{testBindAddress}
return opt
}
func TestMetrics(t *testing.T) {
stats := accounting.GlobalStats()
tests := makeMetricsTestCases(stats)
opt := newMetricsTestOpt()
testMetricsServer(t, tests, &opt)
// Test changing a couple options
stats.Bytes(500)
for i := 0; i < 30; i++ {
require.NoError(t, stats.DeleteFile(context.Background(), 0))
}
stats.Errors(2)
stats.Bytes(324)
tests = makeMetricsTestCases(stats)
testMetricsServer(t, tests, &opt)
}
func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) {
tests = []testRun{{
Name: "Bytes Transferred Metric",
URL: "metrics",
Method: "GET",
Status: http.StatusOK,
Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())),
}, {
Name: "Checked Files Metric",
URL: "metrics",
Method: "GET",
Status: http.StatusOK,
Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())),
}, {
Name: "Errors Metric",
URL: "metrics",
Method: "GET",
Status: http.StatusOK,
Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())),
}, {
Name: "Deleted Files Metric",
URL: "metrics",
Method: "GET",
Status: http.StatusOK,
Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.GetDeletes())),
}, {
Name: "Files Transferred Metric",
URL: "metrics",
Method: "GET",
Status: http.StatusOK,
Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())),
},
}
return
}

View File

@ -18,13 +18,9 @@ import (
"time"
"github.com/go-chi/chi/v5/middleware"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/fs/rc/jobs"
@ -35,21 +31,6 @@ import (
"github.com/skratchdot/open-golang/open"
)
var promHandler http.Handler
func init() {
rcloneCollector := accounting.NewRcloneCollector(context.Background())
prometheus.MustRegister(rcloneCollector)
m := fshttp.NewMetrics("rclone")
for _, c := range m.Collectors() {
prometheus.MustRegister(c)
}
fshttp.DefaultMetrics = m
promHandler = promhttp.Handler()
}
// Start the remote control server if configured
//
// If the server wasn't configured the *Server returned may be nil
@ -376,7 +357,7 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string)
s.serveRemote(w, r, fsMatchResult[2], fsMatchResult[1])
return
case path == "metrics" && s.opt.EnableMetrics:
promHandler.ServeHTTP(w, r)
promHandlerFunc(w, r)
return
case path == "*" && s.opt.Serve:
// Serve /* as the remote listing

View File

@ -15,9 +15,10 @@ import (
"testing"
"time"
"github.com/go-chi/chi/v5"
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/config/configfile"
"github.com/rclone/rclone/fs/rc"
"github.com/stretchr/testify/assert"
@ -115,6 +116,10 @@ func testServer(t *testing.T, tests []testRun, opt *rc.Options) {
require.NoError(t, err)
testURL := rcServer.server.URLs()[0]
mux := rcServer.server.Router()
emulateCalls(t, tests, mux, testURL)
}
func emulateCalls(t *testing.T, tests []testRun, mux chi.Router, testURL string) {
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
t.Helper()
@ -568,61 +573,6 @@ Unknown command
testServer(t, tests, &opt)
}
func TestMetrics(t *testing.T) {
stats := accounting.GlobalStats()
tests := makeMetricsTestCases(stats)
opt := newTestOpt()
opt.EnableMetrics = true
testServer(t, tests, &opt)
// Test changing a couple options
stats.Bytes(500)
for i := 0; i < 30; i++ {
require.NoError(t, stats.DeleteFile(context.Background(), 0))
}
stats.Errors(2)
stats.Bytes(324)
tests = makeMetricsTestCases(stats)
testServer(t, tests, &opt)
}
func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) {
tests = []testRun{{
Name: "Bytes Transferred Metric",
URL: "/metrics",
Method: "GET",
Status: http.StatusOK,
Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())),
}, {
Name: "Checked Files Metric",
URL: "/metrics",
Method: "GET",
Status: http.StatusOK,
Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())),
}, {
Name: "Errors Metric",
URL: "/metrics",
Method: "GET",
Status: http.StatusOK,
Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())),
}, {
Name: "Deleted Files Metric",
URL: "/metrics",
Method: "GET",
Status: http.StatusOK,
Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.GetDeletes())),
}, {
Name: "Files Transferred Metric",
URL: "/metrics",
Method: "GET",
Status: http.StatusOK,
Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())),
},
}
return
}
var matchRemoteDirListing = regexp.MustCompile(`<title>Directory listing of /</title>`)
func TestServingRoot(t *testing.T) {