From 83ab21f00ccdf5ac12ac47a62b692a68422954b6 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 10 Mar 2023 14:25:29 -0500 Subject: [PATCH 01/83] roughed in limits model that incorporates bandwidth limit specs (#235) --- controller/config.go | 17 +++-------- controller/enable.go | 7 +++-- controller/limits/model.go | 51 ++++++++++++++++++++++++++++++++ controller/share.go | 7 +++-- controller/zrokEdgeSdk/client.go | 4 +-- 5 files changed, 65 insertions(+), 21 deletions(-) create mode 100644 controller/limits/model.go diff --git a/controller/config.go b/controller/config.go index df68e46e..7be959f0 100644 --- a/controller/config.go +++ b/controller/config.go @@ -1,6 +1,7 @@ package controller import ( + "github.com/openziti/zrok/controller/limits" "github.com/openziti/zrok/controller/zrokEdgeSdk" "time" @@ -17,12 +18,12 @@ type Config struct { Endpoint *EndpointConfig Email *EmailConfig Influx *InfluxConfig - Limits *LimitsConfig + Limits *limits.Config Maintenance *MaintenanceConfig Registration *RegistrationConfig ResetPassword *ResetPasswordConfig Store *store.Config - Ziti *zrokEdgeSdk.ZitiConfig + Ziti *zrokEdgeSdk.Config } type AdminConfig struct { @@ -76,19 +77,9 @@ type ResetPasswordMaintenanceConfig struct { BatchLimit int } -const Unlimited = -1 - -type LimitsConfig struct { - Environments int - Shares int -} - func DefaultConfig() *Config { return &Config{ - Limits: &LimitsConfig{ - Environments: Unlimited, - Shares: Unlimited, - }, + Limits: limits.DefaultConfig(), Maintenance: &MaintenanceConfig{ ResetPassword: &ResetPasswordMaintenanceConfig{ ExpirationTimeout: time.Minute * 15, diff --git a/controller/enable.go b/controller/enable.go index 0b321e11..1025c562 100644 --- a/controller/enable.go +++ b/controller/enable.go @@ -5,6 +5,7 @@ import ( "encoding/json" "github.com/go-openapi/runtime/middleware" "github.com/jmoiron/sqlx" + "github.com/openziti/zrok/controller/limits" "github.com/openziti/zrok/controller/store" "github.com/openziti/zrok/controller/zrokEdgeSdk" "github.com/openziti/zrok/rest_model_zrok" @@ -14,10 +15,10 @@ import ( ) type enableHandler struct { - cfg *LimitsConfig + cfg *limits.Config } -func newEnableHandler(cfg *LimitsConfig) *enableHandler { +func newEnableHandler(cfg *limits.Config) *enableHandler { return &enableHandler{cfg: cfg} } @@ -100,7 +101,7 @@ func (h *enableHandler) Handle(params environment.EnableParams, principal *rest_ } func (h *enableHandler) checkLimits(principal *rest_model_zrok.Principal, tx *sqlx.Tx) error { - if !principal.Limitless && h.cfg.Environments > Unlimited { + if !principal.Limitless && h.cfg.Environments > limits.Unlimited { envs, err := str.FindEnvironmentsForAccount(int(principal.ID), tx) if err != nil { return errors.Errorf("unable to find environments for account '%v': %v", principal.Email, err) diff --git a/controller/limits/model.go b/controller/limits/model.go new file mode 100644 index 00000000..7622005e --- /dev/null +++ b/controller/limits/model.go @@ -0,0 +1,51 @@ +package limits + +import "time" + +const Unlimited = -1 + +type Config struct { + Environments int + Shares int + Bandwidth *BandwidthConfig +} + +type BandwidthConfig struct { + PerAccount *BandwidthPerPeriod + PerEnvironment *BandwidthPerPeriod + PerShare *BandwidthPerPeriod +} + +type BandwidthPerPeriod struct { + Period time.Duration + Rx int64 + Tx int64 + Total int64 +} + +func DefaultConfig() *Config { + return &Config{ + Environments: Unlimited, + Shares: Unlimited, + Bandwidth: &BandwidthConfig{ + PerAccount: &BandwidthPerPeriod{ + Period: 365 * (24 * time.Hour), + Rx: Unlimited, + Tx: Unlimited, + Total: Unlimited, + }, + PerEnvironment: &BandwidthPerPeriod{ + Period: 365 * (24 * time.Hour), + Rx: Unlimited, + Tx: Unlimited, + Total: Unlimited, + }, + PerShare: &BandwidthPerPeriod{ + Period: 365 * (24 * time.Hour), + Rx: Unlimited, + Tx: Unlimited, + Total: Unlimited, + }, + }, + } +} diff --git a/controller/share.go b/controller/share.go index a79965ab..a25b604b 100644 --- a/controller/share.go +++ b/controller/share.go @@ -3,6 +3,7 @@ package controller import ( "github.com/go-openapi/runtime/middleware" "github.com/jmoiron/sqlx" + "github.com/openziti/zrok/controller/limits" "github.com/openziti/zrok/controller/store" "github.com/openziti/zrok/controller/zrokEdgeSdk" "github.com/openziti/zrok/rest_model_zrok" @@ -12,10 +13,10 @@ import ( ) type shareHandler struct { - cfg *LimitsConfig + cfg *limits.Config } -func newShareHandler(cfg *LimitsConfig) *shareHandler { +func newShareHandler(cfg *limits.Config) *shareHandler { return &shareHandler{cfg: cfg} } @@ -144,7 +145,7 @@ func (h *shareHandler) Handle(params share.ShareParams, principal *rest_model_zr } func (h *shareHandler) checkLimits(principal *rest_model_zrok.Principal, envs []*store.Environment, tx *sqlx.Tx) error { - if !principal.Limitless && h.cfg.Shares > Unlimited { + if !principal.Limitless && h.cfg.Shares > limits.Unlimited { total := 0 for i := range envs { shrs, err := str.FindSharesForEnvironment(envs[i].Id, tx) diff --git a/controller/zrokEdgeSdk/client.go b/controller/zrokEdgeSdk/client.go index ace4b123..e1890835 100644 --- a/controller/zrokEdgeSdk/client.go +++ b/controller/zrokEdgeSdk/client.go @@ -6,13 +6,13 @@ import ( "github.com/openziti/edge/rest_util" ) -type ZitiConfig struct { +type Config struct { ApiEndpoint string Username string Password string `cf:"+secret"` } -func Client(cfg *ZitiConfig) (*rest_management_api_client.ZitiEdgeManagement, error) { +func Client(cfg *Config) (*rest_management_api_client.ZitiEdgeManagement, error) { caCerts, err := rest_util.GetControllerWellKnownCas(cfg.ApiEndpoint) if err != nil { return nil, err From bc1b42d94628ffce078e064aaa666291d7fbba31 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 10 Mar 2023 14:32:00 -0500 Subject: [PATCH 02/83] incorporate a separate 'warning' and 'limit' threshold (#235) --- controller/limits/model.go | 52 ++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/controller/limits/model.go b/controller/limits/model.go index 7622005e..9f14c2a9 100644 --- a/controller/limits/model.go +++ b/controller/limits/model.go @@ -17,10 +17,15 @@ type BandwidthConfig struct { } type BandwidthPerPeriod struct { - Period time.Duration - Rx int64 - Tx int64 - Total int64 + Period time.Duration + Warning *Bandwidth + Limit *Bandwidth +} + +type Bandwidth struct { + Rx int64 + Tx int64 + Total int64 } func DefaultConfig() *Config { @@ -30,21 +35,42 @@ func DefaultConfig() *Config { Bandwidth: &BandwidthConfig{ PerAccount: &BandwidthPerPeriod{ Period: 365 * (24 * time.Hour), - Rx: Unlimited, - Tx: Unlimited, - Total: Unlimited, + Warning: &Bandwidth{ + Rx: Unlimited, + Tx: Unlimited, + Total: Unlimited, + }, + Limit: &Bandwidth{ + Rx: Unlimited, + Tx: Unlimited, + Total: Unlimited, + }, }, PerEnvironment: &BandwidthPerPeriod{ Period: 365 * (24 * time.Hour), - Rx: Unlimited, - Tx: Unlimited, - Total: Unlimited, + Warning: &Bandwidth{ + Rx: Unlimited, + Tx: Unlimited, + Total: Unlimited, + }, + Limit: &Bandwidth{ + Rx: Unlimited, + Tx: Unlimited, + Total: Unlimited, + }, }, PerShare: &BandwidthPerPeriod{ Period: 365 * (24 * time.Hour), - Rx: Unlimited, - Tx: Unlimited, - Total: Unlimited, + Warning: &Bandwidth{ + Rx: Unlimited, + Tx: Unlimited, + Total: Unlimited, + }, + Limit: &Bandwidth{ + Rx: Unlimited, + Tx: Unlimited, + Total: Unlimited, + }, }, }, } From d54fefb0feba47649afb957e6d80ea64cb6dcb41 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 13 Mar 2023 14:19:38 -0400 Subject: [PATCH 03/83] consolidated configuration; 'zrok metrics' and 'zrok ctrl' share config (#269) --- cmd/zrok/adminBootstrap.go | 3 ++- cmd/zrok/adminGc.go | 3 ++- cmd/zrok/controller.go | 3 ++- cmd/zrok/controllerValidate.go | 4 ++-- cmd/zrok/metrics.go | 8 +++++--- controller/bootstrap.go | 7 ++++--- controller/{ => config}/config.go | 15 +++++---------- controller/configuration.go | 5 +++-- controller/controller.go | 12 ++++++++---- controller/env/cf.go | 14 ++++++++++++++ controller/environmentDetail.go | 4 +++- controller/gc.go | 3 ++- controller/invite.go | 5 +++-- controller/maintenance.go | 9 +++++---- controller/metrics/agent.go | 16 +++++++--------- controller/metrics/cf.go | 10 ---------- controller/metrics/config.go | 19 ++++--------------- controller/metrics/fileSource.go | 5 +++++ controller/metrics/websocketSource.go | 5 +++++ controller/shareDetail.go | 4 +++- controller/sparkData.go | 2 +- controller/util.go | 5 +++-- 22 files changed, 88 insertions(+), 73 deletions(-) rename controller/{ => config}/config.go (90%) create mode 100644 controller/env/cf.go delete mode 100644 controller/metrics/cf.go diff --git a/cmd/zrok/adminBootstrap.go b/cmd/zrok/adminBootstrap.go index 878e5296..383b0620 100644 --- a/cmd/zrok/adminBootstrap.go +++ b/cmd/zrok/adminBootstrap.go @@ -3,6 +3,7 @@ package main import ( "github.com/michaelquigley/cf" "github.com/openziti/zrok/controller" + "github.com/openziti/zrok/controller/config" "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -32,7 +33,7 @@ func newAdminBootstrap() *adminBootstrap { func (cmd *adminBootstrap) run(_ *cobra.Command, args []string) { configPath := args[0] - inCfg, err := controller.LoadConfig(configPath) + inCfg, err := config.LoadConfig(configPath) if err != nil { panic(err) } diff --git a/cmd/zrok/adminGc.go b/cmd/zrok/adminGc.go index ff597ec7..4179d558 100644 --- a/cmd/zrok/adminGc.go +++ b/cmd/zrok/adminGc.go @@ -3,6 +3,7 @@ package main import ( "github.com/michaelquigley/cf" "github.com/openziti/zrok/controller" + "github.com/openziti/zrok/controller/config" "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -27,7 +28,7 @@ func newAdminGcCommand() *adminGcCommand { } func (gc *adminGcCommand) run(_ *cobra.Command, args []string) { - cfg, err := controller.LoadConfig(args[0]) + cfg, err := config.LoadConfig(args[0]) if err != nil { panic(err) } diff --git a/cmd/zrok/controller.go b/cmd/zrok/controller.go index 6e498797..68250dd4 100644 --- a/cmd/zrok/controller.go +++ b/cmd/zrok/controller.go @@ -3,6 +3,7 @@ package main import ( "github.com/michaelquigley/cf" "github.com/openziti/zrok/controller" + "github.com/openziti/zrok/controller/config" "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -31,7 +32,7 @@ func newControllerCommand() *controllerCommand { } func (cmd *controllerCommand) run(_ *cobra.Command, args []string) { - cfg, err := controller.LoadConfig(args[0]) + cfg, err := config.LoadConfig(args[0]) if err != nil { panic(err) } diff --git a/cmd/zrok/controllerValidate.go b/cmd/zrok/controllerValidate.go index a09d106f..bf632c29 100644 --- a/cmd/zrok/controllerValidate.go +++ b/cmd/zrok/controllerValidate.go @@ -2,7 +2,7 @@ package main import ( "github.com/michaelquigley/cf" - "github.com/openziti/zrok/controller" + "github.com/openziti/zrok/controller/config" "github.com/openziti/zrok/tui" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -28,7 +28,7 @@ func newControllerValidateCommand() *controllerValidateCommand { } func (cmd *controllerValidateCommand) run(_ *cobra.Command, args []string) { - cfg, err := controller.LoadConfig(args[0]) + cfg, err := config.LoadConfig(args[0]) if err != nil { tui.Error("controller config validation failed", err) } diff --git a/cmd/zrok/metrics.go b/cmd/zrok/metrics.go index 4001c224..fb482276 100644 --- a/cmd/zrok/metrics.go +++ b/cmd/zrok/metrics.go @@ -2,6 +2,8 @@ package main import ( "github.com/michaelquigley/cf" + "github.com/openziti/zrok/controller/config" + "github.com/openziti/zrok/controller/env" "github.com/openziti/zrok/controller/metrics" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -31,13 +33,13 @@ func newMetricsCommand() *metricsCommand { } func (cmd *metricsCommand) run(_ *cobra.Command, args []string) { - cfg, err := metrics.LoadConfig(args[0]) + cfg, err := config.LoadConfig(args[0]) if err != nil { panic(err) } - logrus.Infof(cf.Dump(cfg, metrics.GetCfOptions())) + logrus.Infof(cf.Dump(cfg, env.GetCfOptions())) - ma, err := metrics.Run(cfg) + ma, err := metrics.Run(cfg.Metrics, cfg.Store) if err != nil { panic(err) } diff --git a/controller/bootstrap.go b/controller/bootstrap.go index 729ce523..b413263d 100644 --- a/controller/bootstrap.go +++ b/controller/bootstrap.go @@ -12,7 +12,8 @@ import ( "github.com/openziti/edge/rest_model" rest_model_edge "github.com/openziti/edge/rest_model" "github.com/openziti/sdk-golang/ziti" - config2 "github.com/openziti/sdk-golang/ziti/config" + ziti_config "github.com/openziti/sdk-golang/ziti/config" + zrok_config "github.com/openziti/zrok/controller/config" "github.com/openziti/zrok/controller/store" "github.com/openziti/zrok/controller/zrokEdgeSdk" "github.com/openziti/zrok/model" @@ -22,7 +23,7 @@ import ( "time" ) -func Bootstrap(skipCtrl, skipFrontend bool, inCfg *Config) error { +func Bootstrap(skipCtrl, skipFrontend bool, inCfg *zrok_config.Config) error { cfg = inCfg if v, err := store.Open(cfg.Store); err == nil { @@ -138,7 +139,7 @@ func getIdentityId(identityName string) (string, error) { if err != nil { return "", errors.Wrapf(err, "error opening identity '%v' from zrokdir", identityName) } - zcfg, err := config2.NewFromFile(zif) + zcfg, err := ziti_config.NewFromFile(zif) if err != nil { return "", errors.Wrapf(err, "error loading ziti config from file '%v'", zif) } diff --git a/controller/config.go b/controller/config/config.go similarity index 90% rename from controller/config.go rename to controller/config/config.go index 7be959f0..cdc7c2b7 100644 --- a/controller/config.go +++ b/controller/config/config.go @@ -1,7 +1,9 @@ -package controller +package config import ( + "github.com/openziti/zrok/controller/env" "github.com/openziti/zrok/controller/limits" + "github.com/openziti/zrok/controller/metrics" "github.com/openziti/zrok/controller/zrokEdgeSdk" "time" @@ -17,9 +19,9 @@ type Config struct { Admin *AdminConfig Endpoint *EndpointConfig Email *EmailConfig - Influx *InfluxConfig Limits *limits.Config Maintenance *MaintenanceConfig + Metrics *metrics.Config Registration *RegistrationConfig ResetPassword *ResetPasswordConfig Store *store.Config @@ -53,13 +55,6 @@ type ResetPasswordConfig struct { ResetUrlTemplate string } -type InfluxConfig struct { - Url string - Bucket string - Org string - Token string `cf:"+secret"` -} - type MaintenanceConfig struct { ResetPassword *ResetPasswordMaintenanceConfig Registration *RegistrationMaintenanceConfig @@ -97,7 +92,7 @@ func DefaultConfig() *Config { func LoadConfig(path string) (*Config, error) { cfg := DefaultConfig() - if err := cf.BindYaml(cfg, path, cf.DefaultOptions()); err != nil { + if err := cf.BindYaml(cfg, path, env.GetCfOptions()); err != nil { return nil, errors.Wrapf(err, "error loading controller config '%v'", path) } if cfg.V != ConfigVersion { diff --git a/controller/configuration.go b/controller/configuration.go index 89071813..1cc10cfe 100644 --- a/controller/configuration.go +++ b/controller/configuration.go @@ -3,15 +3,16 @@ package controller import ( "github.com/go-openapi/runtime/middleware" "github.com/openziti/zrok/build" + "github.com/openziti/zrok/controller/config" "github.com/openziti/zrok/rest_model_zrok" "github.com/openziti/zrok/rest_server_zrok/operations/metadata" ) type configurationHandler struct { - cfg *Config + cfg *config.Config } -func newConfigurationHandler(cfg *Config) *configurationHandler { +func newConfigurationHandler(cfg *config.Config) *configurationHandler { return &configurationHandler{ cfg: cfg, } diff --git a/controller/controller.go b/controller/controller.go index 6d5db1a3..f5e2c79a 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -2,6 +2,8 @@ package controller import ( "context" + "github.com/openziti/zrok/controller/config" + "github.com/sirupsen/logrus" "github.com/go-openapi/loads" influxdb2 "github.com/influxdata/influxdb-client-go/v2" @@ -13,11 +15,11 @@ import ( "github.com/pkg/errors" ) -var cfg *Config +var cfg *config.Config var str *store.Store var idb influxdb2.Client -func Run(inCfg *Config) error { +func Run(inCfg *config.Config) error { cfg = inCfg swaggerSpec, err := loads.Embedded(rest_server_zrok.SwaggerJSON, rest_server_zrok.FlatSwaggerJSON) @@ -62,8 +64,10 @@ func Run(inCfg *Config) error { return errors.Wrap(err, "error opening store") } - if cfg.Influx != nil { - idb = influxdb2.NewClient(cfg.Influx.Url, cfg.Influx.Token) + if cfg.Metrics != nil && cfg.Metrics.Influx != nil { + idb = influxdb2.NewClient(cfg.Metrics.Influx.Url, cfg.Metrics.Influx.Token) + } else { + logrus.Warn("skipping influx client; no configuration") } ctx, cancel := context.WithCancel(context.Background()) diff --git a/controller/env/cf.go b/controller/env/cf.go new file mode 100644 index 00000000..e0fb0cf5 --- /dev/null +++ b/controller/env/cf.go @@ -0,0 +1,14 @@ +package env + +import ( + "github.com/michaelquigley/cf" +) + +var cfOpts *cf.Options + +func GetCfOptions() *cf.Options { + if cfOpts == nil { + cfOpts = cf.DefaultOptions() + } + return cfOpts +} diff --git a/controller/environmentDetail.go b/controller/environmentDetail.go index 4b9b5331..145947c4 100644 --- a/controller/environmentDetail.go +++ b/controller/environmentDetail.go @@ -41,11 +41,13 @@ func (h *environmentDetailHandler) Handle(params metadata.GetEnvironmentDetailPa return metadata.NewGetEnvironmentDetailInternalServerError() } var sparkData map[string][]int64 - if cfg.Influx != nil { + if cfg.Metrics != nil && cfg.Metrics.Influx != nil { sparkData, err = sparkDataForShares(shrs) if err != nil { logrus.Errorf("error querying spark data for shares for user '%v': %v", principal.Email, err) } + } else { + logrus.Debug("skipping spark data for shares; no influx configuration") } for _, shr := range shrs { feEndpoint := "" diff --git a/controller/gc.go b/controller/gc.go index ad58d89c..e0df53b9 100644 --- a/controller/gc.go +++ b/controller/gc.go @@ -8,6 +8,7 @@ import ( "github.com/openziti/edge/rest_management_api_client/service" "github.com/openziti/edge/rest_management_api_client/service_edge_router_policy" "github.com/openziti/edge/rest_management_api_client/service_policy" + zrok_config "github.com/openziti/zrok/controller/config" "github.com/openziti/zrok/controller/store" "github.com/openziti/zrok/controller/zrokEdgeSdk" "github.com/pkg/errors" @@ -16,7 +17,7 @@ import ( "time" ) -func GC(inCfg *Config) error { +func GC(inCfg *zrok_config.Config) error { cfg = inCfg if v, err := store.Open(cfg.Store); err == nil { str = v diff --git a/controller/invite.go b/controller/invite.go index 2bcc207c..ab2f80f0 100644 --- a/controller/invite.go +++ b/controller/invite.go @@ -2,6 +2,7 @@ package controller import ( "github.com/go-openapi/runtime/middleware" + "github.com/openziti/zrok/controller/config" "github.com/openziti/zrok/controller/store" "github.com/openziti/zrok/rest_server_zrok/operations/account" "github.com/openziti/zrok/util" @@ -9,10 +10,10 @@ import ( ) type inviteHandler struct { - cfg *Config + cfg *config.Config } -func newInviteHandler(cfg *Config) *inviteHandler { +func newInviteHandler(cfg *config.Config) *inviteHandler { return &inviteHandler{ cfg: cfg, } diff --git a/controller/maintenance.go b/controller/maintenance.go index 1076b9e8..e9039b7b 100644 --- a/controller/maintenance.go +++ b/controller/maintenance.go @@ -3,6 +3,7 @@ package controller import ( "context" "fmt" + "github.com/openziti/zrok/controller/config" "strings" "time" @@ -11,11 +12,11 @@ import ( ) type maintenanceRegistrationAgent struct { - cfg *RegistrationMaintenanceConfig + cfg *config.RegistrationMaintenanceConfig ctx context.Context } -func newRegistrationMaintenanceAgent(ctx context.Context, cfg *RegistrationMaintenanceConfig) *maintenanceRegistrationAgent { +func newRegistrationMaintenanceAgent(ctx context.Context, cfg *config.RegistrationMaintenanceConfig) *maintenanceRegistrationAgent { return &maintenanceRegistrationAgent{ cfg: cfg, ctx: ctx, @@ -78,11 +79,11 @@ func (ma *maintenanceRegistrationAgent) deleteExpiredAccountRequests() error { } type maintenanceResetPasswordAgent struct { - cfg *ResetPasswordMaintenanceConfig + cfg *config.ResetPasswordMaintenanceConfig ctx context.Context } -func newMaintenanceResetPasswordAgent(ctx context.Context, cfg *ResetPasswordMaintenanceConfig) *maintenanceResetPasswordAgent { +func newMaintenanceResetPasswordAgent(ctx context.Context, cfg *config.ResetPasswordMaintenanceConfig) *maintenanceResetPasswordAgent { return &maintenanceResetPasswordAgent{ cfg: cfg, ctx: ctx, diff --git a/controller/metrics/agent.go b/controller/metrics/agent.go index f6036e16..b01ddc44 100644 --- a/controller/metrics/agent.go +++ b/controller/metrics/agent.go @@ -1,6 +1,7 @@ package metrics import ( + "github.com/openziti/zrok/controller/store" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -11,24 +12,21 @@ type MetricsAgent struct { join chan struct{} } -func Run(cfg *Config) (*MetricsAgent, error) { +func Run(cfg *Config, strCfg *store.Config) (*MetricsAgent, error) { logrus.Info("starting") - if cfg.Store == nil { - return nil, errors.New("no 'store' configured; exiting") - } - cache, err := newShareCache(cfg.Store) + cache, err := newShareCache(strCfg) if err != nil { return nil, errors.Wrap(err, "error creating share cache") } - if cfg.Source == nil { - return nil, errors.New("no 'source' configured; exiting") + if cfg.Strategies == nil || cfg.Strategies.Source == nil { + return nil, errors.New("no 'strategies/source' configured; exiting") } - src, ok := cfg.Source.(Source) + src, ok := cfg.Strategies.Source.(Source) if !ok { - return nil, errors.New("invalid 'source'; exiting") + return nil, errors.New("invalid 'strategies/source'; exiting") } if cfg.Influx == nil { diff --git a/controller/metrics/cf.go b/controller/metrics/cf.go deleted file mode 100644 index e68ae9c4..00000000 --- a/controller/metrics/cf.go +++ /dev/null @@ -1,10 +0,0 @@ -package metrics - -import "github.com/michaelquigley/cf" - -func GetCfOptions() *cf.Options { - opts := cf.DefaultOptions() - opts.AddFlexibleSetter("file", loadFileSourceConfig) - opts.AddFlexibleSetter("websocket", loadWebsocketSourceConfig) - return opts -} diff --git a/controller/metrics/config.go b/controller/metrics/config.go index 4f08f5a1..4cdafcbb 100644 --- a/controller/metrics/config.go +++ b/controller/metrics/config.go @@ -1,15 +1,8 @@ package metrics -import ( - "github.com/michaelquigley/cf" - "github.com/openziti/zrok/controller/store" - "github.com/pkg/errors" -) - type Config struct { - Source interface{} - Influx *InfluxConfig - Store *store.Config + Influx *InfluxConfig + Strategies *StrategiesConfig } type InfluxConfig struct { @@ -19,10 +12,6 @@ type InfluxConfig struct { Token string `cf:"+secret"` } -func LoadConfig(path string) (*Config, error) { - cfg := &Config{} - if err := cf.BindYaml(cfg, path, GetCfOptions()); err != nil { - return nil, errors.Wrapf(err, "error loading config from '%v'", path) - } - return cfg, nil +type StrategiesConfig struct { + Source interface{} } diff --git a/controller/metrics/fileSource.go b/controller/metrics/fileSource.go index b85bd770..67007552 100644 --- a/controller/metrics/fileSource.go +++ b/controller/metrics/fileSource.go @@ -5,11 +5,16 @@ import ( "encoding/json" "github.com/michaelquigley/cf" "github.com/nxadm/tail" + "github.com/openziti/zrok/controller/env" "github.com/pkg/errors" "github.com/sirupsen/logrus" "os" ) +func init() { + env.GetCfOptions().AddFlexibleSetter("file", loadFileSourceConfig) +} + type FileSourceConfig struct { Path string IndexPath string diff --git a/controller/metrics/websocketSource.go b/controller/metrics/websocketSource.go index 4e757c3b..3b1b2898 100644 --- a/controller/metrics/websocketSource.go +++ b/controller/metrics/websocketSource.go @@ -14,6 +14,7 @@ import ( "github.com/openziti/fabric/pb/mgmt_pb" "github.com/openziti/identity" "github.com/openziti/sdk-golang/ziti/constants" + "github.com/openziti/zrok/controller/env" "github.com/pkg/errors" "github.com/sirupsen/logrus" "io" @@ -22,6 +23,10 @@ import ( "time" ) +func init() { + env.GetCfOptions().AddFlexibleSetter("websocket", loadWebsocketSourceConfig) +} + type WebsocketSourceConfig struct { WebsocketEndpoint string ApiEndpoint string diff --git a/controller/shareDetail.go b/controller/shareDetail.go index 453d0eb1..dbccdc1b 100644 --- a/controller/shareDetail.go +++ b/controller/shareDetail.go @@ -43,12 +43,14 @@ func (h *shareDetailHandler) Handle(params metadata.GetShareDetailParams, princi return metadata.NewGetShareDetailNotFound() } var sparkData map[string][]int64 - if cfg.Influx != nil { + if cfg.Metrics != nil && cfg.Metrics.Influx != nil { sparkData, err = sparkDataForShares([]*store.Share{shr}) logrus.Info(sparkData) if err != nil { logrus.Errorf("error querying spark data for share: %v", err) } + } else { + logrus.Debug("skipping spark data; no influx configuration") } feEndpoint := "" if shr.FrontendEndpoint != nil { diff --git a/controller/sparkData.go b/controller/sparkData.go index 60cce565..bc793660 100644 --- a/controller/sparkData.go +++ b/controller/sparkData.go @@ -10,7 +10,7 @@ func sparkDataForShares(shrs []*store.Share) (map[string][]int64, error) { out := make(map[string][]int64) if len(shrs) > 0 { - qapi := idb.QueryAPI(cfg.Influx.Org) + qapi := idb.QueryAPI(cfg.Metrics.Influx.Org) result, err := qapi.Query(context.Background(), sparkFluxQuery(shrs)) if err != nil { diff --git a/controller/util.go b/controller/util.go index 59ee7340..68e55a6c 100644 --- a/controller/util.go +++ b/controller/util.go @@ -3,6 +3,7 @@ package controller import ( errors2 "github.com/go-openapi/errors" "github.com/jaevor/go-nanoid" + "github.com/openziti/zrok/controller/config" "github.com/openziti/zrok/rest_model_zrok" "github.com/sirupsen/logrus" "net/http" @@ -10,10 +11,10 @@ import ( ) type zrokAuthenticator struct { - cfg *Config + cfg *config.Config } -func newZrokAuthenticator(cfg *Config) *zrokAuthenticator { +func newZrokAuthenticator(cfg *config.Config) *zrokAuthenticator { return &zrokAuthenticator{cfg} } From 95ad851e587afe2940ccd11900dd7d8d5837affa Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 13 Mar 2023 14:21:54 -0400 Subject: [PATCH 04/83] reference config no longer needed (#269) --- etc/metrics.yml | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 etc/metrics.yml diff --git a/etc/metrics.yml b/etc/metrics.yml deleted file mode 100644 index 9b28e9bc..00000000 --- a/etc/metrics.yml +++ /dev/null @@ -1,20 +0,0 @@ -# file source -# -source: - type: file - path: /tmp/fabric-usage.log - -# websocket source -# -#source: -# type: websocket -# websocket_endpoint: wss://127.0.0.1:1280/fabric/v1/ws-api -# api_endpoint: https://127.0.0.1:1280 -# username: admin -# password: "" - -influx: - url: "http://127.0.0.1:8086" - bucket: zrok - org: zrok - token: "" \ No newline at end of file From a601600bb4e31d3530f8c8cac4928cdd8cefb47e Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 13 Mar 2023 15:17:07 -0400 Subject: [PATCH 05/83] write circuit data to metrics (#263) --- controller/metrics/influx.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/controller/metrics/influx.go b/controller/metrics/influx.go index b88a4300..a2059ff9 100644 --- a/controller/metrics/influx.go +++ b/controller/metrics/influx.go @@ -23,7 +23,14 @@ func openInfluxDb(cfg *InfluxConfig) *influxDb { func (i *influxDb) Write(u *Usage) error { out := fmt.Sprintf("share: %v, circuit: %v", u.ShareToken, u.ZitiCircuitId) + var pts []*write.Point + circuitPt := influxdb2.NewPoint("circuits", + map[string]string{"share": u.ShareToken}, + map[string]interface{}{"circuit": u.ZitiCircuitId}, + u.IntervalStart) + pts = append(pts, circuitPt) + if u.BackendTx > 0 || u.BackendRx > 0 { pt := influxdb2.NewPoint("xfer", map[string]string{"namespace": "backend", "share": u.ShareToken}, @@ -40,12 +47,12 @@ func (i *influxDb) Write(u *Usage) error { pts = append(pts, pt) out += fmt.Sprintf(" frontend {rx: %v, tx: %v}", util.BytesToSize(u.FrontendRx), util.BytesToSize(u.FrontendTx)) } - if len(pts) > 0 { - if err := i.writeApi.WritePoint(context.Background(), pts...); err == nil { - logrus.Info(out) - } else { - return err - } + + if err := i.writeApi.WritePoint(context.Background(), pts...); err == nil { + logrus.Info(out) + } else { + return err } + return nil } From 66e0c0e479c1bb476f64107e2b72208d115b6761 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 13 Mar 2023 16:20:56 -0400 Subject: [PATCH 06/83] record environment and account ids on metrics records (#235); 'zrok metrics' -> 'zrok ctrl metrics' (#269) --- cmd/zrok/{metrics.go => controllerMetrics.go} | 2 +- controller/metrics/agent.go | 5 +-- controller/metrics/cache.go | 40 +++++++++++++++++++ controller/metrics/influx.go | 9 +++-- controller/metrics/model.go | 4 ++ controller/metrics/shareCache.go | 31 -------------- 6 files changed, 53 insertions(+), 38 deletions(-) rename cmd/zrok/{metrics.go => controllerMetrics.go} (95%) create mode 100644 controller/metrics/cache.go delete mode 100644 controller/metrics/shareCache.go diff --git a/cmd/zrok/metrics.go b/cmd/zrok/controllerMetrics.go similarity index 95% rename from cmd/zrok/metrics.go rename to cmd/zrok/controllerMetrics.go index fb482276..f29d75c9 100644 --- a/cmd/zrok/metrics.go +++ b/cmd/zrok/controllerMetrics.go @@ -14,7 +14,7 @@ import ( ) func init() { - rootCmd.AddCommand(newMetricsCommand().cmd) + controllerCmd.cmd.AddCommand(newMetricsCommand().cmd) } type metricsCommand struct { diff --git a/controller/metrics/agent.go b/controller/metrics/agent.go index b01ddc44..c06ab246 100644 --- a/controller/metrics/agent.go +++ b/controller/metrics/agent.go @@ -8,7 +8,7 @@ import ( type MetricsAgent struct { src Source - cache *shareCache + cache *cache join chan struct{} } @@ -46,8 +46,7 @@ func Run(cfg *Config, strCfg *store.Config) (*MetricsAgent, error) { select { case event := <-events: usage := Ingest(event) - if shrToken, err := cache.getToken(usage.ZitiServiceId); err == nil { - usage.ShareToken = shrToken + if err := cache.addZrokDetail(usage); err == nil { if err := idb.Write(usage); err != nil { logrus.Error(err) } diff --git a/controller/metrics/cache.go b/controller/metrics/cache.go new file mode 100644 index 00000000..6cb2b59e --- /dev/null +++ b/controller/metrics/cache.go @@ -0,0 +1,40 @@ +package metrics + +import ( + "github.com/openziti/zrok/controller/store" + "github.com/pkg/errors" +) + +type cache struct { + str *store.Store +} + +func newShareCache(cfg *store.Config) (*cache, error) { + str, err := store.Open(cfg) + if err != nil { + return nil, errors.Wrap(err, "error opening store") + } + return &cache{str}, nil +} + +func (sc *cache) addZrokDetail(u *Usage) error { + tx, err := sc.str.Begin() + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + shr, err := sc.str.FindShareWithZIdAndDeleted(u.ZitiServiceId, tx) + if err != nil { + return err + } + u.ShareToken = shr.Token + env, err := sc.str.GetEnvironment(shr.EnvironmentId, tx) + if err != nil { + return err + } + u.EnvironmentId = int64(env.Id) + u.AccountId = int64(*env.AccountId) + + return nil +} diff --git a/controller/metrics/influx.go b/controller/metrics/influx.go index a2059ff9..214ed8e5 100644 --- a/controller/metrics/influx.go +++ b/controller/metrics/influx.go @@ -24,16 +24,19 @@ func openInfluxDb(cfg *InfluxConfig) *influxDb { func (i *influxDb) Write(u *Usage) error { out := fmt.Sprintf("share: %v, circuit: %v", u.ShareToken, u.ZitiCircuitId) + envId := fmt.Sprintf("%d", u.EnvironmentId) + acctId := fmt.Sprintf("%d", u.AccountId) + var pts []*write.Point circuitPt := influxdb2.NewPoint("circuits", - map[string]string{"share": u.ShareToken}, + map[string]string{"share": u.ShareToken, "envId": envId, "acctId": acctId}, map[string]interface{}{"circuit": u.ZitiCircuitId}, u.IntervalStart) pts = append(pts, circuitPt) if u.BackendTx > 0 || u.BackendRx > 0 { pt := influxdb2.NewPoint("xfer", - map[string]string{"namespace": "backend", "share": u.ShareToken}, + map[string]string{"namespace": "backend", "share": u.ShareToken, "envId": envId, "acctId": acctId}, map[string]interface{}{"bytesRead": u.BackendRx, "bytesWritten": u.BackendTx}, u.IntervalStart) pts = append(pts, pt) @@ -41,7 +44,7 @@ func (i *influxDb) Write(u *Usage) error { } if u.FrontendTx > 0 || u.FrontendRx > 0 { pt := influxdb2.NewPoint("xfer", - map[string]string{"namespace": "frontend", "share": u.ShareToken}, + map[string]string{"namespace": "frontend", "share": u.ShareToken, "envId": envId, "acctId": acctId}, map[string]interface{}{"bytesRead": u.FrontendRx, "bytesWritten": u.FrontendTx}, u.IntervalStart) pts = append(pts, pt) diff --git a/controller/metrics/model.go b/controller/metrics/model.go index 0dcf13eb..632e8404 100644 --- a/controller/metrics/model.go +++ b/controller/metrics/model.go @@ -12,6 +12,8 @@ type Usage struct { ZitiServiceId string ZitiCircuitId string ShareToken string + EnvironmentId int64 + AccountId int64 FrontendTx int64 FrontendRx int64 BackendTx int64 @@ -25,6 +27,8 @@ func (u Usage) String() string { out += ", " + fmt.Sprintf("service '%v'", u.ZitiServiceId) out += ", " + fmt.Sprintf("circuit '%v'", u.ZitiCircuitId) out += ", " + fmt.Sprintf("share '%v'", u.ShareToken) + out += ", " + fmt.Sprintf("environment '%d'", u.EnvironmentId) + out += ", " + fmt.Sprintf("account '%v'", u.AccountId) out += ", " + fmt.Sprintf("fe {rx %v, tx %v}", util.BytesToSize(u.FrontendRx), util.BytesToSize(u.FrontendTx)) out += ", " + fmt.Sprintf("be {rx %v, tx %v}", util.BytesToSize(u.BackendRx), util.BytesToSize(u.BackendTx)) out += "}" diff --git a/controller/metrics/shareCache.go b/controller/metrics/shareCache.go deleted file mode 100644 index 2789953d..00000000 --- a/controller/metrics/shareCache.go +++ /dev/null @@ -1,31 +0,0 @@ -package metrics - -import ( - "github.com/openziti/zrok/controller/store" - "github.com/pkg/errors" -) - -type shareCache struct { - str *store.Store -} - -func newShareCache(cfg *store.Config) (*shareCache, error) { - str, err := store.Open(cfg) - if err != nil { - return nil, errors.Wrap(err, "error opening store") - } - return &shareCache{str}, nil -} - -func (sc *shareCache) getToken(svcZId string) (string, error) { - tx, err := sc.str.Begin() - if err != nil { - return "", err - } - defer func() { _ = tx.Rollback() }() - shr, err := sc.str.FindShareWithZIdAndDeleted(svcZId, tx) - if err != nil { - return "", err - } - return shr.Token, nil -} From 858872f861f81a0ccede13cec8a33461e244d4b8 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 14 Mar 2023 14:45:44 -0400 Subject: [PATCH 07/83] lint --- controller/limits/{model.go => config.go} | 1 + 1 file changed, 1 insertion(+) rename controller/limits/{model.go => config.go} (98%) diff --git a/controller/limits/model.go b/controller/limits/config.go similarity index 98% rename from controller/limits/model.go rename to controller/limits/config.go index 9f14c2a9..4c044950 100644 --- a/controller/limits/model.go +++ b/controller/limits/config.go @@ -8,6 +8,7 @@ type Config struct { Environments int Shares int Bandwidth *BandwidthConfig + Cycle time.Duration } type BandwidthConfig struct { From 9ca7dfb102e5408a3b47dcc95ed19a4d930b37b8 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 14 Mar 2023 16:23:34 -0400 Subject: [PATCH 08/83] amqp sender (#270) --- controller/metrics/amqpSender.go | 51 ++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 7 +++-- 3 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 controller/metrics/amqpSender.go diff --git a/controller/metrics/amqpSender.go b/controller/metrics/amqpSender.go new file mode 100644 index 00000000..332d5328 --- /dev/null +++ b/controller/metrics/amqpSender.go @@ -0,0 +1,51 @@ +package metrics + +import ( + "context" + "github.com/pkg/errors" + amqp "github.com/rabbitmq/amqp091-go" + "time" +) + +type AmqpSenderConfig struct { + Url string `cf:"+secret"` + Queue string +} + +type AmqpSender struct { + conn *amqp.Connection + ch *amqp.Channel + queue amqp.Queue +} + +func NewAmqpSender(cfg *AmqpSenderConfig) (*AmqpSender, error) { + conn, err := amqp.Dial(cfg.Url) + if err != nil { + return nil, errors.Wrap(err, "error dialing amqp broker") + } + + ch, err := conn.Channel() + if err != nil { + return nil, errors.Wrap(err, "error getting channel from amqp connection") + } + + queue, err := ch.QueueDeclare(cfg.Queue, true, false, false, false, nil) + if err != nil { + return nil, errors.Wrap(err, "error creating amqp queue") + } + + return &AmqpSender{conn, ch, queue}, nil +} + +func (s *AmqpSender) Send(json string) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + err := s.ch.PublishWithContext(ctx, "", s.queue.Name, false, false, amqp.Publishing{ + ContentType: "application/json", + Body: []byte(json), + }) + if err != nil { + return errors.Wrap(err, "error sending") + } + return nil +} diff --git a/go.mod b/go.mod index f1a3ed36..7a4cf38c 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/openziti/identity v1.0.37 github.com/openziti/sdk-golang v0.18.61 github.com/pkg/errors v0.9.1 + github.com/rabbitmq/amqp091-go v1.7.0 github.com/rubenv/sql-migrate v1.1.2 github.com/shirou/gopsutil/v3 v3.23.2 github.com/sirupsen/logrus v1.9.0 @@ -41,7 +42,6 @@ require ( golang.org/x/crypto v0.6.0 golang.org/x/net v0.8.0 golang.org/x/time v0.3.0 - gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 nhooyr.io/websocket v1.8.7 ) diff --git a/go.sum b/go.sum index 47153b47..3878353a 100644 --- a/go.sum +++ b/go.sum @@ -525,6 +525,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:Om github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1 h1:oL4IBbcqwhhNWh31bjOX8C/OCy0zs9906d/VUru+bqg= github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rabbitmq/amqp091-go v1.7.0 h1:V5CF5qPem5OGSnEo8BoSbsDGwejg6VUJsKEdneaoTUo= +github.com/rabbitmq/amqp091-go v1.7.0/go.mod h1:wfClAtY0C7bOHxd3GjmF26jEHn+rR/0B3+YV+Vn9/NI= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -638,6 +640,8 @@ go.opentelemetry.io/otel/sdk v1.11.1 h1:F7KmQgoHljhUuJyA+9BiU+EkJfyX5nVVF4wyzWZp go.opentelemetry.io/otel/trace v1.11.1 h1:ofxdnzsNrGBYXbP7t7zpUK281+go5rF7dvdIZXF8gdQ= go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -913,6 +917,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1031,8 +1036,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= -gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 559a59cd8e15c283b59ff1027cdaccd463807036 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 15 Mar 2023 12:47:26 -0400 Subject: [PATCH 09/83] revamped fileSource+amqpSink (#270) --- controller/metrics2/amqpSink.go | 64 +++++++++++++++ controller/metrics2/fileSource.go | 130 ++++++++++++++++++++++++++++++ controller/metrics2/model.go | 12 +++ 3 files changed, 206 insertions(+) create mode 100644 controller/metrics2/amqpSink.go create mode 100644 controller/metrics2/fileSource.go create mode 100644 controller/metrics2/model.go diff --git a/controller/metrics2/amqpSink.go b/controller/metrics2/amqpSink.go new file mode 100644 index 00000000..545f3433 --- /dev/null +++ b/controller/metrics2/amqpSink.go @@ -0,0 +1,64 @@ +package metrics2 + +import ( + "context" + "github.com/michaelquigley/cf" + "github.com/openziti/zrok/controller/env" + "github.com/pkg/errors" + amqp "github.com/rabbitmq/amqp091-go" + "time" +) + +func init() { + env.GetCfOptions().AddFlexibleSetter("amqpSink", loadAmqpSinkConfig) +} + +type AmqpSinkConfig struct { + Url string `cf:"+secret"` + QueueName string +} + +func loadAmqpSinkConfig(v interface{}, _ *cf.Options) (interface{}, error) { + if submap, ok := v.(map[string]interface{}); ok { + cfg := &AmqpSinkConfig{} + if err := cf.Bind(cfg, submap, cf.DefaultOptions()); err != nil { + return nil, err + } + return newAmqpSink(cfg) + } + return nil, errors.New("invalid config structure for 'amqpSink'") +} + +type amqpSink struct { + conn *amqp.Connection + ch *amqp.Channel + queue amqp.Queue +} + +func newAmqpSink(cfg *AmqpSinkConfig) (*amqpSink, error) { + conn, err := amqp.Dial(cfg.Url) + if err != nil { + return nil, errors.Wrap(err, "error dialing amqp broker") + } + + ch, err := conn.Channel() + if err != nil { + return nil, errors.Wrap(err, "error getting amqp channel") + } + + queue, err := ch.QueueDeclare(cfg.QueueName, true, false, false, false, nil) + if err != nil { + return nil, errors.Wrap(err, "error declaring queue") + } + + return &amqpSink{conn, ch, queue}, nil +} + +func (s *amqpSink) Handle(event ZitiEventJson) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return s.ch.PublishWithContext(ctx, "", s.queue.Name, false, false, amqp.Publishing{ + ContentType: "application/json", + Body: []byte(event), + }) +} diff --git a/controller/metrics2/fileSource.go b/controller/metrics2/fileSource.go new file mode 100644 index 00000000..71a02939 --- /dev/null +++ b/controller/metrics2/fileSource.go @@ -0,0 +1,130 @@ +package metrics2 + +import ( + "encoding/binary" + "github.com/michaelquigley/cf" + "github.com/nxadm/tail" + "github.com/openziti/zrok/controller/env" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "os" +) + +func init() { + env.GetCfOptions().AddFlexibleSetter("fileSource", loadFileSourceConfig) +} + +type FileSourceConfig struct { + Path string + PointerPath string +} + +func loadFileSourceConfig(v interface{}, _ *cf.Options) (interface{}, error) { + if submap, ok := v.(map[string]interface{}); ok { + cfg := &FileSourceConfig{} + if err := cf.Bind(cfg, submap, cf.DefaultOptions()); err != nil { + return nil, err + } + return &fileSource{cfg: cfg}, nil + } + return nil, errors.New("invalid config structure for 'fileSource'") +} + +type fileSource struct { + cfg *FileSourceConfig + ptrF *os.File + t *tail.Tail +} + +func (s *fileSource) Start(events chan ZitiEventJson) (join chan struct{}, err error) { + f, err := os.Open(s.cfg.Path) + if err != nil { + return nil, errors.Wrapf(err, "error opening '%v'", s.cfg.Path) + } + _ = f.Close() + + s.ptrF, err = os.OpenFile(s.pointerPath(), os.O_CREATE|os.O_RDWR, os.ModePerm) + if err != nil { + return nil, errors.Wrapf(err, "error opening pointer '%v'", s.pointerPath()) + } + + ptr, err := s.readPtr() + if err != nil { + return nil, errors.Wrap(err, "error reading pointer") + } + logrus.Infof("retrieved stored position pointer at '%d'", ptr) + + join = make(chan struct{}) + go func() { + s.tail(ptr, events) + close(join) + }() + + return join, nil +} + +func (s *fileSource) Stop() { + if err := s.t.Stop(); err != nil { + logrus.Error(err) + } +} + +func (s *fileSource) tail(ptr int64, events chan ZitiEventJson) { + logrus.Info("started") + defer logrus.Info("stopped") + + var err error + s.t, err = tail.TailFile(s.cfg.Path, tail.Config{ + ReOpen: true, + Follow: true, + Location: &tail.SeekInfo{Offset: ptr}, + }) + if err != nil { + logrus.Error(err) + return + } + + for event := range s.t.Lines { + events <- ZitiEventJson(event.Text) + + if err := s.writePtr(event.SeekInfo.Offset); err != nil { + logrus.Error(err) + } + } +} + +func (s *fileSource) pointerPath() string { + if s.cfg.PointerPath == "" { + return s.cfg.Path + ".ptr" + } else { + return s.cfg.PointerPath + } +} + +func (s *fileSource) readPtr() (int64, error) { + ptr := int64(0) + buf := make([]byte, 8) + if n, err := s.ptrF.Seek(0, 0); err == nil && n == 0 { + if n, err := s.ptrF.Read(buf); err == nil && n == 8 { + ptr = int64(binary.LittleEndian.Uint64(buf)) + return ptr, nil + } else { + return -1, errors.Wrapf(err, "error reading pointer (%d): %v", n, err) + } + } else { + return -1, errors.Wrapf(err, "error seeking pointer (%d): %v", n, err) + } +} + +func (s *fileSource) writePtr(ptr int64) error { + buf := make([]byte, 8) + binary.LittleEndian.PutUint64(buf, uint64(ptr)) + if n, err := s.ptrF.Seek(0, 0); err == nil && n == 0 { + if n, err := s.ptrF.Write(buf); err != nil || n != 8 { + return errors.Wrapf(err, "error writing pointer (%d): %v", n, err) + } + } else { + return errors.Wrapf(err, "error seeking pointer (%d): %v", n, err) + } + return nil +} diff --git a/controller/metrics2/model.go b/controller/metrics2/model.go new file mode 100644 index 00000000..a53cb37f --- /dev/null +++ b/controller/metrics2/model.go @@ -0,0 +1,12 @@ +package metrics2 + +type ZitiEventJson string + +type ZitiEventJsonSource interface { + Start(chan ZitiEventJson) (join chan struct{}, err error) + Stop() +} + +type ZitiEventJsonSink interface { + Handle(event ZitiEventJson) error +} From 182c7bc510f20e3843163707137eafd92f830e54 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 15 Mar 2023 13:49:17 -0400 Subject: [PATCH 10/83] basic amqp bridge (#270) --- cmd/zrok/controllerMetrics.go | 5 +- cmd/zrok/controllerMetricsBridge.go | 61 +++++++++++++++++++++++ controller/config/config.go | 2 + controller/metrics2/bridge.go | 77 +++++++++++++++++++++++++++++ controller/metrics2/fileSource.go | 8 +-- 5 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 cmd/zrok/controllerMetricsBridge.go create mode 100644 controller/metrics2/bridge.go diff --git a/cmd/zrok/controllerMetrics.go b/cmd/zrok/controllerMetrics.go index f29d75c9..7de987c8 100644 --- a/cmd/zrok/controllerMetrics.go +++ b/cmd/zrok/controllerMetrics.go @@ -13,8 +13,11 @@ import ( "time" ) +var metricsCmd *cobra.Command + func init() { - controllerCmd.cmd.AddCommand(newMetricsCommand().cmd) + metricsCmd = newMetricsCommand().cmd + controllerCmd.cmd.AddCommand(metricsCmd) } type metricsCommand struct { diff --git a/cmd/zrok/controllerMetricsBridge.go b/cmd/zrok/controllerMetricsBridge.go new file mode 100644 index 00000000..b67ef129 --- /dev/null +++ b/cmd/zrok/controllerMetricsBridge.go @@ -0,0 +1,61 @@ +package main + +import ( + "github.com/michaelquigley/cf" + "github.com/openziti/zrok/controller/config" + "github.com/openziti/zrok/controller/env" + "github.com/openziti/zrok/controller/metrics2" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "os" + "os/signal" + "syscall" + "time" +) + +func init() { + metricsCmd.AddCommand(newBridgeCommand().cmd) +} + +type bridgeCommand struct { + cmd *cobra.Command +} + +func newBridgeCommand() *bridgeCommand { + cmd := &cobra.Command{ + Use: "bridge ", + Short: "Start a zrok metrics bridge", + Args: cobra.ExactArgs(1), + } + command := &bridgeCommand{cmd} + cmd.Run = command.run + return command +} + +func (cmd *bridgeCommand) run(_ *cobra.Command, args []string) { + cfg, err := config.LoadConfig(args[0]) + if err != nil { + panic(err) + } + logrus.Infof(cf.Dump(cfg, env.GetCfOptions())) + + bridge, err := metrics2.NewBridge(cfg.Bridge) + if err != nil { + panic(err) + } + if _, err = bridge.Start(); err != nil { + panic(err) + } + + c := make(chan os.Signal) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + bridge.Stop() + os.Exit(0) + }() + + for { + time.Sleep(24 * 60 * time.Minute) + } +} diff --git a/controller/config/config.go b/controller/config/config.go index cdc7c2b7..19ec8833 100644 --- a/controller/config/config.go +++ b/controller/config/config.go @@ -4,6 +4,7 @@ import ( "github.com/openziti/zrok/controller/env" "github.com/openziti/zrok/controller/limits" "github.com/openziti/zrok/controller/metrics" + "github.com/openziti/zrok/controller/metrics2" "github.com/openziti/zrok/controller/zrokEdgeSdk" "time" @@ -17,6 +18,7 @@ const ConfigVersion = 2 type Config struct { V int Admin *AdminConfig + Bridge *metrics2.BridgeConfig Endpoint *EndpointConfig Email *EmailConfig Limits *limits.Config diff --git a/controller/metrics2/bridge.go b/controller/metrics2/bridge.go new file mode 100644 index 00000000..be51cbf2 --- /dev/null +++ b/controller/metrics2/bridge.go @@ -0,0 +1,77 @@ +package metrics2 + +import ( + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +type BridgeConfig struct { + Source interface{} + Sink interface{} +} + +type Bridge struct { + src ZitiEventJsonSource + srcJoin chan struct{} + snk ZitiEventJsonSink + events chan ZitiEventJson + close chan struct{} + join chan struct{} +} + +func NewBridge(cfg *BridgeConfig) (*Bridge, error) { + b := &Bridge{ + events: make(chan ZitiEventJson), + join: make(chan struct{}), + close: make(chan struct{}), + } + if v, ok := cfg.Source.(ZitiEventJsonSource); ok { + b.src = v + } else { + return nil, errors.New("invalid source type") + } + if v, ok := cfg.Sink.(ZitiEventJsonSink); ok { + b.snk = v + } else { + return nil, errors.New("invalid sink type") + } + return b, nil +} + +func (b *Bridge) Start() (join chan struct{}, err error) { + if b.srcJoin, err = b.src.Start(b.events); err != nil { + return nil, err + } + + go func() { + logrus.Info("started") + defer logrus.Info("stopped") + defer close(b.join) + + eventLoop: + for { + select { + case eventJson := <-b.events: + logrus.Info(eventJson) + if err := b.snk.Handle(eventJson); err == nil { + logrus.Info("-> %v", eventJson) + } else { + logrus.Error(err) + } + + case <-b.close: + logrus.Info("received close signal") + break eventLoop + } + } + }() + + return b.join, nil +} + +func (b *Bridge) Stop() { + b.src.Stop() + close(b.close) + <-b.srcJoin + <-b.join +} diff --git a/controller/metrics2/fileSource.go b/controller/metrics2/fileSource.go index 71a02939..9ca527a5 100644 --- a/controller/metrics2/fileSource.go +++ b/controller/metrics2/fileSource.go @@ -50,7 +50,7 @@ func (s *fileSource) Start(events chan ZitiEventJson) (join chan struct{}, err e ptr, err := s.readPtr() if err != nil { - return nil, errors.Wrap(err, "error reading pointer") + logrus.Errorf("error reading pointer: %v", err) } logrus.Infof("retrieved stored position pointer at '%d'", ptr) @@ -80,7 +80,7 @@ func (s *fileSource) tail(ptr int64, events chan ZitiEventJson) { Location: &tail.SeekInfo{Offset: ptr}, }) if err != nil { - logrus.Error(err) + logrus.Errorf("error starting tail: %v", err) return } @@ -109,10 +109,10 @@ func (s *fileSource) readPtr() (int64, error) { ptr = int64(binary.LittleEndian.Uint64(buf)) return ptr, nil } else { - return -1, errors.Wrapf(err, "error reading pointer (%d): %v", n, err) + return 0, errors.Wrapf(err, "error reading pointer (%d): %v", n, err) } } else { - return -1, errors.Wrapf(err, "error seeking pointer (%d): %v", n, err) + return 0, errors.Wrapf(err, "error seeking pointer (%d): %v", n, err) } } From b420211c8d9b23bf857b8284ed36910a3cc572d8 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 15 Mar 2023 14:39:25 -0400 Subject: [PATCH 11/83] lint (#270) --- controller/metrics2/bridge.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controller/metrics2/bridge.go b/controller/metrics2/bridge.go index be51cbf2..f91c4219 100644 --- a/controller/metrics2/bridge.go +++ b/controller/metrics2/bridge.go @@ -54,7 +54,7 @@ func (b *Bridge) Start() (join chan struct{}, err error) { case eventJson := <-b.events: logrus.Info(eventJson) if err := b.snk.Handle(eventJson); err == nil { - logrus.Info("-> %v", eventJson) + logrus.Infof("-> %v", eventJson) } else { logrus.Error(err) } From bf43366e40091afe6139d85c73853ddc1ac48916 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 15 Mar 2023 14:52:14 -0400 Subject: [PATCH 12/83] amqpSource (#270) --- controller/metrics2/amqpSource.go | 81 +++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 controller/metrics2/amqpSource.go diff --git a/controller/metrics2/amqpSource.go b/controller/metrics2/amqpSource.go new file mode 100644 index 00000000..772d1685 --- /dev/null +++ b/controller/metrics2/amqpSource.go @@ -0,0 +1,81 @@ +package metrics2 + +import ( + "github.com/michaelquigley/cf" + "github.com/pkg/errors" + amqp "github.com/rabbitmq/amqp091-go" + "github.com/sirupsen/logrus" +) + +type AmqpSourceConfig struct { + Url string `cf:"+secret"` + QueueName string +} + +func loadAmqpSourceConfig(v interface{}, _ *cf.Options) (interface{}, error) { + if submap, ok := v.(map[string]interface{}); ok { + cfg := &AmqpSourceConfig{} + if err := cf.Bind(cfg, submap, cf.DefaultOptions()); err != nil { + return nil, err + } + return newAmqpSource(cfg) + } + return nil, errors.New("invalid config structure for 'amqpSource'") +} + +type amqpSource struct { + conn *amqp.Connection + ch *amqp.Channel + queue amqp.Queue + msgs <-chan amqp.Delivery + join chan struct{} +} + +func newAmqpSource(cfg *AmqpSourceConfig) (*amqpSource, error) { + conn, err := amqp.Dial(cfg.Url) + if err != nil { + return nil, errors.Wrap(err, "error dialing amqp broker") + } + + ch, err := conn.Channel() + if err != nil { + return nil, errors.Wrap(err, "error getting amqp channel") + } + + queue, err := ch.QueueDeclare(cfg.QueueName, true, false, false, false, nil) + if err != nil { + return nil, errors.Wrap(err, "error declaring queue") + } + + msgs, err := ch.Consume(cfg.QueueName, "zrok", true, false, false, false, nil) + if err != nil { + return nil, errors.Wrap(err, "error consuming") + } + + return &amqpSource{ + conn, + ch, + queue, + msgs, + make(chan struct{}), + }, nil +} + +func (s *amqpSource) Start(events chan ZitiEventJson) (join chan struct{}, err error) { + go func() { + logrus.Info("started") + defer logrus.Info("stopped") + for event := range s.msgs { + events <- ZitiEventJson(event.Body) + } + close(s.join) + }() + return s.join, nil +} + +func (s *amqpSource) Stop() { + if err := s.ch.Close(); err != nil { + logrus.Error(err) + } + <-s.join +} From 1e9a57cb81fb24b765aea9535703eca9fb30b7cc Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 15 Mar 2023 15:08:06 -0400 Subject: [PATCH 13/83] usageIngest (#270) --- controller/metrics2/amqpSource.go | 5 ++ controller/metrics2/model.go | 29 ++++++++ controller/metrics2/usageIngest.go | 102 +++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 controller/metrics2/usageIngest.go diff --git a/controller/metrics2/amqpSource.go b/controller/metrics2/amqpSource.go index 772d1685..5e07f5d4 100644 --- a/controller/metrics2/amqpSource.go +++ b/controller/metrics2/amqpSource.go @@ -2,11 +2,16 @@ package metrics2 import ( "github.com/michaelquigley/cf" + "github.com/openziti/zrok/controller/env" "github.com/pkg/errors" amqp "github.com/rabbitmq/amqp091-go" "github.com/sirupsen/logrus" ) +func init() { + env.GetCfOptions().AddFlexibleSetter("amqpSource", loadAmqpSourceConfig) +} + type AmqpSourceConfig struct { Url string `cf:"+secret"` QueueName string diff --git a/controller/metrics2/model.go b/controller/metrics2/model.go index a53cb37f..e533c9ea 100644 --- a/controller/metrics2/model.go +++ b/controller/metrics2/model.go @@ -1,5 +1,34 @@ package metrics2 +type Usage struct { + ProcessedStamp time.Time + IntervalStart time.Time + ZitiServiceId string + ZitiCircuitId string + ShareToken string + EnvironmentId int64 + AccountId int64 + FrontendTx int64 + FrontendRx int64 + BackendTx int64 + BackendRx int64 +} + +func (u Usage) String() string { + out := "Usage {" + out += fmt.Sprintf("processed '%v'", u.ProcessedStamp) + out += ", " + fmt.Sprintf("interval '%v'", u.IntervalStart) + out += ", " + fmt.Sprintf("service '%v'", u.ZitiServiceId) + out += ", " + fmt.Sprintf("circuit '%v'", u.ZitiCircuitId) + out += ", " + fmt.Sprintf("share '%v'", u.ShareToken) + out += ", " + fmt.Sprintf("environment '%d'", u.EnvironmentId) + out += ", " + fmt.Sprintf("account '%v'", u.AccountId) + out += ", " + fmt.Sprintf("fe {rx %v, tx %v}", util.BytesToSize(u.FrontendRx), util.BytesToSize(u.FrontendTx)) + out += ", " + fmt.Sprintf("be {rx %v, tx %v}", util.BytesToSize(u.BackendRx), util.BytesToSize(u.BackendTx)) + out += "}" + return out +} + type ZitiEventJson string type ZitiEventJsonSource interface { diff --git a/controller/metrics2/usageIngest.go b/controller/metrics2/usageIngest.go new file mode 100644 index 00000000..99f50122 --- /dev/null +++ b/controller/metrics2/usageIngest.go @@ -0,0 +1,102 @@ +package metrics2 + +import ( + "encoding/json" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "reflect" + "time" +) + +func Ingest(event ZitiEventJson) (*Usage, error) { + eventMap := make(map[string]interface{}) + if err := json.Unmarshal([]byte(event), &eventMap); err == nil { + u := &Usage{ProcessedStamp: time.Now()} + if ns, found := eventMap["namespace"]; found && ns == "fabric.usage" { + if v, found := eventMap["interval_start_utc"]; found { + if vFloat64, ok := v.(float64); ok { + u.IntervalStart = time.Unix(int64(vFloat64), 0) + } else { + logrus.Error("unable to assert 'interval_start_utc'") + } + } else { + logrus.Error("missing 'interval_start_utc'") + } + if v, found := eventMap["tags"]; found { + if tags, ok := v.(map[string]interface{}); ok { + if v, found := tags["serviceId"]; found { + if vStr, ok := v.(string); ok { + u.ZitiServiceId = vStr + } else { + logrus.Error("unable to assert 'tags/serviceId'") + } + } else { + logrus.Error("missing 'tags/serviceId'") + } + } else { + logrus.Errorf("unable to assert 'tags'") + } + } else { + logrus.Errorf("missing 'tags'") + } + if v, found := eventMap["usage"]; found { + if usage, ok := v.(map[string]interface{}); ok { + if v, found := usage["ingress.tx"]; found { + if vFloat64, ok := v.(float64); ok { + u.FrontendTx = int64(vFloat64) + } else { + logrus.Error("unable to assert 'usage/ingress.tx'") + } + } else { + logrus.Warn("missing 'usage/ingress.tx'") + } + if v, found := usage["ingress.rx"]; found { + if vFloat64, ok := v.(float64); ok { + u.FrontendRx = int64(vFloat64) + } else { + logrus.Error("unable to assert 'usage/ingress.rx") + } + } else { + logrus.Warn("missing 'usage/ingress.rx") + } + if v, found := usage["egress.tx"]; found { + if vFloat64, ok := v.(float64); ok { + u.BackendTx = int64(vFloat64) + } else { + logrus.Error("unable to assert 'usage/egress.tx'") + } + } else { + logrus.Warn("missing 'usage/egress.tx'") + } + if v, found := usage["egress.rx"]; found { + if vFloat64, ok := v.(float64); ok { + u.BackendRx = int64(vFloat64) + } else { + logrus.Error("unable to assert 'usage/egress.rx'") + } + } else { + logrus.Warn("missing 'usage/egress.rx'") + } + } else { + logrus.Errorf("unable to assert 'usage' (%v) %v", reflect.TypeOf(v), event) + } + } else { + logrus.Warnf("missing 'usage'") + } + if v, found := eventMap["circuit_id"]; found { + if vStr, ok := v.(string); ok { + u.ZitiCircuitId = vStr + } else { + logrus.Error("unable to assert 'circuit_id'") + } + } else { + logrus.Warn("missing 'circuit_id'") + } + } else { + logrus.Errorf("not 'fabric.usage'") + } + return u, nil + } else { + return nil, errors.Wrap(err, "error unmarshaling") + } +} From 917226012cc4a9437fe7021e3126ad0e21c7fd3a Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 15 Mar 2023 15:13:21 -0400 Subject: [PATCH 14/83] influxWriter (#270) --- controller/metrics2/config.go | 8 ++++ controller/metrics2/influxWriter.go | 61 +++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 controller/metrics2/config.go create mode 100644 controller/metrics2/influxWriter.go diff --git a/controller/metrics2/config.go b/controller/metrics2/config.go new file mode 100644 index 00000000..f25cd2f5 --- /dev/null +++ b/controller/metrics2/config.go @@ -0,0 +1,8 @@ +package metrics2 + +type InfluxConfig struct { + Url string + Bucket string + Org string + Token string `cf:"+secret"` +} diff --git a/controller/metrics2/influxWriter.go b/controller/metrics2/influxWriter.go new file mode 100644 index 00000000..69772a52 --- /dev/null +++ b/controller/metrics2/influxWriter.go @@ -0,0 +1,61 @@ +package metrics2 + +import ( + "context" + "fmt" + influxdb2 "github.com/influxdata/influxdb-client-go/v2" + "github.com/influxdata/influxdb-client-go/v2/api" + "github.com/influxdata/influxdb-client-go/v2/api/write" + "github.com/openziti/zrok/util" + "github.com/sirupsen/logrus" +) + +type influxWriter struct { + idb influxdb2.Client + writeApi api.WriteAPIBlocking +} + +func openInfluxWriter(cfg *InfluxConfig) *influxWriter { + idb := influxdb2.NewClient(cfg.Url, cfg.Token) + writeApi := idb.WriteAPIBlocking(cfg.Org, cfg.Bucket) + return &influxWriter{idb, writeApi} +} + +func (w *influxWriter) Write(u *Usage) error { + out := fmt.Sprintf("share: %v, circuit: %v", u.ShareToken, u.ZitiCircuitId) + + envId := fmt.Sprintf("%d", u.EnvironmentId) + acctId := fmt.Sprintf("%d", u.AccountId) + + var pts []*write.Point + circuitPt := influxdb2.NewPoint("circuits", + map[string]string{"share": u.ShareToken, "envId": envId, "acctId": acctId}, + map[string]interface{}{"circuit": u.ZitiCircuitId}, + u.IntervalStart) + pts = append(pts, circuitPt) + + if u.BackendTx > 0 || u.BackendRx > 0 { + pt := influxdb2.NewPoint("xfer", + map[string]string{"namespace": "backend", "share": u.ShareToken, "envId": envId, "acctId": acctId}, + map[string]interface{}{"bytesRead": u.BackendRx, "bytesWritten": u.BackendTx}, + u.IntervalStart) + pts = append(pts, pt) + out += fmt.Sprintf(" backend {rx: %v, tx: %v}", util.BytesToSize(u.BackendRx), util.BytesToSize(u.BackendTx)) + } + if u.FrontendTx > 0 || u.FrontendRx > 0 { + pt := influxdb2.NewPoint("xfer", + map[string]string{"namespace": "frontend", "share": u.ShareToken, "envId": envId, "acctId": acctId}, + map[string]interface{}{"bytesRead": u.FrontendRx, "bytesWritten": u.FrontendTx}, + u.IntervalStart) + pts = append(pts, pt) + out += fmt.Sprintf(" frontend {rx: %v, tx: %v}", util.BytesToSize(u.FrontendRx), util.BytesToSize(u.FrontendTx)) + } + + if err := w.writeApi.WritePoint(context.Background(), pts...); err == nil { + logrus.Info(out) + } else { + return err + } + + return nil +} From 8b99d13f409e54b50aeb7c4e3d85bfb06ed622f0 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 15 Mar 2023 15:13:51 -0400 Subject: [PATCH 15/83] lint (#270) --- controller/metrics2/model.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/controller/metrics2/model.go b/controller/metrics2/model.go index e533c9ea..03a480ab 100644 --- a/controller/metrics2/model.go +++ b/controller/metrics2/model.go @@ -1,5 +1,11 @@ package metrics2 +import ( + "fmt" + "github.com/openziti/zrok/util" + "time" +) + type Usage struct { ProcessedStamp time.Time IntervalStart time.Time From 20bd5bbb0950f1adf827875695deb1d8d6dd5309 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 15 Mar 2023 15:16:48 -0400 Subject: [PATCH 16/83] detail cache (#270) --- controller/metrics2/cache.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 controller/metrics2/cache.go diff --git a/controller/metrics2/cache.go b/controller/metrics2/cache.go new file mode 100644 index 00000000..7dc8b047 --- /dev/null +++ b/controller/metrics2/cache.go @@ -0,0 +1,33 @@ +package metrics2 + +import "github.com/openziti/zrok/controller/store" + +type cache struct { + str *store.Store +} + +func newShareCache(str *store.Store) *cache { + return &cache{str} +} + +func (c *cache) addZrokDetail(u *Usage) error { + tx, err := c.str.Begin() + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + shr, err := c.str.FindShareWithZIdAndDeleted(u.ZitiServiceId, tx) + if err != nil { + return err + } + u.ShareToken = shr.Token + env, err := c.str.GetEnvironment(shr.EnvironmentId, tx) + if err != nil { + return err + } + u.EnvironmentId = int64(env.Id) + u.AccountId = int64(*env.AccountId) + + return nil +} From b4d13e15f0a5e8309c91316ed652b6fb5f695127 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 15 Mar 2023 15:38:35 -0400 Subject: [PATCH 17/83] agent (#270) --- controller/metrics2/agent.go | 58 +++++++++++++++++++++++++++++ controller/metrics2/config.go | 4 ++ controller/metrics2/influxWriter.go | 4 +- controller/metrics2/model.go | 4 ++ 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 controller/metrics2/agent.go diff --git a/controller/metrics2/agent.go b/controller/metrics2/agent.go new file mode 100644 index 00000000..cf2b3ff7 --- /dev/null +++ b/controller/metrics2/agent.go @@ -0,0 +1,58 @@ +package metrics2 + +import ( + "github.com/openziti/zrok/controller/store" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +type Agent struct { + events chan ZitiEventJson + src ZitiEventJsonSource + srcJoin chan struct{} + cache *cache + snk UsageSink +} + +func NewAgent(cfg *AgentConfig, str *store.Store, ifxCfg *InfluxConfig) (*Agent, error) { + a := &Agent{} + if v, ok := cfg.Source.(ZitiEventJsonSource); ok { + a.src = v + } else { + return nil, errors.New("invalid event json source") + } + a.cache = newShareCache(str) + a.snk = newInfluxWriter(ifxCfg) + return a, nil +} + +func (a *Agent) Start() error { + a.events = make(chan ZitiEventJson) + srcJoin, err := a.src.Start(a.events) + if err != nil { + return err + } + a.srcJoin = srcJoin + + go func() { + for { + select { + case event := <-a.events: + if usage, err := Ingest(event); err == nil { + if err := a.snk.Handle(usage); err != nil { + logrus.Error(err) + } + } else { + logrus.Error(err) + } + } + } + }() + + return nil +} + +func (a *Agent) Stop() { + a.src.Stop() + close(a.events) +} diff --git a/controller/metrics2/config.go b/controller/metrics2/config.go index f25cd2f5..012de398 100644 --- a/controller/metrics2/config.go +++ b/controller/metrics2/config.go @@ -1,5 +1,9 @@ package metrics2 +type AgentConfig struct { + Source interface{} +} + type InfluxConfig struct { Url string Bucket string diff --git a/controller/metrics2/influxWriter.go b/controller/metrics2/influxWriter.go index 69772a52..a2652302 100644 --- a/controller/metrics2/influxWriter.go +++ b/controller/metrics2/influxWriter.go @@ -15,13 +15,13 @@ type influxWriter struct { writeApi api.WriteAPIBlocking } -func openInfluxWriter(cfg *InfluxConfig) *influxWriter { +func newInfluxWriter(cfg *InfluxConfig) *influxWriter { idb := influxdb2.NewClient(cfg.Url, cfg.Token) writeApi := idb.WriteAPIBlocking(cfg.Org, cfg.Bucket) return &influxWriter{idb, writeApi} } -func (w *influxWriter) Write(u *Usage) error { +func (w *influxWriter) Handle(u *Usage) error { out := fmt.Sprintf("share: %v, circuit: %v", u.ShareToken, u.ZitiCircuitId) envId := fmt.Sprintf("%d", u.EnvironmentId) diff --git a/controller/metrics2/model.go b/controller/metrics2/model.go index 03a480ab..939cb09f 100644 --- a/controller/metrics2/model.go +++ b/controller/metrics2/model.go @@ -35,6 +35,10 @@ func (u Usage) String() string { return out } +type UsageSink interface { + Handle(u *Usage) error +} + type ZitiEventJson string type ZitiEventJsonSource interface { From 89202873bcb605f161bccb9dc068e989d0d4e287 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 15 Mar 2023 16:05:01 -0400 Subject: [PATCH 18/83] incorporate metrics.Agent into the controller (#270) --- cmd/zrok/controller.go | 6 ++++ cmd/zrok/controllerMetrics.go | 62 ----------------------------------- controller/config/config.go | 3 +- controller/controller.go | 12 +++++++ controller/metrics2/agent.go | 5 +++ controller/metrics2/cache.go | 4 ++- controller/metrics2/config.go | 5 +++ 7 files changed, 32 insertions(+), 65 deletions(-) delete mode 100644 cmd/zrok/controllerMetrics.go diff --git a/cmd/zrok/controller.go b/cmd/zrok/controller.go index 68250dd4..2652a25b 100644 --- a/cmd/zrok/controller.go +++ b/cmd/zrok/controller.go @@ -10,8 +10,14 @@ import ( var controllerCmd *controllerCommand +var metricsCmd = &cobra.Command{ + Use: "metrics", + Short: "Metrics related commands", +} + func init() { controllerCmd = newControllerCommand() + controllerCmd.cmd.AddCommand(metricsCmd) rootCmd.AddCommand(controllerCmd.cmd) } diff --git a/cmd/zrok/controllerMetrics.go b/cmd/zrok/controllerMetrics.go deleted file mode 100644 index 7de987c8..00000000 --- a/cmd/zrok/controllerMetrics.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -import ( - "github.com/michaelquigley/cf" - "github.com/openziti/zrok/controller/config" - "github.com/openziti/zrok/controller/env" - "github.com/openziti/zrok/controller/metrics" - "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "os" - "os/signal" - "syscall" - "time" -) - -var metricsCmd *cobra.Command - -func init() { - metricsCmd = newMetricsCommand().cmd - controllerCmd.cmd.AddCommand(metricsCmd) -} - -type metricsCommand struct { - cmd *cobra.Command -} - -func newMetricsCommand() *metricsCommand { - cmd := &cobra.Command{ - Use: "metrics ", - Short: "Start a zrok metrics agent", - Args: cobra.ExactArgs(1), - } - command := &metricsCommand{cmd} - cmd.Run = command.run - return command -} - -func (cmd *metricsCommand) run(_ *cobra.Command, args []string) { - cfg, err := config.LoadConfig(args[0]) - if err != nil { - panic(err) - } - logrus.Infof(cf.Dump(cfg, env.GetCfOptions())) - - ma, err := metrics.Run(cfg.Metrics, cfg.Store) - if err != nil { - panic(err) - } - - c := make(chan os.Signal) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - go func() { - <-c - ma.Stop() - ma.Join() - os.Exit(0) - }() - - for { - time.Sleep(30 * time.Minute) - } -} diff --git a/controller/config/config.go b/controller/config/config.go index 19ec8833..06413296 100644 --- a/controller/config/config.go +++ b/controller/config/config.go @@ -3,7 +3,6 @@ package config import ( "github.com/openziti/zrok/controller/env" "github.com/openziti/zrok/controller/limits" - "github.com/openziti/zrok/controller/metrics" "github.com/openziti/zrok/controller/metrics2" "github.com/openziti/zrok/controller/zrokEdgeSdk" "time" @@ -23,7 +22,7 @@ type Config struct { Email *EmailConfig Limits *limits.Config Maintenance *MaintenanceConfig - Metrics *metrics.Config + Metrics *metrics2.Config Registration *RegistrationConfig ResetPassword *ResetPasswordConfig Store *store.Config diff --git a/controller/controller.go b/controller/controller.go index f5e2c79a..e9e52f24 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -3,6 +3,7 @@ package controller import ( "context" "github.com/openziti/zrok/controller/config" + "github.com/openziti/zrok/controller/metrics2" "github.com/sirupsen/logrus" "github.com/go-openapi/loads" @@ -70,6 +71,17 @@ func Run(inCfg *config.Config) error { logrus.Warn("skipping influx client; no configuration") } + if cfg.Metrics != nil && cfg.Metrics.Agent != nil && cfg.Metrics.Influx != nil { + ma, err := metrics2.NewAgent(cfg.Metrics.Agent, str, cfg.Metrics.Influx) + if err != nil { + return errors.Wrap(err, "error creating metrics agent") + } + if err := ma.Start(); err != nil { + return errors.Wrap(err, "error starting metrics agent") + } + defer func() { ma.Stop() }() + } + ctx, cancel := context.WithCancel(context.Background()) defer func() { cancel() diff --git a/controller/metrics2/agent.go b/controller/metrics2/agent.go index cf2b3ff7..0e299379 100644 --- a/controller/metrics2/agent.go +++ b/controller/metrics2/agent.go @@ -35,10 +35,15 @@ func (a *Agent) Start() error { a.srcJoin = srcJoin go func() { + logrus.Info("started") + defer logrus.Info("stopped") for { select { case event := <-a.events: if usage, err := Ingest(event); err == nil { + if err := a.cache.addZrokDetail(usage); err != nil { + logrus.Error(err) + } if err := a.snk.Handle(usage); err != nil { logrus.Error(err) } diff --git a/controller/metrics2/cache.go b/controller/metrics2/cache.go index 7dc8b047..32acd1c3 100644 --- a/controller/metrics2/cache.go +++ b/controller/metrics2/cache.go @@ -1,6 +1,8 @@ package metrics2 -import "github.com/openziti/zrok/controller/store" +import ( + "github.com/openziti/zrok/controller/store" +) type cache struct { str *store.Store diff --git a/controller/metrics2/config.go b/controller/metrics2/config.go index 012de398..de593f3a 100644 --- a/controller/metrics2/config.go +++ b/controller/metrics2/config.go @@ -1,5 +1,10 @@ package metrics2 +type Config struct { + Influx *InfluxConfig + Agent *AgentConfig +} + type AgentConfig struct { Source interface{} } From 3f7db68ed702b58f42cb7ce6995ab13ac17c69e9 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 15 Mar 2023 16:11:12 -0400 Subject: [PATCH 19/83] updated websocketSource (#270) --- controller/metrics2/websocketSource.go | 154 +++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 controller/metrics2/websocketSource.go diff --git a/controller/metrics2/websocketSource.go b/controller/metrics2/websocketSource.go new file mode 100644 index 00000000..10a4d973 --- /dev/null +++ b/controller/metrics2/websocketSource.go @@ -0,0 +1,154 @@ +package metrics2 + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "github.com/gorilla/websocket" + "github.com/michaelquigley/cf" + "github.com/openziti/channel/v2" + "github.com/openziti/channel/v2/websockets" + "github.com/openziti/edge/rest_util" + "github.com/openziti/fabric/event" + "github.com/openziti/fabric/pb/mgmt_pb" + "github.com/openziti/identity" + "github.com/openziti/sdk-golang/ziti/constants" + "github.com/openziti/zrok/controller/env" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "io" + "net/http" + "net/url" + "time" +) + +func init() { + env.GetCfOptions().AddFlexibleSetter("websocketSource", loadWebsocketSourceConfig) +} + +type WebsocketSourceConfig struct { + WebsocketEndpoint string + ApiEndpoint string + Username string + Password string `cf:"+secret"` +} + +func loadWebsocketSourceConfig(v interface{}, _ *cf.Options) (interface{}, error) { + if submap, ok := v.(map[string]interface{}); ok { + cfg := &WebsocketSourceConfig{} + if err := cf.Bind(cfg, submap, cf.DefaultOptions()); err != nil { + return nil, err + } + return &websocketSource{cfg: cfg}, nil + } + return nil, errors.New("invalid config struture for 'websocketSource'") +} + +type websocketSource struct { + cfg *WebsocketSourceConfig + ch channel.Channel + events chan ZitiEventJson + join chan struct{} +} + +func (s *websocketSource) Start(events chan ZitiEventJson) (join chan struct{}, err error) { + caCerts, err := rest_util.GetControllerWellKnownCas(s.cfg.ApiEndpoint) + if err != nil { + return nil, err + } + caPool := x509.NewCertPool() + for _, ca := range caCerts { + caPool.AddCert(ca) + } + + authenticator := rest_util.NewAuthenticatorUpdb(s.cfg.Username, s.cfg.Password) + authenticator.RootCas = caPool + + apiEndpointUrl, err := url.Parse(s.cfg.ApiEndpoint) + if err != nil { + return nil, err + } + apiSession, err := authenticator.Authenticate(apiEndpointUrl) + if err != nil { + return nil, err + } + + dialer := &websocket.Dialer{ + TLSClientConfig: &tls.Config{ + RootCAs: caPool, + }, + HandshakeTimeout: 5 * time.Second, + } + + conn, resp, err := dialer.Dial(s.cfg.WebsocketEndpoint, http.Header{constants.ZitiSession: []string{*apiSession.Token}}) + if err != nil { + if resp != nil { + if body, rerr := io.ReadAll(resp.Body); rerr == nil { + logrus.Errorf("response body '%v': %v", string(body), err) + } + } else { + logrus.Errorf("no response from websocket dial: %v", err) + } + } + + id := &identity.TokenId{Token: "mgmt"} + underlayFactory := websockets.NewUnderlayFactory(id, conn, nil) + + s.join = make(chan struct{}) + s.events = events + bindHandler := func(binding channel.Binding) error { + binding.AddReceiveHandler(int32(mgmt_pb.ContentType_StreamEventsEventType), s) + binding.AddCloseHandler(channel.CloseHandlerF(func(ch channel.Channel) { + close(s.join) + })) + return nil + } + + s.ch, err = channel.NewChannel("mgmt", underlayFactory, channel.BindHandlerF(bindHandler), nil) + if err != nil { + return nil, err + } + + streamEventsRequest := map[string]interface{}{} + streamEventsRequest["format"] = "json" + streamEventsRequest["subscriptions"] = []*event.Subscription{ + { + Type: "fabric.usage", + Options: map[string]interface{}{ + "version": uint8(3), + }, + }, + } + + msgBytes, err := json.Marshal(streamEventsRequest) + if err != nil { + return nil, err + } + + requestMsg := channel.NewMessage(int32(mgmt_pb.ContentType_StreamEventsRequestType), msgBytes) + responseMsg, err := requestMsg.WithTimeout(5 * time.Second).SendForReply(s.ch) + if err != nil { + return nil, err + } + + if responseMsg.ContentType == channel.ContentTypeResultType { + result := channel.UnmarshalResult(responseMsg) + if result.Success { + logrus.Infof("event stream started: %v", result.Message) + } else { + return nil, errors.Wrap(err, "error starting event streaming") + } + } else { + return nil, errors.Errorf("unexpected response type %v", responseMsg.ContentType) + } + + return s.join, nil +} + +func (s *websocketSource) Stop() { + _ = s.ch.Close() +} + +func (s *websocketSource) HandleReceive(msg *channel.Message, _ channel.Channel) { + s.events <- ZitiEventJson(msg.Body) +} From 86126b3f532a76c23768fd6732929672788845b6 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 15 Mar 2023 16:14:06 -0400 Subject: [PATCH 20/83] metrics2 -> metrics (#270) --- cmd/zrok/controllerMetricsBridge.go | 4 +- controller/config/config.go | 6 +- controller/controller.go | 4 +- controller/metrics/agent.go | 75 ++++----- controller/metrics/amqpSender.go | 51 ------ controller/{metrics2 => metrics}/amqpSink.go | 2 +- .../{metrics2 => metrics}/amqpSource.go | 2 +- controller/{metrics2 => metrics}/bridge.go | 2 +- controller/metrics/cache.go | 17 +- controller/metrics/config.go | 12 +- controller/metrics/fileSource.go | 96 ++++++----- controller/metrics/influx.go | 61 ------- .../{metrics2 => metrics}/influxWriter.go | 2 +- controller/metrics/model.go | 14 +- .../{metrics2 => metrics}/usageIngest.go | 2 +- controller/metrics/usageIngester.go | 95 ----------- controller/metrics/websocketSource.go | 25 +-- controller/metrics2/agent.go | 63 ------- controller/metrics2/cache.go | 35 ---- controller/metrics2/config.go | 17 -- controller/metrics2/fileSource.go | 130 --------------- controller/metrics2/model.go | 51 ------ controller/metrics2/websocketSource.go | 154 ------------------ 23 files changed, 131 insertions(+), 789 deletions(-) delete mode 100644 controller/metrics/amqpSender.go rename controller/{metrics2 => metrics}/amqpSink.go (98%) rename controller/{metrics2 => metrics}/amqpSource.go (99%) rename controller/{metrics2 => metrics}/bridge.go (98%) delete mode 100644 controller/metrics/influx.go rename controller/{metrics2 => metrics}/influxWriter.go (99%) rename controller/{metrics2 => metrics}/usageIngest.go (99%) delete mode 100644 controller/metrics/usageIngester.go delete mode 100644 controller/metrics2/agent.go delete mode 100644 controller/metrics2/cache.go delete mode 100644 controller/metrics2/config.go delete mode 100644 controller/metrics2/fileSource.go delete mode 100644 controller/metrics2/model.go delete mode 100644 controller/metrics2/websocketSource.go diff --git a/cmd/zrok/controllerMetricsBridge.go b/cmd/zrok/controllerMetricsBridge.go index b67ef129..8d9fa28e 100644 --- a/cmd/zrok/controllerMetricsBridge.go +++ b/cmd/zrok/controllerMetricsBridge.go @@ -4,7 +4,7 @@ import ( "github.com/michaelquigley/cf" "github.com/openziti/zrok/controller/config" "github.com/openziti/zrok/controller/env" - "github.com/openziti/zrok/controller/metrics2" + "github.com/openziti/zrok/controller/metrics" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "os" @@ -39,7 +39,7 @@ func (cmd *bridgeCommand) run(_ *cobra.Command, args []string) { } logrus.Infof(cf.Dump(cfg, env.GetCfOptions())) - bridge, err := metrics2.NewBridge(cfg.Bridge) + bridge, err := metrics.NewBridge(cfg.Bridge) if err != nil { panic(err) } diff --git a/controller/config/config.go b/controller/config/config.go index 06413296..d2ac3eb6 100644 --- a/controller/config/config.go +++ b/controller/config/config.go @@ -3,7 +3,7 @@ package config import ( "github.com/openziti/zrok/controller/env" "github.com/openziti/zrok/controller/limits" - "github.com/openziti/zrok/controller/metrics2" + "github.com/openziti/zrok/controller/metrics" "github.com/openziti/zrok/controller/zrokEdgeSdk" "time" @@ -17,12 +17,12 @@ const ConfigVersion = 2 type Config struct { V int Admin *AdminConfig - Bridge *metrics2.BridgeConfig + Bridge *metrics.BridgeConfig Endpoint *EndpointConfig Email *EmailConfig Limits *limits.Config Maintenance *MaintenanceConfig - Metrics *metrics2.Config + Metrics *metrics.Config Registration *RegistrationConfig ResetPassword *ResetPasswordConfig Store *store.Config diff --git a/controller/controller.go b/controller/controller.go index e9e52f24..670bd863 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -3,7 +3,7 @@ package controller import ( "context" "github.com/openziti/zrok/controller/config" - "github.com/openziti/zrok/controller/metrics2" + "github.com/openziti/zrok/controller/metrics" "github.com/sirupsen/logrus" "github.com/go-openapi/loads" @@ -72,7 +72,7 @@ func Run(inCfg *config.Config) error { } if cfg.Metrics != nil && cfg.Metrics.Agent != nil && cfg.Metrics.Influx != nil { - ma, err := metrics2.NewAgent(cfg.Metrics.Agent, str, cfg.Metrics.Influx) + ma, err := metrics.NewAgent(cfg.Metrics.Agent, str, cfg.Metrics.Influx) if err != nil { return errors.Wrap(err, "error creating metrics agent") } diff --git a/controller/metrics/agent.go b/controller/metrics/agent.go index c06ab246..89be79e5 100644 --- a/controller/metrics/agent.go +++ b/controller/metrics/agent.go @@ -6,48 +6,45 @@ import ( "github.com/sirupsen/logrus" ) -type MetricsAgent struct { - src Source - cache *cache - join chan struct{} +type Agent struct { + events chan ZitiEventJson + src ZitiEventJsonSource + srcJoin chan struct{} + cache *cache + snk UsageSink } -func Run(cfg *Config, strCfg *store.Config) (*MetricsAgent, error) { - logrus.Info("starting") +func NewAgent(cfg *AgentConfig, str *store.Store, ifxCfg *InfluxConfig) (*Agent, error) { + a := &Agent{} + if v, ok := cfg.Source.(ZitiEventJsonSource); ok { + a.src = v + } else { + return nil, errors.New("invalid event json source") + } + a.cache = newShareCache(str) + a.snk = newInfluxWriter(ifxCfg) + return a, nil +} - cache, err := newShareCache(strCfg) +func (a *Agent) Start() error { + a.events = make(chan ZitiEventJson) + srcJoin, err := a.src.Start(a.events) if err != nil { - return nil, errors.Wrap(err, "error creating share cache") - } - - if cfg.Strategies == nil || cfg.Strategies.Source == nil { - return nil, errors.New("no 'strategies/source' configured; exiting") - } - - src, ok := cfg.Strategies.Source.(Source) - if !ok { - return nil, errors.New("invalid 'strategies/source'; exiting") - } - - if cfg.Influx == nil { - return nil, errors.New("no 'influx' configured; exiting") - } - - idb := openInfluxDb(cfg.Influx) - - events := make(chan map[string]interface{}) - join, err := src.Start(events) - if err != nil { - return nil, errors.Wrap(err, "error starting source") + return err } + a.srcJoin = srcJoin go func() { + logrus.Info("started") + defer logrus.Info("stopped") for { select { - case event := <-events: - usage := Ingest(event) - if err := cache.addZrokDetail(usage); err == nil { - if err := idb.Write(usage); err != nil { + case event := <-a.events: + if usage, err := Ingest(event); err == nil { + if err := a.cache.addZrokDetail(usage); err != nil { + logrus.Error(err) + } + if err := a.snk.Handle(usage); err != nil { logrus.Error(err) } } else { @@ -57,14 +54,10 @@ func Run(cfg *Config, strCfg *store.Config) (*MetricsAgent, error) { } }() - return &MetricsAgent{src: src, join: join}, nil + return nil } -func (ma *MetricsAgent) Stop() { - logrus.Info("stopping") - ma.src.Stop() -} - -func (ma *MetricsAgent) Join() { - <-ma.join +func (a *Agent) Stop() { + a.src.Stop() + close(a.events) } diff --git a/controller/metrics/amqpSender.go b/controller/metrics/amqpSender.go deleted file mode 100644 index 332d5328..00000000 --- a/controller/metrics/amqpSender.go +++ /dev/null @@ -1,51 +0,0 @@ -package metrics - -import ( - "context" - "github.com/pkg/errors" - amqp "github.com/rabbitmq/amqp091-go" - "time" -) - -type AmqpSenderConfig struct { - Url string `cf:"+secret"` - Queue string -} - -type AmqpSender struct { - conn *amqp.Connection - ch *amqp.Channel - queue amqp.Queue -} - -func NewAmqpSender(cfg *AmqpSenderConfig) (*AmqpSender, error) { - conn, err := amqp.Dial(cfg.Url) - if err != nil { - return nil, errors.Wrap(err, "error dialing amqp broker") - } - - ch, err := conn.Channel() - if err != nil { - return nil, errors.Wrap(err, "error getting channel from amqp connection") - } - - queue, err := ch.QueueDeclare(cfg.Queue, true, false, false, false, nil) - if err != nil { - return nil, errors.Wrap(err, "error creating amqp queue") - } - - return &AmqpSender{conn, ch, queue}, nil -} - -func (s *AmqpSender) Send(json string) error { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - err := s.ch.PublishWithContext(ctx, "", s.queue.Name, false, false, amqp.Publishing{ - ContentType: "application/json", - Body: []byte(json), - }) - if err != nil { - return errors.Wrap(err, "error sending") - } - return nil -} diff --git a/controller/metrics2/amqpSink.go b/controller/metrics/amqpSink.go similarity index 98% rename from controller/metrics2/amqpSink.go rename to controller/metrics/amqpSink.go index 545f3433..451bdb7b 100644 --- a/controller/metrics2/amqpSink.go +++ b/controller/metrics/amqpSink.go @@ -1,4 +1,4 @@ -package metrics2 +package metrics import ( "context" diff --git a/controller/metrics2/amqpSource.go b/controller/metrics/amqpSource.go similarity index 99% rename from controller/metrics2/amqpSource.go rename to controller/metrics/amqpSource.go index 5e07f5d4..506a4984 100644 --- a/controller/metrics2/amqpSource.go +++ b/controller/metrics/amqpSource.go @@ -1,4 +1,4 @@ -package metrics2 +package metrics import ( "github.com/michaelquigley/cf" diff --git a/controller/metrics2/bridge.go b/controller/metrics/bridge.go similarity index 98% rename from controller/metrics2/bridge.go rename to controller/metrics/bridge.go index f91c4219..17e8d6e9 100644 --- a/controller/metrics2/bridge.go +++ b/controller/metrics/bridge.go @@ -1,4 +1,4 @@ -package metrics2 +package metrics import ( "github.com/pkg/errors" diff --git a/controller/metrics/cache.go b/controller/metrics/cache.go index 6cb2b59e..e61866b6 100644 --- a/controller/metrics/cache.go +++ b/controller/metrics/cache.go @@ -2,34 +2,29 @@ package metrics import ( "github.com/openziti/zrok/controller/store" - "github.com/pkg/errors" ) type cache struct { str *store.Store } -func newShareCache(cfg *store.Config) (*cache, error) { - str, err := store.Open(cfg) - if err != nil { - return nil, errors.Wrap(err, "error opening store") - } - return &cache{str}, nil +func newShareCache(str *store.Store) *cache { + return &cache{str} } -func (sc *cache) addZrokDetail(u *Usage) error { - tx, err := sc.str.Begin() +func (c *cache) addZrokDetail(u *Usage) error { + tx, err := c.str.Begin() if err != nil { return err } defer func() { _ = tx.Rollback() }() - shr, err := sc.str.FindShareWithZIdAndDeleted(u.ZitiServiceId, tx) + shr, err := c.str.FindShareWithZIdAndDeleted(u.ZitiServiceId, tx) if err != nil { return err } u.ShareToken = shr.Token - env, err := sc.str.GetEnvironment(shr.EnvironmentId, tx) + env, err := c.str.GetEnvironment(shr.EnvironmentId, tx) if err != nil { return err } diff --git a/controller/metrics/config.go b/controller/metrics/config.go index 4cdafcbb..03e4da83 100644 --- a/controller/metrics/config.go +++ b/controller/metrics/config.go @@ -1,8 +1,12 @@ package metrics type Config struct { - Influx *InfluxConfig - Strategies *StrategiesConfig + Influx *InfluxConfig + Agent *AgentConfig +} + +type AgentConfig struct { + Source interface{} } type InfluxConfig struct { @@ -11,7 +15,3 @@ type InfluxConfig struct { Org string Token string `cf:"+secret"` } - -type StrategiesConfig struct { - Source interface{} -} diff --git a/controller/metrics/fileSource.go b/controller/metrics/fileSource.go index 67007552..ac7f86b8 100644 --- a/controller/metrics/fileSource.go +++ b/controller/metrics/fileSource.go @@ -2,7 +2,6 @@ package metrics import ( "encoding/binary" - "encoding/json" "github.com/michaelquigley/cf" "github.com/nxadm/tail" "github.com/openziti/zrok/controller/env" @@ -12,12 +11,12 @@ import ( ) func init() { - env.GetCfOptions().AddFlexibleSetter("file", loadFileSourceConfig) + env.GetCfOptions().AddFlexibleSetter("fileSource", loadFileSourceConfig) } type FileSourceConfig struct { - Path string - IndexPath string + Path string + PointerPath string } func loadFileSourceConfig(v interface{}, _ *cf.Options) (interface{}, error) { @@ -28,36 +27,36 @@ func loadFileSourceConfig(v interface{}, _ *cf.Options) (interface{}, error) { } return &fileSource{cfg: cfg}, nil } - return nil, errors.New("invalid config structure for 'file' source") + return nil, errors.New("invalid config structure for 'fileSource'") } type fileSource struct { - cfg *FileSourceConfig - t *tail.Tail + cfg *FileSourceConfig + ptrF *os.File + t *tail.Tail } -func (s *fileSource) Start(events chan map[string]interface{}) (join chan struct{}, err error) { +func (s *fileSource) Start(events chan ZitiEventJson) (join chan struct{}, err error) { f, err := os.Open(s.cfg.Path) if err != nil { return nil, errors.Wrapf(err, "error opening '%v'", s.cfg.Path) } _ = f.Close() - idxF, err := os.OpenFile(s.indexPath(), os.O_CREATE|os.O_RDWR, os.ModePerm) + s.ptrF, err = os.OpenFile(s.pointerPath(), os.O_CREATE|os.O_RDWR, os.ModePerm) if err != nil { - return nil, errors.Wrapf(err, "error opening '%v'", s.indexPath()) + return nil, errors.Wrapf(err, "error opening pointer '%v'", s.pointerPath()) } - pos := int64(0) - posBuf := make([]byte, 8) - if n, err := idxF.Read(posBuf); err == nil && n == 8 { - pos = int64(binary.LittleEndian.Uint64(posBuf)) - logrus.Infof("recovered stored position: %d", pos) + ptr, err := s.readPtr() + if err != nil { + logrus.Errorf("error reading pointer: %v", err) } + logrus.Infof("retrieved stored position pointer at '%d'", ptr) join = make(chan struct{}) go func() { - s.tail(pos, events, idxF) + s.tail(ptr, events) close(join) }() @@ -70,43 +69,62 @@ func (s *fileSource) Stop() { } } -func (s *fileSource) tail(pos int64, events chan map[string]interface{}, idxF *os.File) { - logrus.Infof("started") - defer logrus.Infof("stopped") - - posBuf := make([]byte, 8) +func (s *fileSource) tail(ptr int64, events chan ZitiEventJson) { + logrus.Info("started") + defer logrus.Info("stopped") var err error s.t, err = tail.TailFile(s.cfg.Path, tail.Config{ ReOpen: true, Follow: true, - Location: &tail.SeekInfo{Offset: pos}, + Location: &tail.SeekInfo{Offset: ptr}, }) if err != nil { - logrus.Error(err) + logrus.Errorf("error starting tail: %v", err) return } - for line := range s.t.Lines { - event := make(map[string]interface{}) - if err := json.Unmarshal([]byte(line.Text), &event); err == nil { - binary.LittleEndian.PutUint64(posBuf, uint64(line.SeekInfo.Offset)) - if n, err := idxF.Seek(0, 0); err == nil && n == 0 { - if n, err := idxF.Write(posBuf); err != nil || n != 8 { - logrus.Errorf("error writing index (%d): %v", n, err) - } - } - events <- event - } else { - logrus.Errorf("error parsing line #%d: %v", line.Num, err) + for event := range s.t.Lines { + events <- ZitiEventJson(event.Text) + + if err := s.writePtr(event.SeekInfo.Offset); err != nil { + logrus.Error(err) } } } -func (s *fileSource) indexPath() string { - if s.cfg.IndexPath == "" { - return s.cfg.Path + ".idx" +func (s *fileSource) pointerPath() string { + if s.cfg.PointerPath == "" { + return s.cfg.Path + ".ptr" } else { - return s.cfg.IndexPath + return s.cfg.PointerPath } } + +func (s *fileSource) readPtr() (int64, error) { + ptr := int64(0) + buf := make([]byte, 8) + if n, err := s.ptrF.Seek(0, 0); err == nil && n == 0 { + if n, err := s.ptrF.Read(buf); err == nil && n == 8 { + ptr = int64(binary.LittleEndian.Uint64(buf)) + return ptr, nil + } else { + return 0, errors.Wrapf(err, "error reading pointer (%d): %v", n, err) + } + } else { + return 0, errors.Wrapf(err, "error seeking pointer (%d): %v", n, err) + } +} + +func (s *fileSource) writePtr(ptr int64) error { + buf := make([]byte, 8) + binary.LittleEndian.PutUint64(buf, uint64(ptr)) + if n, err := s.ptrF.Seek(0, 0); err == nil && n == 0 { + if n, err := s.ptrF.Write(buf); err != nil || n != 8 { + return errors.Wrapf(err, "error writing pointer (%d): %v", n, err) + } + } else { + return errors.Wrapf(err, "error seeking pointer (%d): %v", n, err) + } + return nil +} diff --git a/controller/metrics/influx.go b/controller/metrics/influx.go deleted file mode 100644 index 214ed8e5..00000000 --- a/controller/metrics/influx.go +++ /dev/null @@ -1,61 +0,0 @@ -package metrics - -import ( - "context" - "fmt" - influxdb2 "github.com/influxdata/influxdb-client-go/v2" - "github.com/influxdata/influxdb-client-go/v2/api" - "github.com/influxdata/influxdb-client-go/v2/api/write" - "github.com/openziti/zrok/util" - "github.com/sirupsen/logrus" -) - -type influxDb struct { - idb influxdb2.Client - writeApi api.WriteAPIBlocking -} - -func openInfluxDb(cfg *InfluxConfig) *influxDb { - idb := influxdb2.NewClient(cfg.Url, cfg.Token) - wapi := idb.WriteAPIBlocking(cfg.Org, cfg.Bucket) - return &influxDb{idb, wapi} -} - -func (i *influxDb) Write(u *Usage) error { - out := fmt.Sprintf("share: %v, circuit: %v", u.ShareToken, u.ZitiCircuitId) - - envId := fmt.Sprintf("%d", u.EnvironmentId) - acctId := fmt.Sprintf("%d", u.AccountId) - - var pts []*write.Point - circuitPt := influxdb2.NewPoint("circuits", - map[string]string{"share": u.ShareToken, "envId": envId, "acctId": acctId}, - map[string]interface{}{"circuit": u.ZitiCircuitId}, - u.IntervalStart) - pts = append(pts, circuitPt) - - if u.BackendTx > 0 || u.BackendRx > 0 { - pt := influxdb2.NewPoint("xfer", - map[string]string{"namespace": "backend", "share": u.ShareToken, "envId": envId, "acctId": acctId}, - map[string]interface{}{"bytesRead": u.BackendRx, "bytesWritten": u.BackendTx}, - u.IntervalStart) - pts = append(pts, pt) - out += fmt.Sprintf(" backend {rx: %v, tx: %v}", util.BytesToSize(u.BackendRx), util.BytesToSize(u.BackendTx)) - } - if u.FrontendTx > 0 || u.FrontendRx > 0 { - pt := influxdb2.NewPoint("xfer", - map[string]string{"namespace": "frontend", "share": u.ShareToken, "envId": envId, "acctId": acctId}, - map[string]interface{}{"bytesRead": u.FrontendRx, "bytesWritten": u.FrontendTx}, - u.IntervalStart) - pts = append(pts, pt) - out += fmt.Sprintf(" frontend {rx: %v, tx: %v}", util.BytesToSize(u.FrontendRx), util.BytesToSize(u.FrontendTx)) - } - - if err := i.writeApi.WritePoint(context.Background(), pts...); err == nil { - logrus.Info(out) - } else { - return err - } - - return nil -} diff --git a/controller/metrics2/influxWriter.go b/controller/metrics/influxWriter.go similarity index 99% rename from controller/metrics2/influxWriter.go rename to controller/metrics/influxWriter.go index a2652302..12c09693 100644 --- a/controller/metrics2/influxWriter.go +++ b/controller/metrics/influxWriter.go @@ -1,4 +1,4 @@ -package metrics2 +package metrics import ( "context" diff --git a/controller/metrics/model.go b/controller/metrics/model.go index 632e8404..bda74939 100644 --- a/controller/metrics/model.go +++ b/controller/metrics/model.go @@ -35,11 +35,17 @@ func (u Usage) String() string { return out } -type Source interface { - Start(chan map[string]interface{}) (chan struct{}, error) +type UsageSink interface { + Handle(u *Usage) error +} + +type ZitiEventJson string + +type ZitiEventJsonSource interface { + Start(chan ZitiEventJson) (join chan struct{}, err error) Stop() } -type Ingester interface { - Ingest(msg map[string]interface{}) error +type ZitiEventJsonSink interface { + Handle(event ZitiEventJson) error } diff --git a/controller/metrics2/usageIngest.go b/controller/metrics/usageIngest.go similarity index 99% rename from controller/metrics2/usageIngest.go rename to controller/metrics/usageIngest.go index 99f50122..d22a3a57 100644 --- a/controller/metrics2/usageIngest.go +++ b/controller/metrics/usageIngest.go @@ -1,4 +1,4 @@ -package metrics2 +package metrics import ( "encoding/json" diff --git a/controller/metrics/usageIngester.go b/controller/metrics/usageIngester.go deleted file mode 100644 index 80db3412..00000000 --- a/controller/metrics/usageIngester.go +++ /dev/null @@ -1,95 +0,0 @@ -package metrics - -import ( - "github.com/sirupsen/logrus" - "reflect" - "time" -) - -func Ingest(event map[string]interface{}) *Usage { - u := &Usage{ProcessedStamp: time.Now()} - if ns, found := event["namespace"]; found && ns == "fabric.usage" { - if v, found := event["interval_start_utc"]; found { - if vFloat64, ok := v.(float64); ok { - u.IntervalStart = time.Unix(int64(vFloat64), 0) - } else { - logrus.Error("unable to assert 'interval_start_utc'") - } - } else { - logrus.Error("missing 'interval_start_utc'") - } - if v, found := event["tags"]; found { - if tags, ok := v.(map[string]interface{}); ok { - if v, found := tags["serviceId"]; found { - if vStr, ok := v.(string); ok { - u.ZitiServiceId = vStr - } else { - logrus.Error("unable to assert 'tags/serviceId'") - } - } else { - logrus.Error("missing 'tags/serviceId'") - } - } else { - logrus.Errorf("unable to assert 'tags'") - } - } else { - logrus.Errorf("missing 'tags'") - } - if v, found := event["usage"]; found { - if usage, ok := v.(map[string]interface{}); ok { - if v, found := usage["ingress.tx"]; found { - if vFloat64, ok := v.(float64); ok { - u.FrontendTx = int64(vFloat64) - } else { - logrus.Error("unable to assert 'usage/ingress.tx'") - } - } else { - logrus.Warn("missing 'usage/ingress.tx'") - } - if v, found := usage["ingress.rx"]; found { - if vFloat64, ok := v.(float64); ok { - u.FrontendRx = int64(vFloat64) - } else { - logrus.Error("unable to assert 'usage/ingress.rx") - } - } else { - logrus.Warn("missing 'usage/ingress.rx") - } - if v, found := usage["egress.tx"]; found { - if vFloat64, ok := v.(float64); ok { - u.BackendTx = int64(vFloat64) - } else { - logrus.Error("unable to assert 'usage/egress.tx'") - } - } else { - logrus.Warn("missing 'usage/egress.tx'") - } - if v, found := usage["egress.rx"]; found { - if vFloat64, ok := v.(float64); ok { - u.BackendRx = int64(vFloat64) - } else { - logrus.Error("unable to assert 'usage/egress.rx'") - } - } else { - logrus.Warn("missing 'usage/egress.rx'") - } - } else { - logrus.Errorf("unable to assert 'usage' (%v) %v", reflect.TypeOf(v), event) - } - } else { - logrus.Warnf("missing 'usage'") - } - if v, found := event["circuit_id"]; found { - if vStr, ok := v.(string); ok { - u.ZitiCircuitId = vStr - } else { - logrus.Error("unable to assert 'circuit_id'") - } - } else { - logrus.Warn("missing 'circuit_id'") - } - } else { - logrus.Errorf("not 'fabric.usage'") - } - return u -} diff --git a/controller/metrics/websocketSource.go b/controller/metrics/websocketSource.go index 3b1b2898..e6b66ac6 100644 --- a/controller/metrics/websocketSource.go +++ b/controller/metrics/websocketSource.go @@ -1,7 +1,6 @@ package metrics import ( - "bytes" "crypto/tls" "crypto/x509" "encoding/json" @@ -24,14 +23,14 @@ import ( ) func init() { - env.GetCfOptions().AddFlexibleSetter("websocket", loadWebsocketSourceConfig) + env.GetCfOptions().AddFlexibleSetter("websocketSource", loadWebsocketSourceConfig) } type WebsocketSourceConfig struct { WebsocketEndpoint string ApiEndpoint string Username string - Password string + Password string `cf:"+secret"` } func loadWebsocketSourceConfig(v interface{}, _ *cf.Options) (interface{}, error) { @@ -42,17 +41,17 @@ func loadWebsocketSourceConfig(v interface{}, _ *cf.Options) (interface{}, error } return &websocketSource{cfg: cfg}, nil } - return nil, errors.New("invalid config structure for 'websocket' source") + return nil, errors.New("invalid config struture for 'websocketSource'") } type websocketSource struct { cfg *WebsocketSourceConfig ch channel.Channel - events chan map[string]interface{} + events chan ZitiEventJson join chan struct{} } -func (s *websocketSource) Start(events chan map[string]interface{}) (chan struct{}, error) { +func (s *websocketSource) Start(events chan ZitiEventJson) (join chan struct{}, err error) { caCerts, err := rest_util.GetControllerWellKnownCas(s.cfg.ApiEndpoint) if err != nil { return nil, err @@ -151,17 +150,5 @@ func (s *websocketSource) Stop() { } func (s *websocketSource) HandleReceive(msg *channel.Message, _ channel.Channel) { - decoder := json.NewDecoder(bytes.NewReader(msg.Body)) - for { - ev := make(map[string]interface{}) - err := decoder.Decode(&ev) - if err == io.EOF { - break - } - if err == nil { - s.events <- ev - } else { - logrus.Errorf("error parsing '%v': %v", string(msg.Body), err) - } - } + s.events <- ZitiEventJson(msg.Body) } diff --git a/controller/metrics2/agent.go b/controller/metrics2/agent.go deleted file mode 100644 index 0e299379..00000000 --- a/controller/metrics2/agent.go +++ /dev/null @@ -1,63 +0,0 @@ -package metrics2 - -import ( - "github.com/openziti/zrok/controller/store" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -type Agent struct { - events chan ZitiEventJson - src ZitiEventJsonSource - srcJoin chan struct{} - cache *cache - snk UsageSink -} - -func NewAgent(cfg *AgentConfig, str *store.Store, ifxCfg *InfluxConfig) (*Agent, error) { - a := &Agent{} - if v, ok := cfg.Source.(ZitiEventJsonSource); ok { - a.src = v - } else { - return nil, errors.New("invalid event json source") - } - a.cache = newShareCache(str) - a.snk = newInfluxWriter(ifxCfg) - return a, nil -} - -func (a *Agent) Start() error { - a.events = make(chan ZitiEventJson) - srcJoin, err := a.src.Start(a.events) - if err != nil { - return err - } - a.srcJoin = srcJoin - - go func() { - logrus.Info("started") - defer logrus.Info("stopped") - for { - select { - case event := <-a.events: - if usage, err := Ingest(event); err == nil { - if err := a.cache.addZrokDetail(usage); err != nil { - logrus.Error(err) - } - if err := a.snk.Handle(usage); err != nil { - logrus.Error(err) - } - } else { - logrus.Error(err) - } - } - } - }() - - return nil -} - -func (a *Agent) Stop() { - a.src.Stop() - close(a.events) -} diff --git a/controller/metrics2/cache.go b/controller/metrics2/cache.go deleted file mode 100644 index 32acd1c3..00000000 --- a/controller/metrics2/cache.go +++ /dev/null @@ -1,35 +0,0 @@ -package metrics2 - -import ( - "github.com/openziti/zrok/controller/store" -) - -type cache struct { - str *store.Store -} - -func newShareCache(str *store.Store) *cache { - return &cache{str} -} - -func (c *cache) addZrokDetail(u *Usage) error { - tx, err := c.str.Begin() - if err != nil { - return err - } - defer func() { _ = tx.Rollback() }() - - shr, err := c.str.FindShareWithZIdAndDeleted(u.ZitiServiceId, tx) - if err != nil { - return err - } - u.ShareToken = shr.Token - env, err := c.str.GetEnvironment(shr.EnvironmentId, tx) - if err != nil { - return err - } - u.EnvironmentId = int64(env.Id) - u.AccountId = int64(*env.AccountId) - - return nil -} diff --git a/controller/metrics2/config.go b/controller/metrics2/config.go deleted file mode 100644 index de593f3a..00000000 --- a/controller/metrics2/config.go +++ /dev/null @@ -1,17 +0,0 @@ -package metrics2 - -type Config struct { - Influx *InfluxConfig - Agent *AgentConfig -} - -type AgentConfig struct { - Source interface{} -} - -type InfluxConfig struct { - Url string - Bucket string - Org string - Token string `cf:"+secret"` -} diff --git a/controller/metrics2/fileSource.go b/controller/metrics2/fileSource.go deleted file mode 100644 index 9ca527a5..00000000 --- a/controller/metrics2/fileSource.go +++ /dev/null @@ -1,130 +0,0 @@ -package metrics2 - -import ( - "encoding/binary" - "github.com/michaelquigley/cf" - "github.com/nxadm/tail" - "github.com/openziti/zrok/controller/env" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "os" -) - -func init() { - env.GetCfOptions().AddFlexibleSetter("fileSource", loadFileSourceConfig) -} - -type FileSourceConfig struct { - Path string - PointerPath string -} - -func loadFileSourceConfig(v interface{}, _ *cf.Options) (interface{}, error) { - if submap, ok := v.(map[string]interface{}); ok { - cfg := &FileSourceConfig{} - if err := cf.Bind(cfg, submap, cf.DefaultOptions()); err != nil { - return nil, err - } - return &fileSource{cfg: cfg}, nil - } - return nil, errors.New("invalid config structure for 'fileSource'") -} - -type fileSource struct { - cfg *FileSourceConfig - ptrF *os.File - t *tail.Tail -} - -func (s *fileSource) Start(events chan ZitiEventJson) (join chan struct{}, err error) { - f, err := os.Open(s.cfg.Path) - if err != nil { - return nil, errors.Wrapf(err, "error opening '%v'", s.cfg.Path) - } - _ = f.Close() - - s.ptrF, err = os.OpenFile(s.pointerPath(), os.O_CREATE|os.O_RDWR, os.ModePerm) - if err != nil { - return nil, errors.Wrapf(err, "error opening pointer '%v'", s.pointerPath()) - } - - ptr, err := s.readPtr() - if err != nil { - logrus.Errorf("error reading pointer: %v", err) - } - logrus.Infof("retrieved stored position pointer at '%d'", ptr) - - join = make(chan struct{}) - go func() { - s.tail(ptr, events) - close(join) - }() - - return join, nil -} - -func (s *fileSource) Stop() { - if err := s.t.Stop(); err != nil { - logrus.Error(err) - } -} - -func (s *fileSource) tail(ptr int64, events chan ZitiEventJson) { - logrus.Info("started") - defer logrus.Info("stopped") - - var err error - s.t, err = tail.TailFile(s.cfg.Path, tail.Config{ - ReOpen: true, - Follow: true, - Location: &tail.SeekInfo{Offset: ptr}, - }) - if err != nil { - logrus.Errorf("error starting tail: %v", err) - return - } - - for event := range s.t.Lines { - events <- ZitiEventJson(event.Text) - - if err := s.writePtr(event.SeekInfo.Offset); err != nil { - logrus.Error(err) - } - } -} - -func (s *fileSource) pointerPath() string { - if s.cfg.PointerPath == "" { - return s.cfg.Path + ".ptr" - } else { - return s.cfg.PointerPath - } -} - -func (s *fileSource) readPtr() (int64, error) { - ptr := int64(0) - buf := make([]byte, 8) - if n, err := s.ptrF.Seek(0, 0); err == nil && n == 0 { - if n, err := s.ptrF.Read(buf); err == nil && n == 8 { - ptr = int64(binary.LittleEndian.Uint64(buf)) - return ptr, nil - } else { - return 0, errors.Wrapf(err, "error reading pointer (%d): %v", n, err) - } - } else { - return 0, errors.Wrapf(err, "error seeking pointer (%d): %v", n, err) - } -} - -func (s *fileSource) writePtr(ptr int64) error { - buf := make([]byte, 8) - binary.LittleEndian.PutUint64(buf, uint64(ptr)) - if n, err := s.ptrF.Seek(0, 0); err == nil && n == 0 { - if n, err := s.ptrF.Write(buf); err != nil || n != 8 { - return errors.Wrapf(err, "error writing pointer (%d): %v", n, err) - } - } else { - return errors.Wrapf(err, "error seeking pointer (%d): %v", n, err) - } - return nil -} diff --git a/controller/metrics2/model.go b/controller/metrics2/model.go deleted file mode 100644 index 939cb09f..00000000 --- a/controller/metrics2/model.go +++ /dev/null @@ -1,51 +0,0 @@ -package metrics2 - -import ( - "fmt" - "github.com/openziti/zrok/util" - "time" -) - -type Usage struct { - ProcessedStamp time.Time - IntervalStart time.Time - ZitiServiceId string - ZitiCircuitId string - ShareToken string - EnvironmentId int64 - AccountId int64 - FrontendTx int64 - FrontendRx int64 - BackendTx int64 - BackendRx int64 -} - -func (u Usage) String() string { - out := "Usage {" - out += fmt.Sprintf("processed '%v'", u.ProcessedStamp) - out += ", " + fmt.Sprintf("interval '%v'", u.IntervalStart) - out += ", " + fmt.Sprintf("service '%v'", u.ZitiServiceId) - out += ", " + fmt.Sprintf("circuit '%v'", u.ZitiCircuitId) - out += ", " + fmt.Sprintf("share '%v'", u.ShareToken) - out += ", " + fmt.Sprintf("environment '%d'", u.EnvironmentId) - out += ", " + fmt.Sprintf("account '%v'", u.AccountId) - out += ", " + fmt.Sprintf("fe {rx %v, tx %v}", util.BytesToSize(u.FrontendRx), util.BytesToSize(u.FrontendTx)) - out += ", " + fmt.Sprintf("be {rx %v, tx %v}", util.BytesToSize(u.BackendRx), util.BytesToSize(u.BackendTx)) - out += "}" - return out -} - -type UsageSink interface { - Handle(u *Usage) error -} - -type ZitiEventJson string - -type ZitiEventJsonSource interface { - Start(chan ZitiEventJson) (join chan struct{}, err error) - Stop() -} - -type ZitiEventJsonSink interface { - Handle(event ZitiEventJson) error -} diff --git a/controller/metrics2/websocketSource.go b/controller/metrics2/websocketSource.go deleted file mode 100644 index 10a4d973..00000000 --- a/controller/metrics2/websocketSource.go +++ /dev/null @@ -1,154 +0,0 @@ -package metrics2 - -import ( - "crypto/tls" - "crypto/x509" - "encoding/json" - "github.com/gorilla/websocket" - "github.com/michaelquigley/cf" - "github.com/openziti/channel/v2" - "github.com/openziti/channel/v2/websockets" - "github.com/openziti/edge/rest_util" - "github.com/openziti/fabric/event" - "github.com/openziti/fabric/pb/mgmt_pb" - "github.com/openziti/identity" - "github.com/openziti/sdk-golang/ziti/constants" - "github.com/openziti/zrok/controller/env" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "io" - "net/http" - "net/url" - "time" -) - -func init() { - env.GetCfOptions().AddFlexibleSetter("websocketSource", loadWebsocketSourceConfig) -} - -type WebsocketSourceConfig struct { - WebsocketEndpoint string - ApiEndpoint string - Username string - Password string `cf:"+secret"` -} - -func loadWebsocketSourceConfig(v interface{}, _ *cf.Options) (interface{}, error) { - if submap, ok := v.(map[string]interface{}); ok { - cfg := &WebsocketSourceConfig{} - if err := cf.Bind(cfg, submap, cf.DefaultOptions()); err != nil { - return nil, err - } - return &websocketSource{cfg: cfg}, nil - } - return nil, errors.New("invalid config struture for 'websocketSource'") -} - -type websocketSource struct { - cfg *WebsocketSourceConfig - ch channel.Channel - events chan ZitiEventJson - join chan struct{} -} - -func (s *websocketSource) Start(events chan ZitiEventJson) (join chan struct{}, err error) { - caCerts, err := rest_util.GetControllerWellKnownCas(s.cfg.ApiEndpoint) - if err != nil { - return nil, err - } - caPool := x509.NewCertPool() - for _, ca := range caCerts { - caPool.AddCert(ca) - } - - authenticator := rest_util.NewAuthenticatorUpdb(s.cfg.Username, s.cfg.Password) - authenticator.RootCas = caPool - - apiEndpointUrl, err := url.Parse(s.cfg.ApiEndpoint) - if err != nil { - return nil, err - } - apiSession, err := authenticator.Authenticate(apiEndpointUrl) - if err != nil { - return nil, err - } - - dialer := &websocket.Dialer{ - TLSClientConfig: &tls.Config{ - RootCAs: caPool, - }, - HandshakeTimeout: 5 * time.Second, - } - - conn, resp, err := dialer.Dial(s.cfg.WebsocketEndpoint, http.Header{constants.ZitiSession: []string{*apiSession.Token}}) - if err != nil { - if resp != nil { - if body, rerr := io.ReadAll(resp.Body); rerr == nil { - logrus.Errorf("response body '%v': %v", string(body), err) - } - } else { - logrus.Errorf("no response from websocket dial: %v", err) - } - } - - id := &identity.TokenId{Token: "mgmt"} - underlayFactory := websockets.NewUnderlayFactory(id, conn, nil) - - s.join = make(chan struct{}) - s.events = events - bindHandler := func(binding channel.Binding) error { - binding.AddReceiveHandler(int32(mgmt_pb.ContentType_StreamEventsEventType), s) - binding.AddCloseHandler(channel.CloseHandlerF(func(ch channel.Channel) { - close(s.join) - })) - return nil - } - - s.ch, err = channel.NewChannel("mgmt", underlayFactory, channel.BindHandlerF(bindHandler), nil) - if err != nil { - return nil, err - } - - streamEventsRequest := map[string]interface{}{} - streamEventsRequest["format"] = "json" - streamEventsRequest["subscriptions"] = []*event.Subscription{ - { - Type: "fabric.usage", - Options: map[string]interface{}{ - "version": uint8(3), - }, - }, - } - - msgBytes, err := json.Marshal(streamEventsRequest) - if err != nil { - return nil, err - } - - requestMsg := channel.NewMessage(int32(mgmt_pb.ContentType_StreamEventsRequestType), msgBytes) - responseMsg, err := requestMsg.WithTimeout(5 * time.Second).SendForReply(s.ch) - if err != nil { - return nil, err - } - - if responseMsg.ContentType == channel.ContentTypeResultType { - result := channel.UnmarshalResult(responseMsg) - if result.Success { - logrus.Infof("event stream started: %v", result.Message) - } else { - return nil, errors.Wrap(err, "error starting event streaming") - } - } else { - return nil, errors.Errorf("unexpected response type %v", responseMsg.ContentType) - } - - return s.join, nil -} - -func (s *websocketSource) Stop() { - _ = s.ch.Close() -} - -func (s *websocketSource) HandleReceive(msg *channel.Message, _ channel.Channel) { - s.events <- ZitiEventJson(msg.Body) -} From 192a49fe197c812c6f088c5107fe040e3f15ea57 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 16 Mar 2023 15:05:39 -0400 Subject: [PATCH 21/83] wire in the new 'limits.Agent' infrastructure; extend the 'metrics.Agent' to support additional 'metrics.UsageSink' instances (#271) --- controller/controller.go | 13 +++++++++++++ controller/limits/agent.go | 27 +++++++++++++++++++++++++++ controller/limits/config.go | 2 +- controller/metrics/agent.go | 14 ++++++++++---- 4 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 controller/limits/agent.go diff --git a/controller/controller.go b/controller/controller.go index 670bd863..ab91ab97 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -3,6 +3,7 @@ package controller import ( "context" "github.com/openziti/zrok/controller/config" + "github.com/openziti/zrok/controller/limits" "github.com/openziti/zrok/controller/metrics" "github.com/sirupsen/logrus" @@ -80,6 +81,18 @@ func Run(inCfg *config.Config) error { return errors.Wrap(err, "error starting metrics agent") } defer func() { ma.Stop() }() + + if cfg.Limits != nil && cfg.Limits.Enforcing { + la, err := limits.NewAgent(cfg.Limits, cfg.Metrics.Influx, cfg.Ziti, str) + if err != nil { + return errors.Wrap(err, "error creating limits agent") + } + ma.AddUsageSink(la) + if err := la.Start(); err != nil { + return errors.Wrap(err, "error starting limits agent") + } + defer func() { la.Stop() }() + } } ctx, cancel := context.WithCancel(context.Background()) diff --git a/controller/limits/agent.go b/controller/limits/agent.go new file mode 100644 index 00000000..d69193d6 --- /dev/null +++ b/controller/limits/agent.go @@ -0,0 +1,27 @@ +package limits + +import ( + "github.com/openziti/zrok/controller/metrics" + "github.com/openziti/zrok/controller/store" + "github.com/openziti/zrok/controller/zrokEdgeSdk" + "github.com/sirupsen/logrus" +) + +type Agent struct { +} + +func NewAgent(cfg *Config, ifxCfg *metrics.InfluxConfig, zCfg *zrokEdgeSdk.Config, str *store.Store) (*Agent, error) { + return &Agent{}, nil +} + +func (a *Agent) Start() error { + return nil +} + +func (a *Agent) Stop() { +} + +func (a *Agent) Handle(u *metrics.Usage) error { + logrus.Infof("handling: %v", u) + return nil +} diff --git a/controller/limits/config.go b/controller/limits/config.go index 4c044950..e3d10cff 100644 --- a/controller/limits/config.go +++ b/controller/limits/config.go @@ -8,7 +8,7 @@ type Config struct { Environments int Shares int Bandwidth *BandwidthConfig - Cycle time.Duration + Enforcing bool } type BandwidthConfig struct { diff --git a/controller/metrics/agent.go b/controller/metrics/agent.go index 89be79e5..ce10c8fb 100644 --- a/controller/metrics/agent.go +++ b/controller/metrics/agent.go @@ -11,7 +11,7 @@ type Agent struct { src ZitiEventJsonSource srcJoin chan struct{} cache *cache - snk UsageSink + snks []UsageSink } func NewAgent(cfg *AgentConfig, str *store.Store, ifxCfg *InfluxConfig) (*Agent, error) { @@ -22,10 +22,14 @@ func NewAgent(cfg *AgentConfig, str *store.Store, ifxCfg *InfluxConfig) (*Agent, return nil, errors.New("invalid event json source") } a.cache = newShareCache(str) - a.snk = newInfluxWriter(ifxCfg) + a.snks = append(a.snks, newInfluxWriter(ifxCfg)) return a, nil } +func (a *Agent) AddUsageSink(snk UsageSink) { + a.snks = append(a.snks, snk) +} + func (a *Agent) Start() error { a.events = make(chan ZitiEventJson) srcJoin, err := a.src.Start(a.events) @@ -44,8 +48,10 @@ func (a *Agent) Start() error { if err := a.cache.addZrokDetail(usage); err != nil { logrus.Error(err) } - if err := a.snk.Handle(usage); err != nil { - logrus.Error(err) + for _, snk := range a.snks { + if err := snk.Handle(usage); err != nil { + logrus.Error(err) + } } } else { logrus.Error(err) From 1af440fa37e410e1305071225530c30d4f33c76d Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 17 Mar 2023 11:46:28 -0400 Subject: [PATCH 22/83] limits journals ddl (#273) --- .../postgresql/009_v0_4_0_limits_journals.sql | 27 +++++++++++++++++++ .../sqlite3/009_v0_4_0_limits_journals.sql | 25 +++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 controller/store/sql/postgresql/009_v0_4_0_limits_journals.sql create mode 100644 controller/store/sql/sqlite3/009_v0_4_0_limits_journals.sql diff --git a/controller/store/sql/postgresql/009_v0_4_0_limits_journals.sql b/controller/store/sql/postgresql/009_v0_4_0_limits_journals.sql new file mode 100644 index 00000000..0b1d75b5 --- /dev/null +++ b/controller/store/sql/postgresql/009_v0_4_0_limits_journals.sql @@ -0,0 +1,27 @@ +-- +migrate Up + +create type limit_action_type as enum ('clear', 'warning', 'limit'); + +create table account_limit_journal ( + id serial primary key, + account_id integer references accounts(id), + action limit_action_type not null, + created_at timestamptz not null default(current_timestamp), + updated_at timestamptz not null default(current_timestamp) +); + +create table environment_limit_journal ( + id serial primary key, + environment_id integer references environments(id), + action limit_action_type not null, + created_at timestamptz not null default(current_timestamp), + updated_at timestamptz not null default(current_timestamp) +); + +create table share_limit_journal ( + id serial primary key, + share_id integer references shares(id), + action limit_action_type not null, + created_at timestamptz not null default(current_timestamp), + updated_at timestamptz not null default(current_timestamp) +); \ No newline at end of file diff --git a/controller/store/sql/sqlite3/009_v0_4_0_limits_journals.sql b/controller/store/sql/sqlite3/009_v0_4_0_limits_journals.sql new file mode 100644 index 00000000..a76c45b5 --- /dev/null +++ b/controller/store/sql/sqlite3/009_v0_4_0_limits_journals.sql @@ -0,0 +1,25 @@ +-- +migrate Up + +create table account_limit_journal ( + id serial primary key, + account_id integer references accounts(id), + action limit_action_type not null, + created_at timestamptz not null default(current_timestamp), + updated_at timestamptz not null default(current_timestamp) +); + +create table environment_limit_journal ( + id serial primary key, + environment_id integer references environments(id), + action limit_action_type not null, + created_at timestamptz not null default(current_timestamp), + updated_at timestamptz not null default(current_timestamp) +); + +create table share_limit_journal ( + id serial primary key, + share_id integer references shares(id), + action limit_action_type not null, + created_at timestamptz not null default(current_timestamp), + updated_at timestamptz not null default(current_timestamp) +); \ No newline at end of file From 0fa682e764029fe6ff631fffbdbbf930a2504b7f Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 17 Mar 2023 12:02:18 -0400 Subject: [PATCH 23/83] [account|environment|share] limit journals (#273) --- controller/store/accountLimitJournal.go | 32 +++++++++++++++++++++ controller/store/environmentLimitJournal.go | 32 +++++++++++++++++++++ controller/store/shareLimitJournal.go | 32 +++++++++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 controller/store/accountLimitJournal.go create mode 100644 controller/store/environmentLimitJournal.go create mode 100644 controller/store/shareLimitJournal.go diff --git a/controller/store/accountLimitJournal.go b/controller/store/accountLimitJournal.go new file mode 100644 index 00000000..3760dc03 --- /dev/null +++ b/controller/store/accountLimitJournal.go @@ -0,0 +1,32 @@ +package store + +import ( + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" +) + +type AccountLimitJournal struct { + Model + AccountId int + Action string +} + +func (self *Store) CreateAccountLimitJournal(j *AccountLimitJournal, tx *sqlx.Tx) (int, error) { + stmt, err := tx.Prepare("insert into account_limit_journal (account_id, action) values ($1, $2) returning id") + if err != nil { + return 0, errors.Wrap(err, "error preparing account_limit_journal insert statement") + } + var id int + if err := stmt.QueryRow(j.AccountId, j.AccountId).Scan(&id); err != nil { + return 0, errors.Wrap(err, "error executing account_limit_journal insert statement") + } + return id, nil +} + +func (self *Store) FindLatestAccountJournal(acctId int, tx *sqlx.Tx) (*AccountLimitJournal, error) { + j := &AccountLimitJournal{} + if err := tx.QueryRowx("select * from account_limit_journal where account_id = $1", acctId).StructScan(j); err != nil { + return nil, errors.Wrap(err, "error finding account_limit_journal by account_id") + } + return j, nil +} diff --git a/controller/store/environmentLimitJournal.go b/controller/store/environmentLimitJournal.go new file mode 100644 index 00000000..c721022c --- /dev/null +++ b/controller/store/environmentLimitJournal.go @@ -0,0 +1,32 @@ +package store + +import ( + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" +) + +type EnvironmentLimitJournal struct { + Model + EnvironmentId int + Action string +} + +func (self *Store) CreateEnvironmentLimitJournal(j *EnvironmentLimitJournal, tx *sqlx.Tx) (int, error) { + stmt, err := tx.Prepare("insert into environment_limit_journal (environment_id, action) values ($1, $2) returning id") + if err != nil { + return 0, errors.Wrap(err, "error preparing environment_limit_journal insert statement") + } + var id int + if err := stmt.QueryRow(j.EnvironmentId, j.Action).Scan(&id); err != nil { + return 0, errors.Wrap(err, "error executing environment_limit_journal insert statement") + } + return id, nil +} + +func (self *Store) FindLatestEnvironmentLimitJournal(envId int, tx *sqlx.Tx) (*EnvironmentLimitJournal, error) { + j := &EnvironmentLimitJournal{} + if err := tx.QueryRowx("select * from environment_limit_journal where environment_id = $1", envId).StructScan(j); err != nil { + return nil, errors.Wrap(err, "error finding environment_limit_journal by environment_id") + } + return j, nil +} diff --git a/controller/store/shareLimitJournal.go b/controller/store/shareLimitJournal.go new file mode 100644 index 00000000..f8ac92ea --- /dev/null +++ b/controller/store/shareLimitJournal.go @@ -0,0 +1,32 @@ +package store + +import ( + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" +) + +type ShareLimitJournal struct { + Model + ShareId int + Action string +} + +func (self *Store) CreateShareLimitJournal(j *ShareLimitJournal, tx *sqlx.Tx) (int, error) { + stmt, err := tx.Prepare("insert into share_limit_journal (share_id, action) values ($1, $2) returning id") + if err != nil { + return 0, errors.Wrap(err, "error preparing share_limit_journal insert statement") + } + var id int + if err := stmt.QueryRow(j.ShareId, j.Action).Scan(&id); err != nil { + return 0, errors.Wrap(err, "error executing share_limit_journal insert statement") + } + return id, nil +} + +func (self *Store) FindLatestShareLimitJournal(shrId int, tx *sqlx.Tx) (*ShareLimitJournal, error) { + j := &ShareLimitJournal{} + if err := tx.QueryRowx("select * from share_limit_journal where share_id = $1", shrId).StructScan(j); err != nil { + return nil, errors.Wrap(err, "error finding share_limit_journal by share_id") + } + return j, nil +} From b69237e9cc1cc63d3be8aa0927cb9091b00ffc18 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 17 Mar 2023 12:57:26 -0400 Subject: [PATCH 24/83] include rx|tx byte counts in limits journals (#273) --- controller/store/accountLimitJournal.go | 6 ++++-- controller/store/environmentLimitJournal.go | 6 ++++-- controller/store/shareLimitJournal.go | 6 ++++-- .../store/sql/postgresql/009_v0_4_0_limits_journals.sql | 6 ++++++ controller/store/sql/sqlite3/009_v0_4_0_limits_journals.sql | 6 ++++++ 5 files changed, 24 insertions(+), 6 deletions(-) diff --git a/controller/store/accountLimitJournal.go b/controller/store/accountLimitJournal.go index 3760dc03..ca86a93f 100644 --- a/controller/store/accountLimitJournal.go +++ b/controller/store/accountLimitJournal.go @@ -8,16 +8,18 @@ import ( type AccountLimitJournal struct { Model AccountId int + RxBytes int64 + TxBytes int64 Action string } func (self *Store) CreateAccountLimitJournal(j *AccountLimitJournal, tx *sqlx.Tx) (int, error) { - stmt, err := tx.Prepare("insert into account_limit_journal (account_id, action) values ($1, $2) returning id") + stmt, err := tx.Prepare("insert into account_limit_journal (account_id, rx_bytes, tx_bytes, action) values ($1, $2, $3, $4) returning id") if err != nil { return 0, errors.Wrap(err, "error preparing account_limit_journal insert statement") } var id int - if err := stmt.QueryRow(j.AccountId, j.AccountId).Scan(&id); err != nil { + if err := stmt.QueryRow(j.AccountId, j.RxBytes, j.TxBytes, j.AccountId).Scan(&id); err != nil { return 0, errors.Wrap(err, "error executing account_limit_journal insert statement") } return id, nil diff --git a/controller/store/environmentLimitJournal.go b/controller/store/environmentLimitJournal.go index c721022c..077d03ac 100644 --- a/controller/store/environmentLimitJournal.go +++ b/controller/store/environmentLimitJournal.go @@ -8,16 +8,18 @@ import ( type EnvironmentLimitJournal struct { Model EnvironmentId int + RxBytes int64 + TxBytes int64 Action string } func (self *Store) CreateEnvironmentLimitJournal(j *EnvironmentLimitJournal, tx *sqlx.Tx) (int, error) { - stmt, err := tx.Prepare("insert into environment_limit_journal (environment_id, action) values ($1, $2) returning id") + stmt, err := tx.Prepare("insert into environment_limit_journal (environment_id, rx_bytes, tx_bytes, action) values ($1, $2, $3, $4) returning id") if err != nil { return 0, errors.Wrap(err, "error preparing environment_limit_journal insert statement") } var id int - if err := stmt.QueryRow(j.EnvironmentId, j.Action).Scan(&id); err != nil { + if err := stmt.QueryRow(j.EnvironmentId, j.RxBytes, j.TxBytes, j.Action).Scan(&id); err != nil { return 0, errors.Wrap(err, "error executing environment_limit_journal insert statement") } return id, nil diff --git a/controller/store/shareLimitJournal.go b/controller/store/shareLimitJournal.go index f8ac92ea..fe97733a 100644 --- a/controller/store/shareLimitJournal.go +++ b/controller/store/shareLimitJournal.go @@ -8,16 +8,18 @@ import ( type ShareLimitJournal struct { Model ShareId int + RxBytes int64 + TxBytes int64 Action string } func (self *Store) CreateShareLimitJournal(j *ShareLimitJournal, tx *sqlx.Tx) (int, error) { - stmt, err := tx.Prepare("insert into share_limit_journal (share_id, action) values ($1, $2) returning id") + stmt, err := tx.Prepare("insert into share_limit_journal (share_id, rx_bytes, tx_bytes, action) values ($1, $2, $3, $4) returning id") if err != nil { return 0, errors.Wrap(err, "error preparing share_limit_journal insert statement") } var id int - if err := stmt.QueryRow(j.ShareId, j.Action).Scan(&id); err != nil { + if err := stmt.QueryRow(j.ShareId, j.RxBytes, j.TxBytes, j.Action).Scan(&id); err != nil { return 0, errors.Wrap(err, "error executing share_limit_journal insert statement") } return id, nil diff --git a/controller/store/sql/postgresql/009_v0_4_0_limits_journals.sql b/controller/store/sql/postgresql/009_v0_4_0_limits_journals.sql index 0b1d75b5..9238c2cc 100644 --- a/controller/store/sql/postgresql/009_v0_4_0_limits_journals.sql +++ b/controller/store/sql/postgresql/009_v0_4_0_limits_journals.sql @@ -5,6 +5,8 @@ create type limit_action_type as enum ('clear', 'warning', 'limit'); create table account_limit_journal ( id serial primary key, account_id integer references accounts(id), + rx_bytes bigint not null, + tx_bytes bigint not null, action limit_action_type not null, created_at timestamptz not null default(current_timestamp), updated_at timestamptz not null default(current_timestamp) @@ -13,6 +15,8 @@ create table account_limit_journal ( create table environment_limit_journal ( id serial primary key, environment_id integer references environments(id), + rx_bytes bigint not null, + tx_bytes bigint not null, action limit_action_type not null, created_at timestamptz not null default(current_timestamp), updated_at timestamptz not null default(current_timestamp) @@ -21,6 +25,8 @@ create table environment_limit_journal ( create table share_limit_journal ( id serial primary key, share_id integer references shares(id), + rx_bytes bigint not null, + tx_bytes bigint not null, action limit_action_type not null, created_at timestamptz not null default(current_timestamp), updated_at timestamptz not null default(current_timestamp) diff --git a/controller/store/sql/sqlite3/009_v0_4_0_limits_journals.sql b/controller/store/sql/sqlite3/009_v0_4_0_limits_journals.sql index a76c45b5..e9b14ebd 100644 --- a/controller/store/sql/sqlite3/009_v0_4_0_limits_journals.sql +++ b/controller/store/sql/sqlite3/009_v0_4_0_limits_journals.sql @@ -3,6 +3,8 @@ create table account_limit_journal ( id serial primary key, account_id integer references accounts(id), + rx_bytes bigint not null, + tx_bytes bigint not null, action limit_action_type not null, created_at timestamptz not null default(current_timestamp), updated_at timestamptz not null default(current_timestamp) @@ -11,6 +13,8 @@ create table account_limit_journal ( create table environment_limit_journal ( id serial primary key, environment_id integer references environments(id), + rx_bytes bigint not null, + tx_bytes bigint not null, action limit_action_type not null, created_at timestamptz not null default(current_timestamp), updated_at timestamptz not null default(current_timestamp) @@ -19,6 +23,8 @@ create table environment_limit_journal ( create table share_limit_journal ( id serial primary key, share_id integer references shares(id), + rx_bytes bigint not null, + tx_bytes bigint not null, action limit_action_type not null, created_at timestamptz not null default(current_timestamp), updated_at timestamptz not null default(current_timestamp) From 9418195150da254f1698ae0afad2bc52e9de6f98 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 17 Mar 2023 13:13:33 -0400 Subject: [PATCH 25/83] more limits.Agent elaboration (#271) --- controller/controller.go | 4 +--- controller/limits/agent.go | 39 ++++++++++++++++++++++++++++--- controller/limits/config.go | 3 +++ controller/limits/influxReader.go | 18 ++++++++++++++ 4 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 controller/limits/influxReader.go diff --git a/controller/controller.go b/controller/controller.go index ab91ab97..f02ecd33 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -88,9 +88,7 @@ func Run(inCfg *config.Config) error { return errors.Wrap(err, "error creating limits agent") } ma.AddUsageSink(la) - if err := la.Start(); err != nil { - return errors.Wrap(err, "error starting limits agent") - } + la.Start() defer func() { la.Stop() }() } } diff --git a/controller/limits/agent.go b/controller/limits/agent.go index d69193d6..683bf27c 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -5,23 +5,56 @@ import ( "github.com/openziti/zrok/controller/store" "github.com/openziti/zrok/controller/zrokEdgeSdk" "github.com/sirupsen/logrus" + "time" ) type Agent struct { + cfg *Config + ifx *influxReader + zCfg *zrokEdgeSdk.Config + str *store.Store + close chan struct{} + join chan struct{} } func NewAgent(cfg *Config, ifxCfg *metrics.InfluxConfig, zCfg *zrokEdgeSdk.Config, str *store.Store) (*Agent, error) { - return &Agent{}, nil + return &Agent{ + cfg: cfg, + ifx: newInfluxReader(ifxCfg), + zCfg: zCfg, + str: str, + close: make(chan struct{}), + join: make(chan struct{}), + }, nil } -func (a *Agent) Start() error { - return nil +func (a *Agent) Start() { + go a.run() } func (a *Agent) Stop() { + close(a.close) + <-a.join } func (a *Agent) Handle(u *metrics.Usage) error { logrus.Infof("handling: %v", u) return nil } + +func (a *Agent) run() { + logrus.Info("started") + defer logrus.Info("stopped") + +mainLoop: + for { + select { + case <-time.After(a.cfg.Cycle): + logrus.Info("insepection cycle") + + case <-a.close: + close(a.join) + break mainLoop + } + } +} diff --git a/controller/limits/config.go b/controller/limits/config.go index e3d10cff..7117f97e 100644 --- a/controller/limits/config.go +++ b/controller/limits/config.go @@ -8,6 +8,7 @@ type Config struct { Environments int Shares int Bandwidth *BandwidthConfig + Cycle time.Duration Enforcing bool } @@ -74,5 +75,7 @@ func DefaultConfig() *Config { }, }, }, + Enforcing: false, + Cycle: 15 * time.Minute, } } diff --git a/controller/limits/influxReader.go b/controller/limits/influxReader.go new file mode 100644 index 00000000..8ee7cbdb --- /dev/null +++ b/controller/limits/influxReader.go @@ -0,0 +1,18 @@ +package limits + +import ( + influxdb2 "github.com/influxdata/influxdb-client-go/v2" + "github.com/influxdata/influxdb-client-go/v2/api" + "github.com/openziti/zrok/controller/metrics" +) + +type influxReader struct { + idb influxdb2.Client + queryApi api.QueryAPI +} + +func newInfluxReader(cfg *metrics.InfluxConfig) *influxReader { + idb := influxdb2.NewClient(cfg.Url, cfg.Token) + queryApi := idb.QueryAPI(cfg.Org) + return &influxReader{idb, queryApi} +} From df56e49fab3865c46e463791d43c6ffdc09eccf5 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 17 Mar 2023 13:51:12 -0400 Subject: [PATCH 26/83] flux query influx for duration totals (#271) --- controller/limits/agent.go | 10 +++++++ controller/limits/influxReader.go | 49 ++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 683bf27c..6f33ed5c 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -4,6 +4,7 @@ import ( "github.com/openziti/zrok/controller/metrics" "github.com/openziti/zrok/controller/store" "github.com/openziti/zrok/controller/zrokEdgeSdk" + "github.com/openziti/zrok/util" "github.com/sirupsen/logrus" "time" ) @@ -39,6 +40,15 @@ func (a *Agent) Stop() { func (a *Agent) Handle(u *metrics.Usage) error { logrus.Infof("handling: %v", u) + rxTotal, err := a.ifx.totalRxForShare(u.ShareToken, 24*time.Hour) + if err != nil { + logrus.Error(err) + } + txTotal, err := a.ifx.totalTxForShare(u.ShareToken, 24*time.Hour) + if err != nil { + logrus.Error(err) + } + logrus.Infof("'%v': {rx: %v, tx: %v}", u.ShareToken, util.BytesToSize(rxTotal), util.BytesToSize(txTotal)) return nil } diff --git a/controller/limits/influxReader.go b/controller/limits/influxReader.go index 8ee7cbdb..c0791c7a 100644 --- a/controller/limits/influxReader.go +++ b/controller/limits/influxReader.go @@ -1,12 +1,19 @@ package limits import ( + "context" + "fmt" influxdb2 "github.com/influxdata/influxdb-client-go/v2" "github.com/influxdata/influxdb-client-go/v2/api" "github.com/openziti/zrok/controller/metrics" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "strings" + "time" ) type influxReader struct { + cfg *metrics.InfluxConfig idb influxdb2.Client queryApi api.QueryAPI } @@ -14,5 +21,45 @@ type influxReader struct { func newInfluxReader(cfg *metrics.InfluxConfig) *influxReader { idb := influxdb2.NewClient(cfg.Url, cfg.Token) queryApi := idb.QueryAPI(cfg.Org) - return &influxReader{idb, queryApi} + return &influxReader{cfg, idb, queryApi} +} + +func (r *influxReader) totalRxForShare(shrToken string, duration time.Duration) (int64, error) { + query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + + fmt.Sprintf("|> range(start: -%v)\n", duration) + + "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")\n" + + "|> filter(fn: (r) => r[\"_field\"] == \"bytesRead\")\n" + + "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + + fmt.Sprintf("|> filter(fn: (r) => r[\"share\"] == \"%v\")\n", shrToken) + + "|> sum()" + return r.runQueryForSum(query) +} + +func (r *influxReader) totalTxForShare(shrToken string, duration time.Duration) (int64, error) { + query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + + fmt.Sprintf("|> range(start: -%v)\n", duration) + + "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")\n" + + "|> filter(fn: (r) => r[\"_field\"] == \"bytesWritten\")\n" + + "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + + fmt.Sprintf("|> filter(fn: (r) => r[\"share\"] == \"%v\")\n", shrToken) + + "|> sum()" + return r.runQueryForSum(query) +} + +func (r *influxReader) runQueryForSum(query string) (int64, error) { + result, err := r.queryApi.Query(context.Background(), query) + if err != nil { + return -1, err + } + + if result.Next() { + if v, ok := result.Record().Value().(int64); ok { + return v, nil + } else { + return -1, errors.New("error asserting result type") + } + } + + logrus.Warnf("empty read result set for '%v'", strings.ReplaceAll(query, "\n", "")) + return 0, nil } From 96d2f15055952b0df73088baf72fd0dc4a3eb7cc Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 17 Mar 2023 14:04:01 -0400 Subject: [PATCH 27/83] working (but not correct) values for account and environment rx/tx (#271) --- controller/limits/agent.go | 27 ++++++++++++++++--- controller/limits/influxReader.go | 44 +++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 6f33ed5c..e335228d 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -40,15 +40,36 @@ func (a *Agent) Stop() { func (a *Agent) Handle(u *metrics.Usage) error { logrus.Infof("handling: %v", u) - rxTotal, err := a.ifx.totalRxForShare(u.ShareToken, 24*time.Hour) + acctRx, err := a.ifx.totalRxForAccount(u.AccountId, 24*time.Hour) if err != nil { logrus.Error(err) } - txTotal, err := a.ifx.totalTxForShare(u.ShareToken, 24*time.Hour) + acctTx, err := a.ifx.totalTxForAccount(u.AccountId, 24*time.Hour) if err != nil { logrus.Error(err) } - logrus.Infof("'%v': {rx: %v, tx: %v}", u.ShareToken, util.BytesToSize(rxTotal), util.BytesToSize(txTotal)) + envRx, err := a.ifx.totalRxForEnvironment(u.EnvironmentId, 24*time.Hour) + if err != nil { + logrus.Error(err) + } + envTx, err := a.ifx.totalTxForEnvironment(u.EnvironmentId, 24*time.Hour) + if err != nil { + logrus.Error(err) + } + shareRx, err := a.ifx.totalRxForShare(u.ShareToken, 24*time.Hour) + if err != nil { + logrus.Error(err) + } + shareTx, err := a.ifx.totalTxForShare(u.ShareToken, 24*time.Hour) + if err != nil { + logrus.Error(err) + } + logrus.Infof("'%v': acct:{rx: %v, tx: %v}, env:{rx: %v, tx: %v}, share:{rx: %v, tx: %v}", + u.ShareToken, + util.BytesToSize(acctRx), util.BytesToSize(acctTx), + util.BytesToSize(envRx), util.BytesToSize(envTx), + util.BytesToSize(shareRx), util.BytesToSize(shareTx), + ) return nil } diff --git a/controller/limits/influxReader.go b/controller/limits/influxReader.go index c0791c7a..e74d23bf 100644 --- a/controller/limits/influxReader.go +++ b/controller/limits/influxReader.go @@ -24,6 +24,50 @@ func newInfluxReader(cfg *metrics.InfluxConfig) *influxReader { return &influxReader{cfg, idb, queryApi} } +func (r *influxReader) totalRxForAccount(acctId int64, duration time.Duration) (int64, error) { + query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + + fmt.Sprintf("|> range(start: -%v)\n", duration) + + "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")\n" + + "|> filter(fn: (r) => r[\"_field\"] == \"bytesRead\")\n" + + "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + + fmt.Sprintf("|> filter(fn: (r) => r[\"acctId\"] == \"%d\")\n", acctId) + + "|> sum()" + return r.runQueryForSum(query) +} + +func (r *influxReader) totalTxForAccount(acctId int64, duration time.Duration) (int64, error) { + query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + + fmt.Sprintf("|> range(start: -%v)\n", duration) + + "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")\n" + + "|> filter(fn: (r) => r[\"_field\"] == \"bytesWritten\")\n" + + "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + + fmt.Sprintf("|> filter(fn: (r) => r[\"acctId\"] == \"%d\")\n", acctId) + + "|> sum()" + return r.runQueryForSum(query) +} + +func (r *influxReader) totalRxForEnvironment(envId int64, duration time.Duration) (int64, error) { + query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + + fmt.Sprintf("|> range(start: -%v)\n", duration) + + "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")\n" + + "|> filter(fn: (r) => r[\"_field\"] == \"bytesRead\")\n" + + "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + + fmt.Sprintf("|> filter(fn: (r) => r[\"envId\"] == \"%d\")\n", envId) + + "|> sum()" + return r.runQueryForSum(query) +} + +func (r *influxReader) totalTxForEnvironment(envId int64, duration time.Duration) (int64, error) { + query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + + fmt.Sprintf("|> range(start: -%v)\n", duration) + + "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")\n" + + "|> filter(fn: (r) => r[\"_field\"] == \"bytesWritten\")\n" + + "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + + fmt.Sprintf("|> filter(fn: (r) => r[\"envId\"] == \"%d\")\n", envId) + + "|> sum()" + return r.runQueryForSum(query) +} + func (r *influxReader) totalRxForShare(shrToken string, duration time.Duration) (int64, error) { query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + fmt.Sprintf("|> range(start: -%v)\n", duration) + From 8bf6875c3f0d02feb1fadfc0c228b49364879972 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 20 Mar 2023 15:05:45 -0400 Subject: [PATCH 28/83] probably not an optimal approach for this flux query, but seems to be working (#271) --- controller/limits/influxReader.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/controller/limits/influxReader.go b/controller/limits/influxReader.go index e74d23bf..99e3aca0 100644 --- a/controller/limits/influxReader.go +++ b/controller/limits/influxReader.go @@ -31,6 +31,7 @@ func (r *influxReader) totalRxForAccount(acctId int64, duration time.Duration) ( "|> filter(fn: (r) => r[\"_field\"] == \"bytesRead\")\n" + "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + fmt.Sprintf("|> filter(fn: (r) => r[\"acctId\"] == \"%d\")\n", acctId) + + "|> set(key: \"share\", value: \"*\")\n" + "|> sum()" return r.runQueryForSum(query) } @@ -42,6 +43,7 @@ func (r *influxReader) totalTxForAccount(acctId int64, duration time.Duration) ( "|> filter(fn: (r) => r[\"_field\"] == \"bytesWritten\")\n" + "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + fmt.Sprintf("|> filter(fn: (r) => r[\"acctId\"] == \"%d\")\n", acctId) + + "|> set(key: \"share\", value: \"*\")\n" + "|> sum()" return r.runQueryForSum(query) } @@ -53,6 +55,7 @@ func (r *influxReader) totalRxForEnvironment(envId int64, duration time.Duration "|> filter(fn: (r) => r[\"_field\"] == \"bytesRead\")\n" + "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + fmt.Sprintf("|> filter(fn: (r) => r[\"envId\"] == \"%d\")\n", envId) + + "|> set(key: \"share\", value: \"*\")\n" + "|> sum()" return r.runQueryForSum(query) } @@ -64,6 +67,7 @@ func (r *influxReader) totalTxForEnvironment(envId int64, duration time.Duration "|> filter(fn: (r) => r[\"_field\"] == \"bytesWritten\")\n" + "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + fmt.Sprintf("|> filter(fn: (r) => r[\"envId\"] == \"%d\")\n", envId) + + "|> set(key: \"share\", value: \"*\")\n" + "|> sum()" return r.runQueryForSum(query) } From 40ae2da2c987f352502bb692d3b87f602075091d Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 20 Mar 2023 15:37:21 -0400 Subject: [PATCH 29/83] better account and environment total efficiency and accuracy; 'bytesRead'/'bytesWritten' -> 'rx'/'tx' (#271) --- controller/limits/agent.go | 20 ++------ controller/limits/influxReader.go | 77 ++++++++++-------------------- controller/metrics/influxWriter.go | 4 +- controller/sparkData.go | 6 +-- 4 files changed, 35 insertions(+), 72 deletions(-) diff --git a/controller/limits/agent.go b/controller/limits/agent.go index e335228d..df42b700 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -40,27 +40,15 @@ func (a *Agent) Stop() { func (a *Agent) Handle(u *metrics.Usage) error { logrus.Infof("handling: %v", u) - acctRx, err := a.ifx.totalRxForAccount(u.AccountId, 24*time.Hour) + acctRx, acctTx, err := a.ifx.totalForAccount(u.AccountId, 24*time.Hour) if err != nil { logrus.Error(err) } - acctTx, err := a.ifx.totalTxForAccount(u.AccountId, 24*time.Hour) + envRx, envTx, err := a.ifx.totalForEnvironment(u.EnvironmentId, 24*time.Hour) if err != nil { logrus.Error(err) } - envRx, err := a.ifx.totalRxForEnvironment(u.EnvironmentId, 24*time.Hour) - if err != nil { - logrus.Error(err) - } - envTx, err := a.ifx.totalTxForEnvironment(u.EnvironmentId, 24*time.Hour) - if err != nil { - logrus.Error(err) - } - shareRx, err := a.ifx.totalRxForShare(u.ShareToken, 24*time.Hour) - if err != nil { - logrus.Error(err) - } - shareTx, err := a.ifx.totalTxForShare(u.ShareToken, 24*time.Hour) + shareRx, shareTx, err := a.ifx.totalForShare(u.ShareToken, 24*time.Hour) if err != nil { logrus.Error(err) } @@ -81,7 +69,7 @@ mainLoop: for { select { case <-time.After(a.cfg.Cycle): - logrus.Info("insepection cycle") + logrus.Info("inspection cycle") case <-a.close: close(a.join) diff --git a/controller/limits/influxReader.go b/controller/limits/influxReader.go index 99e3aca0..e2a38333 100644 --- a/controller/limits/influxReader.go +++ b/controller/limits/influxReader.go @@ -24,90 +24,65 @@ func newInfluxReader(cfg *metrics.InfluxConfig) *influxReader { return &influxReader{cfg, idb, queryApi} } -func (r *influxReader) totalRxForAccount(acctId int64, duration time.Duration) (int64, error) { +func (r *influxReader) totalForAccount(acctId int64, duration time.Duration) (int64, int64, error) { query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + fmt.Sprintf("|> range(start: -%v)\n", duration) + "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")\n" + - "|> filter(fn: (r) => r[\"_field\"] == \"bytesRead\")\n" + + "|> filter(fn: (r) => r[\"_field\"] == \"rx\" or r[\"_field\"] == \"tx\")\n" + "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + fmt.Sprintf("|> filter(fn: (r) => r[\"acctId\"] == \"%d\")\n", acctId) + - "|> set(key: \"share\", value: \"*\")\n" + + "|> drop(columns: [\"share\", \"envId\"])\n" + "|> sum()" return r.runQueryForSum(query) } -func (r *influxReader) totalTxForAccount(acctId int64, duration time.Duration) (int64, error) { +func (r *influxReader) totalForEnvironment(envId int64, duration time.Duration) (int64, int64, error) { query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + fmt.Sprintf("|> range(start: -%v)\n", duration) + "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")\n" + - "|> filter(fn: (r) => r[\"_field\"] == \"bytesWritten\")\n" + - "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + - fmt.Sprintf("|> filter(fn: (r) => r[\"acctId\"] == \"%d\")\n", acctId) + - "|> set(key: \"share\", value: \"*\")\n" + - "|> sum()" - return r.runQueryForSum(query) -} - -func (r *influxReader) totalRxForEnvironment(envId int64, duration time.Duration) (int64, error) { - query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + - fmt.Sprintf("|> range(start: -%v)\n", duration) + - "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")\n" + - "|> filter(fn: (r) => r[\"_field\"] == \"bytesRead\")\n" + + "|> filter(fn: (r) => r[\"_field\"] == \"rx\" or r[\"_field\"] == \"tx\")\n" + "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + fmt.Sprintf("|> filter(fn: (r) => r[\"envId\"] == \"%d\")\n", envId) + - "|> set(key: \"share\", value: \"*\")\n" + + "|> drop(columns: [\"share\", \"acctId\"])\n" + "|> sum()" return r.runQueryForSum(query) } -func (r *influxReader) totalTxForEnvironment(envId int64, duration time.Duration) (int64, error) { +func (r *influxReader) totalForShare(shrToken string, duration time.Duration) (int64, int64, error) { query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + fmt.Sprintf("|> range(start: -%v)\n", duration) + "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")\n" + - "|> filter(fn: (r) => r[\"_field\"] == \"bytesWritten\")\n" + - "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + - fmt.Sprintf("|> filter(fn: (r) => r[\"envId\"] == \"%d\")\n", envId) + - "|> set(key: \"share\", value: \"*\")\n" + - "|> sum()" - return r.runQueryForSum(query) -} - -func (r *influxReader) totalRxForShare(shrToken string, duration time.Duration) (int64, error) { - query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + - fmt.Sprintf("|> range(start: -%v)\n", duration) + - "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")\n" + - "|> filter(fn: (r) => r[\"_field\"] == \"bytesRead\")\n" + + "|> filter(fn: (r) => r[\"_field\"] == \"rx\" or r[\"_field\"] == \"tx\")\n" + "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + fmt.Sprintf("|> filter(fn: (r) => r[\"share\"] == \"%v\")\n", shrToken) + "|> sum()" return r.runQueryForSum(query) } -func (r *influxReader) totalTxForShare(shrToken string, duration time.Duration) (int64, error) { - query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + - fmt.Sprintf("|> range(start: -%v)\n", duration) + - "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")\n" + - "|> filter(fn: (r) => r[\"_field\"] == \"bytesWritten\")\n" + - "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + - fmt.Sprintf("|> filter(fn: (r) => r[\"share\"] == \"%v\")\n", shrToken) + - "|> sum()" - return r.runQueryForSum(query) -} - -func (r *influxReader) runQueryForSum(query string) (int64, error) { +func (r *influxReader) runQueryForSum(query string) (rx int64, tx int64, err error) { result, err := r.queryApi.Query(context.Background(), query) if err != nil { - return -1, err + return -1, -1, err } - if result.Next() { + count := 0 + for result.Next() { if v, ok := result.Record().Value().(int64); ok { - return v, nil + switch result.Record().Field() { + case "tx": + tx = v + case "rx": + rx = v + default: + logrus.Warnf("field '%v'?", result.Record().Field()) + } } else { - return -1, errors.New("error asserting result type") + return -1, -1, errors.New("error asserting value type") } + count++ } - - logrus.Warnf("empty read result set for '%v'", strings.ReplaceAll(query, "\n", "")) - return 0, nil + if count != 2 { + return -1, -1, errors.Errorf("expected 2 results; got '%d' (%v)", count, strings.ReplaceAll(query, "\n", "")) + } + return rx, tx, nil } diff --git a/controller/metrics/influxWriter.go b/controller/metrics/influxWriter.go index 12c09693..154f3e08 100644 --- a/controller/metrics/influxWriter.go +++ b/controller/metrics/influxWriter.go @@ -37,7 +37,7 @@ func (w *influxWriter) Handle(u *Usage) error { if u.BackendTx > 0 || u.BackendRx > 0 { pt := influxdb2.NewPoint("xfer", map[string]string{"namespace": "backend", "share": u.ShareToken, "envId": envId, "acctId": acctId}, - map[string]interface{}{"bytesRead": u.BackendRx, "bytesWritten": u.BackendTx}, + map[string]interface{}{"rx": u.BackendRx, "tx": u.BackendTx}, u.IntervalStart) pts = append(pts, pt) out += fmt.Sprintf(" backend {rx: %v, tx: %v}", util.BytesToSize(u.BackendRx), util.BytesToSize(u.BackendTx)) @@ -45,7 +45,7 @@ func (w *influxWriter) Handle(u *Usage) error { if u.FrontendTx > 0 || u.FrontendRx > 0 { pt := influxdb2.NewPoint("xfer", map[string]string{"namespace": "frontend", "share": u.ShareToken, "envId": envId, "acctId": acctId}, - map[string]interface{}{"bytesRead": u.FrontendRx, "bytesWritten": u.FrontendTx}, + map[string]interface{}{"rx": u.FrontendRx, "tx": u.FrontendTx}, u.IntervalStart) pts = append(pts, pt) out += fmt.Sprintf(" frontend {rx: %v, tx: %v}", util.BytesToSize(u.FrontendRx), util.BytesToSize(u.FrontendTx)) diff --git a/controller/sparkData.go b/controller/sparkData.go index bc793660..02b8b35d 100644 --- a/controller/sparkData.go +++ b/controller/sparkData.go @@ -19,11 +19,11 @@ func sparkDataForShares(shrs []*store.Share) (map[string][]int64, error) { for result.Next() { combinedRate := int64(0) - readRate := result.Record().ValueByKey("bytesRead") + readRate := result.Record().ValueByKey("tx") if readRate != nil { combinedRate += readRate.(int64) } - writeRate := result.Record().ValueByKey("bytesWritten") + writeRate := result.Record().ValueByKey("tx") if writeRate != nil { combinedRate += writeRate.(int64) } @@ -48,7 +48,7 @@ func sparkFluxQuery(shrs []*store.Share) string { query := "read = from(bucket: \"zrok\")" + "|> range(start: -5m)" + "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")" + - "|> filter(fn: (r) => r[\"_field\"] == \"bytesRead\" or r[\"_field\"] == \"bytesWritten\")" + + "|> filter(fn: (r) => r[\"_field\"] == \"rx\" or r[\"_field\"] == \"tx\")" + "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")" + shrFilter + "|> aggregateWindow(every: 5s, fn: sum, createEmpty: true)\n" + From 7360598df566e521b897cf592cca4699d803384d Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 21 Mar 2023 12:38:31 -0400 Subject: [PATCH 30/83] naming (#271) --- controller/limits/agent.go | 6 +++--- controller/limits/influxReader.go | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/controller/limits/agent.go b/controller/limits/agent.go index df42b700..5d9508eb 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -40,15 +40,15 @@ func (a *Agent) Stop() { func (a *Agent) Handle(u *metrics.Usage) error { logrus.Infof("handling: %v", u) - acctRx, acctTx, err := a.ifx.totalForAccount(u.AccountId, 24*time.Hour) + acctRx, acctTx, err := a.ifx.totalRxTxForAccount(u.AccountId, 24*time.Hour) if err != nil { logrus.Error(err) } - envRx, envTx, err := a.ifx.totalForEnvironment(u.EnvironmentId, 24*time.Hour) + envRx, envTx, err := a.ifx.totalRxTxForEnvironment(u.EnvironmentId, 24*time.Hour) if err != nil { logrus.Error(err) } - shareRx, shareTx, err := a.ifx.totalForShare(u.ShareToken, 24*time.Hour) + shareRx, shareTx, err := a.ifx.totalRxTxForShare(u.ShareToken, 24*time.Hour) if err != nil { logrus.Error(err) } diff --git a/controller/limits/influxReader.go b/controller/limits/influxReader.go index e2a38333..22b8c4d1 100644 --- a/controller/limits/influxReader.go +++ b/controller/limits/influxReader.go @@ -24,7 +24,7 @@ func newInfluxReader(cfg *metrics.InfluxConfig) *influxReader { return &influxReader{cfg, idb, queryApi} } -func (r *influxReader) totalForAccount(acctId int64, duration time.Duration) (int64, int64, error) { +func (r *influxReader) totalRxTxForAccount(acctId int64, duration time.Duration) (int64, int64, error) { query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + fmt.Sprintf("|> range(start: -%v)\n", duration) + "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")\n" + @@ -33,10 +33,10 @@ func (r *influxReader) totalForAccount(acctId int64, duration time.Duration) (in fmt.Sprintf("|> filter(fn: (r) => r[\"acctId\"] == \"%d\")\n", acctId) + "|> drop(columns: [\"share\", \"envId\"])\n" + "|> sum()" - return r.runQueryForSum(query) + return r.runQueryForRxTx(query) } -func (r *influxReader) totalForEnvironment(envId int64, duration time.Duration) (int64, int64, error) { +func (r *influxReader) totalRxTxForEnvironment(envId int64, duration time.Duration) (int64, int64, error) { query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + fmt.Sprintf("|> range(start: -%v)\n", duration) + "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")\n" + @@ -45,10 +45,10 @@ func (r *influxReader) totalForEnvironment(envId int64, duration time.Duration) fmt.Sprintf("|> filter(fn: (r) => r[\"envId\"] == \"%d\")\n", envId) + "|> drop(columns: [\"share\", \"acctId\"])\n" + "|> sum()" - return r.runQueryForSum(query) + return r.runQueryForRxTx(query) } -func (r *influxReader) totalForShare(shrToken string, duration time.Duration) (int64, int64, error) { +func (r *influxReader) totalRxTxForShare(shrToken string, duration time.Duration) (int64, int64, error) { query := fmt.Sprintf("from(bucket: \"%v\")\n", r.cfg.Bucket) + fmt.Sprintf("|> range(start: -%v)\n", duration) + "|> filter(fn: (r) => r[\"_measurement\"] == \"xfer\")\n" + @@ -56,10 +56,10 @@ func (r *influxReader) totalForShare(shrToken string, duration time.Duration) (i "|> filter(fn: (r) => r[\"namespace\"] == \"backend\")\n" + fmt.Sprintf("|> filter(fn: (r) => r[\"share\"] == \"%v\")\n", shrToken) + "|> sum()" - return r.runQueryForSum(query) + return r.runQueryForRxTx(query) } -func (r *influxReader) runQueryForSum(query string) (rx int64, tx int64, err error) { +func (r *influxReader) runQueryForRxTx(query string) (rx int64, tx int64, err error) { result, err := r.queryApi.Query(context.Background(), query) if err != nil { return -1, -1, err From 73ef98d7a67415ad8c137a2efd119cdc19a8e418 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 21 Mar 2023 13:05:22 -0400 Subject: [PATCH 31/83] queue-based internal design for limits.Agent (#271) --- controller/limits/agent.go | 82 ++++++++++++++++++++++++++++--------- controller/limits/config.go | 58 +++++++++----------------- 2 files changed, 82 insertions(+), 58 deletions(-) diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 5d9508eb..2f9185c5 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -14,6 +14,7 @@ type Agent struct { ifx *influxReader zCfg *zrokEdgeSdk.Config str *store.Store + queue chan *metrics.Usage close chan struct{} join chan struct{} } @@ -24,6 +25,7 @@ func NewAgent(cfg *Config, ifxCfg *metrics.InfluxConfig, zCfg *zrokEdgeSdk.Confi ifx: newInfluxReader(ifxCfg), zCfg: zCfg, str: str, + queue: make(chan *metrics.Usage, 1024), close: make(chan struct{}), join: make(chan struct{}), }, nil @@ -39,25 +41,8 @@ func (a *Agent) Stop() { } func (a *Agent) Handle(u *metrics.Usage) error { - logrus.Infof("handling: %v", u) - acctRx, acctTx, err := a.ifx.totalRxTxForAccount(u.AccountId, 24*time.Hour) - if err != nil { - logrus.Error(err) - } - envRx, envTx, err := a.ifx.totalRxTxForEnvironment(u.EnvironmentId, 24*time.Hour) - if err != nil { - logrus.Error(err) - } - shareRx, shareTx, err := a.ifx.totalRxTxForShare(u.ShareToken, 24*time.Hour) - if err != nil { - logrus.Error(err) - } - logrus.Infof("'%v': acct:{rx: %v, tx: %v}, env:{rx: %v, tx: %v}, share:{rx: %v, tx: %v}", - u.ShareToken, - util.BytesToSize(acctRx), util.BytesToSize(acctTx), - util.BytesToSize(envRx), util.BytesToSize(envTx), - util.BytesToSize(shareRx), util.BytesToSize(shareTx), - ) + logrus.Debugf("handling: %v", u) + a.queue <- u return nil } @@ -68,6 +53,9 @@ func (a *Agent) run() { mainLoop: for { select { + case usage := <-a.queue: + a.enforce(usage) + case <-time.After(a.cfg.Cycle): logrus.Info("inspection cycle") @@ -77,3 +65,59 @@ mainLoop: } } } + +func (a *Agent) enforce(u *metrics.Usage) { + acctPeriod := 24 * time.Hour + acctLimit := DefaultBandwidthPerPeriod() + if a.cfg.Bandwidth != nil && a.cfg.Bandwidth.PerAccount != nil { + acctLimit = a.cfg.Bandwidth.PerAccount + } + if acctLimit.Period > 0 { + acctPeriod = acctLimit.Period + } + acctRx, acctTx, err := a.ifx.totalRxTxForAccount(u.AccountId, acctPeriod) + if err != nil { + logrus.Error(err) + } + if acctLimit.Warning.Rx != Unlimited && acctRx > acctLimit.Warning.Rx { + logrus.Warnf("'%v': account over rx warning limit '%v' at '%v'", u.ShareToken, util.BytesToSize(acctLimit.Warning.Rx), util.BytesToSize(acctRx)) + } + if acctLimit.Warning.Tx != Unlimited && acctTx > acctLimit.Warning.Tx { + logrus.Warnf("'%v': account over tx warning limit '%v' at '%v'", u.ShareToken, util.BytesToSize(acctLimit.Warning.Tx), util.BytesToSize(acctTx)) + } + if acctLimit.Warning.Total != Unlimited && acctTx+acctRx > acctLimit.Warning.Total { + logrus.Warnf("'%v': account over total warning limit '%v' at '%v'", u.ShareToken, util.BytesToSize(acctLimit.Warning.Total), util.BytesToSize(acctRx+acctTx)) + } + + envPeriod := 24 * time.Hour + envLimit := DefaultBandwidthPerPeriod() + if a.cfg.Bandwidth != nil && a.cfg.Bandwidth.PerEnvironment != nil { + envLimit = a.cfg.Bandwidth.PerEnvironment + } + if envLimit.Period > 0 { + envPeriod = envLimit.Period + } + envRx, envTx, err := a.ifx.totalRxTxForEnvironment(u.EnvironmentId, envPeriod) + if err != nil { + logrus.Error(err) + } + + sharePeriod := 24 * time.Hour + shareLimit := DefaultBandwidthPerPeriod() + if a.cfg.Bandwidth != nil && a.cfg.Bandwidth.PerShare != nil { + shareLimit = a.cfg.Bandwidth.PerShare + } + if shareLimit.Period > 0 { + sharePeriod = shareLimit.Period + } + shareRx, shareTx, err := a.ifx.totalRxTxForShare(u.ShareToken, sharePeriod) + if err != nil { + logrus.Error(err) + } + logrus.Infof("'%v': acct:{rx: %v, tx: %v}/%v, env:{rx: %v, tx: %v}/%v, share:{rx: %v, tx: %v}/%v", + u.ShareToken, + util.BytesToSize(acctRx), util.BytesToSize(acctTx), acctPeriod, + util.BytesToSize(envRx), util.BytesToSize(envTx), envPeriod, + util.BytesToSize(shareRx), util.BytesToSize(shareTx), sharePeriod, + ) +} diff --git a/controller/limits/config.go b/controller/limits/config.go index 7117f97e..4e6fbbe1 100644 --- a/controller/limits/config.go +++ b/controller/limits/config.go @@ -30,50 +30,30 @@ type Bandwidth struct { Total int64 } +func DefaultBandwidthPerPeriod() *BandwidthPerPeriod { + return &BandwidthPerPeriod{ + Period: 15 * (24 * time.Hour), + Warning: &Bandwidth{ + Rx: Unlimited, + Tx: Unlimited, + Total: Unlimited, + }, + Limit: &Bandwidth{ + Rx: Unlimited, + Tx: Unlimited, + Total: Unlimited, + }, + } +} + func DefaultConfig() *Config { return &Config{ Environments: Unlimited, Shares: Unlimited, Bandwidth: &BandwidthConfig{ - PerAccount: &BandwidthPerPeriod{ - Period: 365 * (24 * time.Hour), - Warning: &Bandwidth{ - Rx: Unlimited, - Tx: Unlimited, - Total: Unlimited, - }, - Limit: &Bandwidth{ - Rx: Unlimited, - Tx: Unlimited, - Total: Unlimited, - }, - }, - PerEnvironment: &BandwidthPerPeriod{ - Period: 365 * (24 * time.Hour), - Warning: &Bandwidth{ - Rx: Unlimited, - Tx: Unlimited, - Total: Unlimited, - }, - Limit: &Bandwidth{ - Rx: Unlimited, - Tx: Unlimited, - Total: Unlimited, - }, - }, - PerShare: &BandwidthPerPeriod{ - Period: 365 * (24 * time.Hour), - Warning: &Bandwidth{ - Rx: Unlimited, - Tx: Unlimited, - Total: Unlimited, - }, - Limit: &Bandwidth{ - Rx: Unlimited, - Tx: Unlimited, - Total: Unlimited, - }, - }, + PerAccount: DefaultBandwidthPerPeriod(), + PerEnvironment: DefaultBandwidthPerPeriod(), + PerShare: DefaultBandwidthPerPeriod(), }, Enforcing: false, Cycle: 15 * time.Minute, From 2ec228c496b878803667bb725dd39c429f648d90 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 21 Mar 2023 14:06:23 -0400 Subject: [PATCH 32/83] basic limits infrastructure for detecting warning and enforcement conditions (#271) --- controller/limits/agent.go | 209 ++++++++++++++++++++++++++++--------- 1 file changed, 161 insertions(+), 48 deletions(-) diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 2f9185c5..5c144473 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -1,10 +1,13 @@ package limits import ( + "fmt" + "github.com/jmoiron/sqlx" "github.com/openziti/zrok/controller/metrics" "github.com/openziti/zrok/controller/store" "github.com/openziti/zrok/controller/zrokEdgeSdk" "github.com/openziti/zrok/util" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "time" ) @@ -66,58 +69,168 @@ mainLoop: } } -func (a *Agent) enforce(u *metrics.Usage) { - acctPeriod := 24 * time.Hour - acctLimit := DefaultBandwidthPerPeriod() - if a.cfg.Bandwidth != nil && a.cfg.Bandwidth.PerAccount != nil { - acctLimit = a.cfg.Bandwidth.PerAccount - } - if acctLimit.Period > 0 { - acctPeriod = acctLimit.Period - } - acctRx, acctTx, err := a.ifx.totalRxTxForAccount(u.AccountId, acctPeriod) +func (a *Agent) enforce(u *metrics.Usage) error { + trx, err := a.str.Begin() if err != nil { - logrus.Error(err) - } - if acctLimit.Warning.Rx != Unlimited && acctRx > acctLimit.Warning.Rx { - logrus.Warnf("'%v': account over rx warning limit '%v' at '%v'", u.ShareToken, util.BytesToSize(acctLimit.Warning.Rx), util.BytesToSize(acctRx)) - } - if acctLimit.Warning.Tx != Unlimited && acctTx > acctLimit.Warning.Tx { - logrus.Warnf("'%v': account over tx warning limit '%v' at '%v'", u.ShareToken, util.BytesToSize(acctLimit.Warning.Tx), util.BytesToSize(acctTx)) - } - if acctLimit.Warning.Total != Unlimited && acctTx+acctRx > acctLimit.Warning.Total { - logrus.Warnf("'%v': account over total warning limit '%v' at '%v'", u.ShareToken, util.BytesToSize(acctLimit.Warning.Total), util.BytesToSize(acctRx+acctTx)) + return errors.Wrap(err, "error starting transaction") } + defer func() { _ = trx.Rollback() }() - envPeriod := 24 * time.Hour - envLimit := DefaultBandwidthPerPeriod() - if a.cfg.Bandwidth != nil && a.cfg.Bandwidth.PerEnvironment != nil { - envLimit = a.cfg.Bandwidth.PerEnvironment - } - if envLimit.Period > 0 { - envPeriod = envLimit.Period - } - envRx, envTx, err := a.ifx.totalRxTxForEnvironment(u.EnvironmentId, envPeriod) - if err != nil { + if enforce, warning, err := a.checkAccountLimits(u, trx); err == nil { + if enforce { + logrus.Warn("enforcing account limit") + } else if warning { + logrus.Warn("reporting account warning") + } else { + if enforce, warning, err := a.checkEnvironmentLimit(u, trx); err == nil { + if enforce { + logrus.Warn("enforcing environment limit") + } else if warning { + logrus.Warn("reporting environment warning") + } else { + if enforce, warning, err := a.checkShareLimit(u); err == nil { + if enforce { + logrus.Warn("enforcing share limit") + } else if warning { + logrus.Warn("reporting share warning") + } + } else { + logrus.Error(err) + } + } + } else { + logrus.Error(err) + } + } + } else { logrus.Error(err) } - sharePeriod := 24 * time.Hour - shareLimit := DefaultBandwidthPerPeriod() - if a.cfg.Bandwidth != nil && a.cfg.Bandwidth.PerShare != nil { - shareLimit = a.cfg.Bandwidth.PerShare - } - if shareLimit.Period > 0 { - sharePeriod = shareLimit.Period - } - shareRx, shareTx, err := a.ifx.totalRxTxForShare(u.ShareToken, sharePeriod) - if err != nil { - logrus.Error(err) - } - logrus.Infof("'%v': acct:{rx: %v, tx: %v}/%v, env:{rx: %v, tx: %v}/%v, share:{rx: %v, tx: %v}/%v", - u.ShareToken, - util.BytesToSize(acctRx), util.BytesToSize(acctTx), acctPeriod, - util.BytesToSize(envRx), util.BytesToSize(envTx), envPeriod, - util.BytesToSize(shareRx), util.BytesToSize(shareTx), sharePeriod, - ) + return nil +} + +func (a *Agent) checkAccountLimits(u *metrics.Usage, trx *sqlx.Tx) (enforce, warning bool, err error) { + acct, err := a.str.GetAccount(int(u.AccountId), trx) + if err != nil { + return false, false, errors.Wrapf(err, "error getting account '%d'", u.AccountId) + } + + period := 24 * time.Hour + limit := DefaultBandwidthPerPeriod() + if a.cfg.Bandwidth != nil && a.cfg.Bandwidth.PerAccount != nil { + limit = a.cfg.Bandwidth.PerAccount + } + if limit.Period > 0 { + period = limit.Period + } + rx, tx, err := a.ifx.totalRxTxForAccount(u.AccountId, period) + if err != nil { + logrus.Error(err) + } + + enforce, warning = a.checkLimit(limit, rx, tx) + if enforce || warning { + logrus.Warnf("'%v': %v", acct.Email, a.describeLimit(limit, rx, tx)) + } + + return enforce, warning, nil +} + +func (a *Agent) checkEnvironmentLimit(u *metrics.Usage, trx *sqlx.Tx) (enforce, warning bool, err error) { + env, err := a.str.GetEnvironment(int(u.EnvironmentId), trx) + if err != nil { + return false, false, errors.Wrapf(err, "error getting account '%d'", u.EnvironmentId) + } + + period := 24 * time.Hour + limit := DefaultBandwidthPerPeriod() + if a.cfg.Bandwidth != nil && a.cfg.Bandwidth.PerEnvironment != nil { + limit = a.cfg.Bandwidth.PerEnvironment + } + if limit.Period > 0 { + period = limit.Period + } + rx, tx, err := a.ifx.totalRxTxForEnvironment(u.EnvironmentId, period) + if err != nil { + logrus.Error(err) + } + + enforce, warning = a.checkLimit(limit, rx, tx) + if enforce || warning { + logrus.Warnf("'%v': %v", env.ZId, a.describeLimit(limit, rx, tx)) + } + + return enforce, warning, nil +} + +func (a *Agent) checkShareLimit(u *metrics.Usage) (enforce, warning bool, err error) { + period := 24 * time.Hour + limit := DefaultBandwidthPerPeriod() + if a.cfg.Bandwidth != nil && a.cfg.Bandwidth.PerShare != nil { + limit = a.cfg.Bandwidth.PerShare + } + if limit.Period > 0 { + period = limit.Period + } + rx, tx, err := a.ifx.totalRxTxForShare(u.ShareToken, period) + if err != nil { + logrus.Error(err) + } + + enforce, warning = a.checkLimit(limit, rx, tx) + if enforce || warning { + logrus.Warnf("'%v': %v", u.ShareToken, a.describeLimit(limit, rx, tx)) + } + + return enforce, warning, nil +} + +func (a *Agent) checkLimit(cfg *BandwidthPerPeriod, rx, tx int64) (enforce, warning bool) { + if cfg.Limit.Rx != Unlimited && rx > cfg.Limit.Rx { + return true, false + } + if cfg.Limit.Tx != Unlimited && tx > cfg.Limit.Tx { + return true, false + } + if cfg.Limit.Total != Unlimited && rx+tx > cfg.Limit.Total { + return true, false + } + + if cfg.Warning.Rx != Unlimited && rx > cfg.Warning.Rx { + return false, true + } + if cfg.Warning.Tx != Unlimited && tx > cfg.Warning.Tx { + return false, true + } + if cfg.Warning.Total != Unlimited && rx+tx > cfg.Warning.Total { + return false, true + } + + return false, false +} + +func (a *Agent) describeLimit(cfg *BandwidthPerPeriod, rx, tx int64) string { + out := "" + + if cfg.Limit.Rx != Unlimited && rx > cfg.Limit.Rx { + out += fmt.Sprintf("['%v' over rx limit '%v']", util.BytesToSize(rx), util.BytesToSize(cfg.Limit.Rx)) + } + if cfg.Limit.Tx != Unlimited && tx > cfg.Limit.Tx { + out += fmt.Sprintf("['%v' over tx limit '%v']", util.BytesToSize(tx), util.BytesToSize(cfg.Limit.Tx)) + } + if cfg.Limit.Total != Unlimited && rx+tx > cfg.Limit.Total { + out += fmt.Sprintf("['%v' over total limit '%v']", util.BytesToSize(rx+tx), util.BytesToSize(cfg.Limit.Total)) + } + + if cfg.Warning.Rx != Unlimited && rx > cfg.Warning.Rx { + out += fmt.Sprintf("['%v' over rx warning '%v']", util.BytesToSize(rx), util.BytesToSize(cfg.Warning.Rx)) + } + if cfg.Warning.Tx != Unlimited && tx > cfg.Warning.Tx { + out += fmt.Sprintf("['%v' over tx warning '%v']", util.BytesToSize(tx), util.BytesToSize(cfg.Warning.Tx)) + } + if cfg.Warning.Total != Unlimited && rx+tx > cfg.Warning.Total { + out += fmt.Sprintf("['%v' over total warning '%v']", util.BytesToSize(rx+tx), util.BytesToSize(cfg.Warning.Total)) + } + + return out } From 79e9f484dca76ef79f4baba126ffce338a67f3c9 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 21 Mar 2023 16:18:17 -0400 Subject: [PATCH 33/83] centralized environment limits check (#277) --- controller/controller.go | 11 ++++++----- controller/enable.go | 30 ++++++++++++++---------------- controller/limits/agent.go | 19 +++++++++++++++++++ 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/controller/controller.go b/controller/controller.go index f02ecd33..01c61f9e 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -20,6 +20,7 @@ import ( var cfg *config.Config var str *store.Store var idb influxdb2.Client +var limitsAgent *limits.Agent func Run(inCfg *config.Config) error { cfg = inCfg @@ -43,7 +44,7 @@ func Run(inCfg *config.Config) error { api.AdminInviteTokenGenerateHandler = newInviteTokenGenerateHandler() api.AdminListFrontendsHandler = newListFrontendsHandler() api.AdminUpdateFrontendHandler = newUpdateFrontendHandler() - api.EnvironmentEnableHandler = newEnableHandler(cfg.Limits) + api.EnvironmentEnableHandler = newEnableHandler() api.EnvironmentDisableHandler = newDisableHandler() api.MetadataConfigurationHandler = newConfigurationHandler(cfg) api.MetadataGetEnvironmentDetailHandler = newEnvironmentDetailHandler() @@ -83,13 +84,13 @@ func Run(inCfg *config.Config) error { defer func() { ma.Stop() }() if cfg.Limits != nil && cfg.Limits.Enforcing { - la, err := limits.NewAgent(cfg.Limits, cfg.Metrics.Influx, cfg.Ziti, str) + limitsAgent, err = limits.NewAgent(cfg.Limits, cfg.Metrics.Influx, cfg.Ziti, str) if err != nil { return errors.Wrap(err, "error creating limits agent") } - ma.AddUsageSink(la) - la.Start() - defer func() { la.Stop() }() + ma.AddUsageSink(limitsAgent) + limitsAgent.Start() + defer func() { limitsAgent.Stop() }() } } diff --git a/controller/enable.go b/controller/enable.go index 1025c562..cf275d80 100644 --- a/controller/enable.go +++ b/controller/enable.go @@ -4,8 +4,6 @@ import ( "bytes" "encoding/json" "github.com/go-openapi/runtime/middleware" - "github.com/jmoiron/sqlx" - "github.com/openziti/zrok/controller/limits" "github.com/openziti/zrok/controller/store" "github.com/openziti/zrok/controller/zrokEdgeSdk" "github.com/openziti/zrok/rest_model_zrok" @@ -14,12 +12,10 @@ import ( "github.com/sirupsen/logrus" ) -type enableHandler struct { - cfg *limits.Config -} +type enableHandler struct{} -func newEnableHandler(cfg *limits.Config) *enableHandler { - return &enableHandler{cfg: cfg} +func newEnableHandler() *enableHandler { + return &enableHandler{} } func (h *enableHandler) Handle(params environment.EnableParams, principal *rest_model_zrok.Principal) middleware.Responder { @@ -31,7 +27,7 @@ func (h *enableHandler) Handle(params environment.EnableParams, principal *rest_ } defer func() { _ = tx.Rollback() }() - if err := h.checkLimits(principal, tx); err != nil { + if err := h.checkLimits(principal); err != nil { logrus.Errorf("limits error for user '%v': %v", principal.Email, err) return environment.NewEnableUnauthorized() } @@ -100,14 +96,16 @@ func (h *enableHandler) Handle(params environment.EnableParams, principal *rest_ return resp } -func (h *enableHandler) checkLimits(principal *rest_model_zrok.Principal, tx *sqlx.Tx) error { - if !principal.Limitless && h.cfg.Environments > limits.Unlimited { - envs, err := str.FindEnvironmentsForAccount(int(principal.ID), tx) - if err != nil { - return errors.Errorf("unable to find environments for account '%v': %v", principal.Email, err) - } - if len(envs)+1 > h.cfg.Environments { - return errors.Errorf("would exceed environments limit of %d for '%v'", h.cfg.Environments, principal.Email) +func (h *enableHandler) checkLimits(principal *rest_model_zrok.Principal) error { + if !principal.Limitless { + if limitsAgent != nil { + ok, err := limitsAgent.CanCreateEnvironment(int(principal.ID)) + if err != nil { + return errors.Wrapf(err, "error checking limits for '%v'", principal.Email) + } + if !ok { + return errors.Wrapf(err, "environment limit check failed for '%v'", principal.Email) + } } } return nil diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 5c144473..01d07cd8 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -43,6 +43,25 @@ func (a *Agent) Stop() { <-a.join } +func (a *Agent) CanCreateEnvironment(acctId int) (bool, error) { + if a.cfg.Environments > Unlimited { + trx, err := a.str.Begin() + if err != nil { + return false, errors.Wrap(err, "error creating transaction") + } + defer func() { _ = trx.Rollback() }() + + envs, err := a.str.FindEnvironmentsForAccount(acctId, trx) + if err != nil { + return false, err + } + if len(envs)+1 > a.cfg.Environments { + return false, nil + } + } + return true, nil +} + func (a *Agent) Handle(u *metrics.Usage) error { logrus.Debugf("handling: %v", u) a.queue <- u From d0dd04a1416be212f54ba284b64303e1be5ed179 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 21 Mar 2023 16:34:45 -0400 Subject: [PATCH 34/83] share limits check owned by limits.Agent now (#277) --- controller/controller.go | 2 +- controller/enable.go | 21 +++++++++---------- controller/limits/agent.go | 32 +++++++++++++++++++++-------- controller/share.go | 41 ++++++++++++++++---------------------- 4 files changed, 53 insertions(+), 43 deletions(-) diff --git a/controller/controller.go b/controller/controller.go index 01c61f9e..987e2b4a 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -52,7 +52,7 @@ func Run(inCfg *config.Config) error { api.MetadataOverviewHandler = metadata.OverviewHandlerFunc(overviewHandler) api.MetadataVersionHandler = metadata.VersionHandlerFunc(versionHandler) api.ShareAccessHandler = newAccessHandler() - api.ShareShareHandler = newShareHandler(cfg.Limits) + api.ShareShareHandler = newShareHandler() api.ShareUnaccessHandler = newUnaccessHandler() api.ShareUnshareHandler = newUnshareHandler() api.ShareUpdateShareHandler = newUpdateShareHandler() diff --git a/controller/enable.go b/controller/enable.go index cf275d80..56375edf 100644 --- a/controller/enable.go +++ b/controller/enable.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "github.com/go-openapi/runtime/middleware" + "github.com/jmoiron/sqlx" "github.com/openziti/zrok/controller/store" "github.com/openziti/zrok/controller/zrokEdgeSdk" "github.com/openziti/zrok/rest_model_zrok" @@ -20,14 +21,14 @@ func newEnableHandler() *enableHandler { func (h *enableHandler) Handle(params environment.EnableParams, principal *rest_model_zrok.Principal) middleware.Responder { // start transaction early; if it fails, don't bother creating ziti resources - tx, err := str.Begin() + trx, err := str.Begin() if err != nil { logrus.Errorf("error starting transaction for user '%v': %v", principal.Email, err) return environment.NewEnableInternalServerError() } - defer func() { _ = tx.Rollback() }() + defer func() { _ = trx.Rollback() }() - if err := h.checkLimits(principal); err != nil { + if err := h.checkLimits(principal, trx); err != nil { logrus.Errorf("limits error for user '%v': %v", principal.Email, err) return environment.NewEnableUnauthorized() } @@ -67,14 +68,14 @@ func (h *enableHandler) Handle(params environment.EnableParams, principal *rest_ Host: params.Body.Host, Address: realRemoteAddress(params.HTTPRequest), ZId: envZId, - }, tx) + }, trx) if err != nil { logrus.Errorf("error storing created identity for user '%v': %v", principal.Email, err) - _ = tx.Rollback() + _ = trx.Rollback() return environment.NewEnableInternalServerError() } - if err := tx.Commit(); err != nil { + if err := trx.Commit(); err != nil { logrus.Errorf("error committing for user '%v': %v", principal.Email, err) return environment.NewEnableInternalServerError() } @@ -96,15 +97,15 @@ func (h *enableHandler) Handle(params environment.EnableParams, principal *rest_ return resp } -func (h *enableHandler) checkLimits(principal *rest_model_zrok.Principal) error { +func (h *enableHandler) checkLimits(principal *rest_model_zrok.Principal, trx *sqlx.Tx) error { if !principal.Limitless { if limitsAgent != nil { - ok, err := limitsAgent.CanCreateEnvironment(int(principal.ID)) + ok, err := limitsAgent.CanCreateEnvironment(int(principal.ID), trx) if err != nil { - return errors.Wrapf(err, "error checking limits for '%v'", principal.Email) + return errors.Wrapf(err, "error checking environment limits for '%v'", principal.Email) } if !ok { - return errors.Wrapf(err, "environment limit check failed for '%v'", principal.Email) + return errors.Errorf("environment limit check failed for '%v'", principal.Email) } } } diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 01d07cd8..db1ad5ae 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -43,14 +43,8 @@ func (a *Agent) Stop() { <-a.join } -func (a *Agent) CanCreateEnvironment(acctId int) (bool, error) { - if a.cfg.Environments > Unlimited { - trx, err := a.str.Begin() - if err != nil { - return false, errors.Wrap(err, "error creating transaction") - } - defer func() { _ = trx.Rollback() }() - +func (a *Agent) CanCreateEnvironment(acctId int, trx *sqlx.Tx) (bool, error) { + if a.cfg.Enforcing && a.cfg.Environments > Unlimited { envs, err := a.str.FindEnvironmentsForAccount(acctId, trx) if err != nil { return false, err @@ -62,6 +56,28 @@ func (a *Agent) CanCreateEnvironment(acctId int) (bool, error) { return true, nil } +func (a *Agent) CanCreateShare(acctId int, trx *sqlx.Tx) (bool, error) { + if a.cfg.Enforcing && a.cfg.Shares > Unlimited { + envs, err := a.str.FindEnvironmentsForAccount(acctId, trx) + if err != nil { + return false, err + } + total := 0 + for i := range envs { + shrs, err := a.str.FindSharesForEnvironment(envs[i].Id, trx) + if err != nil { + return false, errors.Wrapf(err, "unable to find shares for environment '%v'", envs[i].ZId) + } + total += len(shrs) + if total+1 > a.cfg.Shares { + return false, nil + } + logrus.Infof("total = %d", total) + } + } + return true, nil +} + func (a *Agent) Handle(u *metrics.Usage) error { logrus.Debugf("handling: %v", u) a.queue <- u diff --git a/controller/share.go b/controller/share.go index a25b604b..ec2d9936 100644 --- a/controller/share.go +++ b/controller/share.go @@ -3,7 +3,6 @@ package controller import ( "github.com/go-openapi/runtime/middleware" "github.com/jmoiron/sqlx" - "github.com/openziti/zrok/controller/limits" "github.com/openziti/zrok/controller/store" "github.com/openziti/zrok/controller/zrokEdgeSdk" "github.com/openziti/zrok/rest_model_zrok" @@ -12,27 +11,23 @@ import ( "github.com/sirupsen/logrus" ) -type shareHandler struct { - cfg *limits.Config -} +type shareHandler struct{} -func newShareHandler(cfg *limits.Config) *shareHandler { - return &shareHandler{cfg: cfg} +func newShareHandler() *shareHandler { + return &shareHandler{} } func (h *shareHandler) Handle(params share.ShareParams, principal *rest_model_zrok.Principal) middleware.Responder { - logrus.Infof("handling") - - tx, err := str.Begin() + trx, err := str.Begin() if err != nil { logrus.Errorf("error starting transaction: %v", err) return share.NewShareInternalServerError() } - defer func() { _ = tx.Rollback() }() + defer func() { _ = trx.Rollback() }() envZId := params.Body.EnvZID envId := 0 - envs, err := str.FindEnvironmentsForAccount(int(principal.ID), tx) + envs, err := str.FindEnvironmentsForAccount(int(principal.ID), trx) if err == nil { found := false for _, env := range envs { @@ -52,7 +47,7 @@ func (h *shareHandler) Handle(params share.ShareParams, principal *rest_model_zr return share.NewShareInternalServerError() } - if err := h.checkLimits(principal, envs, tx); err != nil { + if err := h.checkLimits(principal, trx); err != nil { logrus.Errorf("limits error: %v", err) return share.NewShareUnauthorized() } @@ -80,7 +75,7 @@ func (h *shareHandler) Handle(params share.ShareParams, principal *rest_model_zr var frontendZIds []string var frontendTemplates []string for _, frontendSelection := range params.Body.FrontendSelection { - sfe, err := str.FindFrontendPubliclyNamed(frontendSelection, tx) + sfe, err := str.FindFrontendPubliclyNamed(frontendSelection, trx) if err != nil { logrus.Error(err) return share.NewShareNotFound() @@ -126,13 +121,13 @@ func (h *shareHandler) Handle(params share.ShareParams, principal *rest_model_zr sshr.FrontendEndpoint = &sshr.ShareMode } - sid, err := str.CreateShare(envId, sshr, tx) + sid, err := str.CreateShare(envId, sshr, trx) if err != nil { logrus.Errorf("error creating share record: %v", err) return share.NewShareInternalServerError() } - if err := tx.Commit(); err != nil { + if err := trx.Commit(); err != nil { logrus.Errorf("error committing share record: %v", err) return share.NewShareInternalServerError() } @@ -144,17 +139,15 @@ func (h *shareHandler) Handle(params share.ShareParams, principal *rest_model_zr }) } -func (h *shareHandler) checkLimits(principal *rest_model_zrok.Principal, envs []*store.Environment, tx *sqlx.Tx) error { - if !principal.Limitless && h.cfg.Shares > limits.Unlimited { - total := 0 - for i := range envs { - shrs, err := str.FindSharesForEnvironment(envs[i].Id, tx) +func (h *shareHandler) checkLimits(principal *rest_model_zrok.Principal, trx *sqlx.Tx) error { + if !principal.Limitless { + if limitsAgent != nil { + ok, err := limitsAgent.CanCreateShare(int(principal.ID), trx) if err != nil { - return errors.Errorf("unable to find shares for environment '%v': %v", envs[i].ZId, err) + return errors.Wrapf(err, "error checking share limits for '%v'", principal.Email) } - total += len(shrs) - if total+1 > h.cfg.Shares { - return errors.Errorf("would exceed shares limit of %d for '%v'", h.cfg.Shares, principal.Email) + if !ok { + return errors.Errorf("share limit check failed for '%v'", principal.Email) } } } From b103195f88349b90e891db06cac460a6e230123f Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 22 Mar 2023 13:09:21 -0400 Subject: [PATCH 35/83] working out the hitches in the limit journals (#273) --- controller/limits/agent.go | 6 +++ controller/store/accountLimitJournal.go | 14 ++++-- controller/store/accountLimitJournal_test.go | 47 +++++++++++++++++++ .../sqlite3/009_v0_4_0_limits_journals.sql | 18 +++---- 4 files changed, 73 insertions(+), 12 deletions(-) create mode 100644 controller/store/accountLimitJournal_test.go diff --git a/controller/limits/agent.go b/controller/limits/agent.go index db1ad5ae..02401e64 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -114,6 +114,12 @@ func (a *Agent) enforce(u *metrics.Usage) error { if enforce, warning, err := a.checkAccountLimits(u, trx); err == nil { if enforce { logrus.Warn("enforcing account limit") + + alje, err := a.str.FindLatestAccountLimitJournal(int(u.AccountId), trx) + if err != nil { + return err + } + } else if warning { logrus.Warn("reporting account warning") } else { diff --git a/controller/store/accountLimitJournal.go b/controller/store/accountLimitJournal.go index ca86a93f..2c719e73 100644 --- a/controller/store/accountLimitJournal.go +++ b/controller/store/accountLimitJournal.go @@ -19,15 +19,23 @@ func (self *Store) CreateAccountLimitJournal(j *AccountLimitJournal, tx *sqlx.Tx return 0, errors.Wrap(err, "error preparing account_limit_journal insert statement") } var id int - if err := stmt.QueryRow(j.AccountId, j.RxBytes, j.TxBytes, j.AccountId).Scan(&id); err != nil { + if err := stmt.QueryRow(j.AccountId, j.RxBytes, j.TxBytes, j.Action).Scan(&id); err != nil { return 0, errors.Wrap(err, "error executing account_limit_journal insert statement") } return id, nil } -func (self *Store) FindLatestAccountJournal(acctId int, tx *sqlx.Tx) (*AccountLimitJournal, error) { +func (self *Store) IsAccountLimitJournalEmpty(acctId int, tx *sqlx.Tx) (bool, error) { + count := 0 + if err := tx.QueryRowx("select count(0) from account_limit_journal where account_id = $1", acctId).Scan(&count); err != nil { + return false, err + } + return count == 0, nil +} + +func (self *Store) FindLatestAccountLimitJournal(acctId int, tx *sqlx.Tx) (*AccountLimitJournal, error) { j := &AccountLimitJournal{} - if err := tx.QueryRowx("select * from account_limit_journal where account_id = $1", acctId).StructScan(j); err != nil { + if err := tx.QueryRowx("select * from account_limit_journal where account_id = $1 order by created_at desc limit 1", acctId).StructScan(j); err != nil { return nil, errors.Wrap(err, "error finding account_limit_journal by account_id") } return j, nil diff --git a/controller/store/accountLimitJournal_test.go b/controller/store/accountLimitJournal_test.go new file mode 100644 index 00000000..ac12026a --- /dev/null +++ b/controller/store/accountLimitJournal_test.go @@ -0,0 +1,47 @@ +package store + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestAccountLimitJournal(t *testing.T) { + str, err := Open(&Config{Path: ":memory:", Type: "sqlite3"}) + assert.Nil(t, err) + assert.NotNil(t, str) + + trx, err := str.Begin() + assert.Nil(t, err) + assert.NotNil(t, trx) + + aljEmpty, err := str.IsAccountLimitJournalEmpty(1, trx) + assert.Nil(t, err) + assert.True(t, aljEmpty) + + acctId, err := str.CreateAccount(&Account{Email: "nobody@nowehere.com", Salt: "salt", Password: "password", Token: "token", Limitless: false, Deleted: false}, trx) + assert.Nil(t, err) + + _, err = str.CreateAccountLimitJournal(&AccountLimitJournal{AccountId: acctId, RxBytes: 1024, TxBytes: 2048, Action: "warning"}, trx) + assert.Nil(t, err) + + aljEmpty, err = str.IsAccountLimitJournalEmpty(acctId, trx) + assert.Nil(t, err) + assert.False(t, aljEmpty) + + latestAlj, err := str.FindLatestAccountLimitJournal(acctId, trx) + assert.Nil(t, err) + assert.NotNil(t, latestAlj) + assert.Equal(t, int64(1024), latestAlj.RxBytes) + assert.Equal(t, int64(2048), latestAlj.TxBytes) + assert.Equal(t, "warning", latestAlj.Action) + + _, err = str.CreateAccountLimitJournal(&AccountLimitJournal{AccountId: acctId, RxBytes: 2048, TxBytes: 4096, Action: "limit"}, trx) + assert.Nil(t, err) + + latestAlj, err = str.FindLatestAccountLimitJournal(acctId, trx) + assert.Nil(t, err) + assert.NotNil(t, latestAlj) + assert.Equal(t, int64(2048), latestAlj.RxBytes) + assert.Equal(t, int64(4096), latestAlj.TxBytes) + assert.Equal(t, "limit", latestAlj.Action) +} diff --git a/controller/store/sql/sqlite3/009_v0_4_0_limits_journals.sql b/controller/store/sql/sqlite3/009_v0_4_0_limits_journals.sql index e9b14ebd..0c758792 100644 --- a/controller/store/sql/sqlite3/009_v0_4_0_limits_journals.sql +++ b/controller/store/sql/sqlite3/009_v0_4_0_limits_journals.sql @@ -1,31 +1,31 @@ -- +migrate Up create table account_limit_journal ( - id serial primary key, + id integer primary key, account_id integer references accounts(id), rx_bytes bigint not null, tx_bytes bigint not null, action limit_action_type not null, - created_at timestamptz not null default(current_timestamp), - updated_at timestamptz not null default(current_timestamp) + created_at datetime not null default(strftime('%Y-%m-%d %H:%M:%f', 'now')), + updated_at datetime not null default(strftime('%Y-%m-%d %H:%M:%f', 'now')) ); create table environment_limit_journal ( - id serial primary key, + id integer primary key, environment_id integer references environments(id), rx_bytes bigint not null, tx_bytes bigint not null, action limit_action_type not null, - created_at timestamptz not null default(current_timestamp), - updated_at timestamptz not null default(current_timestamp) + created_at datetime not null default(strftime('%Y-%m-%d %H:%M:%f', 'now')), + updated_at datetime not null default(strftime('%Y-%m-%d %H:%M:%f', 'now')) ); create table share_limit_journal ( - id serial primary key, + id integer primary key, share_id integer references shares(id), rx_bytes bigint not null, tx_bytes bigint not null, action limit_action_type not null, - created_at timestamptz not null default(current_timestamp), - updated_at timestamptz not null default(current_timestamp) + created_at datetime not null default(strftime('%Y-%m-%d %H:%M:%f', 'now')), + updated_at datetime not null default(strftime('%Y-%m-%d %H:%M:%f', 'now')) ); \ No newline at end of file From b172ca110005a28b7b8d7a6e1889ca2ebf9e1fbc Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 22 Mar 2023 14:10:07 -0400 Subject: [PATCH 36/83] limit journal logic for limits.Agent (#273, #276) --- controller/limits/agent.go | 189 ++++++++++++++++++- controller/store/accountLimitJournal.go | 14 +- controller/store/accountLimitJournal_test.go | 8 +- controller/store/environmentLimitJournal.go | 18 +- controller/store/model.go | 9 + controller/store/shareLimitJournal.go | 12 +- 6 files changed, 223 insertions(+), 27 deletions(-) create mode 100644 controller/store/model.go diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 02401e64..19bb0a66 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -113,27 +113,198 @@ func (a *Agent) enforce(u *metrics.Usage) error { if enforce, warning, err := a.checkAccountLimits(u, trx); err == nil { if enforce { - logrus.Warn("enforcing account limit") + enforced := false + var enforcedAt time.Time + if empty, err := a.str.IsAccountLimitJournalEmpty(int(u.AccountId), trx); err == nil && !empty { + if latest, err := a.str.FindLatestAccountLimitJournal(int(u.AccountId), trx); err == nil { + enforced = latest.Action == store.LimitAction + enforcedAt = latest.UpdatedAt + } + } - alje, err := a.str.FindLatestAccountLimitJournal(int(u.AccountId), trx) - if err != nil { - return err + if !enforced { + _, err := a.str.CreateAccountLimitJournal(&store.AccountLimitJournal{ + AccountId: int(u.AccountId), + RxBytes: u.BackendRx, + TxBytes: u.BackendTx, + Action: store.LimitAction, + }, trx) + if err != nil { + return err + } + + logrus.Warnf("enforcing account limit for '#%d': %v", u.AccountId, a.describeLimit(a.cfg.Bandwidth.PerAccount, u.BackendRx, u.BackendTx)) + + if err := trx.Commit(); err != nil { + return err + } + } else { + logrus.Debugf("already enforced limit for account '#%d' at %v", u.AccountId, enforcedAt) } } else if warning { - logrus.Warn("reporting account warning") + warned := false + var warnedAt time.Time + if empty, err := a.str.IsAccountLimitJournalEmpty(int(u.AccountId), trx); err == nil && !empty { + if latest, err := a.str.FindLatestAccountLimitJournal(int(u.AccountId), trx); err == nil { + warned = latest.Action == store.WarningAction || latest.Action == store.LimitAction + warnedAt = latest.UpdatedAt + } + } + + if !warned { + _, err := a.str.CreateAccountLimitJournal(&store.AccountLimitJournal{ + AccountId: int(u.AccountId), + RxBytes: u.BackendRx, + TxBytes: u.BackendTx, + Action: store.WarningAction, + }, trx) + if err != nil { + return err + } + + logrus.Warnf("warning account '#%d': %v", u.AccountId, a.describeLimit(a.cfg.Bandwidth.PerAccount, u.BackendRx, u.BackendTx)) + + if err := trx.Commit(); err != nil { + return err + } + } else { + logrus.Debugf("already warned account '#%d' at %v", u.AccountId, warnedAt) + } + } else { if enforce, warning, err := a.checkEnvironmentLimit(u, trx); err == nil { if enforce { - logrus.Warn("enforcing environment limit") + enforced := false + var enforcedAt time.Time + if empty, err := a.str.IsEnvironmentLimitJournalEmpty(int(u.EnvironmentId), trx); err == nil && !empty { + if latest, err := a.str.FindLatestEnvironmentLimitJournal(int(u.EnvironmentId), trx); err == nil { + enforced = latest.Action == store.LimitAction + enforcedAt = latest.UpdatedAt + } + } + + if !enforced { + _, err := a.str.CreateEnvironmentLimitJournal(&store.EnvironmentLimitJournal{ + EnvironmentId: int(u.EnvironmentId), + RxBytes: u.BackendRx, + TxBytes: u.BackendTx, + Action: store.LimitAction, + }, trx) + if err != nil { + return err + } + + logrus.Warnf("enforcing environment limit for environment '#%d': %v", u.EnvironmentId, a.describeLimit(a.cfg.Bandwidth.PerEnvironment, u.BackendRx, u.BackendTx)) + + if err := trx.Commit(); err != nil { + return err + } + } else { + logrus.Debugf("already enforced limit for environment '#%d' at %v", u.EnvironmentId, enforcedAt) + } + } else if warning { - logrus.Warn("reporting environment warning") + warned := false + var warnedAt time.Time + if empty, err := a.str.IsEnvironmentLimitJournalEmpty(int(u.EnvironmentId), trx); err == nil && !empty { + if latest, err := a.str.FindLatestEnvironmentLimitJournal(int(u.EnvironmentId), trx); err == nil { + warned = latest.Action == store.WarningAction || latest.Action == store.LimitAction + warnedAt = latest.UpdatedAt + } + } + + if !warned { + _, err := a.str.CreateEnvironmentLimitJournal(&store.EnvironmentLimitJournal{ + EnvironmentId: int(u.EnvironmentId), + RxBytes: u.BackendRx, + TxBytes: u.BackendTx, + Action: store.WarningAction, + }, trx) + if err != nil { + return err + } + + logrus.Warnf("warning environment '#%d': %v", u.EnvironmentId, a.describeLimit(a.cfg.Bandwidth.PerEnvironment, u.BackendRx, u.BackendTx)) + + if err := trx.Commit(); err != nil { + return err + } + } else { + logrus.Debugf("already warned environment '#%d' at %v", u.EnvironmentId, warnedAt) + } + } else { if enforce, warning, err := a.checkShareLimit(u); err == nil { if enforce { - logrus.Warn("enforcing share limit") + shr, err := a.str.FindShareWithToken(u.ShareToken, trx) + if err != nil { + return err + } + + enforced := false + var enforcedAt time.Time + if empty, err := a.str.IsShareLimitJournalEmpty(shr.Id, trx); err == nil && !empty { + if latest, err := a.str.FindLatestShareLimitJournal(shr.Id, trx); err == nil { + enforced = latest.Action == store.LimitAction + enforcedAt = latest.UpdatedAt + } + } + + if !enforced { + _, err := a.str.CreateShareLimitJournal(&store.ShareLimitJournal{ + ShareId: shr.Id, + RxBytes: u.BackendRx, + TxBytes: u.BackendTx, + Action: store.LimitAction, + }, trx) + if err != nil { + return err + } + + logrus.Warnf("enforcing share limit for share '%v': %v", shr.Token, a.describeLimit(a.cfg.Bandwidth.PerShare, u.BackendRx, u.BackendTx)) + + if err := trx.Commit(); err != nil { + return err + } + } else { + logrus.Debugf("already enforced limit for share '%v' at %v", shr.Token, enforcedAt) + } + } else if warning { - logrus.Warn("reporting share warning") + shr, err := a.str.FindShareWithToken(u.ShareToken, trx) + if err != nil { + return err + } + + warned := false + var warnedAt time.Time + if empty, err := a.str.IsShareLimitJournalEmpty(shr.Id, trx); err == nil && !empty { + if latest, err := a.str.FindLatestShareLimitJournal(shr.Id, trx); err == nil { + warned = latest.Action == store.WarningAction || latest.Action == store.LimitAction + warnedAt = latest.UpdatedAt + } + } + + if !warned { + _, err := a.str.CreateShareLimitJournal(&store.ShareLimitJournal{ + ShareId: shr.Id, + RxBytes: u.BackendRx, + TxBytes: u.BackendTx, + Action: store.WarningAction, + }, trx) + if err != nil { + return err + } + + logrus.Warnf("warning share '%v': %v", shr.Token, a.describeLimit(a.cfg.Bandwidth.PerShare, u.BackendRx, u.BackendTx)) + + if err := trx.Commit(); err != nil { + return err + } + } else { + logrus.Debugf("already warned share '%v' at %v", shr.Token, warnedAt) + } } } else { logrus.Error(err) diff --git a/controller/store/accountLimitJournal.go b/controller/store/accountLimitJournal.go index 2c719e73..9bf6881b 100644 --- a/controller/store/accountLimitJournal.go +++ b/controller/store/accountLimitJournal.go @@ -10,11 +10,11 @@ type AccountLimitJournal struct { AccountId int RxBytes int64 TxBytes int64 - Action string + Action LimitJournalAction } -func (self *Store) CreateAccountLimitJournal(j *AccountLimitJournal, tx *sqlx.Tx) (int, error) { - stmt, err := tx.Prepare("insert into account_limit_journal (account_id, rx_bytes, tx_bytes, action) values ($1, $2, $3, $4) returning id") +func (self *Store) CreateAccountLimitJournal(j *AccountLimitJournal, trx *sqlx.Tx) (int, error) { + stmt, err := trx.Prepare("insert into account_limit_journal (account_id, rx_bytes, tx_bytes, action) values ($1, $2, $3, $4) returning id") if err != nil { return 0, errors.Wrap(err, "error preparing account_limit_journal insert statement") } @@ -25,17 +25,17 @@ func (self *Store) CreateAccountLimitJournal(j *AccountLimitJournal, tx *sqlx.Tx return id, nil } -func (self *Store) IsAccountLimitJournalEmpty(acctId int, tx *sqlx.Tx) (bool, error) { +func (self *Store) IsAccountLimitJournalEmpty(acctId int, trx *sqlx.Tx) (bool, error) { count := 0 - if err := tx.QueryRowx("select count(0) from account_limit_journal where account_id = $1", acctId).Scan(&count); err != nil { + if err := trx.QueryRowx("select count(0) from account_limit_journal where account_id = $1", acctId).Scan(&count); err != nil { return false, err } return count == 0, nil } -func (self *Store) FindLatestAccountLimitJournal(acctId int, tx *sqlx.Tx) (*AccountLimitJournal, error) { +func (self *Store) FindLatestAccountLimitJournal(acctId int, trx *sqlx.Tx) (*AccountLimitJournal, error) { j := &AccountLimitJournal{} - if err := tx.QueryRowx("select * from account_limit_journal where account_id = $1 order by created_at desc limit 1", acctId).StructScan(j); err != nil { + if err := trx.QueryRowx("select * from account_limit_journal where account_id = $1 order by id desc limit 1", acctId).StructScan(j); err != nil { return nil, errors.Wrap(err, "error finding account_limit_journal by account_id") } return j, nil diff --git a/controller/store/accountLimitJournal_test.go b/controller/store/accountLimitJournal_test.go index ac12026a..b4e03532 100644 --- a/controller/store/accountLimitJournal_test.go +++ b/controller/store/accountLimitJournal_test.go @@ -21,7 +21,7 @@ func TestAccountLimitJournal(t *testing.T) { acctId, err := str.CreateAccount(&Account{Email: "nobody@nowehere.com", Salt: "salt", Password: "password", Token: "token", Limitless: false, Deleted: false}, trx) assert.Nil(t, err) - _, err = str.CreateAccountLimitJournal(&AccountLimitJournal{AccountId: acctId, RxBytes: 1024, TxBytes: 2048, Action: "warning"}, trx) + _, err = str.CreateAccountLimitJournal(&AccountLimitJournal{AccountId: acctId, RxBytes: 1024, TxBytes: 2048, Action: WarningAction}, trx) assert.Nil(t, err) aljEmpty, err = str.IsAccountLimitJournalEmpty(acctId, trx) @@ -33,9 +33,9 @@ func TestAccountLimitJournal(t *testing.T) { assert.NotNil(t, latestAlj) assert.Equal(t, int64(1024), latestAlj.RxBytes) assert.Equal(t, int64(2048), latestAlj.TxBytes) - assert.Equal(t, "warning", latestAlj.Action) + assert.Equal(t, WarningAction, latestAlj.Action) - _, err = str.CreateAccountLimitJournal(&AccountLimitJournal{AccountId: acctId, RxBytes: 2048, TxBytes: 4096, Action: "limit"}, trx) + _, err = str.CreateAccountLimitJournal(&AccountLimitJournal{AccountId: acctId, RxBytes: 2048, TxBytes: 4096, Action: LimitAction}, trx) assert.Nil(t, err) latestAlj, err = str.FindLatestAccountLimitJournal(acctId, trx) @@ -43,5 +43,5 @@ func TestAccountLimitJournal(t *testing.T) { assert.NotNil(t, latestAlj) assert.Equal(t, int64(2048), latestAlj.RxBytes) assert.Equal(t, int64(4096), latestAlj.TxBytes) - assert.Equal(t, "limit", latestAlj.Action) + assert.Equal(t, LimitAction, latestAlj.Action) } diff --git a/controller/store/environmentLimitJournal.go b/controller/store/environmentLimitJournal.go index 077d03ac..73cbbd62 100644 --- a/controller/store/environmentLimitJournal.go +++ b/controller/store/environmentLimitJournal.go @@ -10,11 +10,11 @@ type EnvironmentLimitJournal struct { EnvironmentId int RxBytes int64 TxBytes int64 - Action string + Action LimitJournalAction } -func (self *Store) CreateEnvironmentLimitJournal(j *EnvironmentLimitJournal, tx *sqlx.Tx) (int, error) { - stmt, err := tx.Prepare("insert into environment_limit_journal (environment_id, rx_bytes, tx_bytes, action) values ($1, $2, $3, $4) returning id") +func (self *Store) CreateEnvironmentLimitJournal(j *EnvironmentLimitJournal, trx *sqlx.Tx) (int, error) { + stmt, err := trx.Prepare("insert into environment_limit_journal (environment_id, rx_bytes, tx_bytes, action) values ($1, $2, $3, $4) returning id") if err != nil { return 0, errors.Wrap(err, "error preparing environment_limit_journal insert statement") } @@ -25,9 +25,17 @@ func (self *Store) CreateEnvironmentLimitJournal(j *EnvironmentLimitJournal, tx return id, nil } -func (self *Store) FindLatestEnvironmentLimitJournal(envId int, tx *sqlx.Tx) (*EnvironmentLimitJournal, error) { +func (self *Store) IsEnvironmentLimitJournalEmpty(envId int, trx *sqlx.Tx) (bool, error) { + count := 0 + if err := trx.QueryRowx("select count(0) from environment_limit_journal where environment_id = $1", envId).Scan(&count); err != nil { + return false, err + } + return count == 0, nil +} + +func (self *Store) FindLatestEnvironmentLimitJournal(envId int, trx *sqlx.Tx) (*EnvironmentLimitJournal, error) { j := &EnvironmentLimitJournal{} - if err := tx.QueryRowx("select * from environment_limit_journal where environment_id = $1", envId).StructScan(j); err != nil { + if err := trx.QueryRowx("select * from environment_limit_journal where environment_id = $1 order by created_at desc limit 1", envId).StructScan(j); err != nil { return nil, errors.Wrap(err, "error finding environment_limit_journal by environment_id") } return j, nil diff --git a/controller/store/model.go b/controller/store/model.go new file mode 100644 index 00000000..c5dae5eb --- /dev/null +++ b/controller/store/model.go @@ -0,0 +1,9 @@ +package store + +type LimitJournalAction string + +const ( + LimitAction LimitJournalAction = "limit" + WarningAction LimitJournalAction = "warning" + ClearAction LimitJournalAction = "clear" +) diff --git a/controller/store/shareLimitJournal.go b/controller/store/shareLimitJournal.go index fe97733a..8ad3f6df 100644 --- a/controller/store/shareLimitJournal.go +++ b/controller/store/shareLimitJournal.go @@ -10,7 +10,7 @@ type ShareLimitJournal struct { ShareId int RxBytes int64 TxBytes int64 - Action string + Action LimitJournalAction } func (self *Store) CreateShareLimitJournal(j *ShareLimitJournal, tx *sqlx.Tx) (int, error) { @@ -25,9 +25,17 @@ func (self *Store) CreateShareLimitJournal(j *ShareLimitJournal, tx *sqlx.Tx) (i return id, nil } +func (self *Store) IsShareLimitJournalEmpty(shrId int, trx *sqlx.Tx) (bool, error) { + count := 0 + if err := trx.QueryRowx("select count(0) from share_limit_journal where share_id = $1", shrId).Scan(&count); err != nil { + return false, err + } + return count == 0, nil +} + func (self *Store) FindLatestShareLimitJournal(shrId int, tx *sqlx.Tx) (*ShareLimitJournal, error) { j := &ShareLimitJournal{} - if err := tx.QueryRowx("select * from share_limit_journal where share_id = $1", shrId).StructScan(j); err != nil { + if err := tx.QueryRowx("select * from share_limit_journal where share_id = $1 order by created_at desc limit 1", shrId).StructScan(j); err != nil { return nil, errors.Wrap(err, "error finding share_limit_journal by share_id") } return j, nil From 12de429ead6c2f5c11d634f148acfb3f6164570b Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 22 Mar 2023 15:01:05 -0400 Subject: [PATCH 37/83] lint (#276) --- controller/limits/agent.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 19bb0a66..936a9f9c 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -133,7 +133,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { return err } - logrus.Warnf("enforcing account limit for '#%d': %v", u.AccountId, a.describeLimit(a.cfg.Bandwidth.PerAccount, u.BackendRx, u.BackendTx)) + logrus.Warnf("enforcing account limit for '#%d'", u.AccountId) if err := trx.Commit(); err != nil { return err @@ -163,7 +163,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { return err } - logrus.Warnf("warning account '#%d': %v", u.AccountId, a.describeLimit(a.cfg.Bandwidth.PerAccount, u.BackendRx, u.BackendTx)) + logrus.Warnf("warning account '#%d'", u.AccountId) if err := trx.Commit(); err != nil { return err @@ -195,7 +195,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { return err } - logrus.Warnf("enforcing environment limit for environment '#%d': %v", u.EnvironmentId, a.describeLimit(a.cfg.Bandwidth.PerEnvironment, u.BackendRx, u.BackendTx)) + logrus.Warnf("enforcing environment limit for environment '#%d'", u.EnvironmentId) if err := trx.Commit(); err != nil { return err @@ -225,7 +225,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { return err } - logrus.Warnf("warning environment '#%d': %v", u.EnvironmentId, a.describeLimit(a.cfg.Bandwidth.PerEnvironment, u.BackendRx, u.BackendTx)) + logrus.Warnf("warning environment '#%d'", u.EnvironmentId) if err := trx.Commit(); err != nil { return err @@ -262,7 +262,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { return err } - logrus.Warnf("enforcing share limit for share '%v': %v", shr.Token, a.describeLimit(a.cfg.Bandwidth.PerShare, u.BackendRx, u.BackendTx)) + logrus.Warnf("enforcing share limit for share '%v'", shr.Token) if err := trx.Commit(); err != nil { return err @@ -297,7 +297,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { return err } - logrus.Warnf("warning share '%v': %v", shr.Token, a.describeLimit(a.cfg.Bandwidth.PerShare, u.BackendRx, u.BackendTx)) + logrus.Warnf("warning share '%v'", shr.Token) if err := trx.Commit(); err != nil { return err @@ -342,7 +342,7 @@ func (a *Agent) checkAccountLimits(u *metrics.Usage, trx *sqlx.Tx) (enforce, war enforce, warning = a.checkLimit(limit, rx, tx) if enforce || warning { - logrus.Warnf("'%v': %v", acct.Email, a.describeLimit(limit, rx, tx)) + logrus.Debugf("'%v': %v", acct.Email, a.describeLimit(limit, rx, tx)) } return enforce, warning, nil @@ -369,7 +369,7 @@ func (a *Agent) checkEnvironmentLimit(u *metrics.Usage, trx *sqlx.Tx) (enforce, enforce, warning = a.checkLimit(limit, rx, tx) if enforce || warning { - logrus.Warnf("'%v': %v", env.ZId, a.describeLimit(limit, rx, tx)) + logrus.Debugf("'%v': %v", env.ZId, a.describeLimit(limit, rx, tx)) } return enforce, warning, nil @@ -391,7 +391,7 @@ func (a *Agent) checkShareLimit(u *metrics.Usage) (enforce, warning bool, err er enforce, warning = a.checkLimit(limit, rx, tx) if enforce || warning { - logrus.Warnf("'%v': %v", u.ShareToken, a.describeLimit(limit, rx, tx)) + logrus.Debugf("'%v': %v", u.ShareToken, a.describeLimit(limit, rx, tx)) } return enforce, warning, nil From bc5481a2497d84aaf6398efb65e555f90ef48794 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 22 Mar 2023 15:17:27 -0400 Subject: [PATCH 38/83] self -> str --- controller/store/account.go | 12 ++++++------ controller/store/accountLimitJournal.go | 6 +++--- controller/store/account_request.go | 18 +++++++++--------- controller/store/environment.go | 12 ++++++------ controller/store/environmentLimitJournal.go | 6 +++--- controller/store/password_reset_request.go | 14 +++++++------- controller/store/share.go | 16 ++++++++-------- controller/store/shareLimitJournal.go | 6 +++--- controller/store/store.go | 14 +++++++------- 9 files changed, 52 insertions(+), 52 deletions(-) diff --git a/controller/store/account.go b/controller/store/account.go index 9872f943..4ccb7a94 100644 --- a/controller/store/account.go +++ b/controller/store/account.go @@ -15,7 +15,7 @@ type Account struct { Deleted bool } -func (self *Store) CreateAccount(a *Account, tx *sqlx.Tx) (int, error) { +func (str *Store) CreateAccount(a *Account, tx *sqlx.Tx) (int, error) { stmt, err := tx.Prepare("insert into accounts (email, salt, password, token, limitless) values ($1, $2, $3, $4, $5) returning id") if err != nil { return 0, errors.Wrap(err, "error preparing accounts insert statement") @@ -27,7 +27,7 @@ func (self *Store) CreateAccount(a *Account, tx *sqlx.Tx) (int, error) { return id, nil } -func (self *Store) GetAccount(id int, tx *sqlx.Tx) (*Account, error) { +func (str *Store) GetAccount(id int, tx *sqlx.Tx) (*Account, error) { a := &Account{} if err := tx.QueryRowx("select * from accounts where id = $1", id).StructScan(a); err != nil { return nil, errors.Wrap(err, "error selecting account by id") @@ -35,7 +35,7 @@ func (self *Store) GetAccount(id int, tx *sqlx.Tx) (*Account, error) { return a, nil } -func (self *Store) FindAccountWithEmail(email string, tx *sqlx.Tx) (*Account, error) { +func (str *Store) FindAccountWithEmail(email string, tx *sqlx.Tx) (*Account, error) { a := &Account{} if err := tx.QueryRowx("select * from accounts where email = $1 and not deleted", email).StructScan(a); err != nil { return nil, errors.Wrap(err, "error selecting account by email") @@ -43,7 +43,7 @@ func (self *Store) FindAccountWithEmail(email string, tx *sqlx.Tx) (*Account, er return a, nil } -func (self *Store) FindAccountWithEmailAndDeleted(email string, tx *sqlx.Tx) (*Account, error) { +func (str *Store) FindAccountWithEmailAndDeleted(email string, tx *sqlx.Tx) (*Account, error) { a := &Account{} if err := tx.QueryRowx("select * from accounts where email = $1", email).StructScan(a); err != nil { return nil, errors.Wrap(err, "error selecting acount by email") @@ -51,7 +51,7 @@ func (self *Store) FindAccountWithEmailAndDeleted(email string, tx *sqlx.Tx) (*A return a, nil } -func (self *Store) FindAccountWithToken(token string, tx *sqlx.Tx) (*Account, error) { +func (str *Store) FindAccountWithToken(token string, tx *sqlx.Tx) (*Account, error) { a := &Account{} if err := tx.QueryRowx("select * from accounts where token = $1 and not deleted", token).StructScan(a); err != nil { return nil, errors.Wrap(err, "error selecting account by token") @@ -59,7 +59,7 @@ func (self *Store) FindAccountWithToken(token string, tx *sqlx.Tx) (*Account, er return a, nil } -func (self *Store) UpdateAccount(a *Account, tx *sqlx.Tx) (int, error) { +func (str *Store) UpdateAccount(a *Account, tx *sqlx.Tx) (int, error) { stmt, err := tx.Prepare("update accounts set email=$1, salt=$2, password=$3, token=$4, limitless=$5 where id = $6") if err != nil { return 0, errors.Wrap(err, "error preparing accounts update statement") diff --git a/controller/store/accountLimitJournal.go b/controller/store/accountLimitJournal.go index 9bf6881b..10d96464 100644 --- a/controller/store/accountLimitJournal.go +++ b/controller/store/accountLimitJournal.go @@ -13,7 +13,7 @@ type AccountLimitJournal struct { Action LimitJournalAction } -func (self *Store) CreateAccountLimitJournal(j *AccountLimitJournal, trx *sqlx.Tx) (int, error) { +func (str *Store) CreateAccountLimitJournal(j *AccountLimitJournal, trx *sqlx.Tx) (int, error) { stmt, err := trx.Prepare("insert into account_limit_journal (account_id, rx_bytes, tx_bytes, action) values ($1, $2, $3, $4) returning id") if err != nil { return 0, errors.Wrap(err, "error preparing account_limit_journal insert statement") @@ -25,7 +25,7 @@ func (self *Store) CreateAccountLimitJournal(j *AccountLimitJournal, trx *sqlx.T return id, nil } -func (self *Store) IsAccountLimitJournalEmpty(acctId int, trx *sqlx.Tx) (bool, error) { +func (str *Store) IsAccountLimitJournalEmpty(acctId int, trx *sqlx.Tx) (bool, error) { count := 0 if err := trx.QueryRowx("select count(0) from account_limit_journal where account_id = $1", acctId).Scan(&count); err != nil { return false, err @@ -33,7 +33,7 @@ func (self *Store) IsAccountLimitJournalEmpty(acctId int, trx *sqlx.Tx) (bool, e return count == 0, nil } -func (self *Store) FindLatestAccountLimitJournal(acctId int, trx *sqlx.Tx) (*AccountLimitJournal, error) { +func (str *Store) FindLatestAccountLimitJournal(acctId int, trx *sqlx.Tx) (*AccountLimitJournal, error) { j := &AccountLimitJournal{} if err := trx.QueryRowx("select * from account_limit_journal where account_id = $1 order by id desc limit 1", acctId).StructScan(j); err != nil { return nil, errors.Wrap(err, "error finding account_limit_journal by account_id") diff --git a/controller/store/account_request.go b/controller/store/account_request.go index 9b3f1961..edc664b8 100644 --- a/controller/store/account_request.go +++ b/controller/store/account_request.go @@ -17,7 +17,7 @@ type AccountRequest struct { Deleted bool } -func (self *Store) CreateAccountRequest(ar *AccountRequest, tx *sqlx.Tx) (int, error) { +func (str *Store) CreateAccountRequest(ar *AccountRequest, tx *sqlx.Tx) (int, error) { stmt, err := tx.Prepare("insert into account_requests (token, email, source_address) values ($1, $2, $3) returning id") if err != nil { return 0, errors.Wrap(err, "error preparing account_requests insert statement") @@ -29,7 +29,7 @@ func (self *Store) CreateAccountRequest(ar *AccountRequest, tx *sqlx.Tx) (int, e return id, nil } -func (self *Store) GetAccountRequest(id int, tx *sqlx.Tx) (*AccountRequest, error) { +func (str *Store) GetAccountRequest(id int, tx *sqlx.Tx) (*AccountRequest, error) { ar := &AccountRequest{} if err := tx.QueryRowx("select * from account_requests where id = $1", id).StructScan(ar); err != nil { return nil, errors.Wrap(err, "error selecting account_request by id") @@ -37,7 +37,7 @@ func (self *Store) GetAccountRequest(id int, tx *sqlx.Tx) (*AccountRequest, erro return ar, nil } -func (self *Store) FindAccountRequestWithToken(token string, tx *sqlx.Tx) (*AccountRequest, error) { +func (str *Store) FindAccountRequestWithToken(token string, tx *sqlx.Tx) (*AccountRequest, error) { ar := &AccountRequest{} if err := tx.QueryRowx("select * from account_requests where token = $1 and not deleted", token).StructScan(ar); err != nil { return nil, errors.Wrap(err, "error selecting account_request by token") @@ -45,9 +45,9 @@ func (self *Store) FindAccountRequestWithToken(token string, tx *sqlx.Tx) (*Acco return ar, nil } -func (self *Store) FindExpiredAccountRequests(before time.Time, limit int, tx *sqlx.Tx) ([]*AccountRequest, error) { +func (str *Store) FindExpiredAccountRequests(before time.Time, limit int, tx *sqlx.Tx) ([]*AccountRequest, error) { var sql string - switch self.cfg.Type { + switch str.cfg.Type { case "postgres": sql = "select * from account_requests where created_at < $1 and not deleted limit %d for update" @@ -55,7 +55,7 @@ func (self *Store) FindExpiredAccountRequests(before time.Time, limit int, tx *s sql = "select * from account_requests where created_at < $1 and not deleted limit %d" default: - return nil, errors.Errorf("unknown database type '%v'", self.cfg.Type) + return nil, errors.Errorf("unknown database type '%v'", str.cfg.Type) } rows, err := tx.Queryx(fmt.Sprintf(sql, limit), before) @@ -73,7 +73,7 @@ func (self *Store) FindExpiredAccountRequests(before time.Time, limit int, tx *s return ars, nil } -func (self *Store) FindAccountRequestWithEmail(email string, tx *sqlx.Tx) (*AccountRequest, error) { +func (str *Store) FindAccountRequestWithEmail(email string, tx *sqlx.Tx) (*AccountRequest, error) { ar := &AccountRequest{} if err := tx.QueryRowx("select * from account_requests where email = $1 and not deleted", email).StructScan(ar); err != nil { return nil, errors.Wrap(err, "error selecting account_request by email") @@ -81,7 +81,7 @@ func (self *Store) FindAccountRequestWithEmail(email string, tx *sqlx.Tx) (*Acco return ar, nil } -func (self *Store) DeleteAccountRequest(id int, tx *sqlx.Tx) error { +func (str *Store) DeleteAccountRequest(id int, tx *sqlx.Tx) error { stmt, err := tx.Prepare("update account_requests set deleted = true, updated_at = current_timestamp where id = $1") if err != nil { return errors.Wrap(err, "error preparing account_requests delete statement") @@ -93,7 +93,7 @@ func (self *Store) DeleteAccountRequest(id int, tx *sqlx.Tx) error { return nil } -func (self *Store) DeleteMultipleAccountRequests(ids []int, tx *sqlx.Tx) error { +func (str *Store) DeleteMultipleAccountRequests(ids []int, tx *sqlx.Tx) error { if len(ids) == 0 { return nil } diff --git a/controller/store/environment.go b/controller/store/environment.go index 4c3e47c0..4ee783a3 100644 --- a/controller/store/environment.go +++ b/controller/store/environment.go @@ -15,7 +15,7 @@ type Environment struct { Deleted bool } -func (self *Store) CreateEnvironment(accountId int, i *Environment, tx *sqlx.Tx) (int, error) { +func (str *Store) CreateEnvironment(accountId int, i *Environment, tx *sqlx.Tx) (int, error) { stmt, err := tx.Prepare("insert into environments (account_id, description, host, address, z_id) values ($1, $2, $3, $4, $5) returning id") if err != nil { return 0, errors.Wrap(err, "error preparing environments insert statement") @@ -27,7 +27,7 @@ func (self *Store) CreateEnvironment(accountId int, i *Environment, tx *sqlx.Tx) return id, nil } -func (self *Store) CreateEphemeralEnvironment(i *Environment, tx *sqlx.Tx) (int, error) { +func (str *Store) CreateEphemeralEnvironment(i *Environment, tx *sqlx.Tx) (int, error) { stmt, err := tx.Prepare("insert into environments (description, host, address, z_id) values ($1, $2, $3, $4) returning id") if err != nil { return 0, errors.Wrap(err, "error preparing environments (ephemeral) insert statement") @@ -39,7 +39,7 @@ func (self *Store) CreateEphemeralEnvironment(i *Environment, tx *sqlx.Tx) (int, return id, nil } -func (self *Store) GetEnvironment(id int, tx *sqlx.Tx) (*Environment, error) { +func (str *Store) GetEnvironment(id int, tx *sqlx.Tx) (*Environment, error) { i := &Environment{} if err := tx.QueryRowx("select * from environments where id = $1", id).StructScan(i); err != nil { return nil, errors.Wrap(err, "error selecting environment by id") @@ -47,7 +47,7 @@ func (self *Store) GetEnvironment(id int, tx *sqlx.Tx) (*Environment, error) { return i, nil } -func (self *Store) FindEnvironmentsForAccount(accountId int, tx *sqlx.Tx) ([]*Environment, error) { +func (str *Store) FindEnvironmentsForAccount(accountId int, tx *sqlx.Tx) ([]*Environment, error) { rows, err := tx.Queryx("select environments.* from environments where account_id = $1 and not deleted", accountId) if err != nil { return nil, errors.Wrap(err, "error selecting environments by account id") @@ -63,7 +63,7 @@ func (self *Store) FindEnvironmentsForAccount(accountId int, tx *sqlx.Tx) ([]*En return is, nil } -func (self *Store) FindEnvironmentForAccount(envZId string, accountId int, tx *sqlx.Tx) (*Environment, error) { +func (str *Store) FindEnvironmentForAccount(envZId string, accountId int, tx *sqlx.Tx) (*Environment, error) { env := &Environment{} if err := tx.QueryRowx("select environments.* from environments where z_id = $1 and account_id = $2 and not deleted", envZId, accountId).StructScan(env); err != nil { return nil, errors.Wrap(err, "error finding environment by z_id and account_id") @@ -71,7 +71,7 @@ func (self *Store) FindEnvironmentForAccount(envZId string, accountId int, tx *s return env, nil } -func (self *Store) DeleteEnvironment(id int, tx *sqlx.Tx) error { +func (str *Store) DeleteEnvironment(id int, tx *sqlx.Tx) error { stmt, err := tx.Prepare("update environments set updated_at = current_timestamp, deleted = true where id = $1") if err != nil { return errors.Wrap(err, "error preparing environments delete statement") diff --git a/controller/store/environmentLimitJournal.go b/controller/store/environmentLimitJournal.go index 73cbbd62..75794185 100644 --- a/controller/store/environmentLimitJournal.go +++ b/controller/store/environmentLimitJournal.go @@ -13,7 +13,7 @@ type EnvironmentLimitJournal struct { Action LimitJournalAction } -func (self *Store) CreateEnvironmentLimitJournal(j *EnvironmentLimitJournal, trx *sqlx.Tx) (int, error) { +func (str *Store) CreateEnvironmentLimitJournal(j *EnvironmentLimitJournal, trx *sqlx.Tx) (int, error) { stmt, err := trx.Prepare("insert into environment_limit_journal (environment_id, rx_bytes, tx_bytes, action) values ($1, $2, $3, $4) returning id") if err != nil { return 0, errors.Wrap(err, "error preparing environment_limit_journal insert statement") @@ -25,7 +25,7 @@ func (self *Store) CreateEnvironmentLimitJournal(j *EnvironmentLimitJournal, trx return id, nil } -func (self *Store) IsEnvironmentLimitJournalEmpty(envId int, trx *sqlx.Tx) (bool, error) { +func (str *Store) IsEnvironmentLimitJournalEmpty(envId int, trx *sqlx.Tx) (bool, error) { count := 0 if err := trx.QueryRowx("select count(0) from environment_limit_journal where environment_id = $1", envId).Scan(&count); err != nil { return false, err @@ -33,7 +33,7 @@ func (self *Store) IsEnvironmentLimitJournalEmpty(envId int, trx *sqlx.Tx) (bool return count == 0, nil } -func (self *Store) FindLatestEnvironmentLimitJournal(envId int, trx *sqlx.Tx) (*EnvironmentLimitJournal, error) { +func (str *Store) FindLatestEnvironmentLimitJournal(envId int, trx *sqlx.Tx) (*EnvironmentLimitJournal, error) { j := &EnvironmentLimitJournal{} if err := trx.QueryRowx("select * from environment_limit_journal where environment_id = $1 order by created_at desc limit 1", envId).StructScan(j); err != nil { return nil, errors.Wrap(err, "error finding environment_limit_journal by environment_id") diff --git a/controller/store/password_reset_request.go b/controller/store/password_reset_request.go index a5c617c5..a6a7b60d 100644 --- a/controller/store/password_reset_request.go +++ b/controller/store/password_reset_request.go @@ -16,7 +16,7 @@ type PasswordResetRequest struct { Deleted bool } -func (self *Store) CreatePasswordResetRequest(prr *PasswordResetRequest, tx *sqlx.Tx) (int, error) { +func (str *Store) CreatePasswordResetRequest(prr *PasswordResetRequest, tx *sqlx.Tx) (int, error) { stmt, err := tx.Prepare("insert into password_reset_requests (account_id, token) values ($1, $2) ON CONFLICT(account_id) DO UPDATE SET token=$2 returning id") if err != nil { return 0, errors.Wrap(err, "error preparing password_reset_requests insert statement") @@ -28,7 +28,7 @@ func (self *Store) CreatePasswordResetRequest(prr *PasswordResetRequest, tx *sql return id, nil } -func (self *Store) FindPasswordResetRequestWithToken(token string, tx *sqlx.Tx) (*PasswordResetRequest, error) { +func (str *Store) FindPasswordResetRequestWithToken(token string, tx *sqlx.Tx) (*PasswordResetRequest, error) { prr := &PasswordResetRequest{} if err := tx.QueryRowx("select * from password_reset_requests where token = $1 and not deleted", token).StructScan(prr); err != nil { return nil, errors.Wrap(err, "error selecting password_reset_requests by token") @@ -36,16 +36,16 @@ func (self *Store) FindPasswordResetRequestWithToken(token string, tx *sqlx.Tx) return prr, nil } -func (self *Store) FindExpiredPasswordResetRequests(before time.Time, limit int, tx *sqlx.Tx) ([]*PasswordResetRequest, error) { +func (str *Store) FindExpiredPasswordResetRequests(before time.Time, limit int, tx *sqlx.Tx) ([]*PasswordResetRequest, error) { var sql string - switch self.cfg.Type { + switch str.cfg.Type { case "postgres": sql = "select * from password_reset_requests where created_at < $1 and not deleted limit %d for update" case "sqlite3": sql = "select * from password_reset_requests where created_at < $1 and not deleted limit %d" default: - return nil, errors.Errorf("unknown database type '%v'", self.cfg.Type) + return nil, errors.Errorf("unknown database type '%v'", str.cfg.Type) } rows, err := tx.Queryx(fmt.Sprintf(sql, limit), before) @@ -63,7 +63,7 @@ func (self *Store) FindExpiredPasswordResetRequests(before time.Time, limit int, return prrs, nil } -func (self *Store) DeletePasswordResetRequest(id int, tx *sqlx.Tx) error { +func (str *Store) DeletePasswordResetRequest(id int, tx *sqlx.Tx) error { stmt, err := tx.Prepare("update password_reset_requests set updated_at = current_timestamp, deleted = true where id = $1") if err != nil { return errors.Wrap(err, "error preparing password_reset_requests delete statement") @@ -75,7 +75,7 @@ func (self *Store) DeletePasswordResetRequest(id int, tx *sqlx.Tx) error { return nil } -func (self *Store) DeleteMultiplePasswordResetRequests(ids []int, tx *sqlx.Tx) error { +func (str *Store) DeleteMultiplePasswordResetRequests(ids []int, tx *sqlx.Tx) error { if len(ids) == 0 { return nil } diff --git a/controller/store/share.go b/controller/store/share.go index 86e55c61..8161fc2d 100644 --- a/controller/store/share.go +++ b/controller/store/share.go @@ -19,7 +19,7 @@ type Share struct { Deleted bool } -func (self *Store) CreateShare(envId int, shr *Share, tx *sqlx.Tx) (int, error) { +func (str *Store) CreateShare(envId int, shr *Share, tx *sqlx.Tx) (int, error) { stmt, err := tx.Prepare("insert into shares (environment_id, z_id, token, share_mode, backend_mode, frontend_selection, frontend_endpoint, backend_proxy_endpoint, reserved) values ($1, $2, $3, $4, $5, $6, $7, $8, $9) returning id") if err != nil { return 0, errors.Wrap(err, "error preparing shares insert statement") @@ -31,7 +31,7 @@ func (self *Store) CreateShare(envId int, shr *Share, tx *sqlx.Tx) (int, error) return id, nil } -func (self *Store) GetShare(id int, tx *sqlx.Tx) (*Share, error) { +func (str *Store) GetShare(id int, tx *sqlx.Tx) (*Share, error) { shr := &Share{} if err := tx.QueryRowx("select * from shares where id = $1", id).StructScan(shr); err != nil { return nil, errors.Wrap(err, "error selecting share by id") @@ -39,7 +39,7 @@ func (self *Store) GetShare(id int, tx *sqlx.Tx) (*Share, error) { return shr, nil } -func (self *Store) FindAllShares(tx *sqlx.Tx) ([]*Share, error) { +func (str *Store) FindAllShares(tx *sqlx.Tx) ([]*Share, error) { rows, err := tx.Queryx("select * from shares where not deleted order by id") if err != nil { return nil, errors.Wrap(err, "error selecting all shares") @@ -55,7 +55,7 @@ func (self *Store) FindAllShares(tx *sqlx.Tx) ([]*Share, error) { return shrs, nil } -func (self *Store) FindShareWithToken(shrToken string, tx *sqlx.Tx) (*Share, error) { +func (str *Store) FindShareWithToken(shrToken string, tx *sqlx.Tx) (*Share, error) { shr := &Share{} if err := tx.QueryRowx("select * from shares where token = $1 and not deleted", shrToken).StructScan(shr); err != nil { return nil, errors.Wrap(err, "error selecting share by token") @@ -63,7 +63,7 @@ func (self *Store) FindShareWithToken(shrToken string, tx *sqlx.Tx) (*Share, err return shr, nil } -func (self *Store) FindShareWithZIdAndDeleted(zId string, tx *sqlx.Tx) (*Share, error) { +func (str *Store) FindShareWithZIdAndDeleted(zId string, tx *sqlx.Tx) (*Share, error) { shr := &Share{} if err := tx.QueryRowx("select * from shares where z_id = $1", zId).StructScan(shr); err != nil { return nil, errors.Wrap(err, "error selecting share by z_id") @@ -71,7 +71,7 @@ func (self *Store) FindShareWithZIdAndDeleted(zId string, tx *sqlx.Tx) (*Share, return shr, nil } -func (self *Store) FindSharesForEnvironment(envId int, tx *sqlx.Tx) ([]*Share, error) { +func (str *Store) FindSharesForEnvironment(envId int, tx *sqlx.Tx) ([]*Share, error) { rows, err := tx.Queryx("select shares.* from shares where environment_id = $1 and not deleted", envId) if err != nil { return nil, errors.Wrap(err, "error selecting shares by environment id") @@ -87,7 +87,7 @@ func (self *Store) FindSharesForEnvironment(envId int, tx *sqlx.Tx) ([]*Share, e return shrs, nil } -func (self *Store) UpdateShare(shr *Share, tx *sqlx.Tx) error { +func (str *Store) UpdateShare(shr *Share, tx *sqlx.Tx) error { sql := "update shares set z_id = $1, token = $2, share_mode = $3, backend_mode = $4, frontend_selection = $5, frontend_endpoint = $6, backend_proxy_endpoint = $7, reserved = $8, updated_at = current_timestamp where id = $9" stmt, err := tx.Prepare(sql) if err != nil { @@ -100,7 +100,7 @@ func (self *Store) UpdateShare(shr *Share, tx *sqlx.Tx) error { return nil } -func (self *Store) DeleteShare(id int, tx *sqlx.Tx) error { +func (str *Store) DeleteShare(id int, tx *sqlx.Tx) error { stmt, err := tx.Prepare("update shares set updated_at = current_timestamp, deleted = true where id = $1") if err != nil { return errors.Wrap(err, "error preparing shares delete statement") diff --git a/controller/store/shareLimitJournal.go b/controller/store/shareLimitJournal.go index 8ad3f6df..59891206 100644 --- a/controller/store/shareLimitJournal.go +++ b/controller/store/shareLimitJournal.go @@ -13,7 +13,7 @@ type ShareLimitJournal struct { Action LimitJournalAction } -func (self *Store) CreateShareLimitJournal(j *ShareLimitJournal, tx *sqlx.Tx) (int, error) { +func (str *Store) CreateShareLimitJournal(j *ShareLimitJournal, tx *sqlx.Tx) (int, error) { stmt, err := tx.Prepare("insert into share_limit_journal (share_id, rx_bytes, tx_bytes, action) values ($1, $2, $3, $4) returning id") if err != nil { return 0, errors.Wrap(err, "error preparing share_limit_journal insert statement") @@ -25,7 +25,7 @@ func (self *Store) CreateShareLimitJournal(j *ShareLimitJournal, tx *sqlx.Tx) (i return id, nil } -func (self *Store) IsShareLimitJournalEmpty(shrId int, trx *sqlx.Tx) (bool, error) { +func (str *Store) IsShareLimitJournalEmpty(shrId int, trx *sqlx.Tx) (bool, error) { count := 0 if err := trx.QueryRowx("select count(0) from share_limit_journal where share_id = $1", shrId).Scan(&count); err != nil { return false, err @@ -33,7 +33,7 @@ func (self *Store) IsShareLimitJournalEmpty(shrId int, trx *sqlx.Tx) (bool, erro return count == 0, nil } -func (self *Store) FindLatestShareLimitJournal(shrId int, tx *sqlx.Tx) (*ShareLimitJournal, error) { +func (str *Store) FindLatestShareLimitJournal(shrId int, tx *sqlx.Tx) (*ShareLimitJournal, error) { j := &ShareLimitJournal{} if err := tx.QueryRowx("select * from share_limit_journal where share_id = $1 order by created_at desc limit 1", shrId).StructScan(j); err != nil { return nil, errors.Wrap(err, "error finding share_limit_journal by share_id") diff --git a/controller/store/store.go b/controller/store/store.go index 60e98f60..f56e7bf5 100644 --- a/controller/store/store.go +++ b/controller/store/store.go @@ -62,15 +62,15 @@ func Open(cfg *Config) (*Store, error) { return store, nil } -func (self *Store) Begin() (*sqlx.Tx, error) { - return self.db.Beginx() +func (str *Store) Begin() (*sqlx.Tx, error) { + return str.db.Beginx() } -func (self *Store) Close() error { - return self.db.Close() +func (str *Store) Close() error { + return str.db.Close() } -func (self *Store) migrate(cfg *Config) error { +func (str *Store) migrate(cfg *Config) error { switch cfg.Type { case "sqlite3": migrations := &migrate.EmbedFileSystemMigrationSource{ @@ -78,7 +78,7 @@ func (self *Store) migrate(cfg *Config) error { Root: "/", } migrate.SetTable("migrations") - n, err := migrate.Exec(self.db.DB, "sqlite3", migrations, migrate.Up) + n, err := migrate.Exec(str.db.DB, "sqlite3", migrations, migrate.Up) if err != nil { return errors.Wrap(err, "error running migrations") } @@ -90,7 +90,7 @@ func (self *Store) migrate(cfg *Config) error { Root: "/", } migrate.SetTable("migrations") - n, err := migrate.Exec(self.db.DB, "postgres", migrations, migrate.Up) + n, err := migrate.Exec(str.db.DB, "postgres", migrations, migrate.Up) if err != nil { return errors.Wrap(err, "error running migrations") } From 0ed00ebc2c04387f63b0351b99cdb34f41d7eeee Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 22 Mar 2023 15:42:47 -0400 Subject: [PATCH 39/83] to find the current latest journal entries for each account_id (#273) --- controller/store/accountLimitJournal.go | 16 ++++++++++ controller/store/accountLimitJournal_test.go | 32 ++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/controller/store/accountLimitJournal.go b/controller/store/accountLimitJournal.go index 10d96464..66308bbc 100644 --- a/controller/store/accountLimitJournal.go +++ b/controller/store/accountLimitJournal.go @@ -40,3 +40,19 @@ func (str *Store) FindLatestAccountLimitJournal(acctId int, trx *sqlx.Tx) (*Acco } return j, nil } + +func (str *Store) FindAllLatestAccountLimitJournal(trx *sqlx.Tx) ([]*AccountLimitJournal, error) { + rows, err := trx.Queryx("select id, account_id, rx_bytes, tx_bytes, action, created_at, updated_at from account_limit_journal where id in (select max(id) as id from account_limit_journal group by account_id)") + if err != nil { + return nil, errors.Wrap(err, "error selecting distinct account_limit_jounal") + } + var is []*AccountLimitJournal + for rows.Next() { + i := &AccountLimitJournal{} + if err := rows.StructScan(i); err != nil { + return nil, errors.Wrap(err, "error scanning account_limit_journal") + } + is = append(is, i) + } + return is, nil +} diff --git a/controller/store/accountLimitJournal_test.go b/controller/store/accountLimitJournal_test.go index b4e03532..88427dfc 100644 --- a/controller/store/accountLimitJournal_test.go +++ b/controller/store/accountLimitJournal_test.go @@ -45,3 +45,35 @@ func TestAccountLimitJournal(t *testing.T) { assert.Equal(t, int64(4096), latestAlj.TxBytes) assert.Equal(t, LimitAction, latestAlj.Action) } + +func TestFindAllLatestAccountLimitJournal(t *testing.T) { + str, err := Open(&Config{Path: ":memory:", Type: "sqlite3"}) + assert.Nil(t, err) + assert.NotNil(t, str) + + trx, err := str.Begin() + assert.Nil(t, err) + assert.NotNil(t, trx) + + acctId1, err := str.CreateAccount(&Account{Email: "nobody@nowehere.com", Salt: "salt1", Password: "password1", Token: "token1", Limitless: false, Deleted: false}, trx) + assert.Nil(t, err) + + _, err = str.CreateAccountLimitJournal(&AccountLimitJournal{AccountId: acctId1, RxBytes: 2048, TxBytes: 4096, Action: WarningAction}, trx) + assert.Nil(t, err) + _, err = str.CreateAccountLimitJournal(&AccountLimitJournal{AccountId: acctId1, RxBytes: 2048, TxBytes: 4096, Action: ClearAction}, trx) + assert.Nil(t, err) + aljId13, err := str.CreateAccountLimitJournal(&AccountLimitJournal{AccountId: acctId1, RxBytes: 2048, TxBytes: 4096, Action: LimitAction}, trx) + assert.Nil(t, err) + + acctId2, err := str.CreateAccount(&Account{Email: "someone@somewhere.com", Salt: "salt2", Password: "password2", Token: "token2", Limitless: false, Deleted: false}, trx) + assert.Nil(t, err) + + aljId21, err := str.CreateAccountLimitJournal(&AccountLimitJournal{AccountId: acctId2, RxBytes: 2048, TxBytes: 4096, Action: WarningAction}, trx) + assert.Nil(t, err) + + aljs, err := str.FindAllLatestAccountLimitJournal(trx) + assert.Nil(t, err) + assert.Equal(t, 2, len(aljs)) + assert.Equal(t, aljId13, aljs[0].Id) + assert.Equal(t, aljId21, aljs[1].Id) +} From 0bed934976ea07a8a39f1697a74bf35cd7313741 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 23 Mar 2023 15:13:59 -0400 Subject: [PATCH 40/83] limit relax triggers and associated store tweaks (#273) --- controller/limits/agent.go | 198 +++++++++++++++++--- controller/store/accountLimitJournal.go | 19 +- controller/store/environmentLimitJournal.go | 23 +++ controller/store/shareLimitJournal.go | 31 ++- 4 files changed, 234 insertions(+), 37 deletions(-) diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 936a9f9c..0237ca99 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -92,10 +92,14 @@ mainLoop: for { select { case usage := <-a.queue: - a.enforce(usage) + if err := a.enforce(usage); err != nil { + logrus.Errorf("error running enforcement: %v", err) + } case <-time.After(a.cfg.Cycle): - logrus.Info("inspection cycle") + if err := a.relax(); err != nil { + logrus.Errorf("error running relax cycle: %v", err) + } case <-a.close: close(a.join) @@ -111,7 +115,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { } defer func() { _ = trx.Rollback() }() - if enforce, warning, err := a.checkAccountLimits(u, trx); err == nil { + if enforce, warning, err := a.checkAccountLimit(u.AccountId); err == nil { if enforce { enforced := false var enforcedAt time.Time @@ -173,7 +177,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { } } else { - if enforce, warning, err := a.checkEnvironmentLimit(u, trx); err == nil { + if enforce, warning, err := a.checkEnvironmentLimit(u.EnvironmentId); err == nil { if enforce { enforced := false var enforcedAt time.Time @@ -235,7 +239,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { } } else { - if enforce, warning, err := a.checkShareLimit(u); err == nil { + if enforce, warning, err := a.checkShareLimit(u.ShareToken); err == nil { if enforce { shr, err := a.str.FindShareWithToken(u.ShareToken, trx) if err != nil { @@ -321,12 +325,165 @@ func (a *Agent) enforce(u *metrics.Usage) error { return nil } -func (a *Agent) checkAccountLimits(u *metrics.Usage, trx *sqlx.Tx) (enforce, warning bool, err error) { - acct, err := a.str.GetAccount(int(u.AccountId), trx) +func (a *Agent) relax() error { + logrus.Info("relaxing") + + trx, err := a.str.Begin() if err != nil { - return false, false, errors.Wrapf(err, "error getting account '%d'", u.AccountId) + return errors.Wrap(err, "error starting transaction") + } + defer func() { _ = trx.Rollback() }() + + commit := false + + if sljs, err := a.str.FindAllLatestShareLimitJournal(trx); err == nil { + for _, slj := range sljs { + if shr, err := a.str.GetShare(slj.ShareId, trx); err == nil { + switch slj.Action { + case store.WarningAction: + if enforce, warning, err := a.checkShareLimit(shr.Token); err == nil { + if !enforce && !warning { + logrus.Infof("relaxing warning for share '%v'", shr.Token) + + if err := a.str.DeleteShareLimitJournalForShare(shr.Id, trx); err == nil { + commit = true + } else { + logrus.Errorf("error deleting share_limit_journal for '%v'", shr.Token) + } + } else { + logrus.Infof("share '%v' still over limit", shr.Token) + } + } else { + logrus.Errorf("error checking share limit for '%v': %v", shr.Token, err) + } + + case store.LimitAction: + if enforce, warning, err := a.checkShareLimit(shr.Token); err == nil { + if !enforce && !warning { + logrus.Infof("relaxing limit for share '%v'", shr.Token) + + if err := a.str.DeleteShareLimitJournalForShare(shr.Id, trx); err == nil { + commit = true + } else { + logrus.Errorf("error deleting share_limit_journal for '%v': %v", shr.Token, err) + } + } else { + logrus.Infof("share '%v' still over limit", shr.Token) + } + } else { + logrus.Errorf("error checking share limit for '%v': %v", shr.Token, err) + } + } + } else { + logrus.Errorf("error getting share for '#%d': %v", slj.ShareId, err) + } + } + } else { + return err } + if eljs, err := a.str.FindAllLatestEnvironmentLimitJournal(trx); err == nil { + for _, elj := range eljs { + if env, err := a.str.GetEnvironment(elj.EnvironmentId, trx); err == nil { + switch elj.Action { + case store.WarningAction: + if enforce, warning, err := a.checkEnvironmentLimit(int64(elj.EnvironmentId)); err == nil { + if !enforce && !warning { + logrus.Infof("relaxing warning for environment '%v'", env.ZId) + + if err := a.str.DeleteEnvironmentLimitJournalForEnvironment(env.Id, trx); err == nil { + commit = true + } else { + logrus.Errorf("error deleteing environment_limit_journal for '%v': %v", env.ZId, err) + } + } else { + logrus.Infof("environment '%v' still over limit", env.ZId) + } + } else { + logrus.Errorf("error checking environment limit for '%v': %v", env.ZId, err) + } + + case store.LimitAction: + if enforce, warning, err := a.checkEnvironmentLimit(int64(elj.EnvironmentId)); err == nil { + if !enforce && !warning { + logrus.Infof("relaxing limit for environment '%v'", env.ZId) + + if err := a.str.DeleteEnvironmentLimitJournalForEnvironment(env.Id, trx); err == nil { + commit = true + } else { + logrus.Errorf("error deleteing environment_limit_journal for '%v': %v", env.ZId, err) + } + } else { + logrus.Infof("environment '%v' still over limit", env.ZId) + } + } else { + logrus.Errorf("error checking environment limit for '%v': %v", env.ZId, err) + } + } + } else { + logrus.Errorf("error getting environment for '#%d': %v", elj.EnvironmentId, err) + } + } + } else { + return err + } + + if aljs, err := a.str.FindAllLatestAccountLimitJournal(trx); err == nil { + for _, alj := range aljs { + if acct, err := a.str.GetAccount(alj.AccountId, trx); err == nil { + switch alj.Action { + case store.WarningAction: + if enforce, warning, err := a.checkAccountLimit(int64(alj.AccountId)); err == nil { + if !enforce && !warning { + logrus.Infof("relaxing warning for account '%v'", acct.Email) + + if err := a.str.DeleteAccountLimitJournalForAccount(acct.Id, trx); err == nil { + commit = true + } else { + logrus.Errorf("error deleting account_limit_journal for '%v': %v", acct.Email, err) + } + } else { + logrus.Infof("account '%v' still over limit", acct.Email) + } + } else { + logrus.Errorf("error checking account limit for '%v': %v", acct.Email, err) + } + + case store.LimitAction: + if enforce, warning, err := a.checkAccountLimit(int64(alj.AccountId)); err == nil { + if !enforce && !warning { + logrus.Infof("relaxing limit for account '%v'", acct.Email) + + if err := a.str.DeleteAccountLimitJournalForAccount(acct.Id, trx); err == nil { + commit = true + } else { + logrus.Errorf("error deleting account_limit_journal for '%v': %v", acct.Email, err) + } + } else { + logrus.Infof("account '%v' still over limit", acct.Email) + } + } else { + logrus.Errorf("error checking account limit for '%v': %v", acct.Email, err) + } + } + } else { + logrus.Errorf("error getting account for '#%d': %v", alj.AccountId, err) + } + } + } else { + return err + } + + if commit { + if err := trx.Commit(); err != nil { + return err + } + } + + return nil +} + +func (a *Agent) checkAccountLimit(acctId int64) (enforce, warning bool, err error) { period := 24 * time.Hour limit := DefaultBandwidthPerPeriod() if a.cfg.Bandwidth != nil && a.cfg.Bandwidth.PerAccount != nil { @@ -335,25 +492,16 @@ func (a *Agent) checkAccountLimits(u *metrics.Usage, trx *sqlx.Tx) (enforce, war if limit.Period > 0 { period = limit.Period } - rx, tx, err := a.ifx.totalRxTxForAccount(u.AccountId, period) + rx, tx, err := a.ifx.totalRxTxForAccount(acctId, period) if err != nil { logrus.Error(err) } enforce, warning = a.checkLimit(limit, rx, tx) - if enforce || warning { - logrus.Debugf("'%v': %v", acct.Email, a.describeLimit(limit, rx, tx)) - } - return enforce, warning, nil } -func (a *Agent) checkEnvironmentLimit(u *metrics.Usage, trx *sqlx.Tx) (enforce, warning bool, err error) { - env, err := a.str.GetEnvironment(int(u.EnvironmentId), trx) - if err != nil { - return false, false, errors.Wrapf(err, "error getting account '%d'", u.EnvironmentId) - } - +func (a *Agent) checkEnvironmentLimit(envId int64) (enforce, warning bool, err error) { period := 24 * time.Hour limit := DefaultBandwidthPerPeriod() if a.cfg.Bandwidth != nil && a.cfg.Bandwidth.PerEnvironment != nil { @@ -362,20 +510,16 @@ func (a *Agent) checkEnvironmentLimit(u *metrics.Usage, trx *sqlx.Tx) (enforce, if limit.Period > 0 { period = limit.Period } - rx, tx, err := a.ifx.totalRxTxForEnvironment(u.EnvironmentId, period) + rx, tx, err := a.ifx.totalRxTxForEnvironment(envId, period) if err != nil { logrus.Error(err) } enforce, warning = a.checkLimit(limit, rx, tx) - if enforce || warning { - logrus.Debugf("'%v': %v", env.ZId, a.describeLimit(limit, rx, tx)) - } - return enforce, warning, nil } -func (a *Agent) checkShareLimit(u *metrics.Usage) (enforce, warning bool, err error) { +func (a *Agent) checkShareLimit(shrToken string) (enforce, warning bool, err error) { period := 24 * time.Hour limit := DefaultBandwidthPerPeriod() if a.cfg.Bandwidth != nil && a.cfg.Bandwidth.PerShare != nil { @@ -384,14 +528,14 @@ func (a *Agent) checkShareLimit(u *metrics.Usage) (enforce, warning bool, err er if limit.Period > 0 { period = limit.Period } - rx, tx, err := a.ifx.totalRxTxForShare(u.ShareToken, period) + rx, tx, err := a.ifx.totalRxTxForShare(shrToken, period) if err != nil { logrus.Error(err) } enforce, warning = a.checkLimit(limit, rx, tx) if enforce || warning { - logrus.Debugf("'%v': %v", u.ShareToken, a.describeLimit(limit, rx, tx)) + logrus.Debugf("'%v': %v", shrToken, a.describeLimit(limit, rx, tx)) } return enforce, warning, nil diff --git a/controller/store/accountLimitJournal.go b/controller/store/accountLimitJournal.go index 66308bbc..80194efa 100644 --- a/controller/store/accountLimitJournal.go +++ b/controller/store/accountLimitJournal.go @@ -44,15 +44,22 @@ func (str *Store) FindLatestAccountLimitJournal(acctId int, trx *sqlx.Tx) (*Acco func (str *Store) FindAllLatestAccountLimitJournal(trx *sqlx.Tx) ([]*AccountLimitJournal, error) { rows, err := trx.Queryx("select id, account_id, rx_bytes, tx_bytes, action, created_at, updated_at from account_limit_journal where id in (select max(id) as id from account_limit_journal group by account_id)") if err != nil { - return nil, errors.Wrap(err, "error selecting distinct account_limit_jounal") + return nil, errors.Wrap(err, "error selecting all latest account_limit_journal") } - var is []*AccountLimitJournal + var aljs []*AccountLimitJournal for rows.Next() { - i := &AccountLimitJournal{} - if err := rows.StructScan(i); err != nil { + alj := &AccountLimitJournal{} + if err := rows.StructScan(alj); err != nil { return nil, errors.Wrap(err, "error scanning account_limit_journal") } - is = append(is, i) + aljs = append(aljs, alj) } - return is, nil + return aljs, nil +} + +func (str *Store) DeleteAccountLimitJournalForAccount(acctId int, trx *sqlx.Tx) error { + if _, err := trx.Exec("delete from account_limit_journal where account_id = $1", acctId); err != nil { + return errors.Wrapf(err, "error deleting account_limit journal for '#%d'", acctId) + } + return nil } diff --git a/controller/store/environmentLimitJournal.go b/controller/store/environmentLimitJournal.go index 75794185..5a7a2963 100644 --- a/controller/store/environmentLimitJournal.go +++ b/controller/store/environmentLimitJournal.go @@ -40,3 +40,26 @@ func (str *Store) FindLatestEnvironmentLimitJournal(envId int, trx *sqlx.Tx) (*E } return j, nil } + +func (str *Store) FindAllLatestEnvironmentLimitJournal(trx *sqlx.Tx) ([]*EnvironmentLimitJournal, error) { + rows, err := trx.Queryx("select id, environment_id, rx_bytes, tx_bytes, action, created_at, updated_at from environment_limit_journal where id in (select max(id) as id from environment_limit_journal group by environment_id)") + if err != nil { + return nil, errors.Wrap(err, "error selecting all latest environment_limit_journal") + } + var eljs []*EnvironmentLimitJournal + for rows.Next() { + elj := &EnvironmentLimitJournal{} + if err := rows.StructScan(elj); err != nil { + return nil, errors.Wrap(err, "error scanning environment_limit_journal") + } + eljs = append(eljs, elj) + } + return eljs, nil +} + +func (str *Store) DeleteEnvironmentLimitJournalForEnvironment(envId int, trx *sqlx.Tx) error { + if _, err := trx.Exec("delete from environment_limit_journal where environment_id = $1", envId); err != nil { + return errors.Wrapf(err, "error deleteing environment_limit_journal for '#%d'", envId) + } + return nil +} diff --git a/controller/store/shareLimitJournal.go b/controller/store/shareLimitJournal.go index 59891206..7dcc351f 100644 --- a/controller/store/shareLimitJournal.go +++ b/controller/store/shareLimitJournal.go @@ -13,8 +13,8 @@ type ShareLimitJournal struct { Action LimitJournalAction } -func (str *Store) CreateShareLimitJournal(j *ShareLimitJournal, tx *sqlx.Tx) (int, error) { - stmt, err := tx.Prepare("insert into share_limit_journal (share_id, rx_bytes, tx_bytes, action) values ($1, $2, $3, $4) returning id") +func (str *Store) CreateShareLimitJournal(j *ShareLimitJournal, trx *sqlx.Tx) (int, error) { + stmt, err := trx.Prepare("insert into share_limit_journal (share_id, rx_bytes, tx_bytes, action) values ($1, $2, $3, $4) returning id") if err != nil { return 0, errors.Wrap(err, "error preparing share_limit_journal insert statement") } @@ -33,10 +33,33 @@ func (str *Store) IsShareLimitJournalEmpty(shrId int, trx *sqlx.Tx) (bool, error return count == 0, nil } -func (str *Store) FindLatestShareLimitJournal(shrId int, tx *sqlx.Tx) (*ShareLimitJournal, error) { +func (str *Store) FindLatestShareLimitJournal(shrId int, trx *sqlx.Tx) (*ShareLimitJournal, error) { j := &ShareLimitJournal{} - if err := tx.QueryRowx("select * from share_limit_journal where share_id = $1 order by created_at desc limit 1", shrId).StructScan(j); err != nil { + if err := trx.QueryRowx("select * from share_limit_journal where share_id = $1 order by created_at desc limit 1", shrId).StructScan(j); err != nil { return nil, errors.Wrap(err, "error finding share_limit_journal by share_id") } return j, nil } + +func (str *Store) FindAllLatestShareLimitJournal(trx *sqlx.Tx) ([]*ShareLimitJournal, error) { + rows, err := trx.Queryx("select id, share_id, rx_bytes, tx_bytes, action, created_at, updated_at from share_limit_journal where id in (select max(id) as id from share_limit_journal group by share_id)") + if err != nil { + return nil, errors.Wrap(err, "error selecting all latest share_limit_journal") + } + var sljs []*ShareLimitJournal + for rows.Next() { + slj := &ShareLimitJournal{} + if err := rows.StructScan(slj); err != nil { + return nil, errors.Wrap(err, "error scanning share_limit_journal") + } + sljs = append(sljs, slj) + } + return sljs, nil +} + +func (str *Store) DeleteShareLimitJournalForShare(shrId int, trx *sqlx.Tx) error { + if _, err := trx.Exec("delete from share_limit_journal where share_id = $1", shrId); err != nil { + return errors.Wrapf(err, "error deleting share_limit_journal for '#%d'", shrId) + } + return nil +} From c91fc8ac9e21ec670506ff983ef4a1ae6e940df3 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 23 Mar 2023 17:04:10 -0400 Subject: [PATCH 41/83] handlers for account, environment, and strategy (#276) --- controller/limits/config.go | 2 +- controller/limits/model.go | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 controller/limits/model.go diff --git a/controller/limits/config.go b/controller/limits/config.go index 4e6fbbe1..86fea17e 100644 --- a/controller/limits/config.go +++ b/controller/limits/config.go @@ -32,7 +32,7 @@ type Bandwidth struct { func DefaultBandwidthPerPeriod() *BandwidthPerPeriod { return &BandwidthPerPeriod{ - Period: 15 * (24 * time.Hour), + Period: 24 * time.Hour, Warning: &Bandwidth{ Rx: Unlimited, Tx: Unlimited, diff --git a/controller/limits/model.go b/controller/limits/model.go new file mode 100644 index 00000000..2753da32 --- /dev/null +++ b/controller/limits/model.go @@ -0,0 +1,15 @@ +package limits + +import "github.com/openziti/zrok/controller/store" + +type AccountStrategy interface { + HandleAccount(a *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error +} + +type EnvironmentStrategy interface { + HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error +} + +type ShareStrategy interface { + HandleShare(s *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error +} From 14c299ee80285affe38cbdd16e70ce345fc484cc Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 23 Mar 2023 17:07:48 -0400 Subject: [PATCH 42/83] strategy -> action (#276) --- controller/limits/model.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/controller/limits/model.go b/controller/limits/model.go index 2753da32..b4c1a1ad 100644 --- a/controller/limits/model.go +++ b/controller/limits/model.go @@ -2,14 +2,14 @@ package limits import "github.com/openziti/zrok/controller/store" -type AccountStrategy interface { +type AccountAction interface { HandleAccount(a *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error } -type EnvironmentStrategy interface { +type EnvironmentAction interface { HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error } -type ShareStrategy interface { +type ShareAction interface { HandleShare(s *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error } From fdc515487385cf272527b7043c3da3539e853db4 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 23 Mar 2023 17:16:35 -0400 Subject: [PATCH 43/83] necessary actions (#276) --- controller/limits/agent.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 0237ca99..ec2186e5 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -13,13 +13,22 @@ import ( ) type Agent struct { - cfg *Config - ifx *influxReader - zCfg *zrokEdgeSdk.Config - str *store.Store - queue chan *metrics.Usage - close chan struct{} - join chan struct{} + cfg *Config + ifx *influxReader + zCfg *zrokEdgeSdk.Config + str *store.Store + queue chan *metrics.Usage + acctWarningEnforce []AccountAction + acctLimitEnforce []AccountAction + acctLimitRelax []AccountAction + envWarningEnforce []EnvironmentAction + envLimitEnforce []EnvironmentAction + envLimitRelax []EnvironmentAction + shrWarningEnforce []ShareAction + shrLimitEnforce []ShareAction + shrLimitRelax []ShareAction + close chan struct{} + join chan struct{} } func NewAgent(cfg *Config, ifxCfg *metrics.InfluxConfig, zCfg *zrokEdgeSdk.Config, str *store.Store) (*Agent, error) { From 662693c2c9e95ea44cae5cc6af1991f79a47ea7c Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 27 Mar 2023 11:34:29 -0400 Subject: [PATCH 44/83] share action skeletons (#276) --- controller/limits/agent.go | 26 ++++++++++++++++--------- controller/limits/shareLimitAction.go | 21 ++++++++++++++++++++ controller/limits/shareRelaxAction.go | 21 ++++++++++++++++++++ controller/limits/shareWarningAction.go | 21 ++++++++++++++++++++ 4 files changed, 80 insertions(+), 9 deletions(-) create mode 100644 controller/limits/shareLimitAction.go create mode 100644 controller/limits/shareRelaxAction.go create mode 100644 controller/limits/shareWarningAction.go diff --git a/controller/limits/agent.go b/controller/limits/agent.go index ec2186e5..0417ba47 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -32,15 +32,23 @@ type Agent struct { } func NewAgent(cfg *Config, ifxCfg *metrics.InfluxConfig, zCfg *zrokEdgeSdk.Config, str *store.Store) (*Agent, error) { - return &Agent{ - cfg: cfg, - ifx: newInfluxReader(ifxCfg), - zCfg: zCfg, - str: str, - queue: make(chan *metrics.Usage, 1024), - close: make(chan struct{}), - join: make(chan struct{}), - }, nil + edge, err := zrokEdgeSdk.Client(zCfg) + if err != nil { + return nil, err + } + a := &Agent{ + cfg: cfg, + ifx: newInfluxReader(ifxCfg), + zCfg: zCfg, + str: str, + queue: make(chan *metrics.Usage, 1024), + shrWarningEnforce: []ShareAction{newShareWarningAction(str, edge)}, + shrLimitEnforce: []ShareAction{newShareLimitAction(str, edge)}, + shrLimitRelax: []ShareAction{newShareRelaxAction(str, edge)}, + close: make(chan struct{}), + join: make(chan struct{}), + } + return a, nil } func (a *Agent) Start() { diff --git a/controller/limits/shareLimitAction.go b/controller/limits/shareLimitAction.go new file mode 100644 index 00000000..61655ef4 --- /dev/null +++ b/controller/limits/shareLimitAction.go @@ -0,0 +1,21 @@ +package limits + +import ( + "github.com/openziti/edge/rest_management_api_client" + "github.com/openziti/zrok/controller/store" + "github.com/sirupsen/logrus" +) + +type shareLimitAction struct { + str *store.Store + edge *rest_management_api_client.ZitiEdgeManagement +} + +func newShareLimitAction(str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement) *shareLimitAction { + return &shareLimitAction{str, edge} +} + +func (a *shareLimitAction) HandleShare(s *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { + logrus.Infof("limiting '%v'", s.Token) + return nil +} diff --git a/controller/limits/shareRelaxAction.go b/controller/limits/shareRelaxAction.go new file mode 100644 index 00000000..3c805dc7 --- /dev/null +++ b/controller/limits/shareRelaxAction.go @@ -0,0 +1,21 @@ +package limits + +import ( + "github.com/openziti/edge/rest_management_api_client" + "github.com/openziti/zrok/controller/store" + "github.com/sirupsen/logrus" +) + +type shareRelaxAction struct { + str *store.Store + edge *rest_management_api_client.ZitiEdgeManagement +} + +func newShareRelaxAction(str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement) *shareRelaxAction { + return &shareRelaxAction{str, edge} +} + +func (a *shareRelaxAction) HandleShare(s *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { + logrus.Infof("relaxing '%v'", s.Token) + return nil +} diff --git a/controller/limits/shareWarningAction.go b/controller/limits/shareWarningAction.go new file mode 100644 index 00000000..05085c74 --- /dev/null +++ b/controller/limits/shareWarningAction.go @@ -0,0 +1,21 @@ +package limits + +import ( + "github.com/openziti/edge/rest_management_api_client" + "github.com/openziti/zrok/controller/store" + "github.com/sirupsen/logrus" +) + +type shareWarningAction struct { + str *store.Store + edge *rest_management_api_client.ZitiEdgeManagement +} + +func newShareWarningAction(str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement) *shareWarningAction { + return &shareWarningAction{str, edge} +} + +func (a *shareWarningAction) HandleShare(s *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { + logrus.Infof("warning '%v'", s.Token) + return nil +} From 2ab6730e23735fe93a584fb10d6d57d4720c97b0 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 27 Mar 2023 11:43:58 -0400 Subject: [PATCH 45/83] environment action skeletons (#276) --- controller/limits/agent.go | 27 ++++++++++--------- controller/limits/environmentLimitAction.go | 21 +++++++++++++++ controller/limits/environmentRelaxAction.go | 21 +++++++++++++++ controller/limits/environmentWarningAction.go | 21 +++++++++++++++ 4 files changed, 78 insertions(+), 12 deletions(-) create mode 100644 controller/limits/environmentLimitAction.go create mode 100644 controller/limits/environmentRelaxAction.go create mode 100644 controller/limits/environmentWarningAction.go diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 0417ba47..f693c111 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -18,15 +18,15 @@ type Agent struct { zCfg *zrokEdgeSdk.Config str *store.Store queue chan *metrics.Usage - acctWarningEnforce []AccountAction - acctLimitEnforce []AccountAction - acctLimitRelax []AccountAction - envWarningEnforce []EnvironmentAction - envLimitEnforce []EnvironmentAction - envLimitRelax []EnvironmentAction - shrWarningEnforce []ShareAction - shrLimitEnforce []ShareAction - shrLimitRelax []ShareAction + acctWarningActions []AccountAction + acctLimitActions []AccountAction + acctRelaxActions []AccountAction + envWarningActions []EnvironmentAction + envLimitActions []EnvironmentAction + envRelaxActions []EnvironmentAction + shrWarningActions []ShareAction + shrLimitActions []ShareAction + shrRelaxActions []ShareAction close chan struct{} join chan struct{} } @@ -42,9 +42,12 @@ func NewAgent(cfg *Config, ifxCfg *metrics.InfluxConfig, zCfg *zrokEdgeSdk.Confi zCfg: zCfg, str: str, queue: make(chan *metrics.Usage, 1024), - shrWarningEnforce: []ShareAction{newShareWarningAction(str, edge)}, - shrLimitEnforce: []ShareAction{newShareLimitAction(str, edge)}, - shrLimitRelax: []ShareAction{newShareRelaxAction(str, edge)}, + envWarningActions: []EnvironmentAction{newEnvironmentWarningAction(str, edge)}, + envLimitActions: []EnvironmentAction{newEnvironmentLimitAction(str, edge)}, + envRelaxActions: []EnvironmentAction{newEnvironmentRelaxAction(str, edge)}, + shrWarningActions: []ShareAction{newShareWarningAction(str, edge)}, + shrLimitActions: []ShareAction{newShareLimitAction(str, edge)}, + shrRelaxActions: []ShareAction{newShareRelaxAction(str, edge)}, close: make(chan struct{}), join: make(chan struct{}), } diff --git a/controller/limits/environmentLimitAction.go b/controller/limits/environmentLimitAction.go new file mode 100644 index 00000000..c7d32b72 --- /dev/null +++ b/controller/limits/environmentLimitAction.go @@ -0,0 +1,21 @@ +package limits + +import ( + "github.com/openziti/edge/rest_management_api_client" + "github.com/openziti/zrok/controller/store" + "github.com/sirupsen/logrus" +) + +type environmentLimitAction struct { + str *store.Store + edge *rest_management_api_client.ZitiEdgeManagement +} + +func newEnvironmentLimitAction(str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement) *environmentLimitAction { + return &environmentLimitAction{str, edge} +} + +func (a *environmentLimitAction) HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { + logrus.Infof("limiting '%v'", e.ZId) + return nil +} diff --git a/controller/limits/environmentRelaxAction.go b/controller/limits/environmentRelaxAction.go new file mode 100644 index 00000000..de4a833c --- /dev/null +++ b/controller/limits/environmentRelaxAction.go @@ -0,0 +1,21 @@ +package limits + +import ( + "github.com/openziti/edge/rest_management_api_client" + "github.com/openziti/zrok/controller/store" + "github.com/sirupsen/logrus" +) + +type environmentRelaxAction struct { + str *store.Store + edge *rest_management_api_client.ZitiEdgeManagement +} + +func newEnvironmentRelaxAction(str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement) *environmentRelaxAction { + return &environmentRelaxAction{str, edge} +} + +func (a *environmentRelaxAction) HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { + logrus.Infof("relaxing '%v'", e.ZId) + return nil +} diff --git a/controller/limits/environmentWarningAction.go b/controller/limits/environmentWarningAction.go new file mode 100644 index 00000000..b6298aa5 --- /dev/null +++ b/controller/limits/environmentWarningAction.go @@ -0,0 +1,21 @@ +package limits + +import ( + "github.com/openziti/edge/rest_management_api_client" + "github.com/openziti/zrok/controller/store" + "github.com/sirupsen/logrus" +) + +type environmentWarningAction struct { + str *store.Store + edge *rest_management_api_client.ZitiEdgeManagement +} + +func newEnvironmentWarningAction(str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement) *environmentWarningAction { + return &environmentWarningAction{str, edge} +} + +func (a *environmentWarningAction) HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { + logrus.Infof("warning '%v'", e.ZId) + return nil +} From 067d9901d6ee5403573a1c8755a64b259af072e3 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 27 Mar 2023 11:53:18 -0400 Subject: [PATCH 46/83] account action skeletons (#276) --- controller/limits/accountLimitAction.go | 21 ++++++++++++++++ controller/limits/accountRelaxAction.go | 21 ++++++++++++++++ controller/limits/accounttWarningAction.go | 21 ++++++++++++++++ controller/limits/agent.go | 29 ++++++++++++---------- 4 files changed, 79 insertions(+), 13 deletions(-) create mode 100644 controller/limits/accountLimitAction.go create mode 100644 controller/limits/accountRelaxAction.go create mode 100644 controller/limits/accounttWarningAction.go diff --git a/controller/limits/accountLimitAction.go b/controller/limits/accountLimitAction.go new file mode 100644 index 00000000..2b3e5d27 --- /dev/null +++ b/controller/limits/accountLimitAction.go @@ -0,0 +1,21 @@ +package limits + +import ( + "github.com/openziti/edge/rest_management_api_client" + "github.com/openziti/zrok/controller/store" + "github.com/sirupsen/logrus" +) + +type accountLimitAction struct { + str *store.Store + edge *rest_management_api_client.ZitiEdgeManagement +} + +func newAccountLimitAction(str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement) *accountLimitAction { + return &accountLimitAction{str, edge} +} + +func (a *accountLimitAction) HandleAccount(acct *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { + logrus.Infof("limiting '%v'", acct.Email) + return nil +} diff --git a/controller/limits/accountRelaxAction.go b/controller/limits/accountRelaxAction.go new file mode 100644 index 00000000..bac2f4c1 --- /dev/null +++ b/controller/limits/accountRelaxAction.go @@ -0,0 +1,21 @@ +package limits + +import ( + "github.com/openziti/edge/rest_management_api_client" + "github.com/openziti/zrok/controller/store" + "github.com/sirupsen/logrus" +) + +type accountRelaxAction struct { + str *store.Store + edge *rest_management_api_client.ZitiEdgeManagement +} + +func newAccountRelaxAction(str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement) *accountRelaxAction { + return &accountRelaxAction{str, edge} +} + +func (a *accountRelaxAction) HandleAccount(acct *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { + logrus.Infof("relaxing '%v'", acct.Email) + return nil +} diff --git a/controller/limits/accounttWarningAction.go b/controller/limits/accounttWarningAction.go new file mode 100644 index 00000000..0223ed17 --- /dev/null +++ b/controller/limits/accounttWarningAction.go @@ -0,0 +1,21 @@ +package limits + +import ( + "github.com/openziti/edge/rest_management_api_client" + "github.com/openziti/zrok/controller/store" + "github.com/sirupsen/logrus" +) + +type accountWarningAction struct { + str *store.Store + edge *rest_management_api_client.ZitiEdgeManagement +} + +func newAccountWarningAction(str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement) *accountWarningAction { + return &accountWarningAction{str, edge} +} + +func (a *accountWarningAction) HandleAccount(acct *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { + logrus.Infof("warning '%v'", acct.Email) + return nil +} diff --git a/controller/limits/agent.go b/controller/limits/agent.go index f693c111..e2420b53 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -37,19 +37,22 @@ func NewAgent(cfg *Config, ifxCfg *metrics.InfluxConfig, zCfg *zrokEdgeSdk.Confi return nil, err } a := &Agent{ - cfg: cfg, - ifx: newInfluxReader(ifxCfg), - zCfg: zCfg, - str: str, - queue: make(chan *metrics.Usage, 1024), - envWarningActions: []EnvironmentAction{newEnvironmentWarningAction(str, edge)}, - envLimitActions: []EnvironmentAction{newEnvironmentLimitAction(str, edge)}, - envRelaxActions: []EnvironmentAction{newEnvironmentRelaxAction(str, edge)}, - shrWarningActions: []ShareAction{newShareWarningAction(str, edge)}, - shrLimitActions: []ShareAction{newShareLimitAction(str, edge)}, - shrRelaxActions: []ShareAction{newShareRelaxAction(str, edge)}, - close: make(chan struct{}), - join: make(chan struct{}), + cfg: cfg, + ifx: newInfluxReader(ifxCfg), + zCfg: zCfg, + str: str, + queue: make(chan *metrics.Usage, 1024), + acctWarningActions: []AccountAction{newAccountWarningAction(str, edge)}, + acctLimitActions: []AccountAction{newAccountLimitAction(str, edge)}, + acctRelaxActions: []AccountAction{newAccountRelaxAction(str, edge)}, + envWarningActions: []EnvironmentAction{newEnvironmentWarningAction(str, edge)}, + envLimitActions: []EnvironmentAction{newEnvironmentLimitAction(str, edge)}, + envRelaxActions: []EnvironmentAction{newEnvironmentRelaxAction(str, edge)}, + shrWarningActions: []ShareAction{newShareWarningAction(str, edge)}, + shrLimitActions: []ShareAction{newShareLimitAction(str, edge)}, + shrRelaxActions: []ShareAction{newShareRelaxAction(str, edge)}, + close: make(chan struct{}), + join: make(chan struct{}), } return a, nil } From 44cbb8491c2a632738318e78f9ff8ce2c0822dd2 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 27 Mar 2023 13:00:05 -0400 Subject: [PATCH 47/83] action execution logic (#276) --- controller/limits/agent.go | 174 ++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 91 deletions(-) diff --git a/controller/limits/agent.go b/controller/limits/agent.go index e2420b53..226adb90 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -138,7 +138,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { } defer func() { _ = trx.Rollback() }() - if enforce, warning, err := a.checkAccountLimit(u.AccountId); err == nil { + if enforce, warning, rxBytes, txBytes, err := a.checkAccountLimit(u.AccountId); err == nil { if enforce { enforced := false var enforcedAt time.Time @@ -159,9 +159,16 @@ func (a *Agent) enforce(u *metrics.Usage) error { if err != nil { return err } - - logrus.Warnf("enforcing account limit for '#%d'", u.AccountId) - + acct, err := a.str.GetAccount(int(u.AccountId), trx) + if err != nil { + return err + } + // run account limit actions + for _, action := range a.acctLimitActions { + if err := action.HandleAccount(acct, rxBytes, txBytes, a.cfg.Bandwidth.PerAccount); err != nil { + return err + } + } if err := trx.Commit(); err != nil { return err } @@ -189,9 +196,16 @@ func (a *Agent) enforce(u *metrics.Usage) error { if err != nil { return err } - - logrus.Warnf("warning account '#%d'", u.AccountId) - + acct, err := a.str.GetAccount(int(u.AccountId), trx) + if err != nil { + return err + } + // run account warning actions + for _, action := range a.acctWarningActions { + if err := action.HandleAccount(acct, rxBytes, txBytes, a.cfg.Bandwidth.PerAccount); err != nil { + return err + } + } if err := trx.Commit(); err != nil { return err } @@ -200,7 +214,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { } } else { - if enforce, warning, err := a.checkEnvironmentLimit(u.EnvironmentId); err == nil { + if enforce, warning, rxBytes, txBytes, err := a.checkEnvironmentLimit(u.EnvironmentId); err == nil { if enforce { enforced := false var enforcedAt time.Time @@ -221,9 +235,16 @@ func (a *Agent) enforce(u *metrics.Usage) error { if err != nil { return err } - - logrus.Warnf("enforcing environment limit for environment '#%d'", u.EnvironmentId) - + env, err := a.str.GetEnvironment(int(u.EnvironmentId), trx) + if err != nil { + return err + } + // run environment limit actions + for _, action := range a.envLimitActions { + if err := action.HandleEnvironment(env, rxBytes, txBytes, a.cfg.Bandwidth.PerEnvironment); err != nil { + return err + } + } if err := trx.Commit(); err != nil { return err } @@ -251,9 +272,16 @@ func (a *Agent) enforce(u *metrics.Usage) error { if err != nil { return err } - - logrus.Warnf("warning environment '#%d'", u.EnvironmentId) - + env, err := a.str.GetEnvironment(int(u.EnvironmentId), trx) + if err != nil { + return err + } + // run environment warning actions + for _, action := range a.envWarningActions { + if err := action.HandleEnvironment(env, rxBytes, txBytes, a.cfg.Bandwidth.PerEnvironment); err != nil { + return err + } + } if err := trx.Commit(); err != nil { return err } @@ -262,7 +290,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { } } else { - if enforce, warning, err := a.checkShareLimit(u.ShareToken); err == nil { + if enforce, warning, rxBytes, txBytes, err := a.checkShareLimit(u.ShareToken); err == nil { if enforce { shr, err := a.str.FindShareWithToken(u.ShareToken, trx) if err != nil { @@ -288,9 +316,12 @@ func (a *Agent) enforce(u *metrics.Usage) error { if err != nil { return err } - - logrus.Warnf("enforcing share limit for share '%v'", shr.Token) - + // run share limit actions + for _, action := range a.shrLimitActions { + if err := action.HandleShare(shr, rxBytes, txBytes, a.cfg.Bandwidth.PerShare); err != nil { + return err + } + } if err := trx.Commit(); err != nil { return err } @@ -323,9 +354,12 @@ func (a *Agent) enforce(u *metrics.Usage) error { if err != nil { return err } - - logrus.Warnf("warning share '%v'", shr.Token) - + // run share warning actions + for _, action := range a.shrWarningActions { + if err := action.HandleShare(shr, rxBytes, txBytes, a.cfg.Bandwidth.PerShare); err != nil { + return err + } + } if err := trx.Commit(); err != nil { return err } @@ -362,12 +396,15 @@ func (a *Agent) relax() error { if sljs, err := a.str.FindAllLatestShareLimitJournal(trx); err == nil { for _, slj := range sljs { if shr, err := a.str.GetShare(slj.ShareId, trx); err == nil { - switch slj.Action { - case store.WarningAction: - if enforce, warning, err := a.checkShareLimit(shr.Token); err == nil { + if slj.Action == store.WarningAction || slj.Action == store.LimitAction { + if enforce, warning, rxBytes, txBytes, err := a.checkShareLimit(shr.Token); err == nil { if !enforce && !warning { - logrus.Infof("relaxing warning for share '%v'", shr.Token) - + // run relax actions for share + for _, action := range a.shrRelaxActions { + if err := action.HandleShare(shr, rxBytes, txBytes, a.cfg.Bandwidth.PerShare); err != nil { + return err + } + } if err := a.str.DeleteShareLimitJournalForShare(shr.Id, trx); err == nil { commit = true } else { @@ -379,23 +416,6 @@ func (a *Agent) relax() error { } else { logrus.Errorf("error checking share limit for '%v': %v", shr.Token, err) } - - case store.LimitAction: - if enforce, warning, err := a.checkShareLimit(shr.Token); err == nil { - if !enforce && !warning { - logrus.Infof("relaxing limit for share '%v'", shr.Token) - - if err := a.str.DeleteShareLimitJournalForShare(shr.Id, trx); err == nil { - commit = true - } else { - logrus.Errorf("error deleting share_limit_journal for '%v': %v", shr.Token, err) - } - } else { - logrus.Infof("share '%v' still over limit", shr.Token) - } - } else { - logrus.Errorf("error checking share limit for '%v': %v", shr.Token, err) - } } } else { logrus.Errorf("error getting share for '#%d': %v", slj.ShareId, err) @@ -408,29 +428,15 @@ func (a *Agent) relax() error { if eljs, err := a.str.FindAllLatestEnvironmentLimitJournal(trx); err == nil { for _, elj := range eljs { if env, err := a.str.GetEnvironment(elj.EnvironmentId, trx); err == nil { - switch elj.Action { - case store.WarningAction: - if enforce, warning, err := a.checkEnvironmentLimit(int64(elj.EnvironmentId)); err == nil { + if elj.Action == store.WarningAction || elj.Action == store.LimitAction { + if enforce, warning, rxBytes, txBytes, err := a.checkEnvironmentLimit(int64(elj.EnvironmentId)); err == nil { if !enforce && !warning { - logrus.Infof("relaxing warning for environment '%v'", env.ZId) - - if err := a.str.DeleteEnvironmentLimitJournalForEnvironment(env.Id, trx); err == nil { - commit = true - } else { - logrus.Errorf("error deleteing environment_limit_journal for '%v': %v", env.ZId, err) + // run relax actions for environment + for _, action := range a.envRelaxActions { + if err := action.HandleEnvironment(env, rxBytes, txBytes, a.cfg.Bandwidth.PerEnvironment); err != nil { + return err + } } - } else { - logrus.Infof("environment '%v' still over limit", env.ZId) - } - } else { - logrus.Errorf("error checking environment limit for '%v': %v", env.ZId, err) - } - - case store.LimitAction: - if enforce, warning, err := a.checkEnvironmentLimit(int64(elj.EnvironmentId)); err == nil { - if !enforce && !warning { - logrus.Infof("relaxing limit for environment '%v'", env.ZId) - if err := a.str.DeleteEnvironmentLimitJournalForEnvironment(env.Id, trx); err == nil { commit = true } else { @@ -454,29 +460,15 @@ func (a *Agent) relax() error { if aljs, err := a.str.FindAllLatestAccountLimitJournal(trx); err == nil { for _, alj := range aljs { if acct, err := a.str.GetAccount(alj.AccountId, trx); err == nil { - switch alj.Action { - case store.WarningAction: - if enforce, warning, err := a.checkAccountLimit(int64(alj.AccountId)); err == nil { + if alj.Action == store.WarningAction || alj.Action == store.LimitAction { + if enforce, warning, rxBytes, txBytes, err := a.checkAccountLimit(int64(alj.AccountId)); err == nil { if !enforce && !warning { - logrus.Infof("relaxing warning for account '%v'", acct.Email) - - if err := a.str.DeleteAccountLimitJournalForAccount(acct.Id, trx); err == nil { - commit = true - } else { - logrus.Errorf("error deleting account_limit_journal for '%v': %v", acct.Email, err) + // run relax actions for account + for _, action := range a.acctRelaxActions { + if err := action.HandleAccount(acct, rxBytes, txBytes, a.cfg.Bandwidth.PerAccount); err != nil { + return err + } } - } else { - logrus.Infof("account '%v' still over limit", acct.Email) - } - } else { - logrus.Errorf("error checking account limit for '%v': %v", acct.Email, err) - } - - case store.LimitAction: - if enforce, warning, err := a.checkAccountLimit(int64(alj.AccountId)); err == nil { - if !enforce && !warning { - logrus.Infof("relaxing limit for account '%v'", acct.Email) - if err := a.str.DeleteAccountLimitJournalForAccount(acct.Id, trx); err == nil { commit = true } else { @@ -506,7 +498,7 @@ func (a *Agent) relax() error { return nil } -func (a *Agent) checkAccountLimit(acctId int64) (enforce, warning bool, err error) { +func (a *Agent) checkAccountLimit(acctId int64) (enforce, warning bool, rxBytes, txBytes int64, err error) { period := 24 * time.Hour limit := DefaultBandwidthPerPeriod() if a.cfg.Bandwidth != nil && a.cfg.Bandwidth.PerAccount != nil { @@ -521,10 +513,10 @@ func (a *Agent) checkAccountLimit(acctId int64) (enforce, warning bool, err erro } enforce, warning = a.checkLimit(limit, rx, tx) - return enforce, warning, nil + return enforce, warning, rx, tx, nil } -func (a *Agent) checkEnvironmentLimit(envId int64) (enforce, warning bool, err error) { +func (a *Agent) checkEnvironmentLimit(envId int64) (enforce, warning bool, rxBytes, txBytes int64, err error) { period := 24 * time.Hour limit := DefaultBandwidthPerPeriod() if a.cfg.Bandwidth != nil && a.cfg.Bandwidth.PerEnvironment != nil { @@ -539,10 +531,10 @@ func (a *Agent) checkEnvironmentLimit(envId int64) (enforce, warning bool, err e } enforce, warning = a.checkLimit(limit, rx, tx) - return enforce, warning, nil + return enforce, warning, rx, tx, nil } -func (a *Agent) checkShareLimit(shrToken string) (enforce, warning bool, err error) { +func (a *Agent) checkShareLimit(shrToken string) (enforce, warning bool, rxBytes, txBytes int64, err error) { period := 24 * time.Hour limit := DefaultBandwidthPerPeriod() if a.cfg.Bandwidth != nil && a.cfg.Bandwidth.PerShare != nil { @@ -561,7 +553,7 @@ func (a *Agent) checkShareLimit(shrToken string) (enforce, warning bool, err err logrus.Debugf("'%v': %v", shrToken, a.describeLimit(limit, rx, tx)) } - return enforce, warning, nil + return enforce, warning, rx, tx, nil } func (a *Agent) checkLimit(cfg *BandwidthPerPeriod, rx, tx int64) (enforce, warning bool) { From 558606fad326da3acb2b34767d34808c7d3214dd Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 27 Mar 2023 13:51:48 -0400 Subject: [PATCH 48/83] add trx to action interfaces; implement limit and relax for share (#276) --- controller/limits/accountLimitAction.go | 3 ++- controller/limits/accountRelaxAction.go | 3 ++- controller/limits/accounttWarningAction.go | 3 ++- controller/limits/agent.go | 18 ++++++------- controller/limits/environmentLimitAction.go | 3 ++- controller/limits/environmentRelaxAction.go | 3 ++- controller/limits/environmentWarningAction.go | 3 ++- controller/limits/model.go | 11 +++++--- controller/limits/shareLimitAction.go | 15 ++++++++++- controller/limits/shareRelaxAction.go | 26 ++++++++++++++++++- controller/limits/shareWarningAction.go | 3 ++- controller/share.go | 3 +++ 12 files changed, 72 insertions(+), 22 deletions(-) diff --git a/controller/limits/accountLimitAction.go b/controller/limits/accountLimitAction.go index 2b3e5d27..38fb4adb 100644 --- a/controller/limits/accountLimitAction.go +++ b/controller/limits/accountLimitAction.go @@ -1,6 +1,7 @@ package limits import ( + "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/store" "github.com/sirupsen/logrus" @@ -15,7 +16,7 @@ func newAccountLimitAction(str *store.Store, edge *rest_management_api_client.Zi return &accountLimitAction{str, edge} } -func (a *accountLimitAction) HandleAccount(acct *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { +func (a *accountLimitAction) HandleAccount(acct *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { logrus.Infof("limiting '%v'", acct.Email) return nil } diff --git a/controller/limits/accountRelaxAction.go b/controller/limits/accountRelaxAction.go index bac2f4c1..03090537 100644 --- a/controller/limits/accountRelaxAction.go +++ b/controller/limits/accountRelaxAction.go @@ -1,6 +1,7 @@ package limits import ( + "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/store" "github.com/sirupsen/logrus" @@ -15,7 +16,7 @@ func newAccountRelaxAction(str *store.Store, edge *rest_management_api_client.Zi return &accountRelaxAction{str, edge} } -func (a *accountRelaxAction) HandleAccount(acct *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { +func (a *accountRelaxAction) HandleAccount(acct *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { logrus.Infof("relaxing '%v'", acct.Email) return nil } diff --git a/controller/limits/accounttWarningAction.go b/controller/limits/accounttWarningAction.go index 0223ed17..9a79341d 100644 --- a/controller/limits/accounttWarningAction.go +++ b/controller/limits/accounttWarningAction.go @@ -1,6 +1,7 @@ package limits import ( + "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/store" "github.com/sirupsen/logrus" @@ -15,7 +16,7 @@ func newAccountWarningAction(str *store.Store, edge *rest_management_api_client. return &accountWarningAction{str, edge} } -func (a *accountWarningAction) HandleAccount(acct *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { +func (a *accountWarningAction) HandleAccount(acct *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { logrus.Infof("warning '%v'", acct.Email) return nil } diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 226adb90..ba22cec3 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -165,7 +165,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { } // run account limit actions for _, action := range a.acctLimitActions { - if err := action.HandleAccount(acct, rxBytes, txBytes, a.cfg.Bandwidth.PerAccount); err != nil { + if err := action.HandleAccount(acct, rxBytes, txBytes, a.cfg.Bandwidth.PerAccount, trx); err != nil { return err } } @@ -202,7 +202,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { } // run account warning actions for _, action := range a.acctWarningActions { - if err := action.HandleAccount(acct, rxBytes, txBytes, a.cfg.Bandwidth.PerAccount); err != nil { + if err := action.HandleAccount(acct, rxBytes, txBytes, a.cfg.Bandwidth.PerAccount, trx); err != nil { return err } } @@ -241,7 +241,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { } // run environment limit actions for _, action := range a.envLimitActions { - if err := action.HandleEnvironment(env, rxBytes, txBytes, a.cfg.Bandwidth.PerEnvironment); err != nil { + if err := action.HandleEnvironment(env, rxBytes, txBytes, a.cfg.Bandwidth.PerEnvironment, trx); err != nil { return err } } @@ -278,7 +278,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { } // run environment warning actions for _, action := range a.envWarningActions { - if err := action.HandleEnvironment(env, rxBytes, txBytes, a.cfg.Bandwidth.PerEnvironment); err != nil { + if err := action.HandleEnvironment(env, rxBytes, txBytes, a.cfg.Bandwidth.PerEnvironment, trx); err != nil { return err } } @@ -318,7 +318,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { } // run share limit actions for _, action := range a.shrLimitActions { - if err := action.HandleShare(shr, rxBytes, txBytes, a.cfg.Bandwidth.PerShare); err != nil { + if err := action.HandleShare(shr, rxBytes, txBytes, a.cfg.Bandwidth.PerShare, trx); err != nil { return err } } @@ -356,7 +356,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { } // run share warning actions for _, action := range a.shrWarningActions { - if err := action.HandleShare(shr, rxBytes, txBytes, a.cfg.Bandwidth.PerShare); err != nil { + if err := action.HandleShare(shr, rxBytes, txBytes, a.cfg.Bandwidth.PerShare, trx); err != nil { return err } } @@ -401,7 +401,7 @@ func (a *Agent) relax() error { if !enforce && !warning { // run relax actions for share for _, action := range a.shrRelaxActions { - if err := action.HandleShare(shr, rxBytes, txBytes, a.cfg.Bandwidth.PerShare); err != nil { + if err := action.HandleShare(shr, rxBytes, txBytes, a.cfg.Bandwidth.PerShare, trx); err != nil { return err } } @@ -433,7 +433,7 @@ func (a *Agent) relax() error { if !enforce && !warning { // run relax actions for environment for _, action := range a.envRelaxActions { - if err := action.HandleEnvironment(env, rxBytes, txBytes, a.cfg.Bandwidth.PerEnvironment); err != nil { + if err := action.HandleEnvironment(env, rxBytes, txBytes, a.cfg.Bandwidth.PerEnvironment, trx); err != nil { return err } } @@ -465,7 +465,7 @@ func (a *Agent) relax() error { if !enforce && !warning { // run relax actions for account for _, action := range a.acctRelaxActions { - if err := action.HandleAccount(acct, rxBytes, txBytes, a.cfg.Bandwidth.PerAccount); err != nil { + if err := action.HandleAccount(acct, rxBytes, txBytes, a.cfg.Bandwidth.PerAccount, trx); err != nil { return err } } diff --git a/controller/limits/environmentLimitAction.go b/controller/limits/environmentLimitAction.go index c7d32b72..79c78517 100644 --- a/controller/limits/environmentLimitAction.go +++ b/controller/limits/environmentLimitAction.go @@ -1,6 +1,7 @@ package limits import ( + "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/store" "github.com/sirupsen/logrus" @@ -15,7 +16,7 @@ func newEnvironmentLimitAction(str *store.Store, edge *rest_management_api_clien return &environmentLimitAction{str, edge} } -func (a *environmentLimitAction) HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { +func (a *environmentLimitAction) HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { logrus.Infof("limiting '%v'", e.ZId) return nil } diff --git a/controller/limits/environmentRelaxAction.go b/controller/limits/environmentRelaxAction.go index de4a833c..0721b508 100644 --- a/controller/limits/environmentRelaxAction.go +++ b/controller/limits/environmentRelaxAction.go @@ -1,6 +1,7 @@ package limits import ( + "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/store" "github.com/sirupsen/logrus" @@ -15,7 +16,7 @@ func newEnvironmentRelaxAction(str *store.Store, edge *rest_management_api_clien return &environmentRelaxAction{str, edge} } -func (a *environmentRelaxAction) HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { +func (a *environmentRelaxAction) HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { logrus.Infof("relaxing '%v'", e.ZId) return nil } diff --git a/controller/limits/environmentWarningAction.go b/controller/limits/environmentWarningAction.go index b6298aa5..2295a45f 100644 --- a/controller/limits/environmentWarningAction.go +++ b/controller/limits/environmentWarningAction.go @@ -1,6 +1,7 @@ package limits import ( + "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/store" "github.com/sirupsen/logrus" @@ -15,7 +16,7 @@ func newEnvironmentWarningAction(str *store.Store, edge *rest_management_api_cli return &environmentWarningAction{str, edge} } -func (a *environmentWarningAction) HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { +func (a *environmentWarningAction) HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { logrus.Infof("warning '%v'", e.ZId) return nil } diff --git a/controller/limits/model.go b/controller/limits/model.go index b4c1a1ad..c13f8aae 100644 --- a/controller/limits/model.go +++ b/controller/limits/model.go @@ -1,15 +1,18 @@ package limits -import "github.com/openziti/zrok/controller/store" +import ( + "github.com/jmoiron/sqlx" + "github.com/openziti/zrok/controller/store" +) type AccountAction interface { - HandleAccount(a *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error + HandleAccount(a *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error } type EnvironmentAction interface { - HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error + HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error } type ShareAction interface { - HandleShare(s *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error + HandleShare(s *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error } diff --git a/controller/limits/shareLimitAction.go b/controller/limits/shareLimitAction.go index 61655ef4..4daf2551 100644 --- a/controller/limits/shareLimitAction.go +++ b/controller/limits/shareLimitAction.go @@ -1,8 +1,10 @@ package limits import ( + "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/store" + "github.com/openziti/zrok/controller/zrokEdgeSdk" "github.com/sirupsen/logrus" ) @@ -15,7 +17,18 @@ func newShareLimitAction(str *store.Store, edge *rest_management_api_client.Ziti return &shareLimitAction{str, edge} } -func (a *shareLimitAction) HandleShare(s *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { +func (a *shareLimitAction) HandleShare(s *store.Share, _, _ int64, _ *BandwidthPerPeriod, trx *sqlx.Tx) error { logrus.Infof("limiting '%v'", s.Token) + + env, err := a.str.GetEnvironment(s.EnvironmentId, trx) + if err != nil { + return err + } + + if err := zrokEdgeSdk.DeleteServicePolicyDial(env.ZId, s.Token, a.edge); err != nil { + return err + } + logrus.Infof("removed service dial policy for '%v'", s.Token) + return nil } diff --git a/controller/limits/shareRelaxAction.go b/controller/limits/shareRelaxAction.go index 3c805dc7..a13c1a7a 100644 --- a/controller/limits/shareRelaxAction.go +++ b/controller/limits/shareRelaxAction.go @@ -1,8 +1,11 @@ package limits import ( + "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/store" + "github.com/openziti/zrok/controller/zrokEdgeSdk" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -15,7 +18,28 @@ func newShareRelaxAction(str *store.Store, edge *rest_management_api_client.Ziti return &shareRelaxAction{str, edge} } -func (a *shareRelaxAction) HandleShare(s *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { +func (a *shareRelaxAction) HandleShare(s *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { logrus.Infof("relaxing '%v'", s.Token) + + if s.ShareMode == "public" { + env, err := a.str.GetEnvironment(s.EnvironmentId, trx) + if err != nil { + return errors.Wrap(err, "error finding environment") + } + + fe, err := a.str.FindFrontendPubliclyNamed(*s.FrontendSelection, trx) + if err != nil { + return errors.Wrapf(err, "error finding frontend name '%v' for '%v'", *s.FrontendSelection, s.Token) + } + + if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+s.ZId+"-dial", s.ZId, []string{fe.ZId}, zrokEdgeSdk.ZrokShareTags(s.Token).SubTags, a.edge); err != nil { + return errors.Wrapf(err, "error creating dial service policy for '%v'", s.Token) + } + logrus.Infof("added dial service policy for '%v'", s.Token) + + } else if s.ShareMode == "private" { + return errors.New("share relax for private share not implemented") + } + return nil } diff --git a/controller/limits/shareWarningAction.go b/controller/limits/shareWarningAction.go index 05085c74..90119879 100644 --- a/controller/limits/shareWarningAction.go +++ b/controller/limits/shareWarningAction.go @@ -1,6 +1,7 @@ package limits import ( + "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/store" "github.com/sirupsen/logrus" @@ -15,7 +16,7 @@ func newShareWarningAction(str *store.Store, edge *rest_management_api_client.Zi return &shareWarningAction{str, edge} } -func (a *shareWarningAction) HandleShare(s *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod) error { +func (a *shareWarningAction) HandleShare(s *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { logrus.Infof("warning '%v'", s.Token) return nil } diff --git a/controller/share.go b/controller/share.go index ec2d9936..4b5b5b1b 100644 --- a/controller/share.go +++ b/controller/share.go @@ -115,6 +115,9 @@ func (h *shareHandler) Handle(params share.ShareParams, principal *rest_model_zr BackendProxyEndpoint: ¶ms.Body.BackendProxyEndpoint, Reserved: reserved, } + if len(params.Body.FrontendSelection) > 0 { + sshr.FrontendSelection = ¶ms.Body.FrontendSelection[0] + } if len(frontendEndpoints) > 0 { sshr.FrontendEndpoint = &frontendEndpoints[0] } else if sshr.ShareMode == "private" { From a6c2841cf2e3885ca3e8535d525518bf25ea98d0 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 27 Mar 2023 14:01:31 -0400 Subject: [PATCH 49/83] environment limit and relax actions (#276) --- controller/limits/environmentLimitAction.go | 19 ++++++++++-- controller/limits/environmentRelaxAction.go | 34 +++++++++++++++++++-- controller/limits/shareLimitAction.go | 10 +++--- controller/limits/shareRelaxAction.go | 20 ++++++------ 4 files changed, 64 insertions(+), 19 deletions(-) diff --git a/controller/limits/environmentLimitAction.go b/controller/limits/environmentLimitAction.go index 79c78517..ce26cafc 100644 --- a/controller/limits/environmentLimitAction.go +++ b/controller/limits/environmentLimitAction.go @@ -4,6 +4,8 @@ import ( "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/store" + "github.com/openziti/zrok/controller/zrokEdgeSdk" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -16,7 +18,20 @@ func newEnvironmentLimitAction(str *store.Store, edge *rest_management_api_clien return &environmentLimitAction{str, edge} } -func (a *environmentLimitAction) HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { - logrus.Infof("limiting '%v'", e.ZId) +func (a *environmentLimitAction) HandleEnvironment(env *store.Environment, _, _ int64, _ *BandwidthPerPeriod, trx *sqlx.Tx) error { + logrus.Infof("limiting '%v'", env.ZId) + + shrs, err := a.str.FindSharesForEnvironment(env.Id, trx) + if err != nil { + return errors.Wrapf(err, "error finding shares for environment '%v'", env.ZId) + } + + for _, shr := range shrs { + if err := zrokEdgeSdk.DeleteServicePolicyDial(env.ZId, shr.Token, a.edge); err != nil { + return errors.Wrapf(err, "error deleting dial service policy for '%v'", shr.Token) + } + logrus.Infof("removed dial service policy for share '%v' of environment '%v'", shr.Token, env.ZId) + } + return nil } diff --git a/controller/limits/environmentRelaxAction.go b/controller/limits/environmentRelaxAction.go index 0721b508..aa79ade3 100644 --- a/controller/limits/environmentRelaxAction.go +++ b/controller/limits/environmentRelaxAction.go @@ -4,6 +4,8 @@ import ( "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/store" + "github.com/openziti/zrok/controller/zrokEdgeSdk" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -16,7 +18,35 @@ func newEnvironmentRelaxAction(str *store.Store, edge *rest_management_api_clien return &environmentRelaxAction{str, edge} } -func (a *environmentRelaxAction) HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { - logrus.Infof("relaxing '%v'", e.ZId) +func (a *environmentRelaxAction) HandleEnvironment(env *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { + logrus.Infof("relaxing '%v'", env.ZId) + + shrs, err := a.str.FindSharesForEnvironment(env.Id, trx) + if err != nil { + return errors.Wrapf(err, "error finding shares for environment '%v'", env.ZId) + } + + for _, shr := range shrs { + if shr.ShareMode == "public" { + env, err := a.str.GetEnvironment(shr.EnvironmentId, trx) + if err != nil { + return errors.Wrap(err, "error finding environment") + } + + fe, err := a.str.FindFrontendPubliclyNamed(*shr.FrontendSelection, trx) + if err != nil { + return errors.Wrapf(err, "error finding frontend name '%v' for '%v'", *shr.FrontendSelection, shr.Token) + } + + if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+shr.ZId+"-dial", shr.ZId, []string{fe.ZId}, zrokEdgeSdk.ZrokShareTags(shr.Token).SubTags, a.edge); err != nil { + return errors.Wrapf(err, "error creating dial service policy for '%v'", shr.Token) + } + logrus.Infof("added dial service policy for '%v'", shr.Token) + + } else if shr.ShareMode == "private" { + return errors.New("share relax for private share not implemented") + } + } + return nil } diff --git a/controller/limits/shareLimitAction.go b/controller/limits/shareLimitAction.go index 4daf2551..7eefffb2 100644 --- a/controller/limits/shareLimitAction.go +++ b/controller/limits/shareLimitAction.go @@ -17,18 +17,18 @@ func newShareLimitAction(str *store.Store, edge *rest_management_api_client.Ziti return &shareLimitAction{str, edge} } -func (a *shareLimitAction) HandleShare(s *store.Share, _, _ int64, _ *BandwidthPerPeriod, trx *sqlx.Tx) error { - logrus.Infof("limiting '%v'", s.Token) +func (a *shareLimitAction) HandleShare(shr *store.Share, _, _ int64, _ *BandwidthPerPeriod, trx *sqlx.Tx) error { + logrus.Infof("limiting '%v'", shr.Token) - env, err := a.str.GetEnvironment(s.EnvironmentId, trx) + env, err := a.str.GetEnvironment(shr.EnvironmentId, trx) if err != nil { return err } - if err := zrokEdgeSdk.DeleteServicePolicyDial(env.ZId, s.Token, a.edge); err != nil { + if err := zrokEdgeSdk.DeleteServicePolicyDial(env.ZId, shr.Token, a.edge); err != nil { return err } - logrus.Infof("removed service dial policy for '%v'", s.Token) + logrus.Infof("removed dial service policy for '%v'", shr.Token) return nil } diff --git a/controller/limits/shareRelaxAction.go b/controller/limits/shareRelaxAction.go index a13c1a7a..e702cf01 100644 --- a/controller/limits/shareRelaxAction.go +++ b/controller/limits/shareRelaxAction.go @@ -18,26 +18,26 @@ func newShareRelaxAction(str *store.Store, edge *rest_management_api_client.Ziti return &shareRelaxAction{str, edge} } -func (a *shareRelaxAction) HandleShare(s *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { - logrus.Infof("relaxing '%v'", s.Token) +func (a *shareRelaxAction) HandleShare(shr *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { + logrus.Infof("relaxing '%v'", shr.Token) - if s.ShareMode == "public" { - env, err := a.str.GetEnvironment(s.EnvironmentId, trx) + if shr.ShareMode == "public" { + env, err := a.str.GetEnvironment(shr.EnvironmentId, trx) if err != nil { return errors.Wrap(err, "error finding environment") } - fe, err := a.str.FindFrontendPubliclyNamed(*s.FrontendSelection, trx) + fe, err := a.str.FindFrontendPubliclyNamed(*shr.FrontendSelection, trx) if err != nil { - return errors.Wrapf(err, "error finding frontend name '%v' for '%v'", *s.FrontendSelection, s.Token) + return errors.Wrapf(err, "error finding frontend name '%v' for '%v'", *shr.FrontendSelection, shr.Token) } - if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+s.ZId+"-dial", s.ZId, []string{fe.ZId}, zrokEdgeSdk.ZrokShareTags(s.Token).SubTags, a.edge); err != nil { - return errors.Wrapf(err, "error creating dial service policy for '%v'", s.Token) + if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+shr.ZId+"-dial", shr.ZId, []string{fe.ZId}, zrokEdgeSdk.ZrokShareTags(shr.Token).SubTags, a.edge); err != nil { + return errors.Wrapf(err, "error creating dial service policy for '%v'", shr.Token) } - logrus.Infof("added dial service policy for '%v'", s.Token) + logrus.Infof("added dial service policy for '%v'", shr.Token) - } else if s.ShareMode == "private" { + } else if shr.ShareMode == "private" { return errors.New("share relax for private share not implemented") } From 1c544c6c33903a4a39a5ae107b341199d0ba4c82 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 27 Mar 2023 14:06:57 -0400 Subject: [PATCH 50/83] acount limit and relax action implementations (#276) --- controller/limits/accountLimitAction.go | 22 ++++++++++++++ controller/limits/accountRelaxAction.go | 39 ++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/controller/limits/accountLimitAction.go b/controller/limits/accountLimitAction.go index 38fb4adb..919166bc 100644 --- a/controller/limits/accountLimitAction.go +++ b/controller/limits/accountLimitAction.go @@ -4,6 +4,8 @@ import ( "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/store" + "github.com/openziti/zrok/controller/zrokEdgeSdk" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -18,5 +20,25 @@ func newAccountLimitAction(str *store.Store, edge *rest_management_api_client.Zi func (a *accountLimitAction) HandleAccount(acct *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { logrus.Infof("limiting '%v'", acct.Email) + + envs, err := a.str.FindEnvironmentsForAccount(acct.Id, trx) + if err != nil { + return errors.Wrapf(err, "error finding environments for account '%v'", acct.Email) + } + + for _, env := range envs { + shrs, err := a.str.FindSharesForEnvironment(env.Id, trx) + if err != nil { + return errors.Wrapf(err, "error finding shares for environment '%v'", env.ZId) + } + + for _, shr := range shrs { + if err := zrokEdgeSdk.DeleteServicePolicyDial(env.ZId, shr.Token, a.edge); err != nil { + return errors.Wrapf(err, "error deleting dial service policy for '%v'", shr.Token) + } + logrus.Infof("removed dial service policy for share '%v' of environment '%v'", shr.Token, env.ZId) + } + } + return nil } diff --git a/controller/limits/accountRelaxAction.go b/controller/limits/accountRelaxAction.go index 03090537..cea7aa21 100644 --- a/controller/limits/accountRelaxAction.go +++ b/controller/limits/accountRelaxAction.go @@ -4,6 +4,8 @@ import ( "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/store" + "github.com/openziti/zrok/controller/zrokEdgeSdk" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -16,7 +18,42 @@ func newAccountRelaxAction(str *store.Store, edge *rest_management_api_client.Zi return &accountRelaxAction{str, edge} } -func (a *accountRelaxAction) HandleAccount(acct *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { +func (a *accountRelaxAction) HandleAccount(acct *store.Account, _, _ int64, _ *BandwidthPerPeriod, trx *sqlx.Tx) error { logrus.Infof("relaxing '%v'", acct.Email) + + envs, err := a.str.FindEnvironmentsForAccount(acct.Id, trx) + if err != nil { + return errors.Wrapf(err, "error finding environments for account '%v'", acct.Email) + } + + for _, env := range envs { + shrs, err := a.str.FindSharesForEnvironment(env.Id, trx) + if err != nil { + return errors.Wrapf(err, "error finding shares for environment '%v'", env.ZId) + } + + for _, shr := range shrs { + if shr.ShareMode == "public" { + env, err := a.str.GetEnvironment(shr.EnvironmentId, trx) + if err != nil { + return errors.Wrap(err, "error finding environment") + } + + fe, err := a.str.FindFrontendPubliclyNamed(*shr.FrontendSelection, trx) + if err != nil { + return errors.Wrapf(err, "error finding frontend name '%v' for '%v'", *shr.FrontendSelection, shr.Token) + } + + if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+shr.ZId+"-dial", shr.ZId, []string{fe.ZId}, zrokEdgeSdk.ZrokShareTags(shr.Token).SubTags, a.edge); err != nil { + return errors.Wrapf(err, "error creating dial service policy for '%v'", shr.Token) + } + logrus.Infof("added dial service policy for '%v'", shr.Token) + + } else if shr.ShareMode == "private" { + return errors.New("share relax for private share not implemented") + } + } + } + return nil } From bb61bdb664147e04fcefa15c8919bbf6ed4d2315 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 27 Mar 2023 15:07:09 -0400 Subject: [PATCH 51/83] limit warning emails (#276) --- controller/emailUi/embed.go | 2 +- controller/emailUi/limitWarning.gohtml | 153 +++++++++++++++++++++++++ controller/emailUi/limitWarning.gotext | 3 + 3 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 controller/emailUi/limitWarning.gohtml create mode 100644 controller/emailUi/limitWarning.gotext diff --git a/controller/emailUi/embed.go b/controller/emailUi/embed.go index ddb7389a..8707ddec 100644 --- a/controller/emailUi/embed.go +++ b/controller/emailUi/embed.go @@ -2,5 +2,5 @@ package emailUi import "embed" -//go:embed verify.gohtml verify.gotext resetPassword.gohtml resetPassword.gotext +//go:embed verify.gohtml verify.gotext resetPassword.gohtml resetPassword.gotext limitWarning.gohtml limitWarning.gotext var FS embed.FS diff --git a/controller/emailUi/limitWarning.gohtml b/controller/emailUi/limitWarning.gohtml new file mode 100644 index 00000000..4626b954 --- /dev/null +++ b/controller/emailUi/limitWarning.gohtml @@ -0,0 +1,153 @@ + + + + + + + Transfer limit warning! + + + + + + + + + + +
+ +
+

Your account is reaching a transfer size limit, {{ .EmailAddress }}.

+

{{ .Detail }}}

+
+ + + + + + + + +
github.com/openziti/zrok
{{ .Version }}
+

Copyright © 2023 NetFoundry, Inc.

+
+ + \ No newline at end of file diff --git a/controller/emailUi/limitWarning.gotext b/controller/emailUi/limitWarning.gotext new file mode 100644 index 00000000..8a07e07e --- /dev/null +++ b/controller/emailUi/limitWarning.gotext @@ -0,0 +1,3 @@ +Your account is nearing a transfer size limit, {{ .EmailAddress }}! + +{{ .Detail }} \ No newline at end of file From d279fbb8cb16b0af3c6cfa1651e540a911da7efc Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 27 Mar 2023 15:29:25 -0400 Subject: [PATCH 52/83] roughed in limit warning email actions (#276) --- controller/config/config.go | 11 +--- controller/controller.go | 2 +- controller/emailUi/config.go | 9 +++ controller/emailUi/model.go | 24 ++++++++ ...rningAction.go => accountWarningAction.go} | 12 +++- controller/limits/agent.go | 13 +++-- controller/limits/email.go | 58 +++++++++++++++++++ controller/limits/environmentWarningAction.go | 23 ++++++-- controller/limits/shareWarningAction.go | 26 +++++++-- 9 files changed, 152 insertions(+), 26 deletions(-) create mode 100644 controller/emailUi/config.go create mode 100644 controller/emailUi/model.go rename controller/limits/{accounttWarningAction.go => accountWarningAction.go} (51%) create mode 100644 controller/limits/email.go diff --git a/controller/config/config.go b/controller/config/config.go index d2ac3eb6..24422afd 100644 --- a/controller/config/config.go +++ b/controller/config/config.go @@ -1,6 +1,7 @@ package config import ( + "github.com/openziti/zrok/controller/emailUi" "github.com/openziti/zrok/controller/env" "github.com/openziti/zrok/controller/limits" "github.com/openziti/zrok/controller/metrics" @@ -19,7 +20,7 @@ type Config struct { Admin *AdminConfig Bridge *metrics.BridgeConfig Endpoint *EndpointConfig - Email *EmailConfig + Email *emailUi.Config Limits *limits.Config Maintenance *MaintenanceConfig Metrics *metrics.Config @@ -39,14 +40,6 @@ type EndpointConfig struct { Port int } -type EmailConfig struct { - Host string - Port int - Username string - Password string `cf:"+secret"` - From string -} - type RegistrationConfig struct { RegistrationUrlTemplate string TokenStrategy string diff --git a/controller/controller.go b/controller/controller.go index 987e2b4a..1d05e128 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -84,7 +84,7 @@ func Run(inCfg *config.Config) error { defer func() { ma.Stop() }() if cfg.Limits != nil && cfg.Limits.Enforcing { - limitsAgent, err = limits.NewAgent(cfg.Limits, cfg.Metrics.Influx, cfg.Ziti, str) + limitsAgent, err = limits.NewAgent(cfg.Limits, cfg.Metrics.Influx, cfg.Ziti, cfg.Email, str) if err != nil { return errors.Wrap(err, "error creating limits agent") } diff --git a/controller/emailUi/config.go b/controller/emailUi/config.go new file mode 100644 index 00000000..a97f141a --- /dev/null +++ b/controller/emailUi/config.go @@ -0,0 +1,9 @@ +package emailUi + +type Config struct { + Host string + Port int + Username string + Password string `cf:"+secret"` + From string +} diff --git a/controller/emailUi/model.go b/controller/emailUi/model.go new file mode 100644 index 00000000..8e30b336 --- /dev/null +++ b/controller/emailUi/model.go @@ -0,0 +1,24 @@ +package emailUi + +import ( + "bytes" + "github.com/pkg/errors" + "html/template" +) + +type WarningEmail struct { + EmailAddress string + Detail string +} + +func (we WarningEmail) MergeTemplate(filename string) (string, error) { + t, err := template.ParseFS(FS, filename) + if err != nil { + return "", errors.Wrapf(err, "error parsing warning email template '%v'", filename) + } + buf := new(bytes.Buffer) + if err := t.Execute(buf, we); err != nil { + return "", errors.Wrapf(err, "error executing warning email template '%v'", filename) + } + return buf.String(), nil +} diff --git a/controller/limits/accounttWarningAction.go b/controller/limits/accountWarningAction.go similarity index 51% rename from controller/limits/accounttWarningAction.go rename to controller/limits/accountWarningAction.go index 9a79341d..39fb9582 100644 --- a/controller/limits/accounttWarningAction.go +++ b/controller/limits/accountWarningAction.go @@ -3,20 +3,28 @@ package limits import ( "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" + "github.com/openziti/zrok/controller/emailUi" "github.com/openziti/zrok/controller/store" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) type accountWarningAction struct { str *store.Store edge *rest_management_api_client.ZitiEdgeManagement + cfg *emailUi.Config } -func newAccountWarningAction(str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement) *accountWarningAction { - return &accountWarningAction{str, edge} +func newAccountWarningAction(cfg *emailUi.Config, str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement) *accountWarningAction { + return &accountWarningAction{str, edge, cfg} } func (a *accountWarningAction) HandleAccount(acct *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { logrus.Infof("warning '%v'", acct.Email) + + if err := sendLimitWarningEmail(a.cfg, acct.Email, limit, rxBytes, txBytes); err != nil { + return errors.Wrapf(err, "error sending limit warning email to '%v'", acct.Email) + } + return nil } diff --git a/controller/limits/agent.go b/controller/limits/agent.go index ba22cec3..3a8b9915 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -3,6 +3,7 @@ package limits import ( "fmt" "github.com/jmoiron/sqlx" + "github.com/openziti/zrok/controller/emailUi" "github.com/openziti/zrok/controller/metrics" "github.com/openziti/zrok/controller/store" "github.com/openziti/zrok/controller/zrokEdgeSdk" @@ -31,7 +32,7 @@ type Agent struct { join chan struct{} } -func NewAgent(cfg *Config, ifxCfg *metrics.InfluxConfig, zCfg *zrokEdgeSdk.Config, str *store.Store) (*Agent, error) { +func NewAgent(cfg *Config, ifxCfg *metrics.InfluxConfig, zCfg *zrokEdgeSdk.Config, emailCfg *emailUi.Config, str *store.Store) (*Agent, error) { edge, err := zrokEdgeSdk.Client(zCfg) if err != nil { return nil, err @@ -42,13 +43,13 @@ func NewAgent(cfg *Config, ifxCfg *metrics.InfluxConfig, zCfg *zrokEdgeSdk.Confi zCfg: zCfg, str: str, queue: make(chan *metrics.Usage, 1024), - acctWarningActions: []AccountAction{newAccountWarningAction(str, edge)}, + acctWarningActions: []AccountAction{newAccountWarningAction(emailCfg, str, edge)}, acctLimitActions: []AccountAction{newAccountLimitAction(str, edge)}, acctRelaxActions: []AccountAction{newAccountRelaxAction(str, edge)}, - envWarningActions: []EnvironmentAction{newEnvironmentWarningAction(str, edge)}, + envWarningActions: []EnvironmentAction{newEnvironmentWarningAction(emailCfg, str, edge)}, envLimitActions: []EnvironmentAction{newEnvironmentLimitAction(str, edge)}, envRelaxActions: []EnvironmentAction{newEnvironmentRelaxAction(str, edge)}, - shrWarningActions: []ShareAction{newShareWarningAction(str, edge)}, + shrWarningActions: []ShareAction{newShareWarningAction(emailCfg, str, edge)}, shrLimitActions: []ShareAction{newShareLimitAction(str, edge)}, shrRelaxActions: []ShareAction{newShareRelaxAction(str, edge)}, close: make(chan struct{}), @@ -550,7 +551,7 @@ func (a *Agent) checkShareLimit(shrToken string) (enforce, warning bool, rxBytes enforce, warning = a.checkLimit(limit, rx, tx) if enforce || warning { - logrus.Debugf("'%v': %v", shrToken, a.describeLimit(limit, rx, tx)) + logrus.Debugf("'%v': %v", shrToken, describeLimit(limit, rx, tx)) } return enforce, warning, rx, tx, nil @@ -580,7 +581,7 @@ func (a *Agent) checkLimit(cfg *BandwidthPerPeriod, rx, tx int64) (enforce, warn return false, false } -func (a *Agent) describeLimit(cfg *BandwidthPerPeriod, rx, tx int64) string { +func describeLimit(cfg *BandwidthPerPeriod, rx, tx int64) string { out := "" if cfg.Limit.Rx != Unlimited && rx > cfg.Limit.Rx { diff --git a/controller/limits/email.go b/controller/limits/email.go new file mode 100644 index 00000000..56680aff --- /dev/null +++ b/controller/limits/email.go @@ -0,0 +1,58 @@ +package limits + +import ( + "github.com/openziti/zrok/controller/emailUi" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/wneessen/go-mail" +) + +func sendLimitWarningEmail(cfg *emailUi.Config, emailTo string, limit *BandwidthPerPeriod, rxBytes, txBytes int64) error { + emailData := &emailUi.WarningEmail{ + EmailAddress: emailTo, + Detail: describeLimit(limit, rxBytes, txBytes), + } + + plainBody, err := emailData.MergeTemplate("limitWarning.gotext") + if err != nil { + return err + } + htmlBody, err := emailData.MergeTemplate("resetPassword.gohtml") + if err != nil { + return err + } + + msg := mail.NewMsg() + if err := msg.From(cfg.From); err != nil { + return errors.Wrap(err, "failed to set from address in limit warning email") + } + if err := msg.To(emailTo); err != nil { + return errors.Wrap(err, "failed to set to address in limit warning email") + } + + msg.Subject("Limit Warning Notification") + msg.SetDate() + msg.SetMessageID() + msg.SetBulk() + msg.SetImportance(mail.ImportanceHigh) + msg.SetBodyString(mail.TypeTextPlain, plainBody) + msg.SetBodyString(mail.TypeTextHTML, htmlBody) + + client, err := mail.NewClient(cfg.Host, + mail.WithPort(cfg.Port), + mail.WithSMTPAuth(mail.SMTPAuthPlain), + mail.WithUsername(cfg.Username), + mail.WithPassword(cfg.Password), + mail.WithTLSPolicy(mail.TLSMandatory), + ) + + if err != nil { + return errors.Wrap(err, "error creating limit warning email client") + } + if err := client.DialAndSend(msg); err != nil { + return errors.Wrap(err, "error sending limit warning email") + } + + logrus.Infof("limit warning email sent to '%v'", emailTo) + return nil +} diff --git a/controller/limits/environmentWarningAction.go b/controller/limits/environmentWarningAction.go index 2295a45f..c3f25903 100644 --- a/controller/limits/environmentWarningAction.go +++ b/controller/limits/environmentWarningAction.go @@ -3,20 +3,35 @@ package limits import ( "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" + "github.com/openziti/zrok/controller/emailUi" "github.com/openziti/zrok/controller/store" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) type environmentWarningAction struct { str *store.Store edge *rest_management_api_client.ZitiEdgeManagement + cfg *emailUi.Config } -func newEnvironmentWarningAction(str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement) *environmentWarningAction { - return &environmentWarningAction{str, edge} +func newEnvironmentWarningAction(cfg *emailUi.Config, str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement) *environmentWarningAction { + return &environmentWarningAction{str, edge, cfg} } -func (a *environmentWarningAction) HandleEnvironment(e *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { - logrus.Infof("warning '%v'", e.ZId) +func (a *environmentWarningAction) HandleEnvironment(env *store.Environment, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { + logrus.Infof("warning '%v'", env.ZId) + + if env.AccountId != nil { + acct, err := a.str.GetAccount(*env.AccountId, trx) + if err != nil { + return err + } + + if err := sendLimitWarningEmail(a.cfg, acct.Email, limit, rxBytes, txBytes); err != nil { + return errors.Wrapf(err, "error sending limit warning email to '%v'", acct.Email) + } + } + return nil } diff --git a/controller/limits/shareWarningAction.go b/controller/limits/shareWarningAction.go index 90119879..c8ca9e65 100644 --- a/controller/limits/shareWarningAction.go +++ b/controller/limits/shareWarningAction.go @@ -3,20 +3,38 @@ package limits import ( "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" + "github.com/openziti/zrok/controller/emailUi" "github.com/openziti/zrok/controller/store" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) type shareWarningAction struct { str *store.Store edge *rest_management_api_client.ZitiEdgeManagement + cfg *emailUi.Config } -func newShareWarningAction(str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement) *shareWarningAction { - return &shareWarningAction{str, edge} +func newShareWarningAction(cfg *emailUi.Config, str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement) *shareWarningAction { + return &shareWarningAction{str, edge, cfg} } -func (a *shareWarningAction) HandleShare(s *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { - logrus.Infof("warning '%v'", s.Token) +func (a *shareWarningAction) HandleShare(shr *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { + logrus.Infof("warning '%v'", shr.Token) + + env, err := a.str.GetEnvironment(shr.EnvironmentId, trx) + if err != nil { + return err + } + + acct, err := a.str.GetAccount(env.Id, trx) + if err != nil { + return err + } + + if err := sendLimitWarningEmail(a.cfg, acct.Email, limit, rxBytes, txBytes); err != nil { + return errors.Wrapf(err, "error sending limit warning email to '%v'", acct.Email) + } + return nil } From 9a6f6a8e2fc1374b40461a08d2ca68c71906d5b6 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 28 Mar 2023 14:39:42 -0400 Subject: [PATCH 53/83] tweaks, improvements, and minor fixes to limits infrastructure as a result of share limit testing (#276) --- controller/emailUi/model.go | 1 + controller/limits/agent.go | 19 ++++++++++--------- controller/limits/email.go | 4 +++- controller/limits/shareWarningAction.go | 14 ++++++++------ 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/controller/emailUi/model.go b/controller/emailUi/model.go index 8e30b336..829a8372 100644 --- a/controller/emailUi/model.go +++ b/controller/emailUi/model.go @@ -9,6 +9,7 @@ import ( type WarningEmail struct { EmailAddress string Detail string + Version string } func (we WarningEmail) MergeTemplate(filename string) (string, error) { diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 3a8b9915..67e5632e 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -10,6 +10,7 @@ import ( "github.com/openziti/zrok/util" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "reflect" "time" ) @@ -167,7 +168,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { // run account limit actions for _, action := range a.acctLimitActions { if err := action.HandleAccount(acct, rxBytes, txBytes, a.cfg.Bandwidth.PerAccount, trx); err != nil { - return err + return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) } } if err := trx.Commit(); err != nil { @@ -204,7 +205,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { // run account warning actions for _, action := range a.acctWarningActions { if err := action.HandleAccount(acct, rxBytes, txBytes, a.cfg.Bandwidth.PerAccount, trx); err != nil { - return err + return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) } } if err := trx.Commit(); err != nil { @@ -243,7 +244,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { // run environment limit actions for _, action := range a.envLimitActions { if err := action.HandleEnvironment(env, rxBytes, txBytes, a.cfg.Bandwidth.PerEnvironment, trx); err != nil { - return err + return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) } } if err := trx.Commit(); err != nil { @@ -280,7 +281,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { // run environment warning actions for _, action := range a.envWarningActions { if err := action.HandleEnvironment(env, rxBytes, txBytes, a.cfg.Bandwidth.PerEnvironment, trx); err != nil { - return err + return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) } } if err := trx.Commit(); err != nil { @@ -320,7 +321,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { // run share limit actions for _, action := range a.shrLimitActions { if err := action.HandleShare(shr, rxBytes, txBytes, a.cfg.Bandwidth.PerShare, trx); err != nil { - return err + return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) } } if err := trx.Commit(); err != nil { @@ -358,7 +359,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { // run share warning actions for _, action := range a.shrWarningActions { if err := action.HandleShare(shr, rxBytes, txBytes, a.cfg.Bandwidth.PerShare, trx); err != nil { - return err + return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) } } if err := trx.Commit(); err != nil { @@ -403,7 +404,7 @@ func (a *Agent) relax() error { // run relax actions for share for _, action := range a.shrRelaxActions { if err := action.HandleShare(shr, rxBytes, txBytes, a.cfg.Bandwidth.PerShare, trx); err != nil { - return err + return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) } } if err := a.str.DeleteShareLimitJournalForShare(shr.Id, trx); err == nil { @@ -435,7 +436,7 @@ func (a *Agent) relax() error { // run relax actions for environment for _, action := range a.envRelaxActions { if err := action.HandleEnvironment(env, rxBytes, txBytes, a.cfg.Bandwidth.PerEnvironment, trx); err != nil { - return err + return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) } } if err := a.str.DeleteEnvironmentLimitJournalForEnvironment(env.Id, trx); err == nil { @@ -467,7 +468,7 @@ func (a *Agent) relax() error { // run relax actions for account for _, action := range a.acctRelaxActions { if err := action.HandleAccount(acct, rxBytes, txBytes, a.cfg.Bandwidth.PerAccount, trx); err != nil { - return err + return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) } } if err := a.str.DeleteAccountLimitJournalForAccount(acct.Id, trx); err == nil { diff --git a/controller/limits/email.go b/controller/limits/email.go index 56680aff..ce60bc8e 100644 --- a/controller/limits/email.go +++ b/controller/limits/email.go @@ -1,6 +1,7 @@ package limits import ( + "github.com/openziti/zrok/build" "github.com/openziti/zrok/controller/emailUi" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -11,13 +12,14 @@ func sendLimitWarningEmail(cfg *emailUi.Config, emailTo string, limit *Bandwidth emailData := &emailUi.WarningEmail{ EmailAddress: emailTo, Detail: describeLimit(limit, rxBytes, txBytes), + Version: build.String(), } plainBody, err := emailData.MergeTemplate("limitWarning.gotext") if err != nil { return err } - htmlBody, err := emailData.MergeTemplate("resetPassword.gohtml") + htmlBody, err := emailData.MergeTemplate("limitWarning.gohtml") if err != nil { return err } diff --git a/controller/limits/shareWarningAction.go b/controller/limits/shareWarningAction.go index c8ca9e65..6514984a 100644 --- a/controller/limits/shareWarningAction.go +++ b/controller/limits/shareWarningAction.go @@ -27,13 +27,15 @@ func (a *shareWarningAction) HandleShare(shr *store.Share, rxBytes, txBytes int6 return err } - acct, err := a.str.GetAccount(env.Id, trx) - if err != nil { - return err - } + if env.AccountId != nil { + acct, err := a.str.GetAccount(*env.AccountId, trx) + if err != nil { + return err + } - if err := sendLimitWarningEmail(a.cfg, acct.Email, limit, rxBytes, txBytes); err != nil { - return errors.Wrapf(err, "error sending limit warning email to '%v'", acct.Email) + if err := sendLimitWarningEmail(a.cfg, acct.Email, limit, rxBytes, txBytes); err != nil { + return errors.Wrapf(err, "error sending limit warning email to '%v'", acct.Email) + } } return nil From 31819b42ba09fe800a3f5927d6773501166c6edc Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 28 Mar 2023 14:40:00 -0400 Subject: [PATCH 54/83] details about limits testing --- docs/guides/v0.4_limits.md | 694 +++++++++++++++++++++++++++++++++++++ 1 file changed, 694 insertions(+) create mode 100644 docs/guides/v0.4_limits.md diff --git a/docs/guides/v0.4_limits.md b/docs/guides/v0.4_limits.md new file mode 100644 index 00000000..cc23bbbd --- /dev/null +++ b/docs/guides/v0.4_limits.md @@ -0,0 +1,694 @@ +# Testing the Limits + +Consider the following `zrok controller` configuration stanza, describing the limits we'll be using for this testing scenario: + +```yaml +limits: + environments: -1 + shares: -1 + bandwidth: + per_account: + period: 5m + warning: + rx: -1 + tx: -1 + total: -1 + limit: + rx: -1 + tx: -1 + total: -1 + per_environment: + period: 5m + warning: + rx: -1 + tx: -1 + total: -1 + limit: + rx: -1 + tx: -1 + total: -1 + per_share: + period: 5m + warning: + rx: -1 + tx: -1 + total: 1048576 + limit: + rx: -1 + tx: -1 + total: 2097152 + enforcing: true + cycle: 1m +``` + +Any limit values set to `-1` are "unlimited". In this case, we're only enforcing a transfer limit on for shares. This limits configuration will send a warning when a share has transferred more than 1 megabyte in a 5 minute period, and will temporarily deactivate the share when it has transferred more than 2 megabytes in a 5 minute period. + +We're going to use the `zrok test loop public` framework to create a number of `public` shares and generate traffic. Here are the parameters we'll be using: + +``` +$ zrok test loop public -l 7 -i 10000 --min-pacing-ms 100 --max-pacing-ms 1500 +``` + +This configuration will create 7 shares. Each share will perform 10,000 iterations. The delay between iterations will be randomly generated with a floor of 100ms and a ceiling of 1500ms. + +Let's look at the `zrok controller` log for this run: + +First, our `zrok test loop public ` command will create the 7 shares: + +``` +[ 2.047] INFO zrok/controller.(*shareHandler).Handle: added frontend selection 'public' with ziti identity 'rBayMvm7UI' for share '0evcupz5k410' +[ 2.081] INFO zrok/controller.(*shareHandler).Handle: added frontend selection 'public' with ziti identity 'rBayMvm7UI' for share '8k6dnu7x7ag0' +[ 2.082] INFO zrok/controller/zrokEdgeSdk.CreateConfig: created config '19cyxfHo32R6fhVsYHZ84g' for environment 'd.wJYlpt9' +[ 2.083] INFO zrok/controller.(*shareHandler).Handle: added frontend selection 'public' with ziti identity 'rBayMvm7UI' for share '53z6mz4re7tu' +[ 2.086] INFO zrok/controller/zrokEdgeSdk.CreateShareService: created share '0evcupz5k410' (with ziti id '3WHJGqUdxkDtPYLgEL5V3q') for environment 'd.wJYlpt9' +[ 2.090] INFO zrok/controller.(*shareHandler).Handle: added frontend selection 'public' with ziti identity 'rBayMvm7UI' for share '7u9szn30ikh0' +[ 2.090] INFO zrok/controller.(*shareHandler).Handle: added frontend selection 'public' with ziti identity 'rBayMvm7UI' for share 'dh3f3jj7zhig' +[ 2.091] INFO zrok/controller.(*shareHandler).Handle: added frontend selection 'public' with ziti identity 'rBayMvm7UI' for share 'tr7vpyrzvmh0' +[ 2.096] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyBind: created bind service policy '4V8FsgCt63ySkG2pFWG5fz' for service '3WHJGqUdxkDtPYLgEL5V3q' for identity 'd.wJYlpt9' +[ 2.097] INFO zrok/controller/zrokEdgeSdk.CreateConfig: created config '5nG9jM8VNl0uBFcRRt3AvI' for environment 'd.wJYlpt9' +[ 2.098] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '74f2gUotsC7DteqpsWrxp0' for service '3WHJGqUdxkDtPYLgEL5V3q' for identities '[rBayMvm7UI]' +[ 2.099] INFO zrok/controller/zrokEdgeSdk.CreateShareService: created share '8k6dnu7x7ag0' (with ziti id '2J0I9dPe2JGnY1GwjmM6n7') for environment 'd.wJYlpt9' +[ 2.100] INFO zrok/controller/zrokEdgeSdk.CreateShareServiceEdgeRouterPolicy: created service edge router policy '2AqCUMqNtarmglOfhvnkI' for service '3WHJGqUdxkDtPYLgEL5V3q' for environment 'd.wJYlpt9' +[ 2.100] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyBind: created bind service policy '4vT5eEPahgWEVdAuKN91Sd' for service '2J0I9dPe2JGnY1GwjmM6n7' for identity 'd.wJYlpt9' +[ 2.104] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '5UHCkXZabFHeWYHmF01Zoc' for service '2J0I9dPe2JGnY1GwjmM6n7' for identities '[rBayMvm7UI]' +[ 2.106] INFO zrok/controller.(*shareHandler).Handle: recorded share '0evcupz5k410' with id '503' for 'michael@quigley.com' +[ 2.106] INFO zrok/controller/zrokEdgeSdk.CreateConfig: created config '6U3XDGnBjtONN5H6pUze12' for environment 'd.wJYlpt9' +[ 2.108] INFO zrok/controller/zrokEdgeSdk.CreateShareServiceEdgeRouterPolicy: created service edge router policy '2RIKOBMOckfbI2xMSLAKxC' for service '2J0I9dPe2JGnY1GwjmM6n7' for environment 'd.wJYlpt9' +[ 2.109] INFO zrok/controller/zrokEdgeSdk.CreateShareService: created share '53z6mz4re7tu' (with ziti id '2NiotGOyBHBEbFZwbTFJ2u') for environment 'd.wJYlpt9' +[ 2.109] INFO zrok/controller/zrokEdgeSdk.CreateConfig: created config '1FnBhnGNXDe58dwTpbFc1x' for environment 'd.wJYlpt9' +[ 2.109] INFO zrok/controller.(*shareHandler).Handle: recorded share '8k6dnu7x7ag0' with id '504' for 'michael@quigley.com' +[ 2.112] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyBind: created bind service policy 'RRfDaA5kjCqUBVC9LvN1H' for service '2NiotGOyBHBEbFZwbTFJ2u' for identity 'd.wJYlpt9' +[ 2.112] INFO zrok/controller/zrokEdgeSdk.CreateConfig: created config '2gid15nP0GIUVuaFQ15GWV' for environment 'd.wJYlpt9' +[ 2.115] INFO zrok/controller/zrokEdgeSdk.CreateShareService: created share '7u9szn30ikh0' (with ziti id '6FzYnK0RFJmT0rDSP1bzVE') for environment 'd.wJYlpt9' +[ 2.115] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '1oo3LuBKxduKAs1wsKndtW' for service '2NiotGOyBHBEbFZwbTFJ2u' for identities '[rBayMvm7UI]' +[ 2.117] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyBind: created bind service policy '1mabRt9jefSe52CJh6FmhB' for service '6FzYnK0RFJmT0rDSP1bzVE' for identity 'd.wJYlpt9' +[ 2.117] INFO zrok/controller/zrokEdgeSdk.CreateShareServiceEdgeRouterPolicy: created service edge router policy '2CM03d1cNpG4rma38BLzCQ' for service '2NiotGOyBHBEbFZwbTFJ2u' for environment 'd.wJYlpt9' +[ 2.118] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '3dBtc3v2G70aqqDSqujQOy' for service '6FzYnK0RFJmT0rDSP1bzVE' for identities '[rBayMvm7UI]' +[ 2.119] INFO zrok/controller.(*shareHandler).Handle: recorded share '53z6mz4re7tu' with id '505' for 'michael@quigley.com' +[ 2.121] INFO zrok/controller/zrokEdgeSdk.CreateShareServiceEdgeRouterPolicy: created service edge router policy '3xAG26zA9yska3LeZQUJ3N' for service '6FzYnK0RFJmT0rDSP1bzVE' for environment 'd.wJYlpt9' +[ 2.122] INFO zrok/controller.(*shareHandler).Handle: added frontend selection 'public' with ziti identity 'rBayMvm7UI' for share 's0uzz1p7xjrr' +[ 2.124] INFO zrok/controller.(*shareHandler).Handle: recorded share '7u9szn30ikh0' with id '506' for 'michael@quigley.com' +[ 2.128] INFO zrok/controller/zrokEdgeSdk.CreateShareService: created share 'tr7vpyrzvmh0' (with ziti id '7jyiTZ0z2ediD5hZbxu7KH') for environment 'd.wJYlpt9' +[ 2.130] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyBind: created bind service policy '6RwWEoIsb8gBVKJfZP3ur3' for service '7jyiTZ0z2ediD5hZbxu7KH' for identity 'd.wJYlpt9' +[ 2.131] INFO zrok/controller/zrokEdgeSdk.CreateConfig: created config '76iBDASRcxOmGtdwjVHo26' for environment 'd.wJYlpt9' +[ 2.132] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '1cURGP202D8n6fzpzWhcgK' for service '7jyiTZ0z2ediD5hZbxu7KH' for identities '[rBayMvm7UI]' +[ 2.138] INFO zrok/controller/zrokEdgeSdk.CreateShareService: created share 'dh3f3jj7zhig' (with ziti id 'nyKOLlxUWWbCzD7h9Jhjq') for environment 'd.wJYlpt9' +[ 2.139] INFO zrok/controller/zrokEdgeSdk.CreateShareServiceEdgeRouterPolicy: created service edge router policy '2nMZaiChQAPpFnblNn1ljP' for service '7jyiTZ0z2ediD5hZbxu7KH' for environment 'd.wJYlpt9' +[ 2.142] INFO zrok/controller.(*shareHandler).Handle: recorded share 'tr7vpyrzvmh0' with id '507' for 'michael@quigley.com' +[ 2.143] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyBind: created bind service policy '1xF4ky6cDJm63tzlNTqoLC' for service 'nyKOLlxUWWbCzD7h9Jhjq' for identity 'd.wJYlpt9' +[ 2.143] INFO zrok/controller/zrokEdgeSdk.CreateConfig: created config '4AN4sOtdQv99uHmFn3erx4' for environment 'd.wJYlpt9' +[ 2.145] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '7GerqbN5lVfcOng91J2J6I' for service 'nyKOLlxUWWbCzD7h9Jhjq' for identities '[rBayMvm7UI]' +[ 2.145] INFO zrok/controller/zrokEdgeSdk.CreateShareService: created share 's0uzz1p7xjrr' (with ziti id 'KtK5E46HR93YIBrrwUlIN') for environment 'd.wJYlpt9' +[ 2.147] INFO zrok/controller/zrokEdgeSdk.CreateShareServiceEdgeRouterPolicy: created service edge router policy '2ZnnIXSTQ3Zscha1kykqQr' for service 'nyKOLlxUWWbCzD7h9Jhjq' for environment 'd.wJYlpt9' +[ 2.149] INFO zrok/controller.(*shareHandler).Handle: recorded share 'dh3f3jj7zhig' with id '508' for 'michael@quigley.com' +[ 2.155] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyBind: created bind service policy '6oohOQFEo75yl9vnIbyzdj' for service 'KtK5E46HR93YIBrrwUlIN' for identity 'd.wJYlpt9' +[ 2.156] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '7eB3ubrntSHxkeHBCGJcOY' for service 'KtK5E46HR93YIBrrwUlIN' for identities '[rBayMvm7UI]' +[ 2.157] INFO zrok/controller/zrokEdgeSdk.CreateShareServiceEdgeRouterPolicy: created service edge router policy '2CGCz8dcquNvZC0ZUwDZ5F' for service 'KtK5E46HR93YIBrrwUlIN' for environment 'd.wJYlpt9' +[ 2.159] INFO zrok/controller.(*shareHandler).Handle: recorded share 's0uzz1p7xjrr' with id '509' for 'michael@quigley.com' +``` + +Next, we observe metrics being reported from OpenZiti into the `zrok` metrics infrastructure for each of the 7 shares: + +``` +[ 10.183] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 32.4 kB, tx: 32.6 kB} frontend {rx: 32.6 kB, tx: 32.4 kB} +[ 10.192] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 22.5 kB, tx: 22.8 kB} frontend {rx: 22.8 kB, tx: 22.5 kB} +[ 10.196] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 15.1 kB, tx: 15.3 kB} frontend {rx: 15.3 kB, tx: 15.1 kB} +[ 15.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 53.0 kB, tx: 53.4 kB} frontend {rx: 53.4 kB, tx: 53.0 kB} +[ 15.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 50.3 kB, tx: 50.6 kB} frontend {rx: 50.6 kB, tx: 50.3 kB} +[ 15.170] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 46.2 kB, tx: 46.6 kB} frontend {rx: 46.6 kB, tx: 46.2 kB} +[ 15.172] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 26.4 kB, tx: 26.8 kB} frontend {rx: 26.8 kB, tx: 26.4 kB} +[ 20.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 26.9 kB, tx: 27.1 kB} frontend {rx: 27.1 kB, tx: 26.9 kB} +[ 20.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 26.0 kB, tx: 26.2 kB} frontend {rx: 26.2 kB, tx: 26.0 kB} +[ 20.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 67.1 kB, tx: 67.6 kB} frontend {rx: 67.6 kB, tx: 67.1 kB} +[ 25.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 38.1 kB, tx: 38.4 kB} frontend {rx: 38.4 kB, tx: 38.1 kB} +[ 25.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 26.3 kB, tx: 26.7 kB} frontend {rx: 26.7 kB, tx: 26.3 kB} +[ 25.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 18.2 kB, tx: 18.4 kB} frontend {rx: 18.4 kB, tx: 18.2 kB} +[ 25.171] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 22.6 kB, tx: 23.0 kB} frontend {rx: 23.0 kB, tx: 22.6 kB} +[ 30.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 45.1 kB, tx: 45.4 kB} frontend {rx: 45.4 kB, tx: 45.1 kB} +[ 30.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 44.0 kB, tx: 44.3 kB} frontend {rx: 44.3 kB, tx: 44.0 kB} +[ 30.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 65.1 kB, tx: 65.5 kB} frontend {rx: 65.5 kB, tx: 65.1 kB} +[ 35.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 35.9 kB, tx: 36.1 kB} frontend {rx: 36.1 kB, tx: 35.9 kB} +[ 35.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 36.4 kB, tx: 36.9 kB} frontend {rx: 36.9 kB, tx: 36.4 kB} +[ 35.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 28.9 kB, tx: 29.3 kB} frontend {rx: 29.3 kB, tx: 28.9 kB} +[ 35.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 40.9 kB, tx: 41.2 kB} frontend {rx: 41.2 kB, tx: 40.9 kB} +[ 40.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 31.0 kB, tx: 31.3 kB} frontend {rx: 31.3 kB, tx: 31.0 kB} +[ 40.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 28.5 kB, tx: 28.8 kB} frontend {rx: 28.8 kB, tx: 28.5 kB} +[ 40.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 40.4 kB, tx: 40.8 kB} frontend {rx: 40.8 kB, tx: 40.4 kB} +[ 45.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 60.1 kB, tx: 60.4 kB} frontend {rx: 60.4 kB, tx: 60.1 kB} +[ 45.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 64.8 kB, tx: 65.2 kB} frontend {rx: 65.2 kB, tx: 64.8 kB} +[ 45.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 39.2 kB, tx: 39.5 kB} frontend {rx: 39.5 kB, tx: 39.2 kB} +[ 45.170] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 23.9 kB, tx: 24.1 kB} frontend {rx: 24.1 kB, tx: 23.9 kB} +[ 50.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 23.0 kB, tx: 23.2 kB} frontend {rx: 23.2 kB, tx: 23.0 kB} +[ 50.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 41.4 kB, tx: 41.8 kB} frontend {rx: 41.8 kB, tx: 41.4 kB} +[ 50.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 50.8 kB, tx: 51.2 kB} frontend {rx: 51.2 kB, tx: 50.8 kB} +[ 55.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 29.2 kB, tx: 29.5 kB} frontend {rx: 29.5 kB, tx: 29.2 kB} +[ 55.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 27.8 kB, tx: 28.0 kB} frontend {rx: 28.0 kB, tx: 27.8 kB} +[ 55.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 21.7 kB, tx: 21.9 kB} frontend {rx: 21.9 kB, tx: 21.7 kB} +[ 55.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 30.0 kB, tx: 30.3 kB} frontend {rx: 30.3 kB, tx: 30.0 kB} +[ 60.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 43.4 kB, tx: 43.7 kB} frontend {rx: 43.7 kB, tx: 43.4 kB} +[ 60.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 44.7 kB, tx: 44.9 kB} frontend {rx: 44.9 kB, tx: 44.7 kB} +[ 60.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 30.6 kB, tx: 30.8 kB} frontend {rx: 30.8 kB, tx: 30.6 kB} +[ 65.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 41.9 kB, tx: 42.2 kB} frontend {rx: 42.2 kB, tx: 41.9 kB} +[ 65.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 38.1 kB, tx: 38.4 kB} frontend {rx: 38.4 kB, tx: 38.1 kB} +[ 65.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 42.8 kB, tx: 43.3 kB} frontend {rx: 43.3 kB, tx: 42.8 kB} +[ 65.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 28.9 kB, tx: 29.2 kB} frontend {rx: 29.2 kB, tx: 28.9 kB} +[ 70.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 43.6 kB, tx: 43.9 kB} frontend {rx: 43.9 kB, tx: 43.6 kB} +[ 70.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 30.3 kB, tx: 30.7 kB} frontend {rx: 30.7 kB, tx: 30.3 kB} +[ 70.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 58.9 kB, tx: 59.5 kB} frontend {rx: 59.5 kB, tx: 58.9 kB} +[ 75.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 32.5 kB, tx: 32.7 kB} frontend {rx: 32.7 kB, tx: 32.5 kB} +[ 75.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 31.7 kB, tx: 32.2 kB} frontend {rx: 32.2 kB, tx: 31.7 kB} +[ 75.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 42.2 kB, tx: 42.6 kB} frontend {rx: 42.6 kB, tx: 42.2 kB} +[ 75.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 61.7 kB, tx: 62.0 kB} frontend {rx: 62.0 kB, tx: 61.7 kB} +[ 80.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 48.3 kB, tx: 48.7 kB} frontend {rx: 48.7 kB, tx: 48.3 kB} +[ 80.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 54.5 kB, tx: 55.2 kB} frontend {rx: 55.2 kB, tx: 54.5 kB} +[ 80.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 21.3 kB, tx: 21.5 kB} frontend {rx: 21.5 kB, tx: 21.3 kB} +[ 85.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 47.7 kB, tx: 48.1 kB} frontend {rx: 48.1 kB, tx: 47.7 kB} +[ 85.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 27.0 kB, tx: 27.4 kB} frontend {rx: 27.4 kB, tx: 27.0 kB} +[ 85.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 72.9 kB, tx: 73.4 kB} frontend {rx: 73.4 kB, tx: 72.9 kB} +[ 85.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 78.6 kB, tx: 79.1 kB} frontend {rx: 79.1 kB, tx: 78.6 kB} +[ 90.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 58.7 kB, tx: 59.1 kB} frontend {rx: 59.1 kB, tx: 58.7 kB} +[ 90.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 48.9 kB, tx: 49.3 kB} frontend {rx: 49.3 kB, tx: 48.9 kB} +[ 90.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 63.4 kB, tx: 63.7 kB} frontend {rx: 63.7 kB, tx: 63.4 kB} +[ 95.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 59.0 kB, tx: 59.4 kB} frontend {rx: 59.4 kB, tx: 59.0 kB} +[ 95.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 65.9 kB, tx: 66.2 kB} frontend {rx: 66.2 kB, tx: 65.9 kB} +[ 95.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 48.9 kB, tx: 49.3 kB} frontend {rx: 49.3 kB, tx: 48.9 kB} +[ 95.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 27.5 kB, tx: 27.8 kB} frontend {rx: 27.8 kB, tx: 27.5 kB} +[ 100.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 56.3 kB, tx: 56.8 kB} frontend {rx: 56.8 kB, tx: 56.3 kB} +[ 100.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 25.8 kB, tx: 26.2 kB} frontend {rx: 26.2 kB, tx: 25.8 kB} +[ 100.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 33.9 kB, tx: 34.2 kB} frontend {rx: 34.2 kB, tx: 33.9 kB} +[ 105.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 30.7 kB, tx: 31.0 kB} frontend {rx: 31.0 kB, tx: 30.7 kB} +[ 105.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 64.6 kB, tx: 64.9 kB} frontend {rx: 64.9 kB, tx: 64.6 kB} +[ 105.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 49.0 kB, tx: 49.3 kB} frontend {rx: 49.3 kB, tx: 49.0 kB} +[ 105.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 36.2 kB, tx: 36.6 kB} frontend {rx: 36.6 kB, tx: 36.2 kB} +``` + +Our first share receives a bandwidth warning, after transferring more than 1 megabyte: + +``` +[ 105.189] INFO zrok/controller/limits.(*shareWarningAction).HandleShare: warning 'tr7vpyrzvmh0' +[ 106.192] INFO zrok/controller/limits.sendLimitWarningEmail: limit warning email sent to 'michael@quigley.com' +[ 110.162] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 30.6 kB, tx: 30.9 kB} frontend {rx: 30.9 kB, tx: 30.6 kB} +[ 110.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 42.9 kB, tx: 43.3 kB} frontend {rx: 43.3 kB, tx: 42.9 kB} +[ 110.170] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 31.5 kB, tx: 31.7 kB} frontend {rx: 31.7 kB, tx: 31.5 kB} +[ 115.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 45.3 kB, tx: 45.7 kB} frontend {rx: 45.7 kB, tx: 45.3 kB} +[ 115.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 36.4 kB, tx: 36.8 kB} frontend {rx: 36.8 kB, tx: 36.4 kB} +[ 115.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 46.5 kB, tx: 46.9 kB} frontend {rx: 46.9 kB, tx: 46.5 kB} +[ 115.170] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 63.9 kB, tx: 64.4 kB} frontend {rx: 64.4 kB, tx: 63.9 kB} +``` + +More shares start receiving bandwidth warnings: + +``` +[ 115.230] INFO zrok/controller/limits.(*shareWarningAction).HandleShare: warning 'dh3f3jj7zhig' +[ 116.575] INFO zrok/controller/limits.sendLimitWarningEmail: limit warning email sent to 'michael@quigley.com' +[ 120.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 58.4 kB, tx: 58.8 kB} frontend {rx: 58.8 kB, tx: 58.4 kB} +[ 120.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 30.5 kB, tx: 30.8 kB} frontend {rx: 30.8 kB, tx: 30.5 kB} +[ 120.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 29.9 kB, tx: 30.2 kB} frontend {rx: 30.2 kB, tx: 29.9 kB} +[ 120.180] INFO zrok/controller/limits.(*shareWarningAction).HandleShare: warning '53z6mz4re7tu' +[ 122.733] INFO zrok/controller/limits.sendLimitWarningEmail: limit warning email sent to 'michael@quigley.com' +[ 125.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 47.0 kB, tx: 47.3 kB} frontend {rx: 47.3 kB, tx: 47.0 kB} +[ 125.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 55.5 kB, tx: 56.0 kB} frontend {rx: 56.0 kB, tx: 55.5 kB} +[ 125.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 49.6 kB, tx: 49.9 kB} frontend {rx: 49.9 kB, tx: 49.6 kB} +[ 125.170] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 17.6 kB, tx: 17.8 kB} frontend {rx: 17.8 kB, tx: 17.6 kB} +[ 125.211] INFO zrok/controller/limits.(*shareWarningAction).HandleShare: warning 's0uzz1p7xjrr' +[ 126.117] INFO zrok/controller/limits.sendLimitWarningEmail: limit warning email sent to 'michael@quigley.com' +[ 130.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 38.8 kB, tx: 39.0 kB} frontend {rx: 39.0 kB, tx: 38.8 kB} +[ 130.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 76.6 kB, tx: 76.9 kB} frontend {rx: 76.9 kB, tx: 76.6 kB} +[ 130.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 50.1 kB, tx: 50.5 kB} frontend {rx: 50.5 kB, tx: 50.1 kB} +[ 130.178] INFO zrok/controller/limits.(*shareWarningAction).HandleShare: warning '0evcupz5k410' +[ 130.921] INFO zrok/controller/limits.sendLimitWarningEmail: limit warning email sent to 'michael@quigley.com' +[ 135.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 32.8 kB, tx: 33.2 kB} frontend {rx: 33.2 kB, tx: 32.8 kB} +[ 135.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 34.7 kB, tx: 35.0 kB} frontend {rx: 35.0 kB, tx: 34.7 kB} +[ 135.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 38.9 kB, tx: 39.2 kB} frontend {rx: 39.2 kB, tx: 38.9 kB} +[ 135.170] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 51.4 kB, tx: 51.8 kB} frontend {rx: 51.8 kB, tx: 51.4 kB} +[ 140.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 52.8 kB, tx: 53.2 kB} frontend {rx: 53.2 kB, tx: 52.8 kB} +[ 140.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 33.1 kB, tx: 33.4 kB} frontend {rx: 33.4 kB, tx: 33.1 kB} +[ 140.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 35.6 kB, tx: 36.0 kB} frontend {rx: 36.0 kB, tx: 35.6 kB} +[ 145.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 27.5 kB, tx: 27.8 kB} frontend {rx: 27.8 kB, tx: 27.5 kB} +[ 145.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 24.6 kB, tx: 25.1 kB} frontend {rx: 25.1 kB, tx: 24.6 kB} +[ 145.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 30.1 kB, tx: 30.5 kB} frontend {rx: 30.5 kB, tx: 30.1 kB} +[ 145.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 24.7 kB, tx: 25.1 kB} frontend {rx: 25.1 kB, tx: 24.7 kB} +[ 150.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 72.0 kB, tx: 72.4 kB} frontend {rx: 72.4 kB, tx: 72.0 kB} +[ 150.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 31.8 kB, tx: 32.1 kB} frontend {rx: 32.1 kB, tx: 31.8 kB} +[ 150.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 43.7 kB, tx: 43.9 kB} frontend {rx: 43.9 kB, tx: 43.7 kB} +[ 155.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 49.4 kB, tx: 49.8 kB} frontend {rx: 49.8 kB, tx: 49.4 kB} +[ 155.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 46.4 kB, tx: 46.6 kB} frontend {rx: 46.6 kB, tx: 46.4 kB} +[ 155.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 50.7 kB, tx: 51.0 kB} frontend {rx: 51.0 kB, tx: 50.7 kB} +[ 155.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 58.5 kB, tx: 58.9 kB} frontend {rx: 58.9 kB, tx: 58.5 kB} +[ 160.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 43.0 kB, tx: 43.3 kB} frontend {rx: 43.3 kB, tx: 43.0 kB} +[ 160.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 66.0 kB, tx: 66.4 kB} frontend {rx: 66.4 kB, tx: 66.0 kB} +[ 160.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 31.5 kB, tx: 31.9 kB} frontend {rx: 31.9 kB, tx: 31.5 kB} +[ 165.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 60.0 kB, tx: 60.3 kB} frontend {rx: 60.3 kB, tx: 60.0 kB} +[ 165.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 47.3 kB, tx: 47.6 kB} frontend {rx: 47.6 kB, tx: 47.3 kB} +[ 165.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 41.1 kB, tx: 41.3 kB} frontend {rx: 41.3 kB, tx: 41.1 kB} +[ 165.170] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 37.2 kB, tx: 37.5 kB} frontend {rx: 37.5 kB, tx: 37.2 kB} +[ 165.216] INFO zrok/controller/limits.(*shareWarningAction).HandleShare: warning '8k6dnu7x7ag0' +[ 165.930] INFO zrok/controller/limits.sendLimitWarningEmail: limit warning email sent to 'michael@quigley.com' +[ 170.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 43.1 kB, tx: 43.5 kB} frontend {rx: 43.5 kB, tx: 43.1 kB} +[ 170.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 45.4 kB, tx: 45.8 kB} frontend {rx: 45.8 kB, tx: 45.4 kB} +[ 170.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 58.0 kB, tx: 58.3 kB} frontend {rx: 58.3 kB, tx: 58.0 kB} +[ 175.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 63.5 kB, tx: 63.9 kB} frontend {rx: 63.9 kB, tx: 63.5 kB} +[ 175.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 45.0 kB, tx: 45.3 kB} frontend {rx: 45.3 kB, tx: 45.0 kB} +[ 175.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 35.0 kB, tx: 35.2 kB} frontend {rx: 35.2 kB, tx: 35.0 kB} +[ 175.171] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 22.8 kB, tx: 23.2 kB} frontend {rx: 23.2 kB, tx: 22.8 kB} +[ 180.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 42.3 kB, tx: 42.6 kB} frontend {rx: 42.6 kB, tx: 42.3 kB} +[ 180.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 39.6 kB, tx: 40.1 kB} frontend {rx: 40.1 kB, tx: 39.6 kB} +[ 180.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 53.1 kB, tx: 53.4 kB} frontend {rx: 53.4 kB, tx: 53.1 kB} +[ 185.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 74.1 kB, tx: 74.6 kB} frontend {rx: 74.6 kB, tx: 74.1 kB} +[ 185.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 71.0 kB, tx: 71.4 kB} frontend {rx: 71.4 kB, tx: 71.0 kB} +[ 185.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 78.8 kB, tx: 79.2 kB} frontend {rx: 79.2 kB, tx: 78.8 kB} +[ 185.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 37.8 kB, tx: 38.2 kB} frontend {rx: 38.2 kB, tx: 37.8 kB} +[ 185.213] INFO zrok/controller/limits.(*shareWarningAction).HandleShare: warning '7u9szn30ikh0' +[ 186.862] INFO zrok/controller/limits.sendLimitWarningEmail: limit warning email sent to 'michael@quigley.com' +[ 190.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 43.3 kB, tx: 43.8 kB} frontend {rx: 43.8 kB, tx: 43.3 kB} +[ 190.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 39.6 kB, tx: 39.9 kB} frontend {rx: 39.9 kB, tx: 39.6 kB} +[ 190.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 38.6 kB, tx: 38.9 kB} frontend {rx: 38.9 kB, tx: 38.6 kB} +[ 195.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 44.0 kB, tx: 44.4 kB} frontend {rx: 44.4 kB, tx: 44.0 kB} +[ 195.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 45.2 kB, tx: 45.5 kB} frontend {rx: 45.5 kB, tx: 45.2 kB} +[ 195.170] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 67.1 kB, tx: 67.5 kB} frontend {rx: 67.5 kB, tx: 67.1 kB} +[ 195.172] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 44.4 kB, tx: 44.8 kB} frontend {rx: 44.8 kB, tx: 44.4 kB} +[ 200.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 23.7 kB, tx: 23.9 kB} frontend {rx: 23.9 kB, tx: 23.7 kB} +[ 200.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 46.7 kB, tx: 47.1 kB} frontend {rx: 47.1 kB, tx: 46.7 kB} +[ 200.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 48.7 kB, tx: 49.1 kB} frontend {rx: 49.1 kB, tx: 48.7 kB} +[ 205.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 47.8 kB, tx: 48.1 kB} frontend {rx: 48.1 kB, tx: 47.8 kB} +[ 205.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 57.2 kB, tx: 57.6 kB} frontend {rx: 57.6 kB, tx: 57.2 kB} +[ 205.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 47.7 kB, tx: 47.9 kB} frontend {rx: 47.9 kB, tx: 47.7 kB} +[ 205.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 38.9 kB, tx: 39.3 kB} frontend {rx: 39.3 kB, tx: 38.9 kB} +[ 210.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 37.5 kB, tx: 37.8 kB} frontend {rx: 37.8 kB, tx: 37.5 kB} +[ 210.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 15.3 kB, tx: 15.5 kB} frontend {rx: 15.5 kB, tx: 15.3 kB} +[ 210.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 41.3 kB, tx: 41.5 kB} frontend {rx: 41.5 kB, tx: 41.3 kB} +[ 215.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 28.0 kB, tx: 28.4 kB} frontend {rx: 28.4 kB, tx: 28.0 kB} +[ 215.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 42.5 kB, tx: 42.8 kB} frontend {rx: 42.8 kB, tx: 42.5 kB} +[ 215.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 47.6 kB, tx: 48.0 kB} frontend {rx: 48.0 kB, tx: 47.6 kB} +[ 215.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 43.4 kB, tx: 43.8 kB} frontend {rx: 43.8 kB, tx: 43.4 kB} +[ 220.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 19.4 kB, tx: 19.7 kB} frontend {rx: 19.7 kB, tx: 19.4 kB} +[ 220.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 37.1 kB, tx: 37.4 kB} frontend {rx: 37.4 kB, tx: 37.1 kB} +[ 220.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 69.1 kB, tx: 69.5 kB} frontend {rx: 69.5 kB, tx: 69.1 kB} +``` + +Our first share crosses the 2 megabyte boundary and the system limits its ability to transfer additional data by removing its dial service policy: + +``` +[ 220.195] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: limiting 'dh3f3jj7zhig' +[ 220.211] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '7GerqbN5lVfcOng91J2J6I' for environment 'd.wJYlpt9' +[ 220.211] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: removed dial service policy for 'dh3f3jj7zhig' +[ 225.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 45.1 kB, tx: 45.5 kB} frontend {rx: 45.5 kB, tx: 45.1 kB} +[ 225.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 26.9 kB, tx: 27.3 kB} frontend {rx: 27.3 kB, tx: 26.9 kB} +[ 225.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: fNXXHVSuw backend {rx: 40.9 kB, tx: 41.0 kB} frontend {rx: 41.0 kB, tx: 40.9 kB} +[ 225.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 28.8 kB, tx: 29.1 kB} frontend {rx: 29.1 kB, tx: 28.8 kB} +[ 230.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 55.4 kB, tx: 55.8 kB} frontend {rx: 55.8 kB, tx: 55.4 kB} +[ 230.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 44.8 kB, tx: 45.2 kB} frontend {rx: 45.2 kB, tx: 44.8 kB} +[ 230.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 52.2 kB, tx: 52.5 kB} frontend {rx: 52.5 kB, tx: 52.2 kB} +[ 235.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 53.2 kB, tx: 53.6 kB} frontend {rx: 53.6 kB, tx: 53.2 kB} +[ 235.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 43.1 kB, tx: 43.4 kB} frontend {rx: 43.4 kB, tx: 43.1 kB} +[ 235.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: ONzzjVS0w backend {rx: 45.9 kB, tx: 46.2 kB} frontend {rx: 46.2 kB, tx: 45.9 kB} +[ 240.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 50.6 kB, tx: 51.0 kB} frontend {rx: 51.0 kB, tx: 50.6 kB} +[ 240.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: cNzzH4i0w backend {rx: 49.7 kB, tx: 50.0 kB} frontend {rx: 50.0 kB, tx: 49.7 kB} +[ 240.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 37.3 kB, tx: 37.6 kB} frontend {rx: 37.6 kB, tx: 37.3 kB} +``` + +More shares become limited and are prevented from transferring data. Notice the metrics output reducing in the logs. As more shares become limited, we're naturally seeing less data transfer occurring on the OpenZiti network: + +``` +[ 240.188] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: limiting '0evcupz5k410' +[ 240.203] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '74f2gUotsC7DteqpsWrxp0' for environment 'd.wJYlpt9' +[ 240.203] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: removed dial service policy for '0evcupz5k410' +[ 245.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 28.2 kB, tx: 28.5 kB} frontend {rx: 28.5 kB, tx: 28.2 kB} +[ 245.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: dev-h4iuwD backend {rx: 47.3 kB, tx: 47.9 kB} frontend {rx: 47.9 kB, tx: 47.3 kB} +[ 245.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 48.2 kB, tx: 48.5 kB} frontend {rx: 48.5 kB, tx: 48.2 kB} +[ 245.194] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: limiting '53z6mz4re7tu' +[ 245.196] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '1oo3LuBKxduKAs1wsKndtW' for environment 'd.wJYlpt9' +[ 245.197] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: removed dial service policy for '53z6mz4re7tu' +[ 250.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: dev-h4iuwD backend {rx: 33.2 kB, tx: 33.5 kB} frontend {rx: 33.5 kB, tx: 33.2 kB} +[ 250.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 48.2 kB, tx: 48.4 kB} frontend {rx: 48.4 kB, tx: 48.2 kB} +[ 250.191] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: limiting 's0uzz1p7xjrr' +[ 250.194] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '7eB3ubrntSHxkeHBCGJcOY' for environment 'd.wJYlpt9' +[ 250.194] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: removed dial service policy for 's0uzz1p7xjrr' +[ 255.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 35.8 kB, tx: 36.0 kB} frontend {rx: 36.0 kB, tx: 35.8 kB} +[ 255.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: RZzXHVSuw backend {rx: 50.4 kB, tx: 50.6 kB} frontend {rx: 50.6 kB, tx: 50.4 kB} +[ 255.179] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: limiting 'tr7vpyrzvmh0' +[ 255.182] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '1cURGP202D8n6fzpzWhcgK' for environment 'd.wJYlpt9' +[ 255.182] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: removed dial service policy for 'tr7vpyrzvmh0' +[ 260.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 37.3 kB, tx: 47.7 kB} frontend {rx: 47.7 kB, tx: 37.3 kB} +[ 260.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: MZXXjVi0w backend {rx: 69.2 kB, tx: 69.7 kB} frontend {rx: 69.7 kB, tx: 69.2 kB} +[ 265.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 59.7 kB, tx: 60.1 kB} frontend {rx: 60.1 kB, tx: 59.7 kB} +[ 270.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 46.9 kB, tx: 47.2 kB} frontend {rx: 47.2 kB, tx: 46.9 kB} +[ 275.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 32.9 kB, tx: 33.2 kB} frontend {rx: 33.2 kB, tx: 32.9 kB} +[ 280.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 43.2 kB, tx: 43.7 kB} frontend {rx: 43.7 kB, tx: 43.2 kB} +[ 285.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 50.0 kB, tx: 50.4 kB} frontend {rx: 50.4 kB, tx: 50.0 kB} +[ 290.162] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: SNXXjViuwU backend {rx: 51.8 kB, tx: 52.3 kB} frontend {rx: 52.3 kB, tx: 51.8 kB} +``` + +By this point, we're seeing very little traffic on the OpenZiti network: + +``` +[ 290.176] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: limiting '8k6dnu7x7ag0' +[ 290.190] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '5UHCkXZabFHeWYHmF01Zoc' for environment 'd.wJYlpt9' +[ 290.191] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: removed dial service policy for '8k6dnu7x7ag0' +[ 295.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: iNzXj4S0r backend {rx: 44.0 kB, tx: 44.4 kB} frontend {rx: 44.4 kB, tx: 44.0 kB} +[ 295.178] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: limiting '7u9szn30ikh0' +[ 295.181] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '3dBtc3v2G70aqqDSqujQOy' for environment 'd.wJYlpt9' +[ 295.181] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: removed dial service policy for '7u9szn30ikh0' +``` +Notice the timestamps on the log messages. There have been no metrics messages for 60 seconds. + +The limits agent runs a periodic process to look for limited resources to re-enable. It produces messages like this when there are no resources to re-enable: + +``` +[ 355.183] INFO zrok/controller/limits.(*Agent).relax: relaxing +[ 355.188] INFO zrok/controller/limits.(*Agent).relax: share 'dh3f3jj7zhig' still over limit +[ 355.192] INFO zrok/controller/limits.(*Agent).relax: share '0evcupz5k410' still over limit +[ 355.196] INFO zrok/controller/limits.(*Agent).relax: share '53z6mz4re7tu' still over limit +[ 355.199] INFO zrok/controller/limits.(*Agent).relax: share 's0uzz1p7xjrr' still over limit +[ 355.203] INFO zrok/controller/limits.(*Agent).relax: share 'tr7vpyrzvmh0' still over limit +[ 355.207] INFO zrok/controller/limits.(*Agent).relax: share '8k6dnu7x7ag0' still over limit +[ 355.220] INFO zrok/controller/limits.(*Agent).relax: share '7u9szn30ikh0' still over limit +[ 415.223] INFO zrok/controller/limits.(*Agent).relax: relaxing +[ 415.228] INFO zrok/controller/limits.(*Agent).relax: share 'dh3f3jj7zhig' still over limit +[ 415.232] INFO zrok/controller/limits.(*Agent).relax: share '0evcupz5k410' still over limit +[ 415.236] INFO zrok/controller/limits.(*Agent).relax: share '53z6mz4re7tu' still over limit +[ 415.240] INFO zrok/controller/limits.(*Agent).relax: share 's0uzz1p7xjrr' still over limit +[ 415.245] INFO zrok/controller/limits.(*Agent).relax: share 'tr7vpyrzvmh0' still over limit +[ 415.250] INFO zrok/controller/limits.(*Agent).relax: share '8k6dnu7x7ag0' still over limit +[ 415.253] INFO zrok/controller/limits.(*Agent).relax: share '7u9szn30ikh0' still over limit +``` +Enough time has finally passed that the agent is able to remove the restrictions on some of the services: + +``` +[ 475.255] INFO zrok/controller/limits.(*Agent).relax: relaxing +[ 475.260] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: relaxing 'dh3f3jj7zhig' +[ 475.274] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '3LQG2ptwUxIuWtRzTLAqAc' for service 'nyKOLlxUWWbCzD7h9Jhjq' for identities '[rBayMvm7UI]' +[ 475.274] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: added dial service policy for 'dh3f3jj7zhig' +[ 475.279] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: relaxing '0evcupz5k410' +[ 475.281] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '4BPqQhFsGGmoBsqFDIWlWA' for service '3WHJGqUdxkDtPYLgEL5V3q' for identities '[rBayMvm7UI]' +[ 475.281] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: added dial service policy for '0evcupz5k410' +[ 475.285] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: relaxing '53z6mz4re7tu' +[ 475.287] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '64Kz6F7CluxH1drfyMkzDx' for service '2NiotGOyBHBEbFZwbTFJ2u' for identities '[rBayMvm7UI]' +[ 475.287] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: added dial service policy for '53z6mz4re7tu' +[ 475.292] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: relaxing 's0uzz1p7xjrr' +[ 475.295] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '6MZ8i9sqvom96P70P24FJQ' for service 'KtK5E46HR93YIBrrwUlIN' for identities '[rBayMvm7UI]' +[ 475.295] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: added dial service policy for 's0uzz1p7xjrr' +[ 475.299] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: relaxing 'tr7vpyrzvmh0' +[ 475.301] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '1kfuMP2APitf3qC2tsOC1b' for service '7jyiTZ0z2ediD5hZbxu7KH' for identities '[rBayMvm7UI]' +[ 475.301] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: added dial service policy for 'tr7vpyrzvmh0' +[ 475.305] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: relaxing '8k6dnu7x7ag0' +[ 475.308] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '12jWOvjIIuvYRW9vXfkRKw' for service '2J0I9dPe2JGnY1GwjmM6n7' for identities '[rBayMvm7UI]' +[ 475.308] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: added dial service policy for '8k6dnu7x7ag0' +[ 475.313] INFO zrok/controller/limits.(*Agent).relax: share '7u9szn30ikh0' still over limit +``` +And notice that we're now starting to see traffic on those shares again: + +``` +[ 485.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 59.7 kB, tx: 60.0 kB} frontend {rx: 60.0 kB, tx: 59.7 kB} +[ 485.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 50.1 kB, tx: 50.4 kB} frontend {rx: 50.4 kB, tx: 50.1 kB} +[ 485.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 80.3 kB, tx: 80.7 kB} frontend {rx: 80.7 kB, tx: 80.3 kB} +[ 485.200] INFO zrok/controller/limits.(*shareWarningAction).HandleShare: warning '8k6dnu7x7ag0' +[ 486.095] INFO zrok/controller/limits.sendLimitWarningEmail: limit warning email sent to 'michael@quigley.com' +[ 490.162] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 40.6 kB, tx: 40.9 kB} frontend {rx: 40.9 kB, tx: 40.6 kB} +[ 490.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 44.5 kB, tx: 45.0 kB} frontend {rx: 45.0 kB, tx: 44.5 kB} +[ 490.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 60.7 kB, tx: 61.1 kB} frontend {rx: 61.1 kB, tx: 60.7 kB} +[ 495.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 45.3 kB, tx: 45.6 kB} frontend {rx: 45.6 kB, tx: 45.3 kB} +[ 495.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 42.0 kB, tx: 42.4 kB} frontend {rx: 42.4 kB, tx: 42.0 kB} +[ 495.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 33.3 kB, tx: 33.8 kB} frontend {rx: 33.8 kB, tx: 33.3 kB} +[ 500.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 28.2 kB, tx: 28.5 kB} frontend {rx: 28.5 kB, tx: 28.2 kB} +[ 500.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 40.0 kB, tx: 40.3 kB} frontend {rx: 40.3 kB, tx: 40.0 kB} +[ 500.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 53.6 kB, tx: 54.0 kB} frontend {rx: 54.0 kB, tx: 53.6 kB} +[ 505.201] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 38.4 kB, tx: 38.6 kB} frontend {rx: 38.6 kB, tx: 38.4 kB} +[ 505.208] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 33.8 kB, tx: 34.2 kB} frontend {rx: 34.2 kB, tx: 33.8 kB} +[ 505.210] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 39.7 kB, tx: 40.0 kB} frontend {rx: 40.0 kB, tx: 39.7 kB} +[ 510.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 74.0 kB, tx: 74.5 kB} frontend {rx: 74.5 kB, tx: 74.0 kB} +[ 510.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 51.5 kB, tx: 51.8 kB} frontend {rx: 51.8 kB, tx: 51.5 kB} +[ 510.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 33.5 kB, tx: 33.9 kB} frontend {rx: 33.9 kB, tx: 33.5 kB} +[ 515.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 62.7 kB, tx: 63.0 kB} frontend {rx: 63.0 kB, tx: 62.7 kB} +[ 515.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 32.5 kB, tx: 32.9 kB} frontend {rx: 32.9 kB, tx: 32.5 kB} +[ 515.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 47.4 kB, tx: 47.7 kB} frontend {rx: 47.7 kB, tx: 47.4 kB} +[ 520.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 48.2 kB, tx: 48.5 kB} frontend {rx: 48.5 kB, tx: 48.2 kB} +[ 520.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 45.8 kB, tx: 46.1 kB} frontend {rx: 46.1 kB, tx: 45.8 kB} +[ 520.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 34.2 kB, tx: 34.4 kB} frontend {rx: 34.4 kB, tx: 34.2 kB} +[ 525.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 35.0 kB, tx: 35.4 kB} frontend {rx: 35.4 kB, tx: 35.0 kB} +[ 525.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 40.1 kB, tx: 40.4 kB} frontend {rx: 40.4 kB, tx: 40.1 kB} +[ 525.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 24.7 kB, tx: 25.0 kB} frontend {rx: 25.0 kB, tx: 24.7 kB} +[ 530.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 57.3 kB, tx: 57.9 kB} frontend {rx: 57.9 kB, tx: 57.3 kB} +[ 530.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 51.4 kB, tx: 51.7 kB} frontend {rx: 51.7 kB, tx: 51.4 kB} +[ 530.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 40.6 kB, tx: 41.0 kB} frontend {rx: 41.0 kB, tx: 40.6 kB} +[ 535.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 41.5 kB, tx: 41.9 kB} frontend {rx: 41.9 kB, tx: 41.5 kB} +[ 535.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 61.4 kB, tx: 61.9 kB} frontend {rx: 61.9 kB, tx: 61.4 kB} +[ 535.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 33.3 kB, tx: 33.6 kB} frontend {rx: 33.6 kB, tx: 33.3 kB} +[ 540.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 37.2 kB, tx: 37.5 kB} frontend {rx: 37.5 kB, tx: 37.2 kB} +[ 540.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 53.8 kB, tx: 54.3 kB} frontend {rx: 54.3 kB, tx: 53.8 kB} +[ 540.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 75.8 kB, tx: 76.4 kB} frontend {rx: 76.4 kB, tx: 75.8 kB} +[ 545.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 69.9 kB, tx: 70.2 kB} frontend {rx: 70.2 kB, tx: 69.9 kB} +[ 545.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 24.9 kB, tx: 25.2 kB} frontend {rx: 25.2 kB, tx: 24.9 kB} +[ 545.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 39.4 kB, tx: 39.6 kB} frontend {rx: 39.6 kB, tx: 39.4 kB} +[ 550.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 64.0 kB, tx: 64.3 kB} frontend {rx: 64.3 kB, tx: 64.0 kB} +[ 550.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 39.2 kB, tx: 39.6 kB} frontend {rx: 39.6 kB, tx: 39.2 kB} +[ 550.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 47.6 kB, tx: 47.9 kB} frontend {rx: 47.9 kB, tx: 47.6 kB} +[ 555.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 54.2 kB, tx: 54.8 kB} frontend {rx: 54.8 kB, tx: 54.2 kB} +[ 555.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 44.5 kB, tx: 44.8 kB} frontend {rx: 44.8 kB, tx: 44.5 kB} +[ 555.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 66.3 kB, tx: 66.7 kB} frontend {rx: 66.7 kB, tx: 66.3 kB} +[ 560.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 18.9 kB, tx: 19.2 kB} frontend {rx: 19.2 kB, tx: 18.9 kB} +[ 560.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 51.5 kB, tx: 51.8 kB} frontend {rx: 51.8 kB, tx: 51.5 kB} +[ 560.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 36.3 kB, tx: 36.7 kB} frontend {rx: 36.7 kB, tx: 36.3 kB} +[ 565.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 70.7 kB, tx: 71.0 kB} frontend {rx: 71.0 kB, tx: 70.7 kB} +[ 565.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 58.7 kB, tx: 59.1 kB} frontend {rx: 59.1 kB, tx: 58.7 kB} +[ 565.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 33.8 kB, tx: 34.0 kB} frontend {rx: 34.0 kB, tx: 33.8 kB} +[ 570.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 35.4 kB, tx: 35.7 kB} frontend {rx: 35.7 kB, tx: 35.4 kB} +[ 570.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 47.2 kB, tx: 47.6 kB} frontend {rx: 47.6 kB, tx: 47.2 kB} +[ 570.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 75.5 kB, tx: 75.8 kB} frontend {rx: 75.8 kB, tx: 75.5 kB} +[ 575.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 34.3 kB, tx: 34.6 kB} frontend {rx: 34.6 kB, tx: 34.3 kB} +[ 575.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 36.2 kB, tx: 36.6 kB} frontend {rx: 36.6 kB, tx: 36.2 kB} +[ 575.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 53.9 kB, tx: 54.2 kB} frontend {rx: 54.2 kB, tx: 53.9 kB} +[ 575.178] INFO zrok/controller/limits.(*shareWarningAction).HandleShare: warning '53z6mz4re7tu' +[ 575.953] INFO zrok/controller/limits.sendLimitWarningEmail: limit warning email sent to 'michael@quigley.com' +[ 580.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 50.3 kB, tx: 50.7 kB} frontend {rx: 50.7 kB, tx: 50.3 kB} +[ 580.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 55.7 kB, tx: 56.1 kB} frontend {rx: 56.1 kB, tx: 55.7 kB} +[ 580.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 41.0 kB, tx: 41.3 kB} frontend {rx: 41.3 kB, tx: 41.0 kB} +[ 585.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 32.3 kB, tx: 32.6 kB} frontend {rx: 32.6 kB, tx: 32.3 kB} +[ 585.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 18.5 kB, tx: 18.8 kB} frontend {rx: 18.8 kB, tx: 18.5 kB} +[ 585.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 43.2 kB, tx: 43.6 kB} frontend {rx: 43.6 kB, tx: 43.2 kB} +[ 590.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 32.8 kB, tx: 33.0 kB} frontend {rx: 33.0 kB, tx: 32.8 kB} +[ 590.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 63.4 kB, tx: 63.7 kB} frontend {rx: 63.7 kB, tx: 63.4 kB} +[ 590.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 18.1 kB, tx: 18.3 kB} frontend {rx: 18.3 kB, tx: 18.1 kB} +[ 590.208] INFO zrok/controller/limits.(*shareWarningAction).HandleShare: warning 'tr7vpyrzvmh0' +[ 591.168] INFO zrok/controller/limits.sendLimitWarningEmail: limit warning email sent to 'michael@quigley.com' +[ 595.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 34.1 kB, tx: 34.5 kB} frontend {rx: 34.5 kB, tx: 34.1 kB} +[ 595.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 46.0 kB, tx: 46.3 kB} frontend {rx: 46.3 kB, tx: 46.0 kB} +[ 595.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 49.1 kB, tx: 49.4 kB} frontend {rx: 49.4 kB, tx: 49.1 kB} +[ 600.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 34.0 kB, tx: 34.3 kB} frontend {rx: 34.3 kB, tx: 34.0 kB} +[ 600.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 46.6 kB, tx: 47.1 kB} frontend {rx: 47.1 kB, tx: 46.6 kB} +[ 600.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 23.2 kB, tx: 23.5 kB} frontend {rx: 23.5 kB, tx: 23.2 kB} +[ 600.189] INFO zrok/controller/limits.(*shareWarningAction).HandleShare: warning 's0uzz1p7xjrr' +[ 600.949] INFO zrok/controller/limits.sendLimitWarningEmail: limit warning email sent to 'michael@quigley.com' +[ 605.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 61.5 kB, tx: 61.8 kB} frontend {rx: 61.8 kB, tx: 61.5 kB} +[ 605.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 38.3 kB, tx: 38.7 kB} frontend {rx: 38.7 kB, tx: 38.3 kB} +[ 605.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 35.1 kB, tx: 35.5 kB} frontend {rx: 35.5 kB, tx: 35.1 kB} +[ 610.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 37.7 kB, tx: 38.1 kB} frontend {rx: 38.1 kB, tx: 37.7 kB} +[ 610.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 50.4 kB, tx: 50.7 kB} frontend {rx: 50.7 kB, tx: 50.4 kB} +[ 610.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 27.9 kB, tx: 28.2 kB} frontend {rx: 28.2 kB, tx: 27.9 kB} +[ 615.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 25.7 kB, tx: 26.0 kB} frontend {rx: 26.0 kB, tx: 25.7 kB} +[ 615.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 32.2 kB, tx: 32.5 kB} frontend {rx: 32.5 kB, tx: 32.2 kB} +[ 615.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 47.2 kB, tx: 47.6 kB} frontend {rx: 47.6 kB, tx: 47.2 kB} +[ 620.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 35.7 kB, tx: 36.2 kB} frontend {rx: 36.2 kB, tx: 35.7 kB} +[ 620.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 60.6 kB, tx: 60.9 kB} frontend {rx: 60.9 kB, tx: 60.6 kB} +[ 620.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 51.8 kB, tx: 52.3 kB} frontend {rx: 52.3 kB, tx: 51.8 kB} +[ 620.178] INFO zrok/controller/limits.(*shareWarningAction).HandleShare: warning 'dh3f3jj7zhig' +[ 620.929] INFO zrok/controller/limits.sendLimitWarningEmail: limit warning email sent to 'michael@quigley.com' +[ 625.162] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 41.6 kB, tx: 42.0 kB} frontend {rx: 42.0 kB, tx: 41.6 kB} +[ 625.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 46.4 kB, tx: 46.7 kB} frontend {rx: 46.7 kB, tx: 46.4 kB} +[ 625.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 48.4 kB, tx: 48.7 kB} frontend {rx: 48.7 kB, tx: 48.4 kB} +[ 630.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 29.3 kB, tx: 29.5 kB} frontend {rx: 29.5 kB, tx: 29.3 kB} +[ 630.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 49.7 kB, tx: 50.2 kB} frontend {rx: 50.2 kB, tx: 49.7 kB} +[ 630.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 47.7 kB, tx: 48.0 kB} frontend {rx: 48.0 kB, tx: 47.7 kB} +[ 635.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 69.2 kB, tx: 69.6 kB} frontend {rx: 69.6 kB, tx: 69.2 kB} +[ 635.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 45.8 kB, tx: 46.2 kB} frontend {rx: 46.2 kB, tx: 45.8 kB} +[ 635.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 38.6 kB, tx: 39.1 kB} frontend {rx: 39.1 kB, tx: 38.6 kB} +[ 640.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 41.7 kB, tx: 42.0 kB} frontend {rx: 42.0 kB, tx: 41.7 kB} +[ 640.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 27.5 kB, tx: 28.0 kB} frontend {rx: 28.0 kB, tx: 27.5 kB} +[ 640.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 28.4 kB, tx: 28.7 kB} frontend {rx: 28.7 kB, tx: 28.4 kB} +[ 645.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 39.8 kB, tx: 40.0 kB} frontend {rx: 40.0 kB, tx: 39.8 kB} +[ 645.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 52.2 kB, tx: 52.5 kB} frontend {rx: 52.5 kB, tx: 52.2 kB} +[ 645.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 39.3 kB, tx: 39.6 kB} frontend {rx: 39.6 kB, tx: 39.3 kB} +[ 645.300] INFO zrok/controller/limits.(*shareWarningAction).HandleShare: warning '0evcupz5k410' +[ 647.031] INFO zrok/controller/limits.sendLimitWarningEmail: limit warning email sent to 'michael@quigley.com' +[ 650.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 36.3 kB, tx: 36.7 kB} frontend {rx: 36.7 kB, tx: 36.3 kB} +[ 650.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 50.7 kB, tx: 51.0 kB} frontend {rx: 51.0 kB, tx: 50.7 kB} +[ 650.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 48.5 kB, tx: 48.8 kB} frontend {rx: 48.8 kB, tx: 48.5 kB} +[ 655.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 24.6 kB, tx: 24.9 kB} frontend {rx: 24.9 kB, tx: 24.6 kB} +[ 655.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 45.6 kB, tx: 46.0 kB} frontend {rx: 46.0 kB, tx: 45.6 kB} +[ 655.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 51.8 kB, tx: 52.1 kB} frontend {rx: 52.1 kB, tx: 51.8 kB} +[ 655.284] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: limiting '53z6mz4re7tu' +[ 655.299] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '64Kz6F7CluxH1drfyMkzDx' for environment 'd.wJYlpt9' +[ 655.299] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: removed dial service policy for '53z6mz4re7tu' +[ 660.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 70.9 kB, tx: 71.4 kB} frontend {rx: 71.4 kB, tx: 70.9 kB} +[ 660.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: CjNBpViur backend {rx: 49.0 kB, tx: 49.5 kB} frontend {rx: 49.5 kB, tx: 49.0 kB} +[ 660.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 36.2 kB, tx: 36.5 kB} frontend {rx: 36.5 kB, tx: 36.2 kB} +[ 665.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 48.3 kB, tx: 48.7 kB} frontend {rx: 48.7 kB, tx: 48.3 kB} +[ 665.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 42.1 kB, tx: 42.4 kB} frontend {rx: 42.4 kB, tx: 42.1 kB} +[ 665.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 47.4 kB, tx: 47.7 kB} frontend {rx: 47.7 kB, tx: 47.4 kB} +[ 670.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 40.2 kB, tx: 40.6 kB} frontend {rx: 40.6 kB, tx: 40.2 kB} +[ 670.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 62.1 kB, tx: 62.4 kB} frontend {rx: 62.4 kB, tx: 62.1 kB} +[ 675.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 13.8 kB, tx: 14.1 kB} frontend {rx: 14.1 kB, tx: 13.8 kB} +[ 675.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 36.6 kB, tx: 36.8 kB} frontend {rx: 36.8 kB, tx: 36.6 kB} +[ 675.168] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 35.3 kB, tx: 35.6 kB} frontend {rx: 35.6 kB, tx: 35.3 kB} +[ 680.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 55.3 kB, tx: 55.8 kB} frontend {rx: 55.8 kB, tx: 55.3 kB} +[ 680.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 46.6 kB, tx: 46.9 kB} frontend {rx: 46.9 kB, tx: 46.6 kB} +[ 685.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 56.2 kB, tx: 56.5 kB} frontend {rx: 56.5 kB, tx: 56.2 kB} +[ 685.172] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 42.4 kB, tx: 42.8 kB} frontend {rx: 42.8 kB, tx: 42.4 kB} +[ 685.175] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 46.8 kB, tx: 47.1 kB} frontend {rx: 47.1 kB, tx: 46.8 kB} +[ 690.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 72.4 kB, tx: 72.8 kB} frontend {rx: 72.8 kB, tx: 72.4 kB} +[ 690.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 58.5 kB, tx: 58.7 kB} frontend {rx: 58.7 kB, tx: 58.5 kB} +[ 695.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 21.1 kB, tx: 21.4 kB} frontend {rx: 21.4 kB, tx: 21.1 kB} +[ 695.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 41.5 kB, tx: 41.8 kB} frontend {rx: 41.8 kB, tx: 41.5 kB} +[ 695.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 49.5 kB, tx: 49.8 kB} frontend {rx: 49.8 kB, tx: 49.5 kB} +[ 700.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 28.6 kB, tx: 28.9 kB} frontend {rx: 28.9 kB, tx: 28.6 kB} +[ 700.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 58.6 kB, tx: 59.0 kB} frontend {rx: 59.0 kB, tx: 58.6 kB} +[ 700.193] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: limiting 's0uzz1p7xjrr' +[ 700.208] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '6MZ8i9sqvom96P70P24FJQ' for environment 'd.wJYlpt9' +[ 700.208] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: removed dial service policy for 's0uzz1p7xjrr' +[ 705.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 40.1 kB, tx: 40.6 kB} frontend {rx: 40.6 kB, tx: 40.1 kB} +[ 705.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 55.7 kB, tx: 56.1 kB} frontend {rx: 56.1 kB, tx: 55.7 kB} +[ 705.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: OJyBpVS0w backend {rx: 46.0 kB, tx: 46.2 kB} frontend {rx: 46.2 kB, tx: 46.0 kB} +[ 710.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 20.4 kB, tx: 20.6 kB} frontend {rx: 20.6 kB, tx: 20.4 kB} +[ 710.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 74.0 kB, tx: 74.4 kB} frontend {rx: 74.4 kB, tx: 74.0 kB} +[ 710.178] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: limiting 'tr7vpyrzvmh0' +[ 710.192] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '1kfuMP2APitf3qC2tsOC1b' for environment 'd.wJYlpt9' +[ 710.192] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: removed dial service policy for 'tr7vpyrzvmh0' +[ 715.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 59.7 kB, tx: 60.2 kB} frontend {rx: 60.2 kB, tx: 59.7 kB} +[ 715.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: LdZLhVSuw backend {rx: 67.5 kB, tx: 67.8 kB} frontend {rx: 67.8 kB, tx: 67.5 kB} +[ 720.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 43.7 kB, tx: 44.1 kB} frontend {rx: 44.1 kB, tx: 43.7 kB} +[ 725.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 36.4 kB, tx: 36.6 kB} frontend {rx: 36.6 kB, tx: 36.4 kB} +[ 725.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 72.2 kB, tx: 72.7 kB} frontend {rx: 72.7 kB, tx: 72.2 kB} +[ 730.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 41.0 kB, tx: 41.4 kB} frontend {rx: 41.4 kB, tx: 41.0 kB} +[ 735.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 31.3 kB, tx: 31.6 kB} frontend {rx: 31.6 kB, tx: 31.3 kB} +[ 735.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 39.1 kB, tx: 39.4 kB} frontend {rx: 39.4 kB, tx: 39.1 kB} +[ 740.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: dh3f3jj7zhig, circuit: 5TNBhVS0r backend {rx: 30.7 kB, tx: 31.0 kB} frontend {rx: 31.0 kB, tx: 30.7 kB} +[ 740.177] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: limiting 'dh3f3jj7zhig' +[ 740.192] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '3LQG2ptwUxIuWtRzTLAqAc' for environment 'd.wJYlpt9' +[ 740.192] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: removed dial service policy for 'dh3f3jj7zhig' +[ 745.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 38.2 kB, tx: 38.5 kB} frontend {rx: 38.5 kB, tx: 38.2 kB} +[ 745.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: qwlLhVS0w backend {rx: 42.3 kB, tx: 42.7 kB} frontend {rx: 42.7 kB, tx: 42.3 kB} +[ 745.192] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: limiting '8k6dnu7x7ag0' +[ 745.195] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '12jWOvjIIuvYRW9vXfkRKw' for environment 'd.wJYlpt9' +[ 745.195] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: removed dial service policy for '8k6dnu7x7ag0' +[ 750.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 36.4 kB, tx: 36.7 kB} frontend {rx: 36.7 kB, tx: 36.4 kB} +[ 760.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: fMNBpVi0w backend {rx: 57.5 kB, tx: 58.0 kB} frontend {rx: 58.0 kB, tx: 57.5 kB} +[ 760.178] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: limiting '0evcupz5k410' +[ 760.194] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '4BPqQhFsGGmoBsqFDIWlWA' for environment 'd.wJYlpt9' +[ 760.194] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: removed dial service policy for '0evcupz5k410' +[ 820.195] INFO zrok/controller/limits.(*Agent).relax: relaxing +[ 820.200] ERROR zrok/controller/limits.(*Agent).checkShareLimit: expected 2 results; got '0' (from(bucket: "zrok")|> range(start: -5m0s)|> filter(fn: (r) => r["_measurement"] == "xfer")|> filter(fn: (r) => r["_field"] == "rx" or r["_field"] == "tx")|> filter(fn: (r) => r["namespace"] == "backend")|> filter(fn: (r) => r["share"] == "7u9szn30ikh0")|> sum()) +[ 820.201] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: relaxing '7u9szn30ikh0' +[ 820.215] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '4yz1WSGg04BeARMuVkmxf7' for service '6FzYnK0RFJmT0rDSP1bzVE' for identities '[rBayMvm7UI]' +[ 820.215] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: added dial service policy for '7u9szn30ikh0' +[ 820.219] INFO zrok/controller/limits.(*Agent).relax: share '53z6mz4re7tu' still over limit +[ 820.223] INFO zrok/controller/limits.(*Agent).relax: share 's0uzz1p7xjrr' still over limit +[ 820.227] INFO zrok/controller/limits.(*Agent).relax: share 'tr7vpyrzvmh0' still over limit +[ 820.231] INFO zrok/controller/limits.(*Agent).relax: share 'dh3f3jj7zhig' still over limit +[ 820.236] INFO zrok/controller/limits.(*Agent).relax: share '8k6dnu7x7ag0' still over limit +[ 820.240] INFO zrok/controller/limits.(*Agent).relax: share '0evcupz5k410' still over limit +[ 830.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 61.6 kB, tx: 61.9 kB} frontend {rx: 61.9 kB, tx: 61.6 kB} +[ 840.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 61.4 kB, tx: 61.7 kB} frontend {rx: 61.7 kB, tx: 61.4 kB} +[ 850.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 30.4 kB, tx: 30.7 kB} frontend {rx: 30.7 kB, tx: 30.4 kB} +[ 860.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 52.6 kB, tx: 53.0 kB} frontend {rx: 53.0 kB, tx: 52.6 kB} +[ 870.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 42.2 kB, tx: 42.6 kB} frontend {rx: 42.6 kB, tx: 42.2 kB} +[ 880.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 27.1 kB, tx: 27.4 kB} frontend {rx: 27.4 kB, tx: 27.1 kB} +[ 890.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 49.9 kB, tx: 50.3 kB} frontend {rx: 50.3 kB, tx: 49.9 kB} +[ 900.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 51.5 kB, tx: 51.8 kB} frontend {rx: 51.8 kB, tx: 51.5 kB} +[ 910.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 44.0 kB, tx: 44.5 kB} frontend {rx: 44.5 kB, tx: 44.0 kB} +[ 920.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 43.5 kB, tx: 43.8 kB} frontend {rx: 43.8 kB, tx: 43.5 kB} +[ 930.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 61.3 kB, tx: 61.7 kB} frontend {rx: 61.7 kB, tx: 61.3 kB} +[ 930.177] INFO zrok/controller/limits.(*shareWarningAction).HandleShare: warning '7u9szn30ikh0' +[ 931.057] INFO zrok/controller/limits.sendLimitWarningEmail: limit warning email sent to 'michael@quigley.com' +[ 940.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 30.2 kB, tx: 30.5 kB} frontend {rx: 30.5 kB, tx: 30.2 kB} +[ 950.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 56.2 kB, tx: 56.6 kB} frontend {rx: 56.6 kB, tx: 56.2 kB} +[ 960.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 73.1 kB, tx: 73.6 kB} frontend {rx: 73.6 kB, tx: 73.1 kB} +[ 970.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 35.1 kB, tx: 35.4 kB} frontend {rx: 35.4 kB, tx: 35.1 kB} +[ 980.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 63.6 kB, tx: 64.0 kB} frontend {rx: 64.0 kB, tx: 63.6 kB} +[ 990.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 46.6 kB, tx: 47.0 kB} frontend {rx: 47.0 kB, tx: 46.6 kB} +[1000.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 36.8 kB, tx: 37.3 kB} frontend {rx: 37.3 kB, tx: 36.8 kB} +[1010.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 24.5 kB, tx: 24.9 kB} frontend {rx: 24.9 kB, tx: 24.5 kB} +[1020.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 47.3 kB, tx: 47.7 kB} frontend {rx: 47.7 kB, tx: 47.3 kB} +[1030.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 29.6 kB, tx: 29.9 kB} frontend {rx: 29.9 kB, tx: 29.6 kB} +[1040.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 48.7 kB, tx: 49.1 kB} frontend {rx: 49.1 kB, tx: 48.7 kB} +[1050.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 7u9szn30ikh0, circuit: ESgSh4i0r backend {rx: 41.8 kB, tx: 42.0 kB} frontend {rx: 42.0 kB, tx: 41.8 kB} +[1050.284] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: limiting '7u9szn30ikh0' +[1050.300] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '4yz1WSGg04BeARMuVkmxf7' for environment 'd.wJYlpt9' +[1050.300] INFO zrok/controller/limits.(*shareLimitAction).HandleShare: removed dial service policy for '7u9szn30ikh0' +[1110.301] INFO zrok/controller/limits.(*Agent).relax: relaxing +[1110.307] ERROR zrok/controller/limits.(*Agent).checkShareLimit: expected 2 results; got '0' (from(bucket: "zrok")|> range(start: -5m0s)|> filter(fn: (r) => r["_measurement"] == "xfer")|> filter(fn: (r) => r["_field"] == "rx" or r["_field"] == "tx")|> filter(fn: (r) => r["namespace"] == "backend")|> filter(fn: (r) => r["share"] == "53z6mz4re7tu")|> sum()) +[1110.307] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: relaxing '53z6mz4re7tu' +[1110.321] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy 'WxOiC60VDWvHHlbtcaJ6D' for service '2NiotGOyBHBEbFZwbTFJ2u' for identities '[rBayMvm7UI]' +[1110.321] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: added dial service policy for '53z6mz4re7tu' +[1110.325] ERROR zrok/controller/limits.(*Agent).checkShareLimit: expected 2 results; got '0' (from(bucket: "zrok")|> range(start: -5m0s)|> filter(fn: (r) => r["_measurement"] == "xfer")|> filter(fn: (r) => r["_field"] == "rx" or r["_field"] == "tx")|> filter(fn: (r) => r["namespace"] == "backend")|> filter(fn: (r) => r["share"] == "s0uzz1p7xjrr")|> sum()) +[1110.325] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: relaxing 's0uzz1p7xjrr' +[1110.327] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '2ubWYvKo2EOnrn1U4MQ4Cu' for service 'KtK5E46HR93YIBrrwUlIN' for identities '[rBayMvm7UI]' +[1110.327] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: added dial service policy for 's0uzz1p7xjrr' +[1110.331] ERROR zrok/controller/limits.(*Agent).checkShareLimit: expected 2 results; got '0' (from(bucket: "zrok")|> range(start: -5m0s)|> filter(fn: (r) => r["_measurement"] == "xfer")|> filter(fn: (r) => r["_field"] == "rx" or r["_field"] == "tx")|> filter(fn: (r) => r["namespace"] == "backend")|> filter(fn: (r) => r["share"] == "tr7vpyrzvmh0")|> sum()) +[1110.331] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: relaxing 'tr7vpyrzvmh0' +[1110.343] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '1Q2DMHZ9AFsBA8D2SNzC4l' for service '7jyiTZ0z2ediD5hZbxu7KH' for identities '[rBayMvm7UI]' +[1110.343] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: added dial service policy for 'tr7vpyrzvmh0' +[1110.348] ERROR zrok/controller/limits.(*Agent).checkShareLimit: expected 2 results; got '0' (from(bucket: "zrok")|> range(start: -5m0s)|> filter(fn: (r) => r["_measurement"] == "xfer")|> filter(fn: (r) => r["_field"] == "rx" or r["_field"] == "tx")|> filter(fn: (r) => r["namespace"] == "backend")|> filter(fn: (r) => r["share"] == "dh3f3jj7zhig")|> sum()) +[1110.349] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: relaxing 'dh3f3jj7zhig' +[1110.351] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy 'BrG9wKvUsajfhPkVfz44g' for service 'nyKOLlxUWWbCzD7h9Jhjq' for identities '[rBayMvm7UI]' +[1110.351] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: added dial service policy for 'dh3f3jj7zhig' +[1110.356] ERROR zrok/controller/limits.(*Agent).checkShareLimit: expected 2 results; got '0' (from(bucket: "zrok")|> range(start: -5m0s)|> filter(fn: (r) => r["_measurement"] == "xfer")|> filter(fn: (r) => r["_field"] == "rx" or r["_field"] == "tx")|> filter(fn: (r) => r["namespace"] == "backend")|> filter(fn: (r) => r["share"] == "8k6dnu7x7ag0")|> sum()) +[1110.356] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: relaxing '8k6dnu7x7ag0' +[1110.364] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy '1kbYWDgPbtk0JYjIPsRGOC' for service '2J0I9dPe2JGnY1GwjmM6n7' for identities '[rBayMvm7UI]' +[1110.364] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: added dial service policy for '8k6dnu7x7ag0' +[1110.372] ERROR zrok/controller/limits.(*Agent).checkShareLimit: expected 2 results; got '0' (from(bucket: "zrok")|> range(start: -5m0s)|> filter(fn: (r) => r["_measurement"] == "xfer")|> filter(fn: (r) => r["_field"] == "rx" or r["_field"] == "tx")|> filter(fn: (r) => r["namespace"] == "backend")|> filter(fn: (r) => r["share"] == "0evcupz5k410")|> sum()) +[1110.372] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: relaxing '0evcupz5k410' +[1110.374] INFO zrok/controller/zrokEdgeSdk.CreateServicePolicyDial: created dial service policy 'E30643mY9M6vU6bQSQHa9' for service '3WHJGqUdxkDtPYLgEL5V3q' for identities '[rBayMvm7UI]' +[1110.374] INFO zrok/controller/limits.(*shareRelaxAction).HandleShare: added dial service policy for '0evcupz5k410' +[1110.378] INFO zrok/controller/limits.(*Agent).relax: share '7u9szn30ikh0' still over limit +[1115.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: gaSGp4i0r backend {rx: 3.4 kB, tx: 3.4 kB} frontend {rx: 3.4 kB, tx: 3.4 kB} +[1120.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: gaSGp4i0r backend {rx: 26.5 kB, tx: 26.7 kB} frontend {rx: 26.7 kB, tx: 26.5 kB} +[1120.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: nESTh4iur backend {rx: 65.6 kB, tx: 66.1 kB} frontend {rx: 66.1 kB, tx: 65.6 kB} +[1120.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: GGiTp4S0w backend {rx: 53.5 kB, tx: 54.0 kB} frontend {rx: 54.0 kB, tx: 53.5 kB} +[1125.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: gaSGp4i0r backend {rx: 43.7 kB, tx: 44.1 kB} frontend {rx: 44.1 kB, tx: 43.7 kB} +[1125.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: fSiGh4iur backend {rx: 17.8 kB, tx: 18.0 kB} frontend {rx: 18.0 kB, tx: 17.8 kB} +[1125.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 0evcupz5k410, circuit: k4SGhVSuw backend {rx: 51.9 kB, tx: 52.3 kB} frontend {rx: 52.3 kB, tx: 51.9 kB} +[1130.163] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: gaSGp4i0r backend {rx: 50.9 kB, tx: 51.2 kB} frontend {rx: 51.2 kB, tx: 50.9 kB} +[1130.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: GGiTp4S0w backend {rx: 48.6 kB, tx: 49.0 kB} frontend {rx: 49.0 kB, tx: 48.6 kB} +[1130.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: fSiGh4iur backend {rx: 37.8 kB, tx: 38.0 kB} frontend {rx: 38.0 kB, tx: 37.8 kB} +[1135.164] INFO zrok/controller/metrics.(*influxWriter).Handle: share: tr7vpyrzvmh0, circuit: gaSGp4i0r backend {rx: 57.6 kB, tx: 58.1 kB} frontend {rx: 58.1 kB, tx: 57.6 kB} +[1135.166] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: fSiGh4iur backend {rx: 43.6 kB, tx: 44.1 kB} frontend {rx: 44.1 kB, tx: 43.6 kB} +[1135.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: GGiTp4S0w backend {rx: 51.0 kB, tx: 51.4 kB} frontend {rx: 51.4 kB, tx: 51.0 kB} +[1140.165] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 8k6dnu7x7ag0, circuit: nESTh4iur backend {rx: 28.4 kB, tx: 28.6 kB} frontend {rx: 28.6 kB, tx: 28.4 kB} +[1140.167] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 53z6mz4re7tu, circuit: fSiGh4iur backend {rx: 31.6 kB, tx: 32.0 kB} frontend {rx: 32.0 kB, tx: 31.6 kB} +[1140.169] INFO zrok/controller/metrics.(*influxWriter).Handle: share: s0uzz1p7xjrr, circuit: GGiTp4S0w backend {rx: 23.8 kB, tx: 24.2 kB} frontend {rx: 24.2 kB, tx: 23.8 kB} +[1141.514] INFO zrok/controller/zrokEdgeSdk.DeleteServiceEdgeRouterPolicy: deleted service edge router policy '2RIKOBMOckfbI2xMSLAKxC' for environment 'd.wJYlpt9' +[1141.517] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '1kbYWDgPbtk0JYjIPsRGOC' for environment 'd.wJYlpt9' +[1141.519] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '4vT5eEPahgWEVdAuKN91Sd' for environment 'd.wJYlpt9' +[1141.521] INFO zrok/controller/zrokEdgeSdk.DeleteConfig: deleted config '5nG9jM8VNl0uBFcRRt3AvI' for 'd.wJYlpt9' +[1141.522] INFO zrok/controller/zrokEdgeSdk.DeleteService: deleted service '2J0I9dPe2JGnY1GwjmM6n7' for environment 'd.wJYlpt9' +[1141.599] INFO zrok/controller/zrokEdgeSdk.DeleteServiceEdgeRouterPolicy: deleted service edge router policy '2CM03d1cNpG4rma38BLzCQ' for environment 'd.wJYlpt9' +[1141.602] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy 'WxOiC60VDWvHHlbtcaJ6D' for environment 'd.wJYlpt9' +[1141.635] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy 'RRfDaA5kjCqUBVC9LvN1H' for environment 'd.wJYlpt9' +[1141.639] INFO zrok/controller/zrokEdgeSdk.DeleteConfig: deleted config '6U3XDGnBjtONN5H6pUze12' for 'd.wJYlpt9' +[1141.645] INFO zrok/controller/zrokEdgeSdk.DeleteService: deleted service '2NiotGOyBHBEbFZwbTFJ2u' for environment 'd.wJYlpt9' +[1141.701] INFO zrok/controller/zrokEdgeSdk.DeleteServiceEdgeRouterPolicy: deleted service edge router policy '2ZnnIXSTQ3Zscha1kykqQr' for environment 'd.wJYlpt9' +[1141.704] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy 'BrG9wKvUsajfhPkVfz44g' for environment 'd.wJYlpt9' +[1141.706] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '1xF4ky6cDJm63tzlNTqoLC' for environment 'd.wJYlpt9' +[1141.707] INFO zrok/controller/zrokEdgeSdk.DeleteConfig: deleted config '76iBDASRcxOmGtdwjVHo26' for 'd.wJYlpt9' +[1141.708] INFO zrok/controller/zrokEdgeSdk.DeleteService: deleted service 'nyKOLlxUWWbCzD7h9Jhjq' for environment 'd.wJYlpt9' +[1141.926] INFO zrok/controller/zrokEdgeSdk.DeleteServiceEdgeRouterPolicy: deleted service edge router policy '3xAG26zA9yska3LeZQUJ3N' for environment 'd.wJYlpt9' +[1141.927] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: did not find a service policy +[1141.929] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '1mabRt9jefSe52CJh6FmhB' for environment 'd.wJYlpt9' +[1141.931] INFO zrok/controller/zrokEdgeSdk.DeleteConfig: deleted config '2gid15nP0GIUVuaFQ15GWV' for 'd.wJYlpt9' +[1141.932] INFO zrok/controller/zrokEdgeSdk.DeleteService: deleted service '6FzYnK0RFJmT0rDSP1bzVE' for environment 'd.wJYlpt9' +[1142.053] INFO zrok/controller/zrokEdgeSdk.DeleteServiceEdgeRouterPolicy: deleted service edge router policy '2nMZaiChQAPpFnblNn1ljP' for environment 'd.wJYlpt9' +[1142.056] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '1Q2DMHZ9AFsBA8D2SNzC4l' for environment 'd.wJYlpt9' +[1142.058] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '6RwWEoIsb8gBVKJfZP3ur3' for environment 'd.wJYlpt9' +[1142.064] INFO zrok/controller/zrokEdgeSdk.DeleteConfig: deleted config '1FnBhnGNXDe58dwTpbFc1x' for 'd.wJYlpt9' +[1142.066] INFO zrok/controller/zrokEdgeSdk.DeleteService: deleted service '7jyiTZ0z2ediD5hZbxu7KH' for environment 'd.wJYlpt9' +[1142.320] INFO zrok/controller/zrokEdgeSdk.DeleteServiceEdgeRouterPolicy: deleted service edge router policy '2AqCUMqNtarmglOfhvnkI' for environment 'd.wJYlpt9' +[1142.324] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy 'E30643mY9M6vU6bQSQHa9' for environment 'd.wJYlpt9' +[1142.326] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '4V8FsgCt63ySkG2pFWG5fz' for environment 'd.wJYlpt9' +[1142.329] INFO zrok/controller/zrokEdgeSdk.DeleteConfig: deleted config '19cyxfHo32R6fhVsYHZ84g' for 'd.wJYlpt9' +[1142.330] INFO zrok/controller/zrokEdgeSdk.DeleteService: deleted service '3WHJGqUdxkDtPYLgEL5V3q' for environment 'd.wJYlpt9' +[1142.701] INFO zrok/controller/zrokEdgeSdk.DeleteServiceEdgeRouterPolicy: deleted service edge router policy '2CGCz8dcquNvZC0ZUwDZ5F' for environment 'd.wJYlpt9' +[1142.704] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '2ubWYvKo2EOnrn1U4MQ4Cu' for environment 'd.wJYlpt9' +[1142.708] INFO zrok/controller/zrokEdgeSdk.DeleteServicePolicy: deleted service policy '6oohOQFEo75yl9vnIbyzdj' for environment 'd.wJYlpt9' +[1142.709] INFO zrok/controller/zrokEdgeSdk.DeleteConfig: deleted config '4AN4sOtdQv99uHmFn3erx4' for 'd.wJYlpt9' +[1142.710] INFO zrok/controller/zrokEdgeSdk.DeleteService: deleted service 'KtK5E46HR93YIBrrwUlIN' for environment 'd.wJYlpt9' +``` \ No newline at end of file From bd4ce22d971e1bd322da0dcc4d16fb2b8c73c473 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 28 Mar 2023 15:05:51 -0400 Subject: [PATCH 55/83] don't try to replace policies for deleted shares (#276) --- controller/limits/accountRelaxAction.go | 32 +++++++++++---------- controller/limits/environmentRelaxAction.go | 32 +++++++++++---------- controller/limits/shareRelaxAction.go | 32 +++++++++++---------- 3 files changed, 51 insertions(+), 45 deletions(-) diff --git a/controller/limits/accountRelaxAction.go b/controller/limits/accountRelaxAction.go index cea7aa21..5fe2045e 100644 --- a/controller/limits/accountRelaxAction.go +++ b/controller/limits/accountRelaxAction.go @@ -33,24 +33,26 @@ func (a *accountRelaxAction) HandleAccount(acct *store.Account, _, _ int64, _ *B } for _, shr := range shrs { - if shr.ShareMode == "public" { - env, err := a.str.GetEnvironment(shr.EnvironmentId, trx) - if err != nil { - return errors.Wrap(err, "error finding environment") - } + if !shr.Deleted { + if shr.ShareMode == "public" { + env, err := a.str.GetEnvironment(shr.EnvironmentId, trx) + if err != nil { + return errors.Wrap(err, "error finding environment") + } - fe, err := a.str.FindFrontendPubliclyNamed(*shr.FrontendSelection, trx) - if err != nil { - return errors.Wrapf(err, "error finding frontend name '%v' for '%v'", *shr.FrontendSelection, shr.Token) - } + fe, err := a.str.FindFrontendPubliclyNamed(*shr.FrontendSelection, trx) + if err != nil { + return errors.Wrapf(err, "error finding frontend name '%v' for '%v'", *shr.FrontendSelection, shr.Token) + } - if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+shr.ZId+"-dial", shr.ZId, []string{fe.ZId}, zrokEdgeSdk.ZrokShareTags(shr.Token).SubTags, a.edge); err != nil { - return errors.Wrapf(err, "error creating dial service policy for '%v'", shr.Token) - } - logrus.Infof("added dial service policy for '%v'", shr.Token) + if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+shr.ZId+"-dial", shr.ZId, []string{fe.ZId}, zrokEdgeSdk.ZrokShareTags(shr.Token).SubTags, a.edge); err != nil { + return errors.Wrapf(err, "error creating dial service policy for '%v'", shr.Token) + } + logrus.Infof("added dial service policy for '%v'", shr.Token) - } else if shr.ShareMode == "private" { - return errors.New("share relax for private share not implemented") + } else if shr.ShareMode == "private" { + return errors.New("share relax for private share not implemented") + } } } } diff --git a/controller/limits/environmentRelaxAction.go b/controller/limits/environmentRelaxAction.go index aa79ade3..c867d0db 100644 --- a/controller/limits/environmentRelaxAction.go +++ b/controller/limits/environmentRelaxAction.go @@ -27,24 +27,26 @@ func (a *environmentRelaxAction) HandleEnvironment(env *store.Environment, rxByt } for _, shr := range shrs { - if shr.ShareMode == "public" { - env, err := a.str.GetEnvironment(shr.EnvironmentId, trx) - if err != nil { - return errors.Wrap(err, "error finding environment") - } + if !shr.Deleted { + if shr.ShareMode == "public" { + env, err := a.str.GetEnvironment(shr.EnvironmentId, trx) + if err != nil { + return errors.Wrap(err, "error finding environment") + } - fe, err := a.str.FindFrontendPubliclyNamed(*shr.FrontendSelection, trx) - if err != nil { - return errors.Wrapf(err, "error finding frontend name '%v' for '%v'", *shr.FrontendSelection, shr.Token) - } + fe, err := a.str.FindFrontendPubliclyNamed(*shr.FrontendSelection, trx) + if err != nil { + return errors.Wrapf(err, "error finding frontend name '%v' for '%v'", *shr.FrontendSelection, shr.Token) + } - if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+shr.ZId+"-dial", shr.ZId, []string{fe.ZId}, zrokEdgeSdk.ZrokShareTags(shr.Token).SubTags, a.edge); err != nil { - return errors.Wrapf(err, "error creating dial service policy for '%v'", shr.Token) - } - logrus.Infof("added dial service policy for '%v'", shr.Token) + if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+shr.ZId+"-dial", shr.ZId, []string{fe.ZId}, zrokEdgeSdk.ZrokShareTags(shr.Token).SubTags, a.edge); err != nil { + return errors.Wrapf(err, "error creating dial service policy for '%v'", shr.Token) + } + logrus.Infof("added dial service policy for '%v'", shr.Token) - } else if shr.ShareMode == "private" { - return errors.New("share relax for private share not implemented") + } else if shr.ShareMode == "private" { + return errors.New("share relax for private share not implemented") + } } } diff --git a/controller/limits/shareRelaxAction.go b/controller/limits/shareRelaxAction.go index e702cf01..4e0a0340 100644 --- a/controller/limits/shareRelaxAction.go +++ b/controller/limits/shareRelaxAction.go @@ -21,24 +21,26 @@ func newShareRelaxAction(str *store.Store, edge *rest_management_api_client.Ziti func (a *shareRelaxAction) HandleShare(shr *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { logrus.Infof("relaxing '%v'", shr.Token) - if shr.ShareMode == "public" { - env, err := a.str.GetEnvironment(shr.EnvironmentId, trx) - if err != nil { - return errors.Wrap(err, "error finding environment") - } + if !shr.Deleted { + if shr.ShareMode == "public" { + env, err := a.str.GetEnvironment(shr.EnvironmentId, trx) + if err != nil { + return errors.Wrap(err, "error finding environment") + } - fe, err := a.str.FindFrontendPubliclyNamed(*shr.FrontendSelection, trx) - if err != nil { - return errors.Wrapf(err, "error finding frontend name '%v' for '%v'", *shr.FrontendSelection, shr.Token) - } + fe, err := a.str.FindFrontendPubliclyNamed(*shr.FrontendSelection, trx) + if err != nil { + return errors.Wrapf(err, "error finding frontend name '%v' for '%v'", *shr.FrontendSelection, shr.Token) + } - if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+shr.ZId+"-dial", shr.ZId, []string{fe.ZId}, zrokEdgeSdk.ZrokShareTags(shr.Token).SubTags, a.edge); err != nil { - return errors.Wrapf(err, "error creating dial service policy for '%v'", shr.Token) - } - logrus.Infof("added dial service policy for '%v'", shr.Token) + if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+shr.ZId+"-dial", shr.ZId, []string{fe.ZId}, zrokEdgeSdk.ZrokShareTags(shr.Token).SubTags, a.edge); err != nil { + return errors.Wrapf(err, "error creating dial service policy for '%v'", shr.Token) + } + logrus.Infof("added dial service policy for '%v'", shr.Token) - } else if shr.ShareMode == "private" { - return errors.New("share relax for private share not implemented") + } else if shr.ShareMode == "private" { + return errors.New("share relax for private share not implemented") + } } return nil From e2d0b7990e6cc417ca865902965b6acc888e329e Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 28 Mar 2023 15:06:19 -0400 Subject: [PATCH 56/83] empty metrics data sets are not an error (#280) --- controller/limits/influxReader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controller/limits/influxReader.go b/controller/limits/influxReader.go index 22b8c4d1..d8b392c6 100644 --- a/controller/limits/influxReader.go +++ b/controller/limits/influxReader.go @@ -81,7 +81,7 @@ func (r *influxReader) runQueryForRxTx(query string) (rx int64, tx int64, err er } count++ } - if count != 2 { + if count != 0 && count != 2 { return -1, -1, errors.Errorf("expected 2 results; got '%d' (%v)", count, strings.ReplaceAll(query, "\n", "")) } return rx, tx, nil From b8aec46548f4e3d48bd95d04a0620af8949b8634 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 28 Mar 2023 15:19:01 -0400 Subject: [PATCH 57/83] debug lint (#276) --- controller/limits/agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 67e5632e..fd58a8f4 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -385,7 +385,7 @@ func (a *Agent) enforce(u *metrics.Usage) error { } func (a *Agent) relax() error { - logrus.Info("relaxing") + logrus.Debug("relaxing") trx, err := a.str.Begin() if err != nil { From f9dc0f6ba17e3ae555e4c703e70ed85f52baec20 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 29 Mar 2023 13:29:12 -0400 Subject: [PATCH 58/83] check limit journals for creating environments or shares (#281) --- controller/limits/agent.go | 68 +++++++++++++++++++++++++++++--------- controller/share.go | 6 ++-- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/controller/limits/agent.go b/controller/limits/agent.go index fd58a8f4..9ec418a3 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -69,35 +69,71 @@ func (a *Agent) Stop() { } func (a *Agent) CanCreateEnvironment(acctId int, trx *sqlx.Tx) (bool, error) { - if a.cfg.Enforcing && a.cfg.Environments > Unlimited { - envs, err := a.str.FindEnvironmentsForAccount(acctId, trx) + if a.cfg.Enforcing { + alj, err := a.str.FindLatestAccountLimitJournal(acctId, trx) if err != nil { return false, err } - if len(envs)+1 > a.cfg.Environments { + if alj.Action == store.LimitAction { return false, nil } + + if a.cfg.Environments > Unlimited { + envs, err := a.str.FindEnvironmentsForAccount(acctId, trx) + if err != nil { + return false, err + } + if len(envs)+1 > a.cfg.Environments { + return false, nil + } + } } return true, nil } -func (a *Agent) CanCreateShare(acctId int, trx *sqlx.Tx) (bool, error) { - if a.cfg.Enforcing && a.cfg.Shares > Unlimited { - envs, err := a.str.FindEnvironmentsForAccount(acctId, trx) - if err != nil { - return false, err - } - total := 0 - for i := range envs { - shrs, err := a.str.FindSharesForEnvironment(envs[i].Id, trx) +func (a *Agent) CanCreateShare(acctId, envId int, trx *sqlx.Tx) (bool, error) { + if a.cfg.Enforcing { + if empty, err := a.str.IsAccountLimitJournalEmpty(acctId, trx); err == nil && !empty { + alj, err := a.str.FindLatestAccountLimitJournal(acctId, trx) if err != nil { - return false, errors.Wrapf(err, "unable to find shares for environment '%v'", envs[i].ZId) + return false, err } - total += len(shrs) - if total+1 > a.cfg.Shares { + if alj.Action == store.LimitAction { return false, nil } - logrus.Infof("total = %d", total) + } else if err != nil { + return false, err + } + + if empty, err := a.str.IsEnvironmentLimitJournalEmpty(envId, trx); err == nil && !empty { + elj, err := a.str.FindLatestEnvironmentLimitJournal(envId, trx) + if err != nil { + return false, err + } + if elj.Action == store.LimitAction { + return false, nil + } + } else if err != nil { + return false, err + } + + if a.cfg.Shares > Unlimited { + envs, err := a.str.FindEnvironmentsForAccount(acctId, trx) + if err != nil { + return false, err + } + total := 0 + for i := range envs { + shrs, err := a.str.FindSharesForEnvironment(envs[i].Id, trx) + if err != nil { + return false, errors.Wrapf(err, "unable to find shares for environment '%v'", envs[i].ZId) + } + total += len(shrs) + if total+1 > a.cfg.Shares { + return false, nil + } + logrus.Infof("total = %d", total) + } } } return true, nil diff --git a/controller/share.go b/controller/share.go index 4b5b5b1b..bdfc6964 100644 --- a/controller/share.go +++ b/controller/share.go @@ -47,7 +47,7 @@ func (h *shareHandler) Handle(params share.ShareParams, principal *rest_model_zr return share.NewShareInternalServerError() } - if err := h.checkLimits(principal, trx); err != nil { + if err := h.checkLimits(envId, principal, trx); err != nil { logrus.Errorf("limits error: %v", err) return share.NewShareUnauthorized() } @@ -142,10 +142,10 @@ func (h *shareHandler) Handle(params share.ShareParams, principal *rest_model_zr }) } -func (h *shareHandler) checkLimits(principal *rest_model_zrok.Principal, trx *sqlx.Tx) error { +func (h *shareHandler) checkLimits(envId int, principal *rest_model_zrok.Principal, trx *sqlx.Tx) error { if !principal.Limitless { if limitsAgent != nil { - ok, err := limitsAgent.CanCreateShare(int(principal.ID), trx) + ok, err := limitsAgent.CanCreateShare(int(principal.ID), envId, trx) if err != nil { return errors.Wrapf(err, "error checking share limits for '%v'", principal.Email) } From 73ea6184c516cb19bdc1919c3d3970f1a375cb89 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 29 Mar 2023 13:53:14 -0400 Subject: [PATCH 59/83] add 'private_share_id' to store 'frontends' table (#278) --- controller/access.go | 6 ++-- .../010_v0_4_0_frontend_private_share.sql | 31 +++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 controller/store/sql/postgresql/010_v0_4_0_frontend_private_share.sql diff --git a/controller/access.go b/controller/access.go index cffd44c7..3be1d064 100644 --- a/controller/access.go +++ b/controller/access.go @@ -45,12 +45,12 @@ func (h *accessHandler) Handle(params share.AccessParams, principal *rest_model_ } shrToken := params.Body.ShrToken - sshr, err := str.FindShareWithToken(shrToken, tx) + shr, err := str.FindShareWithToken(shrToken, tx) if err != nil { logrus.Errorf("error finding share") return share.NewAccessNotFound() } - if sshr == nil { + if shr == nil { logrus.Errorf("unable to find share '%v' for user '%v'", shrToken, principal.Email) return share.NewAccessNotFound() } @@ -76,7 +76,7 @@ func (h *accessHandler) Handle(params share.AccessParams, principal *rest_model_ "zrokFrontendToken": feToken, "zrokShareToken": shrToken, } - if err := zrokEdgeSdk.CreateServicePolicyDial(envZId+"-"+sshr.ZId+"-dial", sshr.ZId, []string{envZId}, addlTags, edge); err != nil { + if err := zrokEdgeSdk.CreateServicePolicyDial(envZId+"-"+shr.ZId+"-dial", shr.ZId, []string{envZId}, addlTags, edge); err != nil { logrus.Errorf("unable to create dial policy for user '%v': %v", principal.Email, err) return share.NewAccessInternalServerError() } diff --git a/controller/store/sql/postgresql/010_v0_4_0_frontend_private_share.sql b/controller/store/sql/postgresql/010_v0_4_0_frontend_private_share.sql new file mode 100644 index 00000000..cabf651d --- /dev/null +++ b/controller/store/sql/postgresql/010_v0_4_0_frontend_private_share.sql @@ -0,0 +1,31 @@ +-- +migrate Up + +alter table frontends rename to frontends_old; +alter sequence frontends_id_seq rename to frontends_id_seq_old; + +create table frontends ( + id serial primary key, + environment_id integer references environments(id), + private_share_id integer references shares(id), + token varchar(32) not null unique, + z_id varchar(32) not null, + url_template varchar(1024), + public_name varchar(64) unique, + reserved boolean not null default(false), + created_at timestamptz not null default(current_timestamp), + updated_at timestamptz not null default(current_timestamp), + deleted boolean not null default(false), +); + +insert into frontends (id, environment_id, token, z_id, url_template, public_name, reserved, created_at, updated_at, deleted) + select id, environment_id, token, z_id, url_template, public_name, reserved, created_at, updated_at, deleted from frontends_old; + +select setval('frontends_id_seq', (select max(id) from frontends)); + +drop table frontends_old; + +alter index frontends_pkey1 rename to frontends_pkey; +alter index frontends_public_name_key1 to frontends_public_name_key; +alter index frontends_token_key1 to frontends_token_key; + +alter table frontends rename constraint frontends_environment_id_fkey1 to frontends_environment_id_fkey; From 98d5d20d3490d3b4c85a20c1bce03ac0fe990ecb Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 29 Mar 2023 13:57:15 -0400 Subject: [PATCH 60/83] sqlite3 'private_share_id' column in 'frontends' table (#278) --- .../store/sql/sqlite3/010_v0_4_0_frontend_private_share.sql | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 controller/store/sql/sqlite3/010_v0_4_0_frontend_private_share.sql diff --git a/controller/store/sql/sqlite3/010_v0_4_0_frontend_private_share.sql b/controller/store/sql/sqlite3/010_v0_4_0_frontend_private_share.sql new file mode 100644 index 00000000..0af63c63 --- /dev/null +++ b/controller/store/sql/sqlite3/010_v0_4_0_frontend_private_share.sql @@ -0,0 +1,3 @@ +-- +migrate Up + +alter table frontends add column private_share_id references shares(id); From dfb35cc5880c41a9d9226173459bad2db371bb2f Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 29 Mar 2023 14:05:44 -0400 Subject: [PATCH 61/83] store implementation for private_share_id (#278) --- controller/store/frontend.go | 39 ++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/controller/store/frontend.go b/controller/store/frontend.go index 8ffea5e6..8d34145e 100644 --- a/controller/store/frontend.go +++ b/controller/store/frontend.go @@ -7,22 +7,23 @@ import ( type Frontend struct { Model - EnvironmentId *int - Token string - ZId string - PublicName *string - UrlTemplate *string - Reserved bool - Deleted bool + EnvironmentId *int + PrivateShareId *int + Token string + ZId string + PublicName *string + UrlTemplate *string + Reserved bool + Deleted bool } func (str *Store) CreateFrontend(envId int, f *Frontend, tx *sqlx.Tx) (int, error) { - stmt, err := tx.Prepare("insert into frontends (environment_id, token, z_id, public_name, url_template, reserved) values ($1, $2, $3, $4, $5, $6) returning id") + stmt, err := tx.Prepare("insert into frontends (environment_id, private_share_id, token, z_id, public_name, url_template, reserved) values ($1, $2, $3, $4, $5, $6, $7) returning id") if err != nil { return 0, errors.Wrap(err, "error preparing frontends insert statement") } var id int - if err := stmt.QueryRow(envId, f.Token, f.ZId, f.PublicName, f.UrlTemplate, f.Reserved).Scan(&id); err != nil { + if err := stmt.QueryRow(envId, f.PrivateShareId, f.Token, f.ZId, f.PublicName, f.UrlTemplate, f.Reserved).Scan(&id); err != nil { return 0, errors.Wrap(err, "error executing frontends insert statement") } return id, nil @@ -104,13 +105,29 @@ func (str *Store) FindPublicFrontends(tx *sqlx.Tx) ([]*Frontend, error) { return frontends, nil } +func (str *Store) FindFrontendsForPrivateShare(shrId int, tx *sqlx.Tx) ([]*Frontend, error) { + rows, err := tx.Queryx("select frontends.* from frontends where private_share_id = $1 and not deleted", shrId) + if err != nil { + return nil, errors.Wrap(err, "error selecting frontends by private_share_id") + } + var is []*Frontend + for rows.Next() { + i := &Frontend{} + if err := rows.StructScan(i); err != nil { + return nil, errors.Wrap(err, "error scanning frontend") + } + is = append(is, i) + } + return is, nil +} + func (str *Store) UpdateFrontend(fe *Frontend, tx *sqlx.Tx) error { - sql := "update frontends set environment_id = $1, token = $2, z_id = $3, public_name = $4, url_template = $5, reserved = $6, updated_at = current_timestamp where id = $7" + sql := "update frontends set environment_id = $1, private_share_id = $2, token = $3, z_id = $4, public_name = $5, url_template = $6, reserved = $7, updated_at = current_timestamp where id = $8" stmt, err := tx.Prepare(sql) if err != nil { return errors.Wrap(err, "error preparing frontends update statement") } - _, err = stmt.Exec(fe.EnvironmentId, fe.Token, fe.ZId, fe.PublicName, fe.UrlTemplate, fe.Reserved, fe.Id) + _, err = stmt.Exec(fe.EnvironmentId, fe.PrivateShareId, fe.Token, fe.ZId, fe.PublicName, fe.UrlTemplate, fe.Reserved, fe.Id) if err != nil { return errors.Wrap(err, "error executing frontends update statement") } From 540e3ffa745bfc86676e218b447a577045c7c495 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 29 Mar 2023 14:21:02 -0400 Subject: [PATCH 62/83] record share identifier for access private frontends (#278) --- controller/access.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controller/access.go b/controller/access.go index 3be1d064..ee2970a5 100644 --- a/controller/access.go +++ b/controller/access.go @@ -61,7 +61,7 @@ func (h *accessHandler) Handle(params share.AccessParams, principal *rest_model_ return share.NewAccessInternalServerError() } - if _, err := str.CreateFrontend(envId, &store.Frontend{Token: feToken, ZId: envZId}, tx); err != nil { + if _, err := str.CreateFrontend(envId, &store.Frontend{PrivateShareId: &shr.Id, Token: feToken, ZId: envZId}, tx); err != nil { logrus.Errorf("error creating frontend record for user '%v': %v", principal.Email, err) return share.NewAccessInternalServerError() } From 3c92b9a8d08552afe5301a201e34d756b5b95cb1 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 29 Mar 2023 15:08:04 -0400 Subject: [PATCH 63/83] share, environment, and account relax actions all support private shares in addition to public shares; consolidated relax code (#278) --- controller/limits/accountRelaxAction.go | 28 +++------ controller/limits/environmentRelaxAction.go | 24 +++---- controller/limits/shareRelaxAction.go | 70 ++++++++++++++++----- 3 files changed, 68 insertions(+), 54 deletions(-) diff --git a/controller/limits/accountRelaxAction.go b/controller/limits/accountRelaxAction.go index 5fe2045e..e73829dd 100644 --- a/controller/limits/accountRelaxAction.go +++ b/controller/limits/accountRelaxAction.go @@ -4,7 +4,6 @@ import ( "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/store" - "github.com/openziti/zrok/controller/zrokEdgeSdk" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -33,25 +32,14 @@ func (a *accountRelaxAction) HandleAccount(acct *store.Account, _, _ int64, _ *B } for _, shr := range shrs { - if !shr.Deleted { - if shr.ShareMode == "public" { - env, err := a.str.GetEnvironment(shr.EnvironmentId, trx) - if err != nil { - return errors.Wrap(err, "error finding environment") - } - - fe, err := a.str.FindFrontendPubliclyNamed(*shr.FrontendSelection, trx) - if err != nil { - return errors.Wrapf(err, "error finding frontend name '%v' for '%v'", *shr.FrontendSelection, shr.Token) - } - - if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+shr.ZId+"-dial", shr.ZId, []string{fe.ZId}, zrokEdgeSdk.ZrokShareTags(shr.Token).SubTags, a.edge); err != nil { - return errors.Wrapf(err, "error creating dial service policy for '%v'", shr.Token) - } - logrus.Infof("added dial service policy for '%v'", shr.Token) - - } else if shr.ShareMode == "private" { - return errors.New("share relax for private share not implemented") + switch shr.ShareMode { + case "public": + if err := relaxPublicShare(a.str, a.edge, shr, trx); err != nil { + return err + } + case "private": + if err := relaxPrivateShare(a.str, a.edge, shr, trx); err != nil { + return err } } } diff --git a/controller/limits/environmentRelaxAction.go b/controller/limits/environmentRelaxAction.go index c867d0db..8d48e589 100644 --- a/controller/limits/environmentRelaxAction.go +++ b/controller/limits/environmentRelaxAction.go @@ -4,7 +4,6 @@ import ( "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/store" - "github.com/openziti/zrok/controller/zrokEdgeSdk" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -28,24 +27,15 @@ func (a *environmentRelaxAction) HandleEnvironment(env *store.Environment, rxByt for _, shr := range shrs { if !shr.Deleted { - if shr.ShareMode == "public" { - env, err := a.str.GetEnvironment(shr.EnvironmentId, trx) - if err != nil { - return errors.Wrap(err, "error finding environment") + switch shr.ShareMode { + case "public": + if err := relaxPublicShare(a.str, a.edge, shr, trx); err != nil { + return err } - - fe, err := a.str.FindFrontendPubliclyNamed(*shr.FrontendSelection, trx) - if err != nil { - return errors.Wrapf(err, "error finding frontend name '%v' for '%v'", *shr.FrontendSelection, shr.Token) + case "private": + if err := relaxPrivateShare(a.str, a.edge, shr, trx); err != nil { + return err } - - if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+shr.ZId+"-dial", shr.ZId, []string{fe.ZId}, zrokEdgeSdk.ZrokShareTags(shr.Token).SubTags, a.edge); err != nil { - return errors.Wrapf(err, "error creating dial service policy for '%v'", shr.Token) - } - logrus.Infof("added dial service policy for '%v'", shr.Token) - - } else if shr.ShareMode == "private" { - return errors.New("share relax for private share not implemented") } } } diff --git a/controller/limits/shareRelaxAction.go b/controller/limits/shareRelaxAction.go index 4e0a0340..511ec49b 100644 --- a/controller/limits/shareRelaxAction.go +++ b/controller/limits/shareRelaxAction.go @@ -18,30 +18,66 @@ func newShareRelaxAction(str *store.Store, edge *rest_management_api_client.Ziti return &shareRelaxAction{str, edge} } -func (a *shareRelaxAction) HandleShare(shr *store.Share, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { +func (a *shareRelaxAction) HandleShare(shr *store.Share, _, _ int64, _ *BandwidthPerPeriod, trx *sqlx.Tx) error { logrus.Infof("relaxing '%v'", shr.Token) if !shr.Deleted { - if shr.ShareMode == "public" { - env, err := a.str.GetEnvironment(shr.EnvironmentId, trx) - if err != nil { - return errors.Wrap(err, "error finding environment") + switch shr.ShareMode { + case "public": + if err := relaxPublicShare(a.str, a.edge, shr, trx); err != nil { + return err } - - fe, err := a.str.FindFrontendPubliclyNamed(*shr.FrontendSelection, trx) - if err != nil { - return errors.Wrapf(err, "error finding frontend name '%v' for '%v'", *shr.FrontendSelection, shr.Token) + case "private": + if err := relaxPrivateShare(a.str, a.edge, shr, trx); err != nil { + return err } - - if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+shr.ZId+"-dial", shr.ZId, []string{fe.ZId}, zrokEdgeSdk.ZrokShareTags(shr.Token).SubTags, a.edge); err != nil { - return errors.Wrapf(err, "error creating dial service policy for '%v'", shr.Token) - } - logrus.Infof("added dial service policy for '%v'", shr.Token) - - } else if shr.ShareMode == "private" { - return errors.New("share relax for private share not implemented") } } return nil } + +func relaxPublicShare(str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement, shr *store.Share, trx *sqlx.Tx) error { + env, err := str.GetEnvironment(shr.EnvironmentId, trx) + if err != nil { + return errors.Wrap(err, "error finding environment") + } + + fe, err := str.FindFrontendPubliclyNamed(*shr.FrontendSelection, trx) + if err != nil { + return errors.Wrapf(err, "error finding frontend name '%v' for '%v'", *shr.FrontendSelection, shr.Token) + } + + if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+shr.ZId+"-dial", shr.ZId, []string{fe.ZId}, zrokEdgeSdk.ZrokShareTags(shr.Token).SubTags, edge); err != nil { + return errors.Wrapf(err, "error creating dial service policy for '%v'", shr.Token) + } + logrus.Infof("added dial service policy for '%v'", shr.Token) + return nil +} + +func relaxPrivateShare(str *store.Store, edge *rest_management_api_client.ZitiEdgeManagement, shr *store.Share, trx *sqlx.Tx) error { + fes, err := str.FindFrontendsForPrivateShare(shr.Id, trx) + if err != nil { + return errors.Wrapf(err, "error finding frontends for share '%v'", shr.Token) + } + for _, fe := range fes { + if fe.EnvironmentId != nil { + env, err := str.GetEnvironment(*fe.EnvironmentId, trx) + if err != nil { + return errors.Wrapf(err, "error getting environment for frontend '%v'", fe.Token) + } + + addlTags := map[string]interface{}{ + "zrokEnvironmentZId": env.ZId, + "zrokFrontendToken": fe.Token, + "zrokShareToken": shr.Token, + } + if err := zrokEdgeSdk.CreateServicePolicyDial(env.ZId+"-"+shr.ZId+"-dial", shr.ZId, []string{env.ZId}, addlTags, edge); err != nil { + return errors.Wrapf(err, "unable to create dial policy for frontend '%v'", fe.Token) + } + + logrus.Infof("added dial service policy for share '%v' to private frontend '%v'", shr.Token, fe.Token) + } + } + return nil +} From d3be3195bfe3685b041dd2a51502bf05151dfaae Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 29 Mar 2023 16:14:57 -0400 Subject: [PATCH 64/83] improved warning email text (#279) --- controller/emailUi/limitWarning.gohtml | 4 ++-- controller/limits/accountWarningAction.go | 20 ++++++++++++++++++- controller/limits/email.go | 4 ++-- controller/limits/environmentWarningAction.go | 20 ++++++++++++++++++- controller/limits/shareWarningAction.go | 20 ++++++++++++++++++- 5 files changed, 61 insertions(+), 7 deletions(-) diff --git a/controller/emailUi/limitWarning.gohtml b/controller/emailUi/limitWarning.gohtml index 4626b954..232dd10d 100644 --- a/controller/emailUi/limitWarning.gohtml +++ b/controller/emailUi/limitWarning.gohtml @@ -135,8 +135,8 @@
-

Your account is reaching a transfer size limit, {{ .EmailAddress }}.

-

{{ .Detail }}}

+

Your account is reaching a transfer limit, {{ .EmailAddress }}.

+

{{ .Detail }}

diff --git a/controller/limits/accountWarningAction.go b/controller/limits/accountWarningAction.go index 39fb9582..018e9909 100644 --- a/controller/limits/accountWarningAction.go +++ b/controller/limits/accountWarningAction.go @@ -1,10 +1,12 @@ package limits import ( + "fmt" "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/emailUi" "github.com/openziti/zrok/controller/store" + "github.com/openziti/zrok/util" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -22,7 +24,23 @@ func newAccountWarningAction(cfg *emailUi.Config, str *store.Store, edge *rest_m func (a *accountWarningAction) HandleAccount(acct *store.Account, rxBytes, txBytes int64, limit *BandwidthPerPeriod, trx *sqlx.Tx) error { logrus.Infof("warning '%v'", acct.Email) - if err := sendLimitWarningEmail(a.cfg, acct.Email, limit, rxBytes, txBytes); err != nil { + rxLimit := "(unlimited bytes)" + if limit.Limit.Rx != Unlimited { + rxLimit = util.BytesToSize(limit.Limit.Rx) + } + txLimit := "(unlimited bytes)" + if limit.Limit.Tx != Unlimited { + txLimit = util.BytesToSize(limit.Limit.Tx) + } + totalLimit := "(unlimited bytes)" + if limit.Limit.Total != Unlimited { + totalLimit = util.BytesToSize(limit.Limit.Total) + } + detail := fmt.Sprintf("Your account has received %v and sent %v (for a total of %v), which has triggered a transfer limit warning.", util.BytesToSize(rxBytes), util.BytesToSize(txBytes), util.BytesToSize(rxBytes+txBytes)) + + fmt.Sprintf(" This zrok instance only allows an account to receive %v, send %v, totalling not more than %v for each %v.", rxLimit, txLimit, totalLimit, limit.Period) + + fmt.Sprintf(" If you exceed the transfer limit, access to your shares will be temporarily disabled (until the last %v falls below the transfer limit)", limit.Period) + + if err := sendLimitWarningEmail(a.cfg, acct.Email, detail); err != nil { return errors.Wrapf(err, "error sending limit warning email to '%v'", acct.Email) } diff --git a/controller/limits/email.go b/controller/limits/email.go index ce60bc8e..ca1e349c 100644 --- a/controller/limits/email.go +++ b/controller/limits/email.go @@ -8,10 +8,10 @@ import ( "github.com/wneessen/go-mail" ) -func sendLimitWarningEmail(cfg *emailUi.Config, emailTo string, limit *BandwidthPerPeriod, rxBytes, txBytes int64) error { +func sendLimitWarningEmail(cfg *emailUi.Config, emailTo, detail string) error { emailData := &emailUi.WarningEmail{ EmailAddress: emailTo, - Detail: describeLimit(limit, rxBytes, txBytes), + Detail: detail, Version: build.String(), } diff --git a/controller/limits/environmentWarningAction.go b/controller/limits/environmentWarningAction.go index c3f25903..4dc9c975 100644 --- a/controller/limits/environmentWarningAction.go +++ b/controller/limits/environmentWarningAction.go @@ -1,10 +1,12 @@ package limits import ( + "fmt" "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/emailUi" "github.com/openziti/zrok/controller/store" + "github.com/openziti/zrok/util" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -28,7 +30,23 @@ func (a *environmentWarningAction) HandleEnvironment(env *store.Environment, rxB return err } - if err := sendLimitWarningEmail(a.cfg, acct.Email, limit, rxBytes, txBytes); err != nil { + rxLimit := "unlimited bytes" + if limit.Limit.Rx != Unlimited { + rxLimit = util.BytesToSize(limit.Limit.Rx) + } + txLimit := "unlimited bytes" + if limit.Limit.Tx != Unlimited { + txLimit = util.BytesToSize(limit.Limit.Tx) + } + totalLimit := "unlimited bytes" + if limit.Limit.Total != Unlimited { + totalLimit = util.BytesToSize(limit.Limit.Total) + } + detail := fmt.Sprintf("Your environment '%v' has received %v and sent %v (for a total of %v), which has triggered a transfer limit warning.", env.Description, util.BytesToSize(rxBytes), util.BytesToSize(txBytes), util.BytesToSize(rxBytes+txBytes)) + + fmt.Sprintf(" This zrok instance only allows a share to receive %v, send %v, totalling not more than %v for each %v.", rxLimit, txLimit, totalLimit, limit.Period) + + fmt.Sprintf(" If you exceed the transfer limit, access to your shares will be temporarily disabled (until the last %v falls below the transfer limit).", limit.Period) + + if err := sendLimitWarningEmail(a.cfg, acct.Email, detail); err != nil { return errors.Wrapf(err, "error sending limit warning email to '%v'", acct.Email) } } diff --git a/controller/limits/shareWarningAction.go b/controller/limits/shareWarningAction.go index 6514984a..c8f8bce9 100644 --- a/controller/limits/shareWarningAction.go +++ b/controller/limits/shareWarningAction.go @@ -1,10 +1,12 @@ package limits import ( + "fmt" "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/emailUi" "github.com/openziti/zrok/controller/store" + "github.com/openziti/zrok/util" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -33,7 +35,23 @@ func (a *shareWarningAction) HandleShare(shr *store.Share, rxBytes, txBytes int6 return err } - if err := sendLimitWarningEmail(a.cfg, acct.Email, limit, rxBytes, txBytes); err != nil { + rxLimit := "unlimited bytes" + if limit.Limit.Rx != Unlimited { + rxLimit = util.BytesToSize(limit.Limit.Rx) + } + txLimit := "unlimited bytes" + if limit.Limit.Tx != Unlimited { + txLimit = util.BytesToSize(limit.Limit.Tx) + } + totalLimit := "unlimited bytes" + if limit.Limit.Total != Unlimited { + totalLimit = util.BytesToSize(limit.Limit.Total) + } + detail := fmt.Sprintf("Your share '%v' has received %v and sent %v (for a total of %v), which has triggered a transfer limit warning.", shr.Token, util.BytesToSize(rxBytes), util.BytesToSize(txBytes), util.BytesToSize(rxBytes+txBytes)) + + fmt.Sprintf(" This zrok instance only allows a share to receive %v, send %v, totalling not more than %v for each %v.", rxLimit, txLimit, totalLimit, limit.Period) + + fmt.Sprintf(" If you exceed the transfer limit, access to your shares will be temporarily disabled (until the last %v falls below the transfer limit).", limit.Period) + + if err := sendLimitWarningEmail(a.cfg, acct.Email, detail); err != nil { return errors.Wrapf(err, "error sending limit warning email to '%v'", acct.Email) } } From 69990447c96c8726ce3bd39a154d54a6921bade2 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 29 Mar 2023 16:24:52 -0400 Subject: [PATCH 65/83] warning graphic for limit email (#279); only run relax actions when reverting a limit not a warning (#276) --- controller/emailUi/limitWarning.gohtml | 2 +- controller/limits/agent.go | 30 +++++++++++++++----------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/controller/emailUi/limitWarning.gohtml b/controller/emailUi/limitWarning.gohtml index 232dd10d..c74dcfde 100644 --- a/controller/emailUi/limitWarning.gohtml +++ b/controller/emailUi/limitWarning.gohtml @@ -132,7 +132,7 @@

Your account is reaching a transfer limit, {{ .EmailAddress }}.

diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 9ec418a3..18b194a6 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -437,10 +437,12 @@ func (a *Agent) relax() error { if slj.Action == store.WarningAction || slj.Action == store.LimitAction { if enforce, warning, rxBytes, txBytes, err := a.checkShareLimit(shr.Token); err == nil { if !enforce && !warning { - // run relax actions for share - for _, action := range a.shrRelaxActions { - if err := action.HandleShare(shr, rxBytes, txBytes, a.cfg.Bandwidth.PerShare, trx); err != nil { - return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) + if slj.Action == store.LimitAction { + // run relax actions for share + for _, action := range a.shrRelaxActions { + if err := action.HandleShare(shr, rxBytes, txBytes, a.cfg.Bandwidth.PerShare, trx); err != nil { + return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) + } } } if err := a.str.DeleteShareLimitJournalForShare(shr.Id, trx); err == nil { @@ -469,10 +471,12 @@ func (a *Agent) relax() error { if elj.Action == store.WarningAction || elj.Action == store.LimitAction { if enforce, warning, rxBytes, txBytes, err := a.checkEnvironmentLimit(int64(elj.EnvironmentId)); err == nil { if !enforce && !warning { - // run relax actions for environment - for _, action := range a.envRelaxActions { - if err := action.HandleEnvironment(env, rxBytes, txBytes, a.cfg.Bandwidth.PerEnvironment, trx); err != nil { - return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) + if elj.Action == store.LimitAction { + // run relax actions for environment + for _, action := range a.envRelaxActions { + if err := action.HandleEnvironment(env, rxBytes, txBytes, a.cfg.Bandwidth.PerEnvironment, trx); err != nil { + return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) + } } } if err := a.str.DeleteEnvironmentLimitJournalForEnvironment(env.Id, trx); err == nil { @@ -501,10 +505,12 @@ func (a *Agent) relax() error { if alj.Action == store.WarningAction || alj.Action == store.LimitAction { if enforce, warning, rxBytes, txBytes, err := a.checkAccountLimit(int64(alj.AccountId)); err == nil { if !enforce && !warning { - // run relax actions for account - for _, action := range a.acctRelaxActions { - if err := action.HandleAccount(acct, rxBytes, txBytes, a.cfg.Bandwidth.PerAccount, trx); err != nil { - return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) + if alj.Action == store.LimitAction { + // run relax actions for account + for _, action := range a.acctRelaxActions { + if err := action.HandleAccount(acct, rxBytes, txBytes, a.cfg.Bandwidth.PerAccount, trx); err != nil { + return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) + } } } if err := a.str.DeleteAccountLimitJournalForAccount(acct.Id, trx); err == nil { From 6fc794ea5090b4ce72c98bfc2d89a56e99048a4f Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 29 Mar 2023 17:03:42 -0400 Subject: [PATCH 66/83] email subject (#279); record windowed transfer correctly in journals (#273); properly cycle the relax run when inbound usage is happenign (#276) --- controller/limits/agent.go | 38 ++++++++++++++++++++++++++------------ controller/limits/email.go | 2 +- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/controller/limits/agent.go b/controller/limits/agent.go index 18b194a6..f2ff9d4c 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -149,6 +149,7 @@ func (a *Agent) run() { logrus.Info("started") defer logrus.Info("stopped") + lastCycle := time.Now() mainLoop: for { select { @@ -156,11 +157,18 @@ mainLoop: if err := a.enforce(usage); err != nil { logrus.Errorf("error running enforcement: %v", err) } + if time.Since(lastCycle) > a.cfg.Cycle { + if err := a.relax(); err != nil { + logrus.Errorf("error running relax cycle: %v", err) + } + lastCycle = time.Now() + } case <-time.After(a.cfg.Cycle): if err := a.relax(); err != nil { logrus.Errorf("error running relax cycle: %v", err) } + lastCycle = time.Now() case <-a.close: close(a.join) @@ -190,8 +198,8 @@ func (a *Agent) enforce(u *metrics.Usage) error { if !enforced { _, err := a.str.CreateAccountLimitJournal(&store.AccountLimitJournal{ AccountId: int(u.AccountId), - RxBytes: u.BackendRx, - TxBytes: u.BackendTx, + RxBytes: rxBytes, + TxBytes: txBytes, Action: store.LimitAction, }, trx) if err != nil { @@ -227,8 +235,8 @@ func (a *Agent) enforce(u *metrics.Usage) error { if !warned { _, err := a.str.CreateAccountLimitJournal(&store.AccountLimitJournal{ AccountId: int(u.AccountId), - RxBytes: u.BackendRx, - TxBytes: u.BackendTx, + RxBytes: rxBytes, + TxBytes: txBytes, Action: store.WarningAction, }, trx) if err != nil { @@ -266,8 +274,8 @@ func (a *Agent) enforce(u *metrics.Usage) error { if !enforced { _, err := a.str.CreateEnvironmentLimitJournal(&store.EnvironmentLimitJournal{ EnvironmentId: int(u.EnvironmentId), - RxBytes: u.BackendRx, - TxBytes: u.BackendTx, + RxBytes: rxBytes, + TxBytes: txBytes, Action: store.LimitAction, }, trx) if err != nil { @@ -303,8 +311,8 @@ func (a *Agent) enforce(u *metrics.Usage) error { if !warned { _, err := a.str.CreateEnvironmentLimitJournal(&store.EnvironmentLimitJournal{ EnvironmentId: int(u.EnvironmentId), - RxBytes: u.BackendRx, - TxBytes: u.BackendTx, + RxBytes: rxBytes, + TxBytes: txBytes, Action: store.WarningAction, }, trx) if err != nil { @@ -347,8 +355,8 @@ func (a *Agent) enforce(u *metrics.Usage) error { if !enforced { _, err := a.str.CreateShareLimitJournal(&store.ShareLimitJournal{ ShareId: shr.Id, - RxBytes: u.BackendRx, - TxBytes: u.BackendTx, + RxBytes: rxBytes, + TxBytes: txBytes, Action: store.LimitAction, }, trx) if err != nil { @@ -385,8 +393,8 @@ func (a *Agent) enforce(u *metrics.Usage) error { if !warned { _, err := a.str.CreateShareLimitJournal(&store.ShareLimitJournal{ ShareId: shr.Id, - RxBytes: u.BackendRx, - TxBytes: u.BackendTx, + RxBytes: rxBytes, + TxBytes: txBytes, Action: store.WarningAction, }, trx) if err != nil { @@ -444,6 +452,8 @@ func (a *Agent) relax() error { return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) } } + } else { + logrus.Infof("relaxing warning for '%v'", shr.Token) } if err := a.str.DeleteShareLimitJournalForShare(shr.Id, trx); err == nil { commit = true @@ -478,6 +488,8 @@ func (a *Agent) relax() error { return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) } } + } else { + logrus.Infof("relaxing warning for '%v'", env.ZId) } if err := a.str.DeleteEnvironmentLimitJournalForEnvironment(env.Id, trx); err == nil { commit = true @@ -512,6 +524,8 @@ func (a *Agent) relax() error { return errors.Wrapf(err, "%v", reflect.TypeOf(action).String()) } } + } else { + logrus.Infof("relaxing warning for '%v'", acct.Email) } if err := a.str.DeleteAccountLimitJournalForAccount(acct.Id, trx); err == nil { commit = true diff --git a/controller/limits/email.go b/controller/limits/email.go index ca1e349c..399171af 100644 --- a/controller/limits/email.go +++ b/controller/limits/email.go @@ -32,7 +32,7 @@ func sendLimitWarningEmail(cfg *emailUi.Config, emailTo, detail string) error { return errors.Wrap(err, "failed to set to address in limit warning email") } - msg.Subject("Limit Warning Notification") + msg.Subject("zrok Limit Warning Notification") msg.SetDate() msg.SetMessageID() msg.SetBulk() From 5146ca8f24905284f870dd95d8bb8683c0312fec Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 3 Apr 2023 14:19:43 -0400 Subject: [PATCH 67/83] better (?) limit email formatting (#279) --- controller/emailUi/limitWarning.gohtml | 7 ++-- controller/emailUi/model.go | 2 +- controller/limits/accountWarningAction.go | 9 ++--- controller/limits/email.go | 36 +++++++++++++++++-- controller/limits/environmentWarningAction.go | 9 ++--- controller/limits/shareWarningAction.go | 9 ++--- 6 files changed, 55 insertions(+), 17 deletions(-) diff --git a/controller/emailUi/limitWarning.gohtml b/controller/emailUi/limitWarning.gohtml index c74dcfde..6a676f53 100644 --- a/controller/emailUi/limitWarning.gohtml +++ b/controller/emailUi/limitWarning.gohtml @@ -135,8 +135,11 @@
-

Your account is reaching a transfer limit, {{ .EmailAddress }}.

-

{{ .Detail }}

+

Your account is reaching a transfer limit, {{ .EmailAddress }}.

+
+ +
+ {{ .Detail }}
diff --git a/controller/emailUi/model.go b/controller/emailUi/model.go index 829a8372..a44594c6 100644 --- a/controller/emailUi/model.go +++ b/controller/emailUi/model.go @@ -3,7 +3,7 @@ package emailUi import ( "bytes" "github.com/pkg/errors" - "html/template" + "text/template" ) type WarningEmail struct { diff --git a/controller/limits/accountWarningAction.go b/controller/limits/accountWarningAction.go index 018e9909..a799b423 100644 --- a/controller/limits/accountWarningAction.go +++ b/controller/limits/accountWarningAction.go @@ -1,7 +1,6 @@ package limits import ( - "fmt" "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/emailUi" @@ -36,9 +35,11 @@ func (a *accountWarningAction) HandleAccount(acct *store.Account, rxBytes, txByt if limit.Limit.Total != Unlimited { totalLimit = util.BytesToSize(limit.Limit.Total) } - detail := fmt.Sprintf("Your account has received %v and sent %v (for a total of %v), which has triggered a transfer limit warning.", util.BytesToSize(rxBytes), util.BytesToSize(txBytes), util.BytesToSize(rxBytes+txBytes)) + - fmt.Sprintf(" This zrok instance only allows an account to receive %v, send %v, totalling not more than %v for each %v.", rxLimit, txLimit, totalLimit, limit.Period) + - fmt.Sprintf(" If you exceed the transfer limit, access to your shares will be temporarily disabled (until the last %v falls below the transfer limit)", limit.Period) + + detail := newDetailMessage() + detail = detail.append("Your account has received %v and sent %v (for a total of %v), which has triggered a transfer limit warning.", util.BytesToSize(rxBytes), util.BytesToSize(txBytes), util.BytesToSize(rxBytes+txBytes)) + detail = detail.append("This zrok instance only allows an account to receive %v, send %v, totalling not more than %v for each %v.", rxLimit, txLimit, totalLimit, limit.Period) + detail = detail.append("If you exceed the transfer limit, access to your shares will be temporarily disabled (until the last %v falls below the transfer limit)", limit.Period) if err := sendLimitWarningEmail(a.cfg, acct.Email, detail); err != nil { return errors.Wrapf(err, "error sending limit warning email to '%v'", acct.Email) diff --git a/controller/limits/email.go b/controller/limits/email.go index 399171af..bb2c3aef 100644 --- a/controller/limits/email.go +++ b/controller/limits/email.go @@ -1,6 +1,7 @@ package limits import ( + "fmt" "github.com/openziti/zrok/build" "github.com/openziti/zrok/controller/emailUi" "github.com/pkg/errors" @@ -8,17 +9,48 @@ import ( "github.com/wneessen/go-mail" ) -func sendLimitWarningEmail(cfg *emailUi.Config, emailTo, detail string) error { +type detailMessage struct { + lines []string +} + +func newDetailMessage() *detailMessage { + return &detailMessage{} +} + +func (m *detailMessage) append(msg string, args ...interface{}) *detailMessage { + m.lines = append(m.lines, fmt.Sprintf(msg, args...)) + return m +} + +func (m *detailMessage) html() string { + out := "" + for i := range m.lines { + out += fmt.Sprintf("

%s

\n", m.lines[i]) + } + return out +} + +func (m *detailMessage) plain() string { + out := "" + for i := range m.lines { + out += fmt.Sprintf("%s\n\n", m.lines[i]) + } + return out +} + +func sendLimitWarningEmail(cfg *emailUi.Config, emailTo string, d *detailMessage) error { emailData := &emailUi.WarningEmail{ EmailAddress: emailTo, - Detail: detail, Version: build.String(), } + emailData.Detail = d.plain() plainBody, err := emailData.MergeTemplate("limitWarning.gotext") if err != nil { return err } + + emailData.Detail = d.html() htmlBody, err := emailData.MergeTemplate("limitWarning.gohtml") if err != nil { return err diff --git a/controller/limits/environmentWarningAction.go b/controller/limits/environmentWarningAction.go index 4dc9c975..120af20d 100644 --- a/controller/limits/environmentWarningAction.go +++ b/controller/limits/environmentWarningAction.go @@ -1,7 +1,6 @@ package limits import ( - "fmt" "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/emailUi" @@ -42,9 +41,11 @@ func (a *environmentWarningAction) HandleEnvironment(env *store.Environment, rxB if limit.Limit.Total != Unlimited { totalLimit = util.BytesToSize(limit.Limit.Total) } - detail := fmt.Sprintf("Your environment '%v' has received %v and sent %v (for a total of %v), which has triggered a transfer limit warning.", env.Description, util.BytesToSize(rxBytes), util.BytesToSize(txBytes), util.BytesToSize(rxBytes+txBytes)) + - fmt.Sprintf(" This zrok instance only allows a share to receive %v, send %v, totalling not more than %v for each %v.", rxLimit, txLimit, totalLimit, limit.Period) + - fmt.Sprintf(" If you exceed the transfer limit, access to your shares will be temporarily disabled (until the last %v falls below the transfer limit).", limit.Period) + + detail := newDetailMessage() + detail = detail.append("Your environment '%v' has received %v and sent %v (for a total of %v), which has triggered a transfer limit warning.", env.Description, util.BytesToSize(rxBytes), util.BytesToSize(txBytes), util.BytesToSize(rxBytes+txBytes)) + detail = detail.append("This zrok instance only allows a share to receive %v, send %v, totalling not more than %v for each %v.", rxLimit, txLimit, totalLimit, limit.Period) + detail = detail.append("If you exceed the transfer limit, access to your shares will be temporarily disabled (until the last %v falls below the transfer limit).", limit.Period) if err := sendLimitWarningEmail(a.cfg, acct.Email, detail); err != nil { return errors.Wrapf(err, "error sending limit warning email to '%v'", acct.Email) diff --git a/controller/limits/shareWarningAction.go b/controller/limits/shareWarningAction.go index c8f8bce9..5781a284 100644 --- a/controller/limits/shareWarningAction.go +++ b/controller/limits/shareWarningAction.go @@ -1,7 +1,6 @@ package limits import ( - "fmt" "github.com/jmoiron/sqlx" "github.com/openziti/edge/rest_management_api_client" "github.com/openziti/zrok/controller/emailUi" @@ -47,9 +46,11 @@ func (a *shareWarningAction) HandleShare(shr *store.Share, rxBytes, txBytes int6 if limit.Limit.Total != Unlimited { totalLimit = util.BytesToSize(limit.Limit.Total) } - detail := fmt.Sprintf("Your share '%v' has received %v and sent %v (for a total of %v), which has triggered a transfer limit warning.", shr.Token, util.BytesToSize(rxBytes), util.BytesToSize(txBytes), util.BytesToSize(rxBytes+txBytes)) + - fmt.Sprintf(" This zrok instance only allows a share to receive %v, send %v, totalling not more than %v for each %v.", rxLimit, txLimit, totalLimit, limit.Period) + - fmt.Sprintf(" If you exceed the transfer limit, access to your shares will be temporarily disabled (until the last %v falls below the transfer limit).", limit.Period) + + detail := newDetailMessage() + detail = detail.append("Your share '%v' has received %v and sent %v (for a total of %v), which has triggered a transfer limit warning.", shr.Token, util.BytesToSize(rxBytes), util.BytesToSize(txBytes), util.BytesToSize(rxBytes+txBytes)) + detail = detail.append("This zrok instance only allows a share to receive %v, send %v, totalling not more than %v for each %v.", rxLimit, txLimit, totalLimit, limit.Period) + detail = detail.append("If you exceed the transfer limit, access to your shares will be temporarily disabled (until the last %v falls below the transfer limit).", limit.Period) if err := sendLimitWarningEmail(a.cfg, acct.Email, detail); err != nil { return errors.Wrapf(err, "error sending limit warning email to '%v'", acct.Email) From c2e9016f61b2c0a33393bb4b13fd1527361ea02c Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 3 Apr 2023 14:29:38 -0400 Subject: [PATCH 68/83] updated svgo to v3.0.2; removed unused react-flow-renderer; lint --- ui/package-lock.json | 1186 +++++++++++++++--------------------------- ui/package.json | 4 +- 2 files changed, 428 insertions(+), 762 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index 4a3e25f6..9a7cb891 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -21,12 +21,12 @@ "react-bootstrap": "^2.7.0", "react-data-table-component": "^7.5.2", "react-dom": "^18.2.0", - "react-flow-renderer": "^10.3.12", "react-force-graph": "^1.41.20", "react-router-dom": "^6.4.0", "react-sizeme": "^3.0.2", "react-sparklines": "^1.7.0", - "styled-components": "^5.3.5" + "styled-components": "^5.3.5", + "svgo": "^3.0.2" }, "devDependencies": { "react-scripts": "^5.0.1" @@ -3876,6 +3876,143 @@ "url": "https://github.com/sponsors/gregberge" } }, + "node_modules/@svgr/plugin-svgo/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/@svgr/plugin-svgo/node_modules/css-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-2.1.0.tgz", + "integrity": "sha512-Dqk7LQKpwLoH3VovzZnkzegqNSuAziQyNZUcrdDM401iY+R5NkGBXGmtO05/yaXQziALuPogeG0b7UAgjnTJTQ==", + "dev": true, + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^3.2.1", + "domutils": "^1.7.0", + "nth-check": "^1.0.2" + } + }, + "node_modules/@svgr/plugin-svgo/node_modules/css-tree": { + "version": "1.0.0-alpha.37", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-1.0.0-alpha.37.tgz", + "integrity": "sha512-DMxWJg0rnz7UgxKT0Q1HU/L9BeJI0M6ksor0OgqOnF+aRCDWg/N2641HmVyU9KVIu0OVVWOb2IpC9A+BJRnejg==", + "dev": true, + "dependencies": { + "mdn-data": "2.0.4", + "source-map": "^0.6.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@svgr/plugin-svgo/node_modules/css-what": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-3.4.2.tgz", + "integrity": "sha512-ACUm3L0/jiZTqfzRM3Hi9Q8eZqd6IK37mMWPLz9PJxkLWllYeRf+EHUSHYEtFop2Eqytaq1FizFVh7XfBnXCDQ==", + "dev": true, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/@svgr/plugin-svgo/node_modules/dom-serializer": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.2.2.tgz", + "integrity": "sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g==", + "dev": true, + "dependencies": { + "domelementtype": "^2.0.1", + "entities": "^2.0.0" + } + }, + "node_modules/@svgr/plugin-svgo/node_modules/domutils": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz", + "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==", + "dev": true, + "dependencies": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, + "node_modules/@svgr/plugin-svgo/node_modules/domutils/node_modules/domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==", + "dev": true + }, + "node_modules/@svgr/plugin-svgo/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/@svgr/plugin-svgo/node_modules/mdn-data": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.4.tgz", + "integrity": "sha512-iV3XNKw06j5Q7mi6h+9vbx23Tv7JkjEVgKHW4pimwyDGWm0OIQntJJ+u1C6mg6mK1EaTv42XQ7w76yuzH7M2cA==", + "dev": true + }, + "node_modules/@svgr/plugin-svgo/node_modules/nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "dev": true, + "dependencies": { + "boolbase": "~1.0.0" + } + }, + "node_modules/@svgr/plugin-svgo/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@svgr/plugin-svgo/node_modules/svgo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/svgo/-/svgo-1.3.2.tgz", + "integrity": "sha512-yhy/sQYxR5BkC98CY7o31VGsg014AKLEPxdfhora76l36hD9Rdy5NZA/Ocn6yayNPgSamYdtX2rFJdcv07AYVw==", + "deprecated": "This SVGO version is no longer supported. Upgrade to v2.x.x.", + "dev": true, + "dependencies": { + "chalk": "^2.4.1", + "coa": "^2.0.2", + "css-select": "^2.0.0", + "css-select-base-adapter": "^0.1.1", + "css-tree": "1.0.0-alpha.37", + "csso": "^4.0.2", + "js-yaml": "^3.13.1", + "mkdirp": "~0.5.1", + "object.values": "^1.1.0", + "sax": "~1.2.4", + "stable": "^0.1.8", + "unquote": "~1.1.1", + "util.promisify": "~1.0.0" + }, + "bin": { + "svgo": "bin/svgo" + }, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/@svgr/webpack": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/@svgr/webpack/-/webpack-5.5.0.tgz", @@ -3920,7 +4057,6 @@ "version": "0.2.0", "resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz", "integrity": "sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==", - "dev": true, "engines": { "node": ">=10.13.0" } @@ -4009,228 +4145,6 @@ "@types/node": "*" } }, - "node_modules/@types/d3": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.0.tgz", - "integrity": "sha512-jIfNVK0ZlxcuRDKtRS/SypEyOQ6UHaFQBKv032X45VvxSJ6Yi5G9behy9h6tNTHTDGh5Vq+KbmBjUWLgY4meCA==", - "dependencies": { - "@types/d3-array": "*", - "@types/d3-axis": "*", - "@types/d3-brush": "*", - "@types/d3-chord": "*", - "@types/d3-color": "*", - "@types/d3-contour": "*", - "@types/d3-delaunay": "*", - "@types/d3-dispatch": "*", - "@types/d3-drag": "*", - "@types/d3-dsv": "*", - "@types/d3-ease": "*", - "@types/d3-fetch": "*", - "@types/d3-force": "*", - "@types/d3-format": "*", - "@types/d3-geo": "*", - "@types/d3-hierarchy": "*", - "@types/d3-interpolate": "*", - "@types/d3-path": "*", - "@types/d3-polygon": "*", - "@types/d3-quadtree": "*", - "@types/d3-random": "*", - "@types/d3-scale": "*", - "@types/d3-scale-chromatic": "*", - "@types/d3-selection": "*", - "@types/d3-shape": "*", - "@types/d3-time": "*", - "@types/d3-time-format": "*", - "@types/d3-timer": "*", - "@types/d3-transition": "*", - "@types/d3-zoom": "*" - } - }, - "node_modules/@types/d3-array": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.0.4.tgz", - "integrity": "sha512-nwvEkG9vYOc0Ic7G7kwgviY4AQlTfYGIZ0fqB7CQHXGyYM6nO7kJh5EguSNA3jfh4rq7Sb7eMVq8isuvg2/miQ==" - }, - "node_modules/@types/d3-axis": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-3.0.2.tgz", - "integrity": "sha512-uGC7DBh0TZrU/LY43Fd8Qr+2ja1FKmH07q2FoZFHo1eYl8aj87GhfVoY1saJVJiq24rp1+wpI6BvQJMKgQm8oA==", - "dependencies": { - "@types/d3-selection": "*" - } - }, - "node_modules/@types/d3-brush": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-brush/-/d3-brush-3.0.2.tgz", - "integrity": "sha512-2TEm8KzUG3N7z0TrSKPmbxByBx54M+S9lHoP2J55QuLU0VSQ9mE96EJSAOVNEqd1bbynMjeTS9VHmz8/bSw8rA==", - "dependencies": { - "@types/d3-selection": "*" - } - }, - "node_modules/@types/d3-chord": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-chord/-/d3-chord-3.0.2.tgz", - "integrity": "sha512-abT/iLHD3sGZwqMTX1TYCMEulr+wBd0SzyOQnjYNLp7sngdOHYtNkMRI5v3w5thoN+BWtlHVDx2Osvq6fxhZWw==" - }, - "node_modules/@types/d3-color": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.0.tgz", - "integrity": "sha512-HKuicPHJuvPgCD+np6Se9MQvS6OCbJmOjGvylzMJRlDwUXjKTTXs6Pwgk79O09Vj/ho3u1ofXnhFOaEWWPrlwA==" - }, - "node_modules/@types/d3-contour": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-contour/-/d3-contour-3.0.2.tgz", - "integrity": "sha512-k6/bGDoAGJZnZWaKzeB+9glgXCYGvh6YlluxzBREiVo8f/X2vpTEdgPy9DN7Z2i42PZOZ4JDhVdlTSTSkLDPlQ==", - "dependencies": { - "@types/d3-array": "*", - "@types/geojson": "*" - } - }, - "node_modules/@types/d3-delaunay": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/@types/d3-delaunay/-/d3-delaunay-6.0.1.tgz", - "integrity": "sha512-tLxQ2sfT0p6sxdG75c6f/ekqxjyYR0+LwPrsO1mbC9YDBzPJhs2HbJJRrn8Ez1DBoHRo2yx7YEATI+8V1nGMnQ==" - }, - "node_modules/@types/d3-dispatch": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-dispatch/-/d3-dispatch-3.0.2.tgz", - "integrity": "sha512-rxN6sHUXEZYCKV05MEh4z4WpPSqIw+aP7n9ZN6WYAAvZoEAghEK1WeVZMZcHRBwyaKflU43PCUAJNjFxCzPDjg==" - }, - "node_modules/@types/d3-drag": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-drag/-/d3-drag-3.0.2.tgz", - "integrity": "sha512-qmODKEDvyKWVHcWWCOVcuVcOwikLVsyc4q4EBJMREsoQnR2Qoc2cZQUyFUPgO9q4S3qdSqJKBsuefv+h0Qy+tw==", - "dependencies": { - "@types/d3-selection": "*" - } - }, - "node_modules/@types/d3-dsv": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-3.0.1.tgz", - "integrity": "sha512-76pBHCMTvPLt44wFOieouXcGXWOF0AJCceUvaFkxSZEu4VDUdv93JfpMa6VGNFs01FHfuP4a5Ou68eRG1KBfTw==" - }, - "node_modules/@types/d3-ease": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.0.tgz", - "integrity": "sha512-aMo4eaAOijJjA6uU+GIeW018dvy9+oH5Y2VPPzjjfxevvGQ/oRDs+tfYC9b50Q4BygRR8yE2QCLsrT0WtAVseA==" - }, - "node_modules/@types/d3-fetch": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-fetch/-/d3-fetch-3.0.2.tgz", - "integrity": "sha512-gllwYWozWfbep16N9fByNBDTkJW/SyhH6SGRlXloR7WdtAaBui4plTP+gbUgiEot7vGw/ZZop1yDZlgXXSuzjA==", - "dependencies": { - "@types/d3-dsv": "*" - } - }, - "node_modules/@types/d3-force": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/d3-force/-/d3-force-3.0.4.tgz", - "integrity": "sha512-q7xbVLrWcXvSBBEoadowIUJ7sRpS1yvgMWnzHJggFy5cUZBq2HZL5k/pBSm0GdYWS1vs5/EDwMjSKF55PDY4Aw==" - }, - "node_modules/@types/d3-format": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/d3-format/-/d3-format-3.0.1.tgz", - "integrity": "sha512-5KY70ifCCzorkLuIkDe0Z9YTf9RR2CjBX1iaJG+rgM/cPP+sO+q9YdQ9WdhQcgPj1EQiJ2/0+yUkkziTG6Lubg==" - }, - "node_modules/@types/d3-geo": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@types/d3-geo/-/d3-geo-3.0.3.tgz", - "integrity": "sha512-bK9uZJS3vuDCNeeXQ4z3u0E7OeJZXjUgzFdSOtNtMCJCLvDtWDwfpRVWlyt3y8EvRzI0ccOu9xlMVirawolSCw==", - "dependencies": { - "@types/geojson": "*" - } - }, - "node_modules/@types/d3-hierarchy": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@types/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", - "integrity": "sha512-9hjRTVoZjRFR6xo8igAJyNXQyPX6Aq++Nhb5ebrUF414dv4jr2MitM2fWiOY475wa3Za7TOS2Gh9fmqEhLTt0A==" - }, - "node_modules/@types/d3-interpolate": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.1.tgz", - "integrity": "sha512-jx5leotSeac3jr0RePOH1KdR9rISG91QIE4Q2PYTu4OymLTZfA3SrnURSLzKH48HmXVUru50b8nje4E79oQSQw==", - "dependencies": { - "@types/d3-color": "*" - } - }, - "node_modules/@types/d3-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.0.0.tgz", - "integrity": "sha512-0g/A+mZXgFkQxN3HniRDbXMN79K3CdTpLsevj+PXiTcb2hVyvkZUBg37StmgCQkaD84cUJ4uaDAWq7UJOQy2Tg==" - }, - "node_modules/@types/d3-polygon": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/d3-polygon/-/d3-polygon-3.0.0.tgz", - "integrity": "sha512-D49z4DyzTKXM0sGKVqiTDTYr+DHg/uxsiWDAkNrwXYuiZVd9o9wXZIo+YsHkifOiyBkmSWlEngHCQme54/hnHw==" - }, - "node_modules/@types/d3-quadtree": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-quadtree/-/d3-quadtree-3.0.2.tgz", - "integrity": "sha512-QNcK8Jguvc8lU+4OfeNx+qnVy7c0VrDJ+CCVFS9srBo2GL9Y18CnIxBdTF3v38flrGy5s1YggcoAiu6s4fLQIw==" - }, - "node_modules/@types/d3-random": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/d3-random/-/d3-random-3.0.1.tgz", - "integrity": "sha512-IIE6YTekGczpLYo/HehAy3JGF1ty7+usI97LqraNa8IiDur+L44d0VOjAvFQWJVdZOJHukUJw+ZdZBlgeUsHOQ==" - }, - "node_modules/@types/d3-scale": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.3.tgz", - "integrity": "sha512-PATBiMCpvHJSMtZAMEhc2WyL+hnzarKzI6wAHYjhsonjWJYGq5BXTzQjv4l8m2jO183/4wZ90rKvSeT7o72xNQ==", - "dependencies": { - "@types/d3-time": "*" - } - }, - "node_modules/@types/d3-scale-chromatic": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.0.0.tgz", - "integrity": "sha512-dsoJGEIShosKVRBZB0Vo3C8nqSDqVGujJU6tPznsBJxNJNwMF8utmS83nvCBKQYPpjCzaaHcrf66iTRpZosLPw==" - }, - "node_modules/@types/d3-selection": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.4.tgz", - "integrity": "sha512-ZeykX7286BCyMg9sH5fIAORyCB6hcATPSRQpN47jwBA2bMbAT0s+EvtDP5r1FZYJ95R8QoEE1CKJX+n0/M5Vhg==" - }, - "node_modules/@types/d3-shape": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.1.tgz", - "integrity": "sha512-6Uh86YFF7LGg4PQkuO2oG6EMBRLuW9cbavUW46zkIO5kuS2PfTqo2o9SkgtQzguBHbLgNnU90UNsITpsX1My+A==", - "dependencies": { - "@types/d3-path": "*" - } - }, - "node_modules/@types/d3-time": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.0.tgz", - "integrity": "sha512-sZLCdHvBUcNby1cB6Fd3ZBrABbjz3v1Vm90nysCQ6Vt7vd6e/h9Lt7SiJUoEX0l4Dzc7P5llKyhqSi1ycSf1Hg==" - }, - "node_modules/@types/d3-time-format": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@types/d3-time-format/-/d3-time-format-4.0.0.tgz", - "integrity": "sha512-yjfBUe6DJBsDin2BMIulhSHmr5qNR5Pxs17+oW4DoVPyVIXZ+m6bs7j1UVKP08Emv6jRmYrYqxYzO63mQxy1rw==" - }, - "node_modules/@types/d3-timer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.0.tgz", - "integrity": "sha512-HNB/9GHqu7Fo8AQiugyJbv6ZxYz58wef0esl4Mv828w1ZKpAshw/uFWVDUcIB9KKFeFKoxS3cHY07FFgtTRZ1g==" - }, - "node_modules/@types/d3-transition": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.3.tgz", - "integrity": "sha512-/S90Od8Id1wgQNvIA8iFv9jRhCiZcGhPd2qX0bKF/PS+y0W5CrXKgIiELd2CvG1mlQrWK/qlYh3VxicqG1ZvgA==", - "dependencies": { - "@types/d3-selection": "*" - } - }, - "node_modules/@types/d3-zoom": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-zoom/-/d3-zoom-3.0.2.tgz", - "integrity": "sha512-t09DDJVBI6AkM7N8kuPsnq/3d/ehtRKBN1xSiYjjMCgbiw6HM6Ged5VhvswmhprfKyGvzeTEL/4WBaK9llWvlA==", - "dependencies": { - "@types/d3-interpolate": "*", - "@types/d3-selection": "*" - } - }, "node_modules/@types/eslint": { "version": "8.21.0", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.21.0.tgz", @@ -4280,11 +4194,6 @@ "@types/range-parser": "*" } }, - "node_modules/@types/geojson": { - "version": "7946.0.10", - "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.10.tgz", - "integrity": "sha512-Nmh0K3iWQJzniTuPRcJn5hxXkfB1T1pgB89SBig5PlJQU5yocazeu4jATJlaA0GYFKWMqDdvYemoSnF2pXgLVA==" - }, "node_modules/@types/graceful-fs": { "version": "4.1.6", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.6.tgz", @@ -4415,11 +4324,6 @@ "@types/react": "*" } }, - "node_modules/@types/resize-observer-browser": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/@types/resize-observer-browser/-/resize-observer-browser-0.1.7.tgz", - "integrity": "sha512-G9eN0Sn0ii9PWQ3Vl72jDPgeJwRWhv2Qk/nQkJuWmRmOB4HX3/BhD5SE1dZs/hzPZL/WKnvF0RHdTSG54QJFyg==" - }, "node_modules/@types/resolve": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.17.1.tgz", @@ -6020,8 +5924,7 @@ "node_modules/boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", - "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", - "dev": true + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" }, "node_modules/bootstrap": { "version": "5.2.3", @@ -6384,11 +6287,6 @@ "integrity": "sha512-cOU9usZw8/dXIXKtwa8pM0OTJQuJkxMN6w30csNRUerHfeQ5R6U3kkU/FtJeIf3M202OHfY2U8ccInBG7/xogA==", "dev": true }, - "node_modules/classcat": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/classcat/-/classcat-5.0.4.tgz", - "integrity": "sha512-sbpkOw6z413p+HDGcBENe498WM9woqWHiJxCq7nvmxe9WmrUmqfAcxpIwAiMtM5Q3AhYkzXcNQHqsWq0mND51g==" - }, "node_modules/classnames": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz", @@ -6981,33 +6879,10 @@ "postcss-value-parser": "^4.0.2" } }, - "node_modules/css-tree": { - "version": "1.0.0-alpha.37", - "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-1.0.0-alpha.37.tgz", - "integrity": "sha512-DMxWJg0rnz7UgxKT0Q1HU/L9BeJI0M6ksor0OgqOnF+aRCDWg/N2641HmVyU9KVIu0OVVWOb2IpC9A+BJRnejg==", - "dev": true, - "dependencies": { - "mdn-data": "2.0.4", - "source-map": "^0.6.1" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/css-tree/node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/css-what": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", - "dev": true, "engines": { "node": ">= 6" }, @@ -7770,7 +7645,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", - "dev": true, "funding": [ { "type": "github", @@ -13462,12 +13336,6 @@ "wrappy": "1" } }, - "node_modules/mdn-data": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.4.tgz", - "integrity": "sha512-iV3XNKw06j5Q7mi6h+9vbx23Tv7JkjEVgKHW4pimwyDGWm0OIQntJJ+u1C6mg6mK1EaTv42XQ7w76yuzH7M2cA==", - "dev": true - }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -13884,7 +13752,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", - "dev": true, "dependencies": { "boolbase": "^1.0.0" }, @@ -16294,29 +16161,6 @@ "integrity": "sha512-/6UZ2qgEyH2aqzYZgQPxEnz33NJ2gNsnHA2o5+o4wW9bLM/JYQitNP9xPhsXwC08hMMovfGe/8retsdDsczPRg==", "dev": true }, - "node_modules/react-flow-renderer": { - "version": "10.3.17", - "resolved": "https://registry.npmjs.org/react-flow-renderer/-/react-flow-renderer-10.3.17.tgz", - "integrity": "sha512-bywiqVErlh5kCDqw3x0an5Ur3mT9j9CwJsDwmhmz4i1IgYM1a0SPqqEhClvjX+s5pU4nHjmVaGXWK96pwsiGcQ==", - "deprecated": "react-flow-renderer has been renamed to reactflow, please use this package from now on https://reactflow.dev/docs/guides/migrate-to-v11/", - "dependencies": { - "@babel/runtime": "^7.18.9", - "@types/d3": "^7.4.0", - "@types/resize-observer-browser": "^0.1.7", - "classcat": "^5.0.3", - "d3-drag": "^3.0.0", - "d3-selection": "^3.0.0", - "d3-zoom": "^3.0.0", - "zustand": "^3.7.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "react": "16 || 17 || 18", - "react-dom": "16 || 17 || 18" - } - }, "node_modules/react-force-graph": { "version": "1.41.20", "resolved": "https://registry.npmjs.org/react-force-graph/-/react-force-graph-1.41.20.tgz", @@ -17405,7 +17249,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -17854,114 +17697,132 @@ "dev": true }, "node_modules/svgo": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/svgo/-/svgo-1.3.2.tgz", - "integrity": "sha512-yhy/sQYxR5BkC98CY7o31VGsg014AKLEPxdfhora76l36hD9Rdy5NZA/Ocn6yayNPgSamYdtX2rFJdcv07AYVw==", - "deprecated": "This SVGO version is no longer supported. Upgrade to v2.x.x.", - "dev": true, + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/svgo/-/svgo-3.0.2.tgz", + "integrity": "sha512-Z706C1U2pb1+JGP48fbazf3KxHrWOsLme6Rv7imFBn5EnuanDW1GPaA/P1/dvObE670JDePC3mnj0k0B7P0jjQ==", "dependencies": { - "chalk": "^2.4.1", - "coa": "^2.0.2", - "css-select": "^2.0.0", - "css-select-base-adapter": "^0.1.1", - "css-tree": "1.0.0-alpha.37", - "csso": "^4.0.2", - "js-yaml": "^3.13.1", - "mkdirp": "~0.5.1", - "object.values": "^1.1.0", - "sax": "~1.2.4", - "stable": "^0.1.8", - "unquote": "~1.1.1", - "util.promisify": "~1.0.0" + "@trysound/sax": "0.2.0", + "commander": "^7.2.0", + "css-select": "^5.1.0", + "css-tree": "^2.2.1", + "csso": "^5.0.5", + "picocolors": "^1.0.0" }, "bin": { "svgo": "bin/svgo" }, "engines": { - "node": ">=4.0.0" + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/svgo" } }, - "node_modules/svgo/node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "dependencies": { - "sprintf-js": "~1.0.2" + "node_modules/svgo/node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "engines": { + "node": ">= 10" } }, "node_modules/svgo/node_modules/css-select": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-2.1.0.tgz", - "integrity": "sha512-Dqk7LQKpwLoH3VovzZnkzegqNSuAziQyNZUcrdDM401iY+R5NkGBXGmtO05/yaXQziALuPogeG0b7UAgjnTJTQ==", - "dev": true, + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", "dependencies": { "boolbase": "^1.0.0", - "css-what": "^3.2.1", - "domutils": "^1.7.0", - "nth-check": "^1.0.2" - } - }, - "node_modules/svgo/node_modules/css-what": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-3.4.2.tgz", - "integrity": "sha512-ACUm3L0/jiZTqfzRM3Hi9Q8eZqd6IK37mMWPLz9PJxkLWllYeRf+EHUSHYEtFop2Eqytaq1FizFVh7XfBnXCDQ==", - "dev": true, - "engines": { - "node": ">= 6" + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" }, "funding": { "url": "https://github.com/sponsors/fb55" } }, - "node_modules/svgo/node_modules/dom-serializer": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.2.2.tgz", - "integrity": "sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g==", - "dev": true, + "node_modules/svgo/node_modules/css-tree": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.2.1.tgz", + "integrity": "sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==", "dependencies": { - "domelementtype": "^2.0.1", - "entities": "^2.0.0" + "mdn-data": "2.0.28", + "source-map-js": "^1.0.1" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/svgo/node_modules/csso": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/csso/-/csso-5.0.5.tgz", + "integrity": "sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==", + "dependencies": { + "css-tree": "~2.2.0" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/svgo/node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/svgo/node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" } }, "node_modules/svgo/node_modules/domutils": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz", - "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==", - "dev": true, + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.0.1.tgz", + "integrity": "sha512-z08c1l761iKhDFtfXO04C7kTdPBLi41zwOZl00WS8b5eiaebNpY00HKbztwBq+e3vyqWNwWF3mP9YLUeqIrF+Q==", "dependencies": { - "dom-serializer": "0", - "domelementtype": "1" - } - }, - "node_modules/svgo/node_modules/domutils/node_modules/domelementtype": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", - "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==", - "dev": true - }, - "node_modules/svgo/node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", - "dev": true, - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.1" }, - "bin": { - "js-yaml": "bin/js-yaml.js" + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" } }, - "node_modules/svgo/node_modules/nth-check": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", - "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", - "dev": true, - "dependencies": { - "boolbase": "~1.0.0" + "node_modules/svgo/node_modules/entities": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.4.0.tgz", + "integrity": "sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/svgo/node_modules/mdn-data": { + "version": "2.0.28", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.28.tgz", + "integrity": "sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==" + }, "node_modules/symbol-tree": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", @@ -19798,22 +19659,6 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } - }, - "node_modules/zustand": { - "version": "3.7.2", - "resolved": "https://registry.npmjs.org/zustand/-/zustand-3.7.2.tgz", - "integrity": "sha512-PIJDIZKtokhof+9+60cpockVOq05sJzHCriyvaLBmEJixseQ1a5Kdov6fWZfWOu5SK9c+FhH1jU0tntLxRJYMA==", - "engines": { - "node": ">=12.7.0" - }, - "peerDependencies": { - "react": ">=16.8" - }, - "peerDependenciesMeta": { - "react": { - "optional": true - } - } } }, "dependencies": { @@ -22370,6 +22215,125 @@ "cosmiconfig": "^7.0.0", "deepmerge": "^4.2.2", "svgo": "^1.2.2" + }, + "dependencies": { + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, + "css-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-2.1.0.tgz", + "integrity": "sha512-Dqk7LQKpwLoH3VovzZnkzegqNSuAziQyNZUcrdDM401iY+R5NkGBXGmtO05/yaXQziALuPogeG0b7UAgjnTJTQ==", + "dev": true, + "requires": { + "boolbase": "^1.0.0", + "css-what": "^3.2.1", + "domutils": "^1.7.0", + "nth-check": "^1.0.2" + } + }, + "css-tree": { + "version": "1.0.0-alpha.37", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-1.0.0-alpha.37.tgz", + "integrity": "sha512-DMxWJg0rnz7UgxKT0Q1HU/L9BeJI0M6ksor0OgqOnF+aRCDWg/N2641HmVyU9KVIu0OVVWOb2IpC9A+BJRnejg==", + "dev": true, + "requires": { + "mdn-data": "2.0.4", + "source-map": "^0.6.1" + } + }, + "css-what": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-3.4.2.tgz", + "integrity": "sha512-ACUm3L0/jiZTqfzRM3Hi9Q8eZqd6IK37mMWPLz9PJxkLWllYeRf+EHUSHYEtFop2Eqytaq1FizFVh7XfBnXCDQ==", + "dev": true + }, + "dom-serializer": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.2.2.tgz", + "integrity": "sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g==", + "dev": true, + "requires": { + "domelementtype": "^2.0.1", + "entities": "^2.0.0" + } + }, + "domutils": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz", + "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==", + "dev": true, + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + }, + "dependencies": { + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==", + "dev": true + } + } + }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, + "mdn-data": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.4.tgz", + "integrity": "sha512-iV3XNKw06j5Q7mi6h+9vbx23Tv7JkjEVgKHW4pimwyDGWm0OIQntJJ+u1C6mg6mK1EaTv42XQ7w76yuzH7M2cA==", + "dev": true + }, + "nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "dev": true, + "requires": { + "boolbase": "~1.0.0" + } + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "svgo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/svgo/-/svgo-1.3.2.tgz", + "integrity": "sha512-yhy/sQYxR5BkC98CY7o31VGsg014AKLEPxdfhora76l36hD9Rdy5NZA/Ocn6yayNPgSamYdtX2rFJdcv07AYVw==", + "dev": true, + "requires": { + "chalk": "^2.4.1", + "coa": "^2.0.2", + "css-select": "^2.0.0", + "css-select-base-adapter": "^0.1.1", + "css-tree": "1.0.0-alpha.37", + "csso": "^4.0.2", + "js-yaml": "^3.13.1", + "mkdirp": "~0.5.1", + "object.values": "^1.1.0", + "sax": "~1.2.4", + "stable": "^0.1.8", + "unquote": "~1.1.1", + "util.promisify": "~1.0.0" + } + } } }, "@svgr/webpack": { @@ -22405,8 +22369,7 @@ "@trysound/sax": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz", - "integrity": "sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==", - "dev": true + "integrity": "sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==" }, "@tweenjs/tween.js": { "version": "18.6.4", @@ -22492,228 +22455,6 @@ "@types/node": "*" } }, - "@types/d3": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.0.tgz", - "integrity": "sha512-jIfNVK0ZlxcuRDKtRS/SypEyOQ6UHaFQBKv032X45VvxSJ6Yi5G9behy9h6tNTHTDGh5Vq+KbmBjUWLgY4meCA==", - "requires": { - "@types/d3-array": "*", - "@types/d3-axis": "*", - "@types/d3-brush": "*", - "@types/d3-chord": "*", - "@types/d3-color": "*", - "@types/d3-contour": "*", - "@types/d3-delaunay": "*", - "@types/d3-dispatch": "*", - "@types/d3-drag": "*", - "@types/d3-dsv": "*", - "@types/d3-ease": "*", - "@types/d3-fetch": "*", - "@types/d3-force": "*", - "@types/d3-format": "*", - "@types/d3-geo": "*", - "@types/d3-hierarchy": "*", - "@types/d3-interpolate": "*", - "@types/d3-path": "*", - "@types/d3-polygon": "*", - "@types/d3-quadtree": "*", - "@types/d3-random": "*", - "@types/d3-scale": "*", - "@types/d3-scale-chromatic": "*", - "@types/d3-selection": "*", - "@types/d3-shape": "*", - "@types/d3-time": "*", - "@types/d3-time-format": "*", - "@types/d3-timer": "*", - "@types/d3-transition": "*", - "@types/d3-zoom": "*" - } - }, - "@types/d3-array": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.0.4.tgz", - "integrity": "sha512-nwvEkG9vYOc0Ic7G7kwgviY4AQlTfYGIZ0fqB7CQHXGyYM6nO7kJh5EguSNA3jfh4rq7Sb7eMVq8isuvg2/miQ==" - }, - "@types/d3-axis": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-3.0.2.tgz", - "integrity": "sha512-uGC7DBh0TZrU/LY43Fd8Qr+2ja1FKmH07q2FoZFHo1eYl8aj87GhfVoY1saJVJiq24rp1+wpI6BvQJMKgQm8oA==", - "requires": { - "@types/d3-selection": "*" - } - }, - "@types/d3-brush": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-brush/-/d3-brush-3.0.2.tgz", - "integrity": "sha512-2TEm8KzUG3N7z0TrSKPmbxByBx54M+S9lHoP2J55QuLU0VSQ9mE96EJSAOVNEqd1bbynMjeTS9VHmz8/bSw8rA==", - "requires": { - "@types/d3-selection": "*" - } - }, - "@types/d3-chord": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-chord/-/d3-chord-3.0.2.tgz", - "integrity": "sha512-abT/iLHD3sGZwqMTX1TYCMEulr+wBd0SzyOQnjYNLp7sngdOHYtNkMRI5v3w5thoN+BWtlHVDx2Osvq6fxhZWw==" - }, - "@types/d3-color": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.0.tgz", - "integrity": "sha512-HKuicPHJuvPgCD+np6Se9MQvS6OCbJmOjGvylzMJRlDwUXjKTTXs6Pwgk79O09Vj/ho3u1ofXnhFOaEWWPrlwA==" - }, - "@types/d3-contour": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-contour/-/d3-contour-3.0.2.tgz", - "integrity": "sha512-k6/bGDoAGJZnZWaKzeB+9glgXCYGvh6YlluxzBREiVo8f/X2vpTEdgPy9DN7Z2i42PZOZ4JDhVdlTSTSkLDPlQ==", - "requires": { - "@types/d3-array": "*", - "@types/geojson": "*" - } - }, - "@types/d3-delaunay": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/@types/d3-delaunay/-/d3-delaunay-6.0.1.tgz", - "integrity": "sha512-tLxQ2sfT0p6sxdG75c6f/ekqxjyYR0+LwPrsO1mbC9YDBzPJhs2HbJJRrn8Ez1DBoHRo2yx7YEATI+8V1nGMnQ==" - }, - "@types/d3-dispatch": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-dispatch/-/d3-dispatch-3.0.2.tgz", - "integrity": "sha512-rxN6sHUXEZYCKV05MEh4z4WpPSqIw+aP7n9ZN6WYAAvZoEAghEK1WeVZMZcHRBwyaKflU43PCUAJNjFxCzPDjg==" - }, - "@types/d3-drag": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-drag/-/d3-drag-3.0.2.tgz", - "integrity": "sha512-qmODKEDvyKWVHcWWCOVcuVcOwikLVsyc4q4EBJMREsoQnR2Qoc2cZQUyFUPgO9q4S3qdSqJKBsuefv+h0Qy+tw==", - "requires": { - "@types/d3-selection": "*" - } - }, - "@types/d3-dsv": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-3.0.1.tgz", - "integrity": "sha512-76pBHCMTvPLt44wFOieouXcGXWOF0AJCceUvaFkxSZEu4VDUdv93JfpMa6VGNFs01FHfuP4a5Ou68eRG1KBfTw==" - }, - "@types/d3-ease": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.0.tgz", - "integrity": "sha512-aMo4eaAOijJjA6uU+GIeW018dvy9+oH5Y2VPPzjjfxevvGQ/oRDs+tfYC9b50Q4BygRR8yE2QCLsrT0WtAVseA==" - }, - "@types/d3-fetch": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-fetch/-/d3-fetch-3.0.2.tgz", - "integrity": "sha512-gllwYWozWfbep16N9fByNBDTkJW/SyhH6SGRlXloR7WdtAaBui4plTP+gbUgiEot7vGw/ZZop1yDZlgXXSuzjA==", - "requires": { - "@types/d3-dsv": "*" - } - }, - "@types/d3-force": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/d3-force/-/d3-force-3.0.4.tgz", - "integrity": "sha512-q7xbVLrWcXvSBBEoadowIUJ7sRpS1yvgMWnzHJggFy5cUZBq2HZL5k/pBSm0GdYWS1vs5/EDwMjSKF55PDY4Aw==" - }, - "@types/d3-format": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/d3-format/-/d3-format-3.0.1.tgz", - "integrity": "sha512-5KY70ifCCzorkLuIkDe0Z9YTf9RR2CjBX1iaJG+rgM/cPP+sO+q9YdQ9WdhQcgPj1EQiJ2/0+yUkkziTG6Lubg==" - }, - "@types/d3-geo": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@types/d3-geo/-/d3-geo-3.0.3.tgz", - "integrity": "sha512-bK9uZJS3vuDCNeeXQ4z3u0E7OeJZXjUgzFdSOtNtMCJCLvDtWDwfpRVWlyt3y8EvRzI0ccOu9xlMVirawolSCw==", - "requires": { - "@types/geojson": "*" - } - }, - "@types/d3-hierarchy": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@types/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", - "integrity": "sha512-9hjRTVoZjRFR6xo8igAJyNXQyPX6Aq++Nhb5ebrUF414dv4jr2MitM2fWiOY475wa3Za7TOS2Gh9fmqEhLTt0A==" - }, - "@types/d3-interpolate": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.1.tgz", - "integrity": "sha512-jx5leotSeac3jr0RePOH1KdR9rISG91QIE4Q2PYTu4OymLTZfA3SrnURSLzKH48HmXVUru50b8nje4E79oQSQw==", - "requires": { - "@types/d3-color": "*" - } - }, - "@types/d3-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.0.0.tgz", - "integrity": "sha512-0g/A+mZXgFkQxN3HniRDbXMN79K3CdTpLsevj+PXiTcb2hVyvkZUBg37StmgCQkaD84cUJ4uaDAWq7UJOQy2Tg==" - }, - "@types/d3-polygon": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/d3-polygon/-/d3-polygon-3.0.0.tgz", - "integrity": "sha512-D49z4DyzTKXM0sGKVqiTDTYr+DHg/uxsiWDAkNrwXYuiZVd9o9wXZIo+YsHkifOiyBkmSWlEngHCQme54/hnHw==" - }, - "@types/d3-quadtree": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-quadtree/-/d3-quadtree-3.0.2.tgz", - "integrity": "sha512-QNcK8Jguvc8lU+4OfeNx+qnVy7c0VrDJ+CCVFS9srBo2GL9Y18CnIxBdTF3v38flrGy5s1YggcoAiu6s4fLQIw==" - }, - "@types/d3-random": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/d3-random/-/d3-random-3.0.1.tgz", - "integrity": "sha512-IIE6YTekGczpLYo/HehAy3JGF1ty7+usI97LqraNa8IiDur+L44d0VOjAvFQWJVdZOJHukUJw+ZdZBlgeUsHOQ==" - }, - "@types/d3-scale": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.3.tgz", - "integrity": "sha512-PATBiMCpvHJSMtZAMEhc2WyL+hnzarKzI6wAHYjhsonjWJYGq5BXTzQjv4l8m2jO183/4wZ90rKvSeT7o72xNQ==", - "requires": { - "@types/d3-time": "*" - } - }, - "@types/d3-scale-chromatic": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.0.0.tgz", - "integrity": "sha512-dsoJGEIShosKVRBZB0Vo3C8nqSDqVGujJU6tPznsBJxNJNwMF8utmS83nvCBKQYPpjCzaaHcrf66iTRpZosLPw==" - }, - "@types/d3-selection": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.4.tgz", - "integrity": "sha512-ZeykX7286BCyMg9sH5fIAORyCB6hcATPSRQpN47jwBA2bMbAT0s+EvtDP5r1FZYJ95R8QoEE1CKJX+n0/M5Vhg==" - }, - "@types/d3-shape": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.1.tgz", - "integrity": "sha512-6Uh86YFF7LGg4PQkuO2oG6EMBRLuW9cbavUW46zkIO5kuS2PfTqo2o9SkgtQzguBHbLgNnU90UNsITpsX1My+A==", - "requires": { - "@types/d3-path": "*" - } - }, - "@types/d3-time": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.0.tgz", - "integrity": "sha512-sZLCdHvBUcNby1cB6Fd3ZBrABbjz3v1Vm90nysCQ6Vt7vd6e/h9Lt7SiJUoEX0l4Dzc7P5llKyhqSi1ycSf1Hg==" - }, - "@types/d3-time-format": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@types/d3-time-format/-/d3-time-format-4.0.0.tgz", - "integrity": "sha512-yjfBUe6DJBsDin2BMIulhSHmr5qNR5Pxs17+oW4DoVPyVIXZ+m6bs7j1UVKP08Emv6jRmYrYqxYzO63mQxy1rw==" - }, - "@types/d3-timer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.0.tgz", - "integrity": "sha512-HNB/9GHqu7Fo8AQiugyJbv6ZxYz58wef0esl4Mv828w1ZKpAshw/uFWVDUcIB9KKFeFKoxS3cHY07FFgtTRZ1g==" - }, - "@types/d3-transition": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.3.tgz", - "integrity": "sha512-/S90Od8Id1wgQNvIA8iFv9jRhCiZcGhPd2qX0bKF/PS+y0W5CrXKgIiELd2CvG1mlQrWK/qlYh3VxicqG1ZvgA==", - "requires": { - "@types/d3-selection": "*" - } - }, - "@types/d3-zoom": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/d3-zoom/-/d3-zoom-3.0.2.tgz", - "integrity": "sha512-t09DDJVBI6AkM7N8kuPsnq/3d/ehtRKBN1xSiYjjMCgbiw6HM6Ged5VhvswmhprfKyGvzeTEL/4WBaK9llWvlA==", - "requires": { - "@types/d3-interpolate": "*", - "@types/d3-selection": "*" - } - }, "@types/eslint": { "version": "8.21.0", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.21.0.tgz", @@ -22763,11 +22504,6 @@ "@types/range-parser": "*" } }, - "@types/geojson": { - "version": "7946.0.10", - "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.10.tgz", - "integrity": "sha512-Nmh0K3iWQJzniTuPRcJn5hxXkfB1T1pgB89SBig5PlJQU5yocazeu4jATJlaA0GYFKWMqDdvYemoSnF2pXgLVA==" - }, "@types/graceful-fs": { "version": "4.1.6", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.6.tgz", @@ -22898,11 +22634,6 @@ "@types/react": "*" } }, - "@types/resize-observer-browser": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/@types/resize-observer-browser/-/resize-observer-browser-0.1.7.tgz", - "integrity": "sha512-G9eN0Sn0ii9PWQ3Vl72jDPgeJwRWhv2Qk/nQkJuWmRmOB4HX3/BhD5SE1dZs/hzPZL/WKnvF0RHdTSG54QJFyg==" - }, "@types/resolve": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.17.1.tgz", @@ -24135,8 +23866,7 @@ "boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", - "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", - "dev": true + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" }, "bootstrap": { "version": "5.2.3", @@ -24380,11 +24110,6 @@ "integrity": "sha512-cOU9usZw8/dXIXKtwa8pM0OTJQuJkxMN6w30csNRUerHfeQ5R6U3kkU/FtJeIf3M202OHfY2U8ccInBG7/xogA==", "dev": true }, - "classcat": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/classcat/-/classcat-5.0.4.tgz", - "integrity": "sha512-sbpkOw6z413p+HDGcBENe498WM9woqWHiJxCq7nvmxe9WmrUmqfAcxpIwAiMtM5Q3AhYkzXcNQHqsWq0mND51g==" - }, "classnames": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz", @@ -24822,29 +24547,10 @@ "postcss-value-parser": "^4.0.2" } }, - "css-tree": { - "version": "1.0.0-alpha.37", - "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-1.0.0-alpha.37.tgz", - "integrity": "sha512-DMxWJg0rnz7UgxKT0Q1HU/L9BeJI0M6ksor0OgqOnF+aRCDWg/N2641HmVyU9KVIu0OVVWOb2IpC9A+BJRnejg==", - "dev": true, - "requires": { - "mdn-data": "2.0.4", - "source-map": "^0.6.1" - }, - "dependencies": { - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true - } - } - }, "css-what": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", - "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", - "dev": true + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==" }, "cssdb": { "version": "7.4.1", @@ -25424,8 +25130,7 @@ "domelementtype": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", - "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", - "dev": true + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==" }, "domexception": { "version": "2.0.1", @@ -29659,12 +29364,6 @@ } } }, - "mdn-data": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.4.tgz", - "integrity": "sha512-iV3XNKw06j5Q7mi6h+9vbx23Tv7JkjEVgKHW4pimwyDGWm0OIQntJJ+u1C6mg6mK1EaTv42XQ7w76yuzH7M2cA==", - "dev": true - }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -29984,7 +29683,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", - "dev": true, "requires": { "boolbase": "^1.0.0" } @@ -31591,21 +31289,6 @@ "integrity": "sha512-/6UZ2qgEyH2aqzYZgQPxEnz33NJ2gNsnHA2o5+o4wW9bLM/JYQitNP9xPhsXwC08hMMovfGe/8retsdDsczPRg==", "dev": true }, - "react-flow-renderer": { - "version": "10.3.17", - "resolved": "https://registry.npmjs.org/react-flow-renderer/-/react-flow-renderer-10.3.17.tgz", - "integrity": "sha512-bywiqVErlh5kCDqw3x0an5Ur3mT9j9CwJsDwmhmz4i1IgYM1a0SPqqEhClvjX+s5pU4nHjmVaGXWK96pwsiGcQ==", - "requires": { - "@babel/runtime": "^7.18.9", - "@types/d3": "^7.4.0", - "@types/resize-observer-browser": "^0.1.7", - "classcat": "^5.0.3", - "d3-drag": "^3.0.0", - "d3-selection": "^3.0.0", - "d3-zoom": "^3.0.0", - "zustand": "^3.7.2" - } - }, "react-force-graph": { "version": "1.41.20", "resolved": "https://registry.npmjs.org/react-force-graph/-/react-force-graph-1.41.20.tgz", @@ -32411,8 +32094,7 @@ "source-map-js": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", - "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", - "dev": true + "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==" }, "source-map-loader": { "version": "3.0.2", @@ -32753,99 +32435,89 @@ "dev": true }, "svgo": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/svgo/-/svgo-1.3.2.tgz", - "integrity": "sha512-yhy/sQYxR5BkC98CY7o31VGsg014AKLEPxdfhora76l36hD9Rdy5NZA/Ocn6yayNPgSamYdtX2rFJdcv07AYVw==", - "dev": true, + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/svgo/-/svgo-3.0.2.tgz", + "integrity": "sha512-Z706C1U2pb1+JGP48fbazf3KxHrWOsLme6Rv7imFBn5EnuanDW1GPaA/P1/dvObE670JDePC3mnj0k0B7P0jjQ==", "requires": { - "chalk": "^2.4.1", - "coa": "^2.0.2", - "css-select": "^2.0.0", - "css-select-base-adapter": "^0.1.1", - "css-tree": "1.0.0-alpha.37", - "csso": "^4.0.2", - "js-yaml": "^3.13.1", - "mkdirp": "~0.5.1", - "object.values": "^1.1.0", - "sax": "~1.2.4", - "stable": "^0.1.8", - "unquote": "~1.1.1", - "util.promisify": "~1.0.0" + "@trysound/sax": "0.2.0", + "commander": "^7.2.0", + "css-select": "^5.1.0", + "css-tree": "^2.2.1", + "csso": "^5.0.5", + "picocolors": "^1.0.0" }, "dependencies": { - "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "requires": { - "sprintf-js": "~1.0.2" - } + "commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==" }, "css-select": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-2.1.0.tgz", - "integrity": "sha512-Dqk7LQKpwLoH3VovzZnkzegqNSuAziQyNZUcrdDM401iY+R5NkGBXGmtO05/yaXQziALuPogeG0b7UAgjnTJTQ==", - "dev": true, + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", "requires": { "boolbase": "^1.0.0", - "css-what": "^3.2.1", - "domutils": "^1.7.0", - "nth-check": "^1.0.2" + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" } }, - "css-what": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-3.4.2.tgz", - "integrity": "sha512-ACUm3L0/jiZTqfzRM3Hi9Q8eZqd6IK37mMWPLz9PJxkLWllYeRf+EHUSHYEtFop2Eqytaq1FizFVh7XfBnXCDQ==", - "dev": true + "css-tree": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.2.1.tgz", + "integrity": "sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==", + "requires": { + "mdn-data": "2.0.28", + "source-map-js": "^1.0.1" + } + }, + "csso": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/csso/-/csso-5.0.5.tgz", + "integrity": "sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==", + "requires": { + "css-tree": "~2.2.0" + } }, "dom-serializer": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.2.2.tgz", - "integrity": "sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g==", - "dev": true, + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", "requires": { - "domelementtype": "^2.0.1", - "entities": "^2.0.0" + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + } + }, + "domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "requires": { + "domelementtype": "^2.3.0" } }, "domutils": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz", - "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==", - "dev": true, + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.0.1.tgz", + "integrity": "sha512-z08c1l761iKhDFtfXO04C7kTdPBLi41zwOZl00WS8b5eiaebNpY00HKbztwBq+e3vyqWNwWF3mP9YLUeqIrF+Q==", "requires": { - "dom-serializer": "0", - "domelementtype": "1" - }, - "dependencies": { - "domelementtype": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", - "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==", - "dev": true - } + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.1" } }, - "js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", - "dev": true, - "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - } + "entities": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.4.0.tgz", + "integrity": "sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==" }, - "nth-check": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", - "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", - "dev": true, - "requires": { - "boolbase": "~1.0.0" - } + "mdn-data": { + "version": "2.0.28", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.28.tgz", + "integrity": "sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==" } } }, @@ -34281,12 +33953,6 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==" - }, - "zustand": { - "version": "3.7.2", - "resolved": "https://registry.npmjs.org/zustand/-/zustand-3.7.2.tgz", - "integrity": "sha512-PIJDIZKtokhof+9+60cpockVOq05sJzHCriyvaLBmEJixseQ1a5Kdov6fWZfWOu5SK9c+FhH1jU0tntLxRJYMA==", - "requires": {} } } } diff --git a/ui/package.json b/ui/package.json index a81becf8..c662144d 100644 --- a/ui/package.json +++ b/ui/package.json @@ -16,12 +16,12 @@ "react-bootstrap": "^2.7.0", "react-data-table-component": "^7.5.2", "react-dom": "^18.2.0", - "react-flow-renderer": "^10.3.12", "react-force-graph": "^1.41.20", "react-router-dom": "^6.4.0", "react-sizeme": "^3.0.2", "react-sparklines": "^1.7.0", - "styled-components": "^5.3.5" + "styled-components": "^5.3.5", + "svgo": "^3.0.2" }, "devDependencies": { "react-scripts": "^5.0.1" From ad818e1dfa07e6a3f401515e2fa7680705f63ea9 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 3 Apr 2023 14:33:21 -0400 Subject: [PATCH 69/83] removed spark data log lint --- controller/shareDetail.go | 1 - 1 file changed, 1 deletion(-) diff --git a/controller/shareDetail.go b/controller/shareDetail.go index dbccdc1b..cc018d1f 100644 --- a/controller/shareDetail.go +++ b/controller/shareDetail.go @@ -45,7 +45,6 @@ func (h *shareDetailHandler) Handle(params metadata.GetShareDetailParams, princi var sparkData map[string][]int64 if cfg.Metrics != nil && cfg.Metrics.Influx != nil { sparkData, err = sparkDataForShares([]*store.Share{shr}) - logrus.Info(sparkData) if err != nil { logrus.Errorf("error querying spark data for share: %v", err) } From 8bb9304425515c8b773f8ab1c25dd62b4e08d198 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 4 Apr 2023 13:02:04 -0400 Subject: [PATCH 70/83] metrics and limits docs iteration (#283) --- .../guides/metrics-and-limits/_category_.json | 7 ++ .../metrics-and-limits/configuring-metrics.md | 11 +++ .../images/metrics-architecture-simple.drawio | 70 ++++++++++++++++++ .../images/metrics-architecture-simple.png | Bin 0 -> 33893 bytes .../images/metrics-architecture.drawio | 67 +++++++++++++++++ .../images/metrics-architecture.png | Bin 0 -> 36024 bytes 6 files changed, 155 insertions(+) create mode 100644 docs/guides/metrics-and-limits/_category_.json create mode 100644 docs/guides/metrics-and-limits/configuring-metrics.md create mode 100644 docs/guides/metrics-and-limits/images/metrics-architecture-simple.drawio create mode 100644 docs/guides/metrics-and-limits/images/metrics-architecture-simple.png create mode 100644 docs/guides/metrics-and-limits/images/metrics-architecture.drawio create mode 100644 docs/guides/metrics-and-limits/images/metrics-architecture.png diff --git a/docs/guides/metrics-and-limits/_category_.json b/docs/guides/metrics-and-limits/_category_.json new file mode 100644 index 00000000..f46ea994 --- /dev/null +++ b/docs/guides/metrics-and-limits/_category_.json @@ -0,0 +1,7 @@ +{ + "label": "Metrics and Limits", + "position": 40, + "link": { + "type": "generated-index" + } +} diff --git a/docs/guides/metrics-and-limits/configuring-metrics.md b/docs/guides/metrics-and-limits/configuring-metrics.md new file mode 100644 index 00000000..cc25e3f0 --- /dev/null +++ b/docs/guides/metrics-and-limits/configuring-metrics.md @@ -0,0 +1,11 @@ +# Configuring Metrics + +A fully configured, production-scale `zrok` service instance looks like this: + +![zrok Metrics Architecture](images/metrics-architecture.png) + +`zrok` metrics builds on top of the `fabric.usage` event type from OpenZiti. The OpenZiti controller has a number of way to emit events. The `zrok` controller has several ways to consume `fabric.usage` events. Smaller installations could be configured in these ways: + +![zrok simplified metrics architecture](images/metrics-architecture-simple.png) + +Environments that horizontally scale the `zrok` control plane with multiple controllers should use an AMQP-based queue to "fan out" the metrics workload across the entire control plane. Simpler installations that use a single `zrok` controller can collect `fabric.usage` events from the OpenZiti controller by "tailing" the events log file, or collecting them from the OpenZiti controller's websocket implementation. \ No newline at end of file diff --git a/docs/guides/metrics-and-limits/images/metrics-architecture-simple.drawio b/docs/guides/metrics-and-limits/images/metrics-architecture-simple.drawio new file mode 100644 index 00000000..2661aed5 --- /dev/null +++ b/docs/guides/metrics-and-limits/images/metrics-architecture-simple.drawio @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/guides/metrics-and-limits/images/metrics-architecture-simple.png b/docs/guides/metrics-and-limits/images/metrics-architecture-simple.png new file mode 100644 index 0000000000000000000000000000000000000000..ca9fdf4114ef8aea501157edf3cacda4a09378a0 GIT binary patch literal 33893 zcmeFZ1z45awl@w7ki|l2FaYV2?vz+8x*MdsyE_B{L0V8iQb7?BK}AIA7C}HH1nF); z8vN!W_IB@czO&E$|IhvIckXwe$4Azh?|f&BImVbHeq%+eD$87eU4fyXpj?oXMW~~o zK)`2IIP@H7Nud7G2L7PBtINPqN;=8rQBdeDJfw6z9DS^89V}7kxFt_M(Q$FuIJbJ&^sm`PsPnkgq7nYAPwxaY=&D4z~7|;E#-@xxF*;6$>|0Z#G+J(4C8ujgt+z zSVfS1vF#jJQ>>dl+bcY&Qg-xo~-;DeAe~|ZaW!M zS3sCQj&?StqmPW6sf&%0vxTJt7}&z+WLQoPZZIiwSr)!0Edm^;vsk;G&GSoxhOOV} zf;dm#v-Y&Luyj8i?PMqqXJ-cwTbDmJnmao=0iO9oM^iU9XYW6DvvPJg-QDS|E@0z- zcnR5{VQ%X1x2-z179KVyefc;}mxt^vYiVn3b2>B6*#k$@v%aT2-EB-QoV`z8KW_T< zP)^8rMoc$nXE59^XZGvkvrdjal9mogTAz{ebRNJ<|M7Q!BZk{=lJoy(I6ncnGTcXx zhs$1#Pg_IPQo~Y`^NhQ^OdUK=X?w%g1F$}y1K2Fw%nf|9Mt+iXb^@aB;9v<_keuW0 z;d^!%V8h_t!P{^^&0{t1}{e8#BMEJ*K|DE_UmO#oq+$k&kzvH68# z1WwybPZ`zvw}Fv8|C!SSeNLI-A1@xMY5&ACfA^rkyvNk~{h%zp09&}T*}0!-?ACl%{?8Dx#zDA!_3*!$->e?(d_@^>2M03()sjw_;^ot`OEQ~oy4!Fb9RdV_nZ#6 zHU4xuCk?+GiKUZ;1XA?@<~MgRb$7Ql|D!axog4so46XyXrH`$LF7h**06*{PcfI59 zXWtM=4nOOPKoaomi>jrYEf51Bs;6y0bbXHpLk@)e^2sY4e-T;X314Bq0Gse*1nl(+Rmw-$L@uUn`W8$^N%fa-ZGk zXH%YiKb!J|^8X*EJ8?dnce1)C0Sb?hy^!i;i`CL7bS34Z}fYtrj*g@K5f-XS9fKht(7<@Xu z3K`{`tQtG9%tQ$8N_b=|ozfs3#rsnq6$2vmA z(*q2H0q@?<#F~AWRH)aPvfas7X?tAC9iKb6M6 zROG;+1I+l3Rlx7s>^JSg#qkRt{nu-9-ZQ`GOsxv=oDlPji~seSoR{~H^2Npdrz!t^ zHMzhE?S9eZCk?-AVy?d{Z@;(ySizcOn6d&t6{ZFR0c zn;8FkS>QUmTu#N0^USb16Tg37SpcD@}QIT|J2d@MZN#KINqoF z_ZPMGKL+T3r2)!$>TI8}7T@2p*1w4b%JsLt%|CY|{y%Jia-E9fU*5W0$87TNZh`Xf z{E^#vIsRl0{_AgD9>G6O$;11nDgSM6-4m7oA?M$3-M_EO|3S3-cMBA0b^e7He|pLP zr#j?Zyr*{PnLa!-lK$$dJ=2?K#?*hkN6z)PR?$DF5C76b`0o-%JtffpBg&8SPgW!k z-zlq~vFyKI`El|5(QxJA{L_^Gw#x6!UHe7({UhOWaz=(2;TERuHpe-Bq_+D-%#iJ> zrXC>4fXsG-kOc=asOb)Jg!Yy?r^z)C|AGImE|J-ZlL`OY`TlpmwYiX?Eu?>Us*}0T z66gPkjC&R>JG%zX&iic2e_zHu4T$~1xKhWdPe5O1kUa)}06xUs+1wtamX4|ZOF#*v zmXV?TUpV)KMv7l%JJnL5~7BYE2#L~N0%G&nL*Z3{365~mLwZ7nR2 zIq<)01pJZjk=cP$6oue#V(_1v_~tqbxS#RE84Lbu7M%6@HBx`Zx@UazUvCzi#Lxbl znQgzRgwyjmv&Mcsp0lI)^?1(u{bwG}aTf1?gLgU02mTH$`c+vUKRtsoojf>w0f{R+ z$p@Im`AngpP@>2oBs6`DmLFgrq)i_FWLHoZde+7ikq(zu7SyDD&2p{Hv|0KC(`|Jg z76(%tsa%f2!SqJ5jHY%(Zk{TQhCZEq;*(F0cf_(*M^}!rMXMwCH+F+o8h%t7ueMBX z>})Oj2bb&zzAssaTALh68Z(`{m> z!7|G&_v!Y#B}VW4oW8yDby^;J<@7O~*J<%tg_lAan`eM^!>tQjQ9`1eR`1Q+jd6%fPP#yt(#N!05 zD!4e3I%60R_V=4f8Qj)!8A6^#{y+M39A!qU1O5BUEEV(<2B$ZiTg7|6*I`mz@*U?6 zJ_?{AQ4!sJn3#-)BT(x(-_!KUzQ2>)YrvFFLkp2>p;|>MR3c$POr59C(qxI9yqN zfBS>dT8z(l#KqwhCXHbYmFv@hslg8R;^Zly>pqn zdAuU{Xm?h)b4UM`-Lw{kd@f7sIoXEgiqVzojR&i5jt=*$sss0L+1)@;5h=cvJ>jeX zi4fVa#geK{L-&uoE8jn`doyIZgnjiM4GdHhbGtt3GkIaVkx=E$=+Rb%)7+=rm7=F5 zT!kOn5eTa9RNWVjnF3D9hOT>{yW>>7YvujTclPe|3;2`LMqBsIx$c#+4{Y2$E^Tya zngcJR;7{;$ginqw;SwaAD=TPkex*h1=!G63Z3LCe{G+tD{YL&DM?Cutowi53N0&ZO z!y-(dJdHr|9-Pt=8cWLG$pVePzIE>_=%Z2aKx|>GMLeZ(ZMfA(I)nHJIVX8O4|a)#ar*;puhiwN{#VfxIhVi#1LD zljnz@=~p;8)rO#YrO@Ohgre$PJf460R_*?3jWbn{&#FS6M2`RV%1W013cgy@IipwC zPKHvW=CJuh`LyEGbK@FD=j8d<&Tr@GpYl`1zzYgPh1iju)(BBWHW-xBEo{^1x+%pu ziwRXk)DToxeENz`xC0Sg>r}OD<#~a&n;NCFj1dv@;>JY?l4zk&yaBTq*5b`_78riW zehvK#M%C-R`g!8fA0H{C3CTN^=#@7I9;~-$=Vwe@B|6BF7NBSkOM06uM~8nhz<2Qw z?>!*Ac3HbgHh7nK?NS;pHj?J8Jr6$oa5&8#?Aj%=5$nG@oi@VRLNM10-8!|vWcCqwR;LMq^rllbB@LB#rG zi+(PQ#En3d%&nD*j{ICyL-x-D>+M=J+&4fhFJa}1RsvNTI)VRY`SsV~OhI?DzWFQs zPD=&K4{ijQ?^)48JywPX1NY_&oq*~pkAGg^9|d`oZ~ZZS+NpAyq$Qn-=>mk|fk)D_ z8o&cJoT0L)aNruc8 zdar(4smEr2nP2O*JTxeiq>N7K_PplkZGHm5RNRBz7cEyalr5RNuTjDdc5@@Imcdmw zB|Gml;nH-*INXc8Kx$>0D7z2^x&L}#MCCqL6VO_ZcD|A9Dy?)`&`iyAaCZ$CGfF2a zE!e|zmhdleSwxX9Uf-L`zc#-(Zlhth6Hchi5!N!(aj}gKHrGMzJ!D%rIOIBchkvi< z&gZ!{_gDI8)&~ia(%9k~ui82B5C$C?{9T#!!d^=@R2j=( zE4(v_<-rv8MrLa4zx-;;bbYb!X`1j@ z2wKm#M+2LnslNjv>ofm6=m+zL%wt7$38I;d z)qmaLIQ*uYnq!B@`P(qKTH76~mR}Ql-Bj&tin*x&4Ku1c!@1g)xNEH?XN8}>+%1VF zUPL|4f%lTC>=Y0t{dp)_$IJFQo z8n``pD_&G0Y;DH5w?JvAO2_R`*Z#$TMp9MK;odC_nWa9tYf%uZKv{@yRWD1prcvC@pf&-+jM*_o8z7=c&teyQv>3GBimv%C^fw%!ozja&D6SxDb#$ z2F=V)`8M;0l%SE501Fm#=bT&$;~0nR9L@T>q>uPx64&>a%9o>Ppla(E^3z2ctAFsq z0(;{q*0+~Bh#;4n<`}-xn z<9k}Bam0RYNpw!*^$~Gg=g1gL1~WL#Z%wmGShu3?;;!9fj-Zt%8W2oo4wc1htH3vb zo9s0Re&71)yp5BmF539bvl<;%NG=wj+Os057G~P&KqwmPoYDo2n{%J-_6p%TH?$!tRj+3wq_Z!FOc#LU>14@ z5oEPDUv`5x4yPFh%|rVb7My_TH-i_CGo0~r_M0vyC5OeeTa9=_Ll5PWWoPUkB0}TF z_2kYW$aZDd>ICt`P{U*)?`?`GbhzR;^#X1?7wD*oHumMu{3Nz*VPD?c6+=r}4#l0! zNQna?lf^nj5ww%5UmdVpfbrnR9dvD$Etb6lf&?UnVD>i^8D2 z>4(_4h>jjU=_`tdo;fp_nnZ}vDTQJ>{KmD}4JAam!7J)qPBI6FE(wL40D9}C=t8@p zdYPv^qE8|P$nxfEt}X98GOXE*cX`XZX&MmtiE|{MOt~rv4?~fMR)5?eGCw@~aLIZ` z4n~@kf`dxtOljsl>?$*VEdudX-1%mp%LIDKD=pw zmRTd4CmHfdj)m72SHGuQ^5d-dka3u8|MMA|gk_R|OU4u)ERlMeJZiP*LEz@pmu7&5 z(TbuAG)x%8hF`WN0aK7ZcsZHob!d01x_Y@`XD`QGu;XDKxE=1dG0KX%aQVnhNR0v2 z^xk)@V5CwF*Y)F!)X=&@RS>VF@#q$EM=!{%)awwTMd}s=je2TH=ON)eCR&tkv>)># z-&o}*dmZ$Y7=QfKUlDk+n){k~YHv0=g;~EO7a1g3;J;a`$7b!-PRFizxJ*Tlh@MM? z+u6Cilf1{i-tDR|wDw^&c<4ur|GW0(Ako<8xI(E7@1ROVt;9+se7Ef%8Zy34F{X=s z_iaa0WNX0O>ovu@1I|fIDb6GVB3M((=l;f#qZ$4hrp>1>46|QH-g9sl7*!u&E>5Tk zD>Z6oY+LclwQ1|h%o48)V zhl|FnQmpvGp9d&2)81`)E`Av~-x(s3@k?HmKMdP1CQKU`p9xp35zwFMJ(YN9+ zT2~7qi#ca$D3?wk4Xa~U^nX-Mzr1(1sHOwAEMJ?R8ukeOcFn7d3lm$W;N>V(il~%! zt>;Hwx+=birr$8IHd*`82;LoFp1(Af74TiI0vM{5a>?|1*E$|{bVT4V6Mv#t$$Dpt z&$_wqYCy3?Wp$pd=D<7GFKLD_jZ; zl$WjQ#c?=|A3heKvTi0oDx6_cQa` zEOaxXWC=|5q9bryxPP9fbQZC#cwW6{_tg$}>3D>mFp&ZK@b zjg#G}x&hu{7SDd6UZDyY*QrjtX^aWiHyOlsqgju1Ij%8tlHy^s%1iZ2!YJAC2*WGG zwZh=my0Vn+ELWh~@q^{oRhUu&JDKdkk62&1_4?t@S3FC*C^>zX6ie@L`&=oTKP+eJgzMU6f!y$c|e zcRcrlR#z8-WppG;h+$-=l)?AP(C!A^l4J)bD*M#f^zA2U#aW-RF7wzV5DQW1U{-mP z1i&S61*Z}|c7}`nJaGS@8{I~s0Ey7R^V+Cetwe;@tP3Mz3~A8O;ce6j%#}gR#D-n# z5bejqP1!`#WT7UNImQyMFy_n^4WhWPS7vhx_H@$`av-Y4@H7(^^vL@d3XYm>nJ<40%_|xlm6liAEFse zn0Ci;gKco!Y*R~dIcZbM&El^347zo8apHF~?%=u{l#T}N55`=Xh{WrwUx^zP+nb~D z?v}bZxGg~axFfF?zxGGJv2!)vy;QN?DMIf)Ege1o1mXZqn+*^$csU6zzJjVod7~h2 zu^X!=><1bJ#?Ukw>4tkXh6G83N9UJhWudm){iftF+Th|+MsfNzO9`Ln33pTTX`>*# zx@*DXbIcVFJhlWfwN0}}U7ks~-^CTWwd|Yz!0ag<%qQL44O_9&RQSP^=hB49zWYqa zi^sPD0Hd+UiwtC1mt$|zi6oW#c(HyClr1Ovj%QA8im5z>!u!pYV{-aV0(V#%hZZk^9k$Nsi_a5SR5NmGP4tiJ>kRy97n}MWGnw>V8i4z>en1 z@G%*&y+nMNVvFHh54ph4XnU)`+98=qdpY(v(lVcGeV(eM7*{T-<>C5=`kjeO*m9J9 zRCx7qFD^}8B~6T^i}I^&Z`$@@D64B7!7?Lnc#DB*Ni$Kd$^WxJ@%}`d=-!-c)!YX% zmhHLwst8nPV-{?mP5C6L5JPTU2C8s5?AeVeK@!dN^rw5nKKI5kYsr5J06`7u<}EnZmJz0~L3 zelMl2Bct~nxjWH~zQJs!vVOTwv7`m)+%`j-u*%&DkRysz%f*5bJFlqxI5;+f4AreSV^0O=wD~c|H=+@iJ)w zPd1$~=T&pXn-|QH&YVBjsw(FTGWhVrbMVOT%gZSfNA7T8eh~Dd=TS3(C%lX@q?z?j z^1-iVr)pKbwE6(MMjtp9tnkoqNf1Dpycj?ucB617ReoRTj?un6TkF&*zJRE zAW_dz91}R)yeRk@9-fAxqS$U1Q>SFCP@B`W4;d6QvKP-`=xgQ{~Dw z$vX+ompbC{NqN-OM_{wQXXo5&+$fZgMJeB3snT!NRu7ZmVycezYo&vNTmN$6!{{)T z$*0@WwlfA!%y&PELDAH3-^EWx`F#>D*fiI-iQ%P&)qR9gG|*5u2f*7~$#jboc4J>t z!tihjt|ux+a&wv4#o!xJS57~?$rp8}q^MqBDGKh{#(R5S=Ez$X4{>eqVIJkb4&?mEX^@;YokkrMr{*Pnf$1(BoG%7s27HXtARy)eeGy3 zY~m3S<-)p)C5sWui{7jlE@6oXg`&q9v~J44uNFbBf`Di0n`r-}p6#2sqNOQ~YS@Dl zjJf0w`QrA1B3>Gg#1tgFF7t5&mTXLrpEF{^#uK^=Y=q3Iatuh z#PMkODABgsy|(Gjlh1fkG`tAp+uk-%Z%*&A>rMlYv|@mhpluIdd5$DeDPL(RE-7CJ ziM~94`J9FfDuG&ecf-;C>gwK)Me7#UrMdNPaci>cPqgKrOiV_T1YBzB`zSRLJkeJt z((g#0uH~~s)~*A-;s>%=EA*aTyE{>kN@86HrYLn-t|o}0nzMS^S*mMQ>O?_iIP$pg z$`-~CWUloYp zCUwqJ2=h$1!zk4*FvPHDULTIN^0HSMn?H)~Z4qN$EgM;{&?8R;CL$4P(TkX=!wdF4 z?{UZkKE?6tJ_cNN1F8mvN|~tDVsV$^jUnW)^bY6%-L4*yY30~gPwB9+(D9iL)#te& z1SV95U{lS}-r~xe$+&WovN^#1L~|bI^SeB?jDk?e0{s_3{SicwX$l?(zH9Sy&?hOk|5O z)Ua^odK+bd@$gUWx8CFRiU#2#u9wcS8dnd?LM-pPsuvdx+XK$W`!MX*Q`wIp3ez!g zjo6=83AV$DMTQU~5L`CIyNqMF92Tv38@9EmNp`Lh0u1B1a(AbiO$nO~it=+x&9w(T zMc>)j4YiEzKXh)AiMWWr@subDIBpNKzmYz==bBNGSg`(VUih3n%9F>gh?WI;Cf{(I z`<5S4^}c)!VH}&3*4EqWx*`7Nh_zzaOfNcv*WPN`R zABzt?3@pAnm-&<;u;{TdZ7GZfXyMKoJw`#VD0z@jF|URpXi8fr(`qTZpA@yf*On#{ zxZTRiFako|ALAHNh!il`{h?TdsN7mwYPbSQT==E%(&X8FIenr_xb%Dlx@FY#tbhct zrl`o2i>MUM!aMvMN#vLocLeLu$OB5n*H_`#MUTj9Ud|V8OXrqXE-d!)RTCpQlthbq zqEvd)zj-EEiP|2mWfoB)LF5xOST`9ggCHes0#UIZLy+&VlCvU=ks1fu!@VIAx56{L zNmRh^4ba#l)%z>m{|hN3qq!^H-I^DnzmW9t1^-&=3j=Vk?!>NO<>&2-!!1;QB=ai15E2SsL3c=a8(;{IED zD|8>c=6DL{WFcFueLRL|)>_+vj0z;zUtd=Ic0d1%vbM0phx1eKwV1yi}z)n)JF)UYL9m)xOPU z`1Vp;-MnZTqd4rbcAF=3m7eeIFsZ#%Ns!hL<_-gvdyFPWm{X)qZ>UZ-Vp6TEC;*YW z0fbL(QLp>ZRki7M6AtR=B{!|o*5o%&b;mDVn5Zp*FcXcxlDJiszLa{qudrZr!#BWw zXqzQqG;;mIQ*&{mowm-akPu?jNKcmYsA{#wsLZHe(8D(lK|1N%!8#CqnM+gK5|F%* zN0*rqCGG(nIsyeDjCl830$-rG(5l@Pzw(LD+Lr~Y8-o@Dag?AH7=ng7+;^_tEx^BL za9FgUbFb2XL#;NpUJuinDQS~R+=jt{XLOt%OQ``Yd`^4ty6MkMZ}Vm={K7X{nh?~` z*wTh^qunQ6#{`eD8!qc9!JQ=4ZUK6oyQP~Y1BFD3U>XB~ItqcJ$`I}*sSr^TkugmX zV6eapE()}0AYy1d4$eX9+G{sNy;lrANTe|15yY{%Hw+6>hoXq{2+-q3K}az8RuVA7 z^Us$wz+k?ueaF1g1$gC>vP%%f%(=JpM(8da5sbA-$-vog?eTL97D=Uo3ZRu`cY-+V?c*bmG5Iq|~?ekF(Z2m4vnF~7vrHd{6 zaS8Hy_ybMm&!*>csq;{F$967SdslP&ZQAhL_Rv0!Y;ZpUQIDZpA;S~`Gng&s>d2^< zQ9SQ_ilI3#iy>4h*dmY0G0jpo5qD#OIAQGT%s$6fnXnc@*%q0h`_ZUAhwgn29 zW7x>~;f=-K+2FhF@kj#6BMUJ7NXoS}jM=+#^9>46L)onewk4JxgTUU%btOBN598G! zgsDsJ9t|X_D!+(S!lB|pRymICW%{nnYhS&bm-e6sM z=|($UiL&&t?>d@q=5foIay-?3I8vMM)HNB8pGL+Xxi}BTRNEg36E4Sbd_tU&cn4aad z{~T+*ij3c+ukL=+MLCU~T=sj9v{bmhjFYo5si1?vC4Pj@jM1HFh@HCjeORmvMW9Df zSHNT_&)cemsxb3&nIp0}60+pP9Dymr;nF*>F=B(FaB1B)F6zZ>xry;_9?4nPH$7&B zHw(!5Qq}qwIFDARnp1wH&co}P0|qF15rNb90rPl05r$X9wBz_D<5Ot!)Wi&OLX3)1 zo(gbdsDGWGcosU$4dnG1116cOxo#7+2@E9VmKEF`RzI(QO_PSzp-}mOLAGe1dTx9Q zBRPb2gQ}cjJjj*=EM2iF0_;$>K68ZR^oZhSe| z-C(UmZ;D;>rmRC3LNYOcIfhd;s z+nh61o{>0=C`E+2^^n~yqbHygkE4zQ@oL{r!C74)>*5ZF(!mhSpwOw#Xh}0F81FoE zx46oCHOsFs9lkFc=afnKM3EjJB1%(>PxD2h=d1jIG)h=4J|j-As4N)*HHM!l+*!Y(Zgr>hRrk!>?m}WOQq8(U zT~u@L=j$G2n~P3?0E0Iny&?xTPqm%EOLqj(tA|;PC@4obg;{<@`Y&w~x9OExuK7ey zYMoa|frh$?n*jX576DzhOb~I0+njObc4$*HIg7Tku`I-#08@>Uh@4pQR>SqKNQ>1&4++kBsniz;>-AChFJsWck8zXpL4EhI~Pmo#x~)HDjRtjc(M zLPN6*0WQ(0@W=cFS7vSy$(G7g=Yf>tT3T|BnJZ*uoM^iQ)R!{3 zO;gR>WW&V;X*j%=boE1a)?(E+B$L?CIpUuiy&q~N6IDOl`=O8)M)fNCCSz$OK!Mz? zn#+rU(Zf7$nhTP+_bd^%HD9^)kg14E56IFSpc8R~pGQE1X6iyIv~_oJe>gh=XnU>275i)Ii3R#*ZmfeyLfT&}iPI)^t)`wpcl2QGu0n|A4dhwBvbUl;Yu+y3CP?eC+6}%1?K`$aHeF>#5wq_r zva}7o8K#(k#e$J=%kRg+pbCL`f+Ct(UF3kCy5gh<^(pLQ3U4e>(U39^ixwbGpbWkg zToOcip{mbLQ8Mb#G3@4ZaoqlLJkZ!HyzO?J(Ts?j&u9uqb;A%32es#X0%p|G0{~_c81}e(0)2T%6dHNeuT_0Yqa2QeKM-`rAfZvD zd=B@7*Bc4>d1>=hMGE!`PJ5%Hk_f7kCSTocCUsVf);@a734FMxA0hMY4Eoftqu6|& z(;e91<*2j23XzT@v&DF>x%bYkw)L_{zoEAt-#9WQQTOCF0wEFq;AGeoV#q^G*aGT{ z9grkoex1@HiXwNzt{rFCZ}FM-AVF|=5du*csdhK+1jnRG1OZ5;Y$@pk-|aALwoo&MzM9Fyf=NDUKf9rq9>gWON1wvr1CVgf6zc%KXnf{lhBfVDfNIiad_mt$Ne!$1 zcy6BgWE46n6W|wVFJku8npIhSyh3vx57TSZ@$2hXVPTqd%Ia(Hb;~TG_20PU_FbD0 zFkPQ$asm!ziR)x@0rynM0VW8Ru-`ADhSA+rzkr2$%(sG+rFMPS_`zP7>084T=cWNf z%|3?bZY0%tq~kUb`qjDSy%qz|0}q~2yMi!>0e~tlH>_%sH!pRmh=TJj?E(>@6ai zxV%XXOO$we>jXBIGmPg0vf2*aN8rYJW};r2LwIZVi@x)qSyx`yDsZ?*Vu?JK2D+7i zrNaHav{za9vTca2IM>!Ue`oi1=b^1}!Ygw-;&|!&&RNh*YFKn_Nydrf05lgGDn5o- zv)qld23d4e{0M|eztYLv;?SMYkf<<IUBGPi1P;pxa5*GaqK zUiH^n=9Zt4m(e#Gpra1)QdKa6empgIvc10z4 zPosO3eKY3gZddRyZt#xe_K^MaK^*|%R^R@}Kl&!a>1|sOa7W(*Ri3o}ZM(WculJF` zaKmd}VE0yZiSECNBNrwaQZfX1TJc*z&$WIsDSZDFF9KjIE5U8&np~6}@I6}iXlHUH zhP~S6XG`8zTFpVz;a-1@*MTFz$gl9%?B4lmmEnkVPKOg^uG+?I3eILuf@~rUG67t< z_`K%ZGhkGMF!jek0Eb;cCNTkOezd^f<@z!b=T`zI^9sD@@AUr1mz81nE>Gc=H~FO8 zR>dpwfjZT7(wgVVz2Cr+PoBr@iWE1V3s z0mycE8C;MnAX3j6hWbQcqk~)NdA!)sz~Rw$jW;kJhZOrq4_CaMfH%4o3@%3(fEaXd zA#;z_V>kl&{x{i2n}sXD_HY4seit1h-$(cnLCx9p3$q@D~VQmd9E&Ud_J%U8Y zOLORE?griS-&z^@O0r;ZSyl{7=I`}_N&Kb@v>K$XI1Pxk%;j~=-d zkdF6e)+u=zj7uRpJOU!PNSL)NMZj7p>&zynu})ME8m`C~5;KSCn)4hFBWU$g?oG)S zik~|zUR{RXBYk_ARxhzd%NXYpfo9IR2DZy1rhZdgiUcFYf2|I4q)6*&$CX;nMqpx$ z+~NObW92>t_@s1H@8IZQR?L|iw+;?f+X1BwwlP~tbP5jqqvh6JN{0pHt!VYUOJ&Hc zh3Yn$h)=2iX15G~gG*f)#!8X`n=8mGjd;YC1?;+Q11x2g{qy%thd3i`&z)xWFlcZrXz0xoM09lYkzuy}A zz(Uk|o0$3h(`Z75+l&vbwiNKG5{$r(dW8Az_ZD;It?kZce^tKHyEhpx=7dd08#1zi zG@4S`4DDtP1M+H}-swCydY#-dJX+~)O+$Qd7>xoVZdIY=&(e%UcL;;r%E&kWRl`r$ zH!v3yByl*gF@zjmG?A=atl5ds!!QckwFLDH#m@8b6lh08+}52hX_7UR1)HiBYHqx= zF(tuaVFfn*&(soK+*-5Q2i~t2OQlWfLhwvSLRwvDB`9nsM}Hnw7y{c?wEpj?Juh4vojt!0&-iq6QD^p7gSM+m)bgdBK%v1_A$;XNF7>o6aL|U+oIJ(AIZjwJ__d>}{_)%RqO`OblzgsElnib|kk1Ysqh z6=MZF2hdt8*71RJ3|$}r(Z#}-a3vaPS>;e8nTh?xsX=Z<9v^IF>bR=oxKIZ?#r*wg zJ$`~zu1m+}g)_VDKGL$|mV6d>W-c<40dujQvzPzuC5v;Qq2v53p)<2A#|2CYYf&6M znEvPs^#fccJ8&z(z$Q>_iIttjVf^kKAw5PUV z4gy$@DNxz78VY#MZzG@WBn1j~LV)_j>bC|O6RPljO z9~T$Ee1mbP zp{u%r+}%F(X)v9{7|;+d$h^hkV0mE?A{AYYml_Ps4tnfxsrPWJp|?I*n@U!BtM63V zbb(BOL(R~`{n!i21YH`VU$0EFLQ)d1f?5k2mLw6_aTr2e8+k(QvQM+mZ2sH83CM+h z+9W<>6+I-YFw;B>`i)f}fEd)`y*g*~w7>uj#j!8<&J6Y(DBz6SC`!g{=O8K~>)}Ye zb=5-AQzC-;W)2|4Ad^x9ZzM|w-VDS zF_>TIdpZmJ&RJx+0DuWjzW(NCU*R#Y-r~*YFkm7k?71+7#QOu7hl0*iZPxyWa)cE@ zhkiOWL4o}_5@`ENulkKvo<6U31NOrM`=OUlzFMXk@*Pf%>N~uUDrli>?~GwYM3)Sa zKmZL>@F6TmEPKxHf28^YEi^3lb62GTIL8%08SB5t`YMlLuW9@^0Z1+ty(i^c)t$O;FVO94CN5XkP%Tz>pg6}MH#074Z& zj%>EU6;%Zs@`mNg*{l-aDmwwR0A~-?lgmg(YET(HSrhATg=u{PzW+p78j;)0Ofr{R z)Dqy6g4!{!6<|Lp+|XKGVb2>INst?VjDHCr43ROtj%YP%^s;ai>?KeTw+O1JY-unt zDFR9tyYnOWfz_~zBQ!_Z|IXKabxil2&x?%f{;!b%ReykR9x2uHl8lVmCi-$DDN34R%N-w|zMtWD7Q|u^RhiI7>0dHA790#{JlUA?M zYnr^V=~6k{WBwLEhgyn&62K8qMP^=3FAI|#17KXw=kK@Dt3cu0@JiLH(LmaVDLecK zuFJBX$4WRMz82WW9bs!AgTi#X@LVsN*ttB~TeuV}%7W7=S6)ud>PQhCkn!6ZnxTsx zZff+oGjbN#vFV}9N{5$EAT1dSVy65mLUPxwL2GNai#Ma-mzB&sANinH?~knBc!)-2 z28u!1s;L4spbt?(Wv==*;`gmp)Eu&645g&OkoF5}+D(c#NB|Bj?ekNeF4+cPb1k8{=4|-^EEOwfE3l9p#gJtOrbbv0P@}U-Zknih zCmAFqBvA6iZEe0`f^s?m0A(pAs)&N~>=gCM<54oT12EFZa4WI7fD~H?;uRLWGQNtK67tD^+C5OEkPlp!~X4LNTwp0AM)>0sQ@L zF*Z7C3`CG<=klvzudj|yotc7OVgy1;4}^FZ^otH5_>|)$8)2DO0s?qPB#CruL9F(q z>`ho&{R+CW&^)cbF$laJd37-7sa`SYlc$4n)NvRiD|(BPmqCSRqWKR{&z1hj6u^qK z&9q&=BuYB@Yy*=Y3GDFl2%m<3iJ5a{-@?HYWKkqi4;y~LEe4QIF^+7BtJ7>IxYsum zh{N(I+?-f~1YC?PZt^+|aA3zEz-`u|F?v1}<$mvPg>Q6?{PO--+q2UW&^~|zy5C04 z@JQFmzOb&pPL6~>ir!?vfrN(93gx?C^D$Er02;xXr-ftWE zK1l1%aX$zN#>EKjVbfwYA@c=w-hW;3h9EK{OEgMZrRzKpdWqFaJ^Aq(8jh5oU?&4=H7{1L4;$Y))^ zjZ`ABP*PQ~&6mG*-DNZK%kSEeZS@kQKlFm$n-2-|Fz67szD}G$oKiKA<}` z^!Q%F8j011+EZ7PIj;eYQ9V`Bf{K$Deq8Ow;q4O#g#AF%yZ94E?8iC~gn!m3)HY1H zNiO8Up(uhnlBs12a{4U%yMRKL9k)o)TFzg54rrXZ`|e29*#-l(7kdPrLt8+@8vPa^(Bf3v5=kx@bFJKmb+-?w3na>J7@?G7}#nFrn z0YIH;$gRompz8f&S~SM}Pai;ic|VCQS;iods|c{mzUp2`2Zgzvndyt4pG;oiu`#)C zwYw&qNPF*ax9#|M4C>_4v%?F^y2CQ@On(6VEd;NM?zsVy0D`LdXxlSCEBU{mLeJlK zq~Z*J%?Cg;N%LfMor#PFH>IVB>`<;6TjotLDIn|N2A;WloqKnseaAygg|{h zw1dj2E+lHL8Mfeu2MU{3)Gd{gSRfaTd~z+7H*)b*LNvj#wspo?tgIZ5&y}w;RAq%6 zKeJMD#kp)jjfHL*er4jI$a%`PrEmgNVwaZ(x``w#p_lusPSI}v?ENOAZcHe1m@#@l zzg*B^Dj%M20Uvc6mF(i4nK51#7?z7r`OL+?VJzAJbCgG!RAnrdV^W;o1zz4CiI z^D_tqTr^OY{Qjm}bUm?-fvIiq{YX)~&qXuo$4A*D(q+4UOyr%>r3Q;&5DiT$Hj&Lt zzQPPK4KBC(=%x>T)x!!A>ha`Wuy6M)0hB_qo<-dk^^RJdkrIadC67bp$F59p!^tZx?JJYVr`9v6 zdwbs+oEpZg1O77baUvu^f>Zgzf}9C2l(v*{N0CT3C7sW40TjNy3yZ{X6K|UpaATOd zrlZV`8#mrc5k?%ki|A&{3(fMKxxC0VD90oz0&>Os4~o-?#2O}0)OH#uj>eGqhr5^n zo2$KcsbC%Z7zVVzHggjgUYh@i>!YsdqC-7PMync7(;os}0+#Zod!GY0+{Naji z*20gaIT}`}7%^bmZ0zyKP_A8CEQQ#&_6=&h2oRucIs^dZveC=g$Z)W9D_&2=_Cnu> zYrCO>+UMW~FIGN3)8{XtirN%CUV^lcEGKUkQWIplBk)Y}W+G;i4?ryY^+o<_OvNEL z0RNE>0vmXXtcn!9AU7x=fQ+MRFU7{DhKdsjctO9%n5cy)8u-mW^1|9kU?~9MOujtv zk8!}#X$f5QAN^WP{`?$|iXH_CCWXg&Va%Q-B7J269saA-uuegksOkS|?aafW-2Xmq z%^1ZDV{C&N>!h(SjWxTmoXXme?0c3GijromgGeK5%CT>eEs+*!>>=W$qO575kPagK zzN4OV{m%6~e?I+y3*;hq2W7(N)71E=_u|OC!vFTDsXt&&Pp}hc4K1?PCf(pHHrvKnOT&k+qO6=lx2Inmp z#k;d&_34Z8y%HSLiNX@!=+3tC;R0&sPng$;_r_SUNou}_@W6)hhTcjC>)SPnB*^q~ zp(P|{y+mB7XcHiwAMb^Kpnb38SbrOIcM63m^ye#9st>o5Sy)-a@2osYlY%{fLENi} z`_ITj=a(VHV!){*(z6knh%Rl@Hb`Gz;m52o&wd0j(L>vEt58OTuJqKpH+S}hb*qRl z%&0jm40Nl0vZB5O0j$W+l4ioRXyVc$do$9bF~yE_&f;>W16YOJ*67g{KUYx2Ixd+Y zcM`QyHiLx$j)?6=k0vpFL{$PRrrTO($A6k-7b?eMZ(+xE`V)}bL@f0)s_&f-Hy_nR zIf{qU;aBt88fK603TGZvOsT267oggiQI=z7%{sr8v(_UJ3-jWZW*!Imwzqq+bxKHB zy>p3gr(PSfgAw&e3U(7u#zgppqEUG&>th)6%+gTq?6CFARKt9qODy;5 zOQdVW-@KLbpwW{$KvHsgCL(ey^M_&@t^N9wSQyFxwB9?>sfNO4wQ5@INcQObSV<1t zeu51$p?T=>nJYKp*4RPSg$Ty8c2kpS%uN6WJ7ih&^02_zJ0-R^zKCx{g%s&RhmwF_ zs6H9(3_Kjy^ZP~zN{%n}&wSrm1p)f$!|+7VxmR%zPiP)yyk_5MlY)3cz#A7jmGAj6 zy=@SnycUgKdHgxoGrgg8)gwW45iFsWK(hQcuj+i1j#XxlzK?eP8z(CL;CMw?5}uF&Y5IpM z>ht)-am%@FR2`@*8efV=PE)Rbet34-X~5;h-4Iu^Tzl_@3!GE;!KNe4Pv0SCHPw}P z1IlZG|23;?4<;Nxe{N}_ zAsc#ErK&nw(U|$GjI|H}>}+rr{79aWaKa8sPpiDBS#kia&#F?SKHSi%#XqlpafruOJ-YZvpqp&KCDK3&Z*`c+3qLmFm|ziK?^6@0!41^y!$Bw7eqE zV3VH@!KwQ$!kDtZa!kJM=kE3tn7P{=Gj$g4pDsUE=pvob{P<4Wmy~I)KpdmaL&KXj zSxchSBSRWPi}DYeJvP3?u;GBGWdD)<=cQ}ED+ySzQ`5!d3}~nD<=+59Uu#D^sqM92 z>Wy$x>14opBFjE*lwXesZY(S7i#+Hd$4k^>r{jf)5K?iel+rHmmND~{m~=V0{$PR~ zo3*&P{VRI0e#$!#IcA4DTZ|K(T*UHy%Y;V;UxVu^I|WP9c?n|X!GcQHOFtq&gwyj< z!dcXMT9Yo2l4epj{obnE0=YUl>(O-?DuTjQg7xyPIP^1x7{|@1)pR4*@JJ9^8Xk6q zngxo#GxXE1ZaZ#V#7l;ph%dTH>f|izJa4O=`}gVv^hb(G$ky6|DKEC>a;AsLtY4cbFNmh287=?1^o?H+~Q0P%WWBrNuwIFOVU~C^X^ii z2z5}j@GKB2%ZLN+&0}jB?ThI=5Z1T;6*n$lJI=@QN#8rxo{wwavS-=*(X!O5sDfrt z=Pe}%WH~3fMY(y>A2c}3e>`FTsPk!AJwD0*(DDl^)=tSV_h*z}aO+HB^A%T_Z=SMo zUGb6`&E{Z4@_(YoeKA__ZN1h$V@l!D5{8KgCy}~4VKDX`{(4@wKlJf$0Y<+(ceIbQSwdv2~UHLdxAuSAD6`A5mU6+ z0FVJ4PV4wTzo{;KJMaq8^w=QFUgxfU(a+lVN7MQmf%Q%3r|Bn~iQ~#SSkh9TwO9`u zu3E~03$%CHVJzt~Ytn^SeJtLlR~kx0v?`Fw%c)4Sy3GS@v00RoKXOqisO*O@7(7iO-NFTLD=soM zRf=<$!`RR2`e%Xxc{K4N>N^L9;TP0CjoEv^#-tzv@>)e4-h;2V$mxG14zryJgBxf&p0<7PTLA@ z?(?DX%vJgv8eznPXm*s*w2{GtMk(7F$`zIvKK$W_z?AiyU9)r#<$mt)>NecE(kq_u zS=FtB=>kdWg8^C9cIeo_(JWKO6_>VYR-5@7kYWnPlfDLqT4h)wHbuD9wa*~^(+?8w zuPT!DVWSrZ)n5>ejZKUxQ+n+&XIF4b(S{@s%E?SHqahP-jF5G&8a&yZMQg zfTJ~D9~Q3urA)O^+|yX>NoR%_$%6e;xyF>x;0=&s(A77>3oT$B{+x?)I9>m2#_Q{F zTYvbKDcH^N`L;6#>7x8Rlg=V;Q7xiiItUe5Li~nQo+~(>k1D;0D}FY-H;G`oWj<>( z?IkBK?*vb2)eP&uKN_L@5R?C08Usg*U7y=`OA9Z%h4#AjMBCZsCInm&j!qr40&m7w zz|32f+*08e1Tpk`=v=aqM#(_60eO-ir8;f$haqCNy^8!H6{cV+4^`6a z@6pefw&Wtx=Jw(qXn}t`4_GdGA`2~p*v%u^u=ax-qb33Nw7TQDDIc@rRp8z6-^e94X#uAZ_7ce) zbmmTdt59OmsE~17sbu5|hyCl$n72KjlwR6YCgA)_W2b=j@)n!CxwX@s0B;l5*6wo{ zhwF0ZOK)dg6JYU@k>b#g87Xf5ro#B9Cm3^ig78x{2&-ckfwxnECYND{k5kLgi4XK# zx>gmV0hF7TNJC}G9Z`ay;ZZW^7MXCL+)96*>oTi2dN?neFZ2@6FK?IMJaxz z;ls#=W=m%-!C|!z`F%8}td(<(vcS31mwhdnZfQ;Ux%*>~W;R=imr^L0dv6)B0kf{; zF3PyHI7vek|AW+Y6@&JckEyIR%vj=_gl)x-Yqm!l)&>`_i1S5a(at;}!b*0Hb!wLl z{UgvOtA85AIfDV-@->OAw(Abr-CSB%<*GYgBop2s@c|?bn$jHK1Uh-y`^_0H(>F!< zYnjgzJADk6mg+8huMr)QhJHv_>_@xkygCNruJ6f9$O<@Wmtq> z)EY{*%OA)mNhPQ!ueb0%f|JoCuf(`o9^M|7#Dr^z{xM&-Og=eV=c-}men#|J}}+EcVa0sIYovOil0@FUOGe1}1P@0_uJ>W#%3O zl^(Z%)~rGU;HE&;g2NlO_VE?34h_hZ<(OhBMK- zf-|+7C7H)C2fzZ81qJ-LzcUhF-8tQ^e)=zlT5K5gDDDqV%wG_=Cy^ID9lh9p=(ZZi zTBS61LCI&MiN8${of28kA{dY7b5G(D;+!#C!o_w%Jd0po|1y~3f|qCXf(T-iR-_U# z{0%Gaoqxu$BR|Hd@x@{mM}&b@qeqQ~TwVFi<5=}@@SWV_aiW&cB*A;&EDJh;2Aw0c zE`)A^STX4c`SYF*G!(-)%f9AO4TN2v)ExnRMxni+D}2SYTmY59hROZE zs2rUefXYczW}hvH_Z0I@cHAF@bIT2{f)VypaPn@+6qIeaIqk>51F(^GGA)Maf_Wu4 z3u?Yjk3EW&aM-uQ=F}4Ce+2X*W_l6Sq%#?MLbu&ip2By#0D`bZ|0oa3@AC!LWHmF3 z`|B7q{ovfzsQ?KZm?GH;5JfQ`M{f6{$13}*L=q~_&P2jJ{80}P2oJ?i^S_cg*?qyA21X+7^Da^ zr}1T(hZC1>I0DNuF0JJ|D5B|?!y?}oCE?1n)DO*15_g`>-H%-BlWhm>8Ajo@VKY(Z zM>#dNf>rj;9_8Q;bGs^eNBP55K+AP2xsIr7LiP)(&;y$E?=@Pq7$epRziJ$0rmRy{ zqN-rbKNMLux(Lijnm(WzsxY9Z?Lmhe!e@Dd=D^zfnlFsz0F=4<`Mz(S#2o-S@t^GX z6er8btNVWhl^=(iz~m#bbWJ$>Aj{s*vk%_-<>59?CDy7|z?nqJ3QaHh=<({$a$cE; zK)|ldeSC2{gm_%laP23f_!79+tn(6NZfrXWyq9Y%(LcVuaG*4R{cRo4HhM?tFe`U1 zx&eu{gKEG5wY2Id;K#n2qjO|*C|Ml zz?!-YvV+S?`lyMEcY*yH`pmjZ!~F4v$8SZY!zIdGJaCP+3#t0Ff@AgiPzLLocyq{G zJ#+g9_wQ0;bDqeQ?Y$wZlK%iXdOiOIQkV=s_| z6eV{ofxOSuqmuj@tR^-oa!uMP*lRDz)qC=1iq!ufkYk*q8g}e|H70k(UE1M&TvHld QOyI-ZK0Bir_EdT%j literal 0 HcmV?d00001 diff --git a/docs/guides/metrics-and-limits/images/metrics-architecture.drawio b/docs/guides/metrics-and-limits/images/metrics-architecture.drawio new file mode 100644 index 00000000..13a39db8 --- /dev/null +++ b/docs/guides/metrics-and-limits/images/metrics-architecture.drawio @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/guides/metrics-and-limits/images/metrics-architecture.png b/docs/guides/metrics-and-limits/images/metrics-architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..063c33a7db422f8dd4d402d7fadf72b22de006e9 GIT binary patch literal 36024 zcmdpf1z45YxQesF&JtT`rNHii&l3GsIZucy#%#dg~#DD&xWoNx_>*z$wAwkQ| zE@@?E;^bg!1Ac<{*0!b=_bkjze|}?UW#_obdJFuc#>Pp@A=2%)`NX_h)+} zb2FQ>f$rFPSXfyZ(X!v>WM&0J(QC1>(Q=4^U-A|<&TimOHy$nnu3I<2FHt)?D>F?q zV>t^aM0*ZyW)3by7ddIQJMy&b;^4QHg{2wzL(0s=(iYLh)WOJ=*}@hyXJ=z(V@51i z{Jw*&HE493jg^a;m6?+9{__nF>)9;k4(Idy@l#{Kkm6L_tA77f-+SmY|`9nh^2M1f%KQ_B(Yjw7}vsvxH z#{bX>@j}(a$m%cOYFd~&-T&E^i}h@Ih}P0(7UuWQX8!Giwb6OovzCtcjZAG_e|G=z z>eoa0Nyc+xI@sER;eI)@Uq7EWvUU?UvqI4NoQ!Al0ABk4@B14u9Db9W|H^P4JYouB zZrZolEtR=6RF%wB&BWQxxy!}K%K418cP*R%>vLIw&59X2fM4c_U*fhlK=iGw%)lE2 z=QujKpC1O;FgSP6TMSV12b-H%**b#*6?46B;bf+2X9W12L&6oXB6xS-$=V7${gija zNI+(s%+5LU=iqimzwh`55&kyWe+`H1H%d&cRUoIj^<{he*t5d9G= zx&I5x@SMFfI^$IH-$q8X{6~%xv^nF3zrA{ds{Iqw{MC~J^Zp>$?*2LNR2mAcJ(=fJmwlOs`l{fzHJRWwgGdiCg5BJ&QFUNC!62G1f=fB3po8TQD>I z3uXK#@dWC?*4g2Ql!GpRm3z;0nUj%&x!IXk{;TbNaQWXBn7=CSzc9=>bNn@iaWJzo zaYGX9S&;rE3o*;)W>^PAG)I9~#SkbolWJZDD6eopou zPI>Fx_MK13@uw+&QvN?NB?o|5K$HEZTK_g>&RYLaXn(KeKc@F?{p7-5ER=_Xt>s@@ zs5?OMTbMWkVu}F}16cOYcItQeLKvTa1KY%LuItPS zp#FbO`yoyWp$WASnvD7HXukh>E%!rv0kr@3wA>G#{@dd5yQKdndF-tJURv(YD(4(q zo%7{CU(5Z4VrBimqUC^3`zQP$i9mNin<+b+0cidgfc*T;Jpjx>tKV@3m%TH>Q4vFY zxas)AbwhZWymkOC02llGGx+rb*wc&NQC0#PUd71R*uv?KGH{@OVt?@N=h#2n{iQ(v zJHvkVTW1LIOtrEB4E2M+zev#k65P*rrd0lxay19G&F+kY{?eT~^TLgc&poy?+WZsU zEOrircK$)(a{$8oS18T_A}$Cv{}r0?oI%5Lt?@r^&CVA2w+Q~v#}a=P4gWm*|C*ur zt8D)72O-Y=^#9FNM>u4F!OypHu6xeJ?eD3cf5S656U|@F}>P@3yqcF;M#!I&%ycc(F1_ge-A|b2FICscn~li@r>xk z%FO;_lpmo61nqx>Eq**9NdMc@pQ9K*`Rbn%jX3iu|1M7dfy(}C>i0Yp@T>m$pnf{P z|ABk~Vfz<~$eBa(cSYof()?d9B7fpnj(@!?%ZYIC5r*+rAiab;}oSvk8&i2cLP?Romd#NEmQWJMe}fGGZ*h~Z!4PmGOBED?^Gjj5uu z6Bq^&bViuWpFX(+B25nbaRdweYg1tC=jP^&({2It`(xT)ke@y*40!8NZ|C4_C9liVpR1dUYRrgMl!ewP} z?gYcoWnNIx2GL~EZz;u6Cfsh@MQgnE*T05lgSvYWVu#ovsUJ?pLrlf$V_%%GBI#&Cp z9e|G_CIaPwNWeXC0)1gHlpA6q*_l*kmbu_H6gjr;3!-_TNH`3(_)>a=C=WCNEnd4K zzEZ|eL5$_#jtf0+U)P_E&i7{8$~|T(k&L)9GWqb-dLU1|lygw4ndsL2DIsF3V!yW! z2D&tKtl#3lQWqyc%$pD?VkLe0SyA}y%S@k9I)$|RV4Uie(q5r*w+&;5C$qm<)(Sgx&YaOLWDG)OEIh>lk zFP_%VBe@*#WY@mbVvL3m;i1TgPPGx+mR;kxH(a1yT3|C#y}S9I<+|DZuxnE7QKT#2 zB+3U1b!vKFz09~fKTIVBmk1iRtv|L}c{^18o~3@A>xDCVFikiz@|LyKZ>G`SiDT1@6H{wu1G)3mB2%$`nmpY+;_)N!s~Z^9USrTX&ObJ^1*J(m=3z!NUZCGDe>AtWkJ*ZQ+yq39^) z>CTTi!i3=8p08TT)9OQyP7c=wOU)IzwM%_>Wlg0|3oE|7aqJZw()ZeE{TO@vHOXK( z5KqAlwXtHtCDpp5^-7%?d^TFxYOyb;+-ih5{(W!2ZUZ?ooP#kLANa>gp#@FwR#e*-ibu%e54BzCH_+pl%x4ivX@igiF=a-7E z84{V^yD4lI(?)7`qUwU4g`3SL}HnG)}JPy)$6l8cx8zIEQ64mvOXGF!I*E>-9EUy2w6YRQEaGE_^6=FJn<<)1Z#6;KJADX^lF84qis52BUfGf zDhZ2~^Mc3UUcKT&a#F$rWWv&cP+##juAgYYC0}Z>zb$v{jq7XFEuy4R?HMC2OHs+x|C>2>Dy@oSbsWJC?u!S zWJWXZ4Fd}O0-&+$z3c#NA)d$Ufs%JiAo-WGnwzjsS8LYlbZk;&++*3>{20vvd77xb zHOls9W=o!L+Co?()4Vo2R}J@Hq}qg&N*KjzEc-G@O%<=tGSL;324;U_?wS>;_e*hw zhVJf;*%EUb@V3nLi9`}{j`t=qYVt*2t<_DeC!>~%DqE{R^&t=*>&is0zpa%`aMFQ6 zEvmI2G+=9_q{^5`>Fa|b3s)4_2&{#F&*ySk)>?QmTxoAYIQ8f~vfJCKX_b79oB8^+ z)rYzJ;=ZX(Kv9IJ$yrMFMF<=%mcP4eG)<^ce~^{)@|a2q*4Iv$jzO~j%A|<*d+zfB zy&4zhf)d$jSc7M%%+6ySn;W{%k;`c|JFv%#*D>-Zk3RN&_QB~$-7b0W$cxtg&}+n5 zeT-$hYM?nio)o|QB$qH&%7+LYqrMpa0sE%l3ZY+`OlE2+Dw*3;Oj5r7*jm{Qn3iSX z+vPFa`nRKREeT(LCh@>^pF_IaM*YjWV(=F6qLNdyTkNNTlVt62Di<<5m!C%qg&#E* zbx_yuQ867gediV~+b%PJyrDY=8sOsu#n-1Av?P$wwDt$&*tztGg^Z&8vXBDlQ^01N z%xdY5J-j|+nzg<#igMmXDzpsz0bI__eV`Qe>H@8dNsPprcao9ZTbX|>sI`r|@c4kv z8zsYEC9nY9l zgof~gxkhUV5^U&`g$g@i6ipMc%Y-$7K?YjKSBNLO;MDwAT^}_iuQSHq%&b48J0^7< z|3voi$bP|`yrQXw!gWP^1pHnUbfvo3tXUEVxe;?2tK zJ~_Q{%8bSFs?rF4iCk-WE$(r<0Ai}5#=v~K>KpD@@V&J)r*_jjq*Uqb)A9Q89XU*D zJnMA6S06Sng}G+kN#)vIF72&e`sS8_vjrub1$zJUSHTXZFJgwrJ3}lLLdu#f)!(SN z(eUV_k1#%@a#t+9uK#kcH^W+jn`VxSA2Tj?{{tim(_d2hpF96*!9usr6NmtJD+faqocCKJL!O@ zZ?|=Qe{*4XKU^kB5=FRbN?ISo#k(@4)I{j?XiKlssY`)>W$ucK^Y?yxci-AXAq*Iu z-IoDY7w^PK?$r9jyNg>vGjcM0(Uk-mdqaAz>9q&*spV}%mOLAReb;FqI^6of-KWvb zE>VQ>B+2hulyCZUKDV8$EC2df=W!YV&b`C$`LA6l`LqbcH(^V_Qmu*Ib^Z3S@lHPg zA~z0DHqE|XlO~v%Uk+;`eCxTlx(lpTkr;fONO=3BIB9HY>69LQzS|l)o}%^+r5ha; z-8J+z+?tF2&^L}o#=E$&#l-Fl>8{Jbw(R7v*Y6)ZV2Y46+DQ!x)m9hW?4X`-&ESx^ zf&>$cM<}`PS1vML((aENiK_NG-H_jw&b)Y`e)a{KGq;6sgPJV-U5nR<7EWtn%!jUs zwF@$Cv}oDA5puc@HKGyQx}bX5Kx_@8wLEofyv3SIXM}I5V0P8NI4}5}y{7e8`3gh% zpuxlAfeE#?f+qA#-FAG92Vd4w7`;#0*5scQz?0>*ND>q*Qrvq~HN53{ma#a((pagJ zQ-+mUnR`D+CO({BOI-ZG;*lV{fGc!natD%FID;$qTRzo!dlDv*aVXrAL0NQN z>}#d9WAQd5r6oO}#df#>%M62H=V0VBon*35&2=VS6?^HUO(@_UjN22LO>($p+X0>mo9DRircBHfP%+9sRQ$ zUF_yyPYqu*cenFjS}cV&2tJ{R&FS@QBG&d1)RcvPX}lT`r8;eh`PDezJVu7f+lAG0 zmiIXDx2aiek1G)AxN+l zHNzon7ZdbHEOQLTE8c3p)@Pogu3>wfFo?|PO?RosiGKZ}@+H=0QFu%B3<|le!})fvf<&ZbrQyyH)~-@wQ+d{n@R9i&kenIEbNc~_L9>#d=U}v>B=oNNL!}E z5IcF~r-#94B48-J&_+pOpoQ*=yFW9n_cg_rr&w@05N{c1Bo`NNjc>`-40ji27n`S9 zYecTF6|5E?Yzdg7Y|9*yR2z3u`gzY66~BY`e{Fay=jcYrc3{YIWie#W#W;V#_lltT zlV#bZKpM#;1D`7XZwtC1q~!6!5wPADy<75^m$5o@NcL22EOG!CuN_6mhZV$f9ijqH zNM~Wy*^+md^jxc*m}}U52heux-WSo6J6t0zRb;_>JlUr0%KDwb+s$Nh?#CfjFerNc zt#6UdK2UNNlGjr+-LgWT=h}`*dp$$+XnPx-nNT=tU56vzq?*Py7_;@X5-vJce+bXS zvr4qXG32ZPD)Tl~5Y?{L(Kqd{j}woX==&HRUm8Ys2<@)HM#is!1R3=0Xjloh7780= z_deD7YBH(m`*vNe+ix0!lm4)LJX}^NW@$x^OcOZUrlj^CqaVb?~@(O&bJvc>D)^khe5-) zV{d!BzjIy2t&4k+LyMWG!2q)o!2uf8R7fBvn*smO*wHNxF^S}QiIgjQ7Gk6 zzwoD+vh=Y$y6sPOT)Z8Ohr(P3>Dg*Z@s!393V;o^NaW>yyyW~oJo+Lb4lecN8ojrj z@7jv5-60JRF(rRjb)t@Pz4D{}6v9Q%uEre9UR>kMsrk0ZPr4Y~v-dIj%yFkxt`gc- zmIXYtXATOZ;Rt*)bV;b1K(hON$TlO!*pv4>vck<~R7H^io>zn1u9{#z z?|r(CA!i%&icptjt42PJc!tH^M*Z3%xm^RYji2apPIuED=AMyDdP?0v)i+FD`^bAG zQj`MR$GgiTTLwj)i94xUeIX>hLWfo_Z+XETcP!@HXhq)+i^A$B8Re3gohzn8A1WLz zZHi^hD0Fdp(qYn(>9fvUvae#|2xPY(aJP?1lnJV)Pgy!zR)Km)yp(s(j=p|JyJWm- zPBK&XeWvikIt)gb(=->Q(ev7Mi0EF@okOs6F7}ox8?F6|11zQ<1L~Sp>qCLk)AJlb zL1etRc53U=^#w=h04X~~(LjWrlYTc3mYF|XE8Gqfk-Lv+W94`fB!l5&tgc|He#L7$ zx>R&=tE4YG`q4TxLMNjYS6ROvV%5Bjt{9v3A^&|3#D!b^&3A<8AHPoo6V7 zaMB*noL-Iau?IuCj*%SPbE@xjY+EE*UaR4wC$3P2q$5EWpUK-4%{ZoC6HEDs5yO$) z%ZwC9#$}aEt)PDe*9JBejF!c;rQlvN&O1k!cipB?cY#@B`VxD|eYcTkYW`Hw^pR}R z+D%O?xjvMtq0upyT2!fG$R>r2uavk^Zr{o_(D%96WWTN#qP~y~Ii*l&e^=pM`8vA`u6NkwEc!xFrRBcgy!3u43C-_Nez^f{t_>?~ zjpn$-jlj#wt`L)$=T&7x7O`=y4Yi2$Uhvl9{pLvq9GLx$V7L!hS!MCt0JpHYg z+XC?uaq-*H(4Hz)gOc{@%=ouIX~&Acw~PmAGt*O(3Ao+>~Q-gx+RN3K|ir*&!QePM%5013`A z3Upk`Xrs%G@;JPWxP|{Q$0A26K&8FNTDf9J#N~nn7hUtgHcWd08@v2wj-#qx-C8WI zH8pQi^y*59tZmA|xV@p$ z5__R({&Jy{+r8HY9f!cc8hjPP?73d>Erpep+B2k^DLGmQeqRbs5Xl|t%5=TY-1*Ug zzulW4rtX;G8lL-IkM6ZU>ePVY;b!aT)Aw6RPo|Q*OB6!=_$7-Owd0*{8cPD5w}ARl zn*$3Cm(QqiPCamDYs6pDCp_(-MmJ#$z6`t;wfdCoXzL75l4N7(T9bP+nC+`3{T+Bp z<7akFc>h-2X9?jTyAAjTdj7+1VTclAI%76ds?c~#3i5(aLEroDNjl%O(7?3(J}#FE zw%Mj*LR;nDOGjC%u!{Xy_#Rkzdr+J>xX;p?@9sVAc>czx zd7nRvUh7lrZtPw@jF#!h|2ud5c4PS3H4LV@A=tAY+h}llFH1VK-e(lYVGU=x%vLc) zB-=0l3|eg1Ome3VLc*ox-F#g?PfpQ1M~3_}%-Zn%DZtEWOZT+1qcJnZ?~O_d%{9t* z8{HHLzG&2w#@ka9%7N0uJe?BWrW3lC>qM*_FSL788vCjt`a@_it#=sVd2ATkGa&8+JqLf!^i~OGzGbaz2DH$_( z(oPe(M?SK^_povoGsAoxF$9*sr84Q0>^51rQM(J=RTc^pt6u|~^NqRXPt(lsNxVT0 z8}(D59#BQbQ4x|@i~`bV{_Xu*Pr46COS3iY1b68Ix;y;_Xr0;g;+jg?A8!7L02qJg zd{oE2c5y>!It_Lk8$ZMnWna*)-Ec3}dYlgCd=xqfu z`_9nuQG9)5Kn7LwGukbGCkoAwC3tO=U`Kjk?jl=8jl|pqPO>~`|HL^~yDPs&& z6G%gZSl9*k_3z@lZwPBxTpH}!^zLgU>Cjr;Zn^LPT z%U6|Ud(>TAOYG++E|DdXE>Qf^Bj0lT;&YXCdA}n{uUH?ovN#f62P*I2XhnXTi2c6& zz-Xi|YPwrKM;q8Js%WUlYYegXR#-l`4iwisv z(N(l5+QGW^*)5Mw{}R^cevkuJKZk+LN^yt@6*|2#O4V><9+|M0|3vJ#ETL?6I%VF6ySE+4K+LZph}GsI;iGN85ViXdrc^%Xysc{?M>4uo6|j%t zG6A`6ZqM9q3>*oOw|iJsgns_ud<5cDW94HuOAg++^ila^1-`H1gH@D8IB>OLqWGsf zVvt*M7c#0I#rhgc1k=c7%f5j%=tLqpU8MEKrs~Lqy=Y-J8W1Uy?}M=;!|DSW7YYnz zsBc#)&&0v}qPG0fT}C@4v#Afz>|Zm=>XW%9`l621ZiX%5x?Vo&P=u&$X5!-2DrH3+ zZ>J9ONj#pbxu%veST;j6Unzr$hHat@4cz8HmGXzMx3c2cP07)tc3*v`)I#2_CR%Wn z?xI+-N6?fSjU{EEH7-VqXGeYt?>tSEs!2FGO11Ab;dFI6HqooROH?%mQ`rV7tOUpD z{Q(m-uC6K0acWTS?nbMCOPF!Dm?CLKp@tb|+zN&~iEh+LBPK#^Xo0vEeNrMls95~V ztVtGht7CUu_}nDz!WA~}fzY=Zq=$Ebq~h~Dne-z4Q{~4xZ;r7+aGK|#q53*h_)2zk z48|2EyHA2ueTvm>JXf$PL84%Z-C>dIJ_%>CLP}>#n`=}tBh4Zu6RS-+!zT&fi?ok% z-h|k{2$!OzOYg=#NUXcgqzU)j`TiBlwNv~WVTI(I$ep#x$q6%xn@_@Fq0}hZtU>bo z>l4)Fak6DF3PxhI>6Bz%MuElb_%3`_uIx1#MXG5KR2&`l@R0fZ{NnLxb|d-YmiJcC z8>K^$U+Evnu;&o?g^+o41Y?tDXi3F}ZxiwN?OhVT#3D4#I{WcP>LHS7-M2Z|{No!J zt9U{+a6tBi4+^U$vTXtpNmf2F0_C{jxVjLWB$>(%Nh0=cVJ~((b;1jB0`@%1#_G{h zSiDLydbv~}-&6*hGK}X4GinZ&lq;r^#q;djE2iq++TWO+K+RN}shT>(KDaV^60L`~ zg*lB|IoiF*6QcJ{$)TBPW-Pxb&{q*+C)}#;(GrR?_*@D*Q<1jxuCcVRs$e!&nt=P8 z8vIMVvt*s$$!p+K}>yuxaqsA z$?|+o-=-AWFa%%GY^i_F4HV&lPJL>+sN^zZkEx<1J%=j;&q0O*gcFI~^5J`0rh2}H zv*ux~p}G9!f~gdwL0!i?d!-hiU#xy-kB5-!Uu}{o0B&GVzuzb>AQS}z#M6a{c^yn= znBC?tXfj9(llyKLfE3h*1QNFJj7JH-=7BIGQAZ3CbCw zed?g!-9fUBDgddGgtz!B`v}N@5nro%MiVM6EIS&*1QKp@j~a+=-%U5YwKI&^%a9#` zhHmfN135F8C&VtjZmWksy&^-l74bG0@zxRoC6(_Zls3q5023#gWD|igB=C{Dc}Yfb zL*!b?QK-Bh^1ONs7KtO~8xcPz+lKwHS8(r9jOiyr$!v0OQW-I@B6b)>32qj?^sCWg zBXOg6S`?~dCS+A-@TEx{NmIUw3eQrGmK)4aHT~ix*c?4O%7sgp?F6HUIP*FLujli} z`n=hG^$F@HLDjG_Dvrh5q9AlIClU2JIRJN&5YKvt7I2IP6viJyTt3%Lkyj{t0=BT# zu5=Tz2_%%cBsxw8ZmS_}^K>K+Y4dohW2f(W&#$1w-kt&x%~JCW_cppaPw*{xXX(?V zg`@7qcZ*RpLP=c5?aK%6T2}GwGXzPAWE;524?F`q<)sn{&Ij?;FYukEBohpe0(Jy# zD6{J^z+^lq20jNl{Kc(NlP~V1HXO=1qEa2-vNSLuM8Oe38WR-SCm9|r(&Ii>Nn}Kj;R9ckexE`!EdiJ5d(j56tAR+X$si;028a4~i}d&X2!_7s zdMr*9N*F(+eA3608tX(^ij;tVWR|&DT6mQx;Yf_)Hnban{~(DBA@Az9HWPfeSB9%zC!nV28zM%Csv*>F`uovVZ^f5dPt?H>q8=>d zSRWU+SLJ|KLa?c4>hB-loCspXvQd8);43Z6R~o|9dqRg6D-1Yym|P^w=&Ybs269#v;8Nv)eDt3eGv8 zQN0k|ub*Q5aIK!d=2%@*&kUqj3?HKnen^NS_Ea9y_UM`k$oSRP0x1xqzYV^bWHW^^K%V(GOf?2rPd5d*tkv$_ zmZa&N#shhA#iwd_pisY4%8!LQ+)KkH&(pnkNBn7m5&d#+g)u>zL<~U&lsRYmX2rNM zBI$UQDz_Jff})QFDfEFDl!nxg`-@xi?iWv6*Jc-xKb8-Z2|KR^kcHdf2@1vfJR8Vw zMDshssRV1X-v0K9%X+lzh*%XP3lJY$?;>x9{_fhatcZ?)uyEA>qTc) zzk;$TY6yg}Y;~e$tikWXXO3_9kJPd=QD`kg#VCj&06j^$8uEj5+0uBGBUf~u&1I0} zZx@llRfghY_yGhpEQ(4xhh;^~Wu;YMGoXlr16w4-v2vz`JbToTtNh&RXnzBd*OtotOLToE2EaL@325` zkXK0ye`OIF4-s}<)jfnSy|z9%Iw&{o#MLRgAA5YfHwh9ToH{0voCScUtEG*Xh_&)H z3a#D_Xha;wI6kfiQTW07hYy#=Dr|eKzcucIQvlMa%IU8teB2LY*CLh2{#oq_e5oA7 zhlXcbBMN2W=?34J$q`<^G}O#Fr2F{}5)FK+nWn*y7Bxw)+R2_<;a~(%vSoI!mC&Ga zqi%mDVi4$!mT1ij1y>Q(7cp8Cb|BL=0-|eH!bj@?m{(aF?v~v*s#&ehA9$sDo$;W` zg3tDgfahlC^%YRz;aZ`9%`=7MnmbZv@ws3$T-=PVLse05V4}DXm?IL2$B&Q^ymQr% zGm{(zMAv=OUB9?TCDgdAJV>TT*QRKeB6DxaIi^++_*u>{B6;6B@+w;sP!{T*-(S5H zix(x8!Uh7q?L-yUy`1^*q~v&ate2Ee-{;_K`9>QtYSjDwSE>_OQ&}NoZ%&UEgiGoQ zocfiec;kl%G<9)!Y`#L##OM8V+$D;&y zwfN^^#KV~bGO;$^X7W0|h|Lx#Ewh;z?Gf0L=LgA`<6F=9ZioZ$&K@M?U1?jp^Ee$~ z>(!5J%YY~AhtaO5O=0=l)~s-i8Asgoir8)^e|VMGEa|4syg)FIR*`n;i$m?)H-JS) zLFL6g1o*mxa41$AcEPgBR?5eUlqPc10}+gtEH*2Urm64V8hrVhakBO^wNSoJ%1!2) zWn)mc0Z*A}+`E6MG%vvh$mZhUL(|*QixAu~udNT%)T-M4 zQ}dnE-#nHI7v6bE^Te$Mv0xS27n-(%TP%*HCq2Xcjh;AnkaXuIZd~Fga1;Ab+ zAl2*3l*fX05C9NYnPouw*4}TAUnI*I)THyeBL_d+s7hY6nWB}~{{8?D?dp{O=IJx1*1rH^bn*k;_`VF z;|_AuMCx*{5XD#w&Zrz(%bU6iO?8i0_>CT^`->yFD!3>)e7cM^OH;wrr})O@0vYkW z;0xFSAKfEFb{Sg54GZ56vg4-C=DnhVyb%S1VBXEs@kg2tt&)-5*QXt1K8dggrePQ` z!5mdg9@NhYsDAT+$KUxNWM1o{t^o-FcN6Vk`#>i~4hWGcF(3?v1bpOaPPNNYVFeLk z9>ST)!eK{wneOsLj<9O2?l71stU4MGm;oLOMpJDxn9JUT>zJPFL}EP6!;^!T5lEeq z&-;Rux&TvpDy%%-gKg}ma@(#x(|*jX>p|7h2)Jc>X8twMY~`mXhc!I#rGiog(XY@v zHm}{VIroiLLeznv(K{4i1_%RZF69a+aM=a5Dg45Ry5UlAkQ;SQkw}wEB@h%})e(s3 zk}cu6|B=FTcf>q>iJDs{%tJ=Ye0dJka49gYVoy~F|0o&ImW1w0ie#@5zDgJ}3F?&< z9#R)tSHBJCUnz^HUd#~mtK$7|fLoIhWsk&&!t`&DLW#`2+w3$9N9gYuNH-O=*ExMAd-}7T%?i2$z%tPxnn2 zZM->afHM2T>z!yTK7dv|OY_0RHg?m=jyLD(XrqKya-qoefLD&G5;r|6@5OOHR<|$8 zVebo>3e!}Kb3pI>sPp!LSqQmLC8DzJ4qwbIqd-KW-xE}H=?XjnTJ>X)yKfq(i;*zT zt3p&oOdu-Q3>dvmYaK@m#r`x-eICv%D4jTf~3 zFTSd1O+Q?3L|uJc6b@Sl0yD{?a-*o7fyk4a`N;u(^tN1xe#86;A{ErF3c!gyqmk8W z3=gki*oO0K8M<)YWOJ<*1yHhg9I4JSe~m5*#6#mu3H?6QI~fH<@iRPR zjNWd{5^x_@Pk7sB-ZcIoztE~Pj+R-AC@B5IfTltEmi8qs?-@A9r&qrbWyYf*l(O)8 zrWZR4K7-EUXC9eaN*gTt5KQ8Ejcpip{z12unj0+E8pFTsXtpZ)&E}pDOY}C|u}E@3 zoE)LNSV3L;qid?ni=Q>!H>ZRRt}Mng`j!TDi+@FA+3(ydPoy@ga#qy)n}1P1CIQVU=&YY1DGx zAuosTj#{#~MrHjbyEh6kER1wQn6_^CNM|-HIJ6pHqU^qaT7p5y?1aOoy%SFd^W+)H z8O8x;xJ*=863*he=r0o}{j{tU6vT}Hcrxd5<%oQqQddr9I&{Jdh`tjZHXgaYP_`74 zVg>-tlY-*>i0on3bM0vpE@LrDY}RWfh{{RstXO2mBqj7^p9g~li*)n^lrfk<0mY*m{D>0269!H$803`s7=j^2Ib;s zx{$ggU5BuAkMFPYJ0h6~^_*gjqKs5b2%?8zjLs|tkZbudVQh}b==DByBy`U<=R}2c zB?)idoy%yB>aDNBZU}T~Lc4yIIe&O(1GlS{tk=dU{tlP@0h(H%NI4LdQc0$q9kkTT zyfa4eWe)Jgm;76Ibl({!(+_*kd0dIPb!;Ipf)ZvQtDcixju^5`tn?o-o!|1kRCnTbPB;fi?ixt~>?aRaj zwGs|W6~LRl1XT$_d-JKb)u}}py5+Jvi30FZTkU%uL-JY zSYVG_h&_tRIiCjkx|&?slQr&w-A1l=)1^@Nm-L<68rPxFkNHhb^E4?@DjjFE{WZ}j zaVy#Ka0O5#1i!BOzH|3)=er>ox*h+lGp1cMG#FUmF+HNDm-V1dl@`~&W_Q>m+Sweb z>UKi6ncKMAR}?iao8)A<(0p+y^mW3{!+y#P1#PsuZ(w5w=qjs*K4lF(c6bO-1LTT+Z&M;Z{hDF%n_wS?ixb@R)U#aw^b9Eq=@|L}N64%FDVP?&Fa84H^?Q83UkdTVTVi0cx& zKuy~=t-eAu8JAAGsE&2pM5cJ!ny`Y9tPk+%ZsB$Z1cbxrF^~(lcl7518xUTq#*)6! z*d~$DTZKS5c6A)RxQ--=wCr{Mc><037@=#$pukj^a4=adw7{3ryCX9Ug(gVdH8%%4E|D2WI)C{stOc~~h~uwZVhqS8>BMW=YZemGOKJz-> zW;3w6-5_cUYTip93!-D2Mj@AC(b3>my#_VL7jdwySZxy%*$OB~b7Qp-e)3y*6HbQu z4K8~Dd$ngFNOZT}J>@9_PQ%^BCDIOJWUqPq=iS`O9$)-({n&A)>`421=jq-&mB$7R zUP=vJB)t)m>h=|8c!!n`8WVF2>brGl>+PJf<7t#d01&@;vqkBW*HD%GHmDYFSAx%l zdkt!v^Oi{kO(grMWV?r&^SK44rN*`)h^A=c&Rj~wF9;mL9%)oG7e)Gp*rsoI|Wv$I0paGF0MgJ?*w=RueM0WNpf2pX$YSEr)TK}+a;Sph1P`jWhM>`WxPpz6jY1F->l6u^Kq_t}h1_5nK9%MAn- z|9qMD9m1(R8xe#oF7a)kdVfJcnTX5kY6;F2eG-)3Scx1E`e4o_nSQ}bf8BZ(k$S|f zcS~tUpRBnnkHGY|ENVJTf{p<4q&G93iKhvIwa6oZz@)<~TFG+1WH z7V-pke7(#qH#2z&yb5cy4S=_eqT{FRDrKZWN{ z>6jUyps0%|!uEtAJmeJDSHTsF3|F`LVm7mv;#7q1>_Q7)Q$fPRe5LD?@8`dM2h%Dx z^ov-P>5}Z@Y%8&AxD)3}l~7bZ`oNyV-!zIB>^PZ`)q3U>6hRLHb?;g!7v5A^c{7eQ z!gOWaj9x9cplnfp_U2{KuxTn+Bm7ZAyrPiB+$+I_G{*0@xul9+eL2qzu&)Bea( zn*;t^01JVq-RC-UTMi^lj^R6wt|)J+uwgV;qXjlY^+24e2xG2G{t~%q{R17SS}<5D zd*JpS2mpP)!(A1j3-o0Y$LV_GWPS2QMH2rEaO%_xASS@7LI@l&Vl5z1l?XMhOcglD(Yv^?g!JrttB0e8j_m8v4u! z19?^AbAYF-JF~?@um*GFlKd4Jrm5-C;z9wa3A;Ox+$Ay9j4gai2*#c7=mQvSV7@1P zsUuyn)?>%&(=$nMr^oRF=R_|Ed{x76<#D~ZCcPX9>5O*+{ixT-0r zUHoIy+3^?Ze;8&MglQ%UCos8&8}>Nz=pi!*0wK~37bT~FS6qs?64bH`WMvWTE567m z^0f*i^n+qU85$m)+Feb8K#_6840-`^!nP0)0{k^mzdwhiee>SnG(hcm`3??=-dqk6p#&d2vfdiU| z^U_VrM`l#4j`Q+ML$5PGB(sS_(LgvKP&qP>^J5=$+-1!A(#9kRMxjVSvoC=O9;g-i zJ-+R$?gPS>1)v)IkMEzFy??yhZ~=RmYs`w;E>PqFz*aAB4-nm_L#aRBO|{2!1@~}J zZbP%t!BBbqg^eCi`Ewry%>c$Airvk4d$Tu6o7u^K^TOiAge&D6et3& zzb(F$oggX*X#jVojI*VeCu%B0a8S+yHI~I>D1rLPAj8OMJ*xH;>ZgW?9wt_EYQYcz zJ1Y=6DF=61^nIaz{fIB=U7SDha%sFs z2)#@h!v`q%O10|JI}mCa1^31t08>P8RgA(JvDlXmn3Vt^?tYRY9|0*XM%F-)?}(); zd|f1Ct+L~HU9|#Nr?jj6K7Lb0xEgQs6x$m>l*P6Ngly&5O$0$XJR+7d29hqxSV*^I zDU|?q>h!}}sEenAiAUpT<#ac9*EyF?EW0L$|r z#Eou1TuIY7~3b>o#QnHN=vf?#h%Ad_aV^o!Z27n3mDuaN^;G1cm z6Azcbasb|50ujkAB@ovK&_GaH0OjG)M#w;}a&v60=Ke9bNK$+WlZeLWi0CP-Ljv4H z+@mCbmVt|f;Qqx*+ufH6q{Q4dX<$m9S66E;-9W4gRVAcBl2FfdyrmGtXd|Mi5iSY0s44WG_|Ka$9)L-;<0oS zp;uE|)X!Z^#uv^kwc9E>CB{K|4xO}Yq<$^T7=DObf8Pi4&4G=JLc&F<1Ip+ym?488 zjAjH}w@T$Oj~W02|@1QwA@)qZ9APF!$90@p^vRaYx zycoQ?DwT2cle+@9>+|OkxiVCJ(NAupGbxs5zfP^Ws38*i0dxtSlB%?f^$AQ=g?@TN z3&y$$`Tog5{AU+w&?TG>C61c7Z|K8>ZvxDrw7QD;4GPs@t+cQ)`i(Lk9~lE~SByw6 zr_s+JCcb^6LXQI>;-^7KX6tfbdg7K3YY5RB(LssD}F%wFk zZSj8FE9Y3Bw6es20-yS2pQS`ve=*Lvr^zrf%3YsSF6#tSLCEs+{v1T@0952kz3FO& zz6B0J9wk!F#r|wDP6))R`w78fyS&jz_)=2$@!%F*GdI|Z6Q>rAJ^&L;#}}uH^K{g zp_DPmBBFZM@7to38|>>FsZ~mK9C{=B;|zY>_HnuTH@4i`848NyE2s;_o;Zskg?_$= zdxw&{s=bQoz9G-&FGz^MF=69S&YOMFG3@@Z)-}QrML}U_|EmF`Ojv;E_I6k`gt0!u zM9|8I^#-QGJ4gL!Zafc!L?GFdNR5$7pQ(jb?M?SH~{adQnL)mga{5MZ1QSt%g zhekmN#ui-J7R7IUFGIJ;eSO$?FGgC}h2gz|;n~7F6{^1to`8EM1ponlMW^qJE?TA0 zM%0erodoaGK0s_P>_Qr-7bux<9@10j2SJDrSD1Cb`GB&HE&x#kQ$bP>+)(}il0`NE zuGdA93x64y0QO@UD6n!q=kHo{i`duLTM#$vTIljra%4#;=vCXYzj2qN)5lFw9aABb zYH<71cldlic!f8}$f`Q%ulaGLl>(+SJ6`Xp>o%@vs^VAvnLCRu$4dgH1#aacXBQYMcnY0b3m}gk9 z)E~X!wCHQm%v3H)xOIH{+7hc7-eO3Y4-S0yjDe^wzoO7%x?9$9x7_N{ZZG&hko^36 z0W1(%T4xX=c#F2E2rN!pB=LgJ8o2dN>hUdW@TDU89>2v!&+Tv14iIs8!O75F%bOq= z{dfReJg0C78j+6MCZ>y0Uen1YcgFmBBXg4FO{Ry>_qiK3pUBn*@{>4i#)qMVD+JQy zpL>Q&Oh80N{&D~+wo5%Cde7zbP015nfFoHH-e|#d0lyGe0BdCVC5a{c_+{gKFfQMD z_5{?A$flo?Jvtk@1zEvrhvefMG4r?jWQ!na-RyGkHHp~e5RoiIpr)I5%UH4eYXfvM z%#96Qr&g}c_=;+?(@ECbx7KSpL~KU{IY}SyA5$H^T6n1--Q1lwhDFN9)6|a9&IkaS zJ>x-l;tBZs1@vZW#eEDjOw5%)l3x9tYlhdR3^9-0)2*q-alD~8{Npx)Q{&E?eJoe{ zD0gP}B9>ofWb#pBs!m)%LSC#uqG)@HcCLedvQT_SGa&voq9CH{U#4d)=J>4z)WO^vtjOptnMTvF!C$S}AXDl+N4 zr=g`u&$^?z4fQ1wJ0M?T)Y>Y!3{_{;+WB;QS(sDH&T%?;(d|g@IPfHl<%9mz$5=b~TNin)ZV=KMVwk9fC6H z^4h=Uy0$?f`^|0L(RyE->lFA)OTqhD&cCfuwls55Wglr;rpyvb?0;3zqI-kB=#D zEy{uoL^wPjA%GAj9dq$c10yfj*agq2zd= zn*sH=+l(qFxBVtyP9PU>6XnlhEi-W?v0b|~@5~C7r?4NknzUiTuDUNoSd_Bx&r%5s z?Wvq2f>I-S*i{QY#z4V$92?=D7fteeh`g|6B7d2%En4*R`Q_c=T58RS;i}9cVy_FQV zwOO~7a~>y8V`V^KQs+y&AVM3sZ|*Y1mC<*lx9>c?+3#LUz$D>yiM@PT9#H3VRa}Pz zh(B_gYo&c7o(yu>q@ohQF1-&e9(?sZ_1>TfN+7Kv&(34y3*n$UMgO})nG`hd+iBd@4Q zM0nqH7`yesLq@)c>m3SZ_4doo{Lze8VAd{i{4_E(_WgbpI9`ZN0Uc)}rA#V+Q4 zkd{so@NZW&U`ws=0A%HYlit;__Z7odMQ!Qt0bh{NR#7A?DE#cibZ-QF!%OrLRXjH= zT#Le@f44Rm-sdGq?)^CN3rjd)L!E&O5?n-%99T zo9h{+7fo2Ja`HXG%#c925fFt_Ni5KwRAikKv~JwlJ3BM&c2HEUU69jlvZ@67kHb#P z49ek8IlD|*cFIo?nwOY+=o(^%og9@qUz{Bq1^{M+Ys~Ie?ZUmjKro#coUk95+!s`D z!|QfS;zep!C%JSbRpF_>^gZi(yyOZqsA{Px7ibQ z?R%Vc?=*CN8ZE?;uTq!!iu)IN zTJjsehS-p-Z!ei-{Gz9B0;;uin5)oeW|IIGS(dNO?e%IBO9@HQ`1VuH=43R(3eIVA z*PkhNed8HWw7?Ln==5vUV)d5_q5zV;9;)%7^OIGa`i>+Wf2_hs*@?sfUg}ziqxm$q zav|SHucm^Ry*wq5`kDzA!^jZ9k95`x$lCPLbNR1GX0I^5UUz9EuuIZVPb=i zaDaUmw{AUzUedO=(Vy(pt&eo|H$J<(tk9SOlB`mSi|DFFT&FQ^e-pWOIsb!tv`sg9 zIHNnp^fux8Uu%qs$9PHU_y;bPtfM2W!VShb%8d^L{3%#hA#Dc$Ma<8(N*D!(Vrlj- zVUo5EyP6&^;Q-TU%3CYqv;=wmkE;c<{ci2yPp$w!KrZF>hMcUmGc36!xhk} zzQCzC9eYkn$EkcK3MYE;lGL%OA>Nv_t)|6mk6lT(o-@zA)aax-l$~~NKh+%WL1;+O z+F{y7t?$NJHeM}L!x`}RKe?Xrt3W$tX_O4|4x3B-AT$dINwS#D*re$@H>=!d{gdH~ z`oWBB*?LX(0-tqbF*wQ^)lZU>e$({fgKS@TU?>CmE(*2IM6-SoPg2#n)CFa$5cUkO zG23hGnagpxyWwOx9Fb%f7rPD+)P6z>#?HMu_{+>ZetqzOl*1mL;C5(rmkks}S`nju zhl~AAXwp%0N#F1*Qc{#HOZt<)lC5vGItZWFRfg92>w-L<(G=PdJbj;YEn_*XCM!0X zIwd*R!NM;~Z)%;PU5|ZXZO3O~uG_E0({0S{hi_$Tnt>AbEkHpP238_-NJk^#Pc8Gg zI6jw}5XbR}j$B|0WhVIJ8gV<;No(k2N$WUnmxkE~u^jKbm*z1>S{QyRm>Q#FIqobh z!2o78A;rYPcuma%PKDz1Yj32n3<5AfsqPEoahjLFA=(d~`1%qD;G~N{0Vo#lp(ljJ z)4?N_=Xzu#wbKFTDf9TIC#SuGVskz|mG0&^JMaae!f&-LXeb81!+uzgzsUcf0*;`i z+2^rr-bgA7C3S}O{!9kUjT^VY{5b>=k{e1?nC7DG7EJSjc|>K&fg6 z+-VUAO9pYQx-5F&UGRYw;69N4RZ^Y;RkCp>@tQwa>R8tnhkE*9FNgff-W#S`gudv* z?(6J+l#nxX@i91AC*A<)9uG*iiVH`d?~3-9SI=XL0csfpiaXWE;tJ?j5q8mgjMqOMgsR6ZR-=yYAABhT#UYGPbfR^5 z7Q$rT-8){L%kX7m3FRLF9Q<)!0W`4gpnl?JKdsx5$#VlPOAF|17eP2Asuwt~iD5Ym zZ&1W|Auvumh|Uboym0+vnK0si0@ohemq7ek0`eunnVI$Ue}Sy5`-9C(94iXxWSu(1 zE~d@n{>w{g1@TV+3HAuC;giKOdQw=tB&>itjJ>OoYF`r1pDAJ!zc?5~zK2OxV813N z5m4p=A9c)jASEtc{S*lVQzj{x;0A<2|X9et#Tke7Gm?}d#AZ<&~c@()%HBk|&IanzS(L}JG#(_Xi9@>;#!{^%G z)2UAPM8Z$9Q>eiN9su@~n~oCtE_*=rN|ZJR`Pf$Ua9F%3TEKBVo{{S5ptjq9yUhZ; z2h7JM_G|xR_`K^F)s3f$K1lZ}wr++A`ij1g>X_qodJMiGM;akthMGboH&k^Jp*Gfy z9G4vaY_uQU+%on!2a|OlWI@m*7G22L3)2N#ya0(r`b)14+<8n`Rp(iOO~6;cOLOoT zMfnvb^Aa$IsQF}X{EGs>Fb4xI-H&@xe5p1>s`1>odb%_GpXj}%C!=^NM(V1dfLnxP zCnnQ3UKhi{3156I*5t0c`WK`^hrFTBKPKv@_{;2BDHLGvorg{;rBG7d+E{ea;kuc) zKHJSlQ{R0sR5cr3R5T378h>B4=f&x39}|U<>Q&G$1qAd$FvSJQ*ivvJ1R^yiXhVfd z4?2IDLzu4e)0ejzU|EXjhJ(si@-rL&Zj$eRsK1u({?Zc$Tf~ZZhUDz?*+uX+h`|tO zS_Vz=7}DTWu6z`|wz={R2~!8zEL`Jm-KSLj2mZxjB}|!LvvW&vYq7*?kSad?a!;C~ zPkt4A|JpH@i(9}Zw8D@Ra;+XL*G;4SFhpR`Rw58_s{yHm-)Oc99= zWM9^+LD`J2Y-*}W00|%61w!}g@YmLaJZRP_^fCNLVD2VOa>*ZRe}RJfEfk+0NWU&# z&HEyCNlHNQoB`A;hS&(r+5ZI66Ql>B)9Bdjm9#AH0t>@m&W5co0c-6bc|bvV zGz|drj}V=z(L2<_Z9G31}&$MKbkc645b|Gg;5M0g`A} z$WW7GN-+QsA#Q@4n)A}>N5_hQj;k0qPi?L$)b|CQ@<=G!--?o0fnx$>@9&|!JQQd) zscg{7Eeqd{^m#MR#-mZ+_+n2@s&k%#<{lcK${{4dz#O5-g;(PYPfBQC+?O)z*q&m` zEMRaWWo^E%&3oM6X92hpMOlzcsllkA>PdX7!sgZ+y9Xz*}=1sA}W5I}Qb8 z0=cLJC|CRt+M+oKAO{kBi{3T{uZ99#)txjal#Gv1aNdaFR3NxjZ#u%I97xkmCmo29 zB0sg>d!D`qf#hdYNM7tL5Cg}8sGbbGfuG`Ft|a8R%aB6*a7v5@M7%xZBkPoJq7{&WX$8+^;89x~Ok)5v8z&@rMF2AVH z1qWnr&ifmCraf3hiJ+(-LjIKUdwoW!n0?&JBUUoNo#pnu0rH$C_z3{vGn9{y)bWMW z;W!B=TnEnuvCssg07b414{FtC%32gkwxe+8zv!~tHv{LU#|uuk_Sd0z$%=O2vZopL zoqABus9M#;1|Q1Rg#FnUOCjF52t8j6!wEaMcs!P$!KVMGT(?6b{McOt*mq$Lk$b7( zqF`yN*z5NCqj5CrS}@{S6?foG=HsK@fapPOu7#qhBI;ivvKa=Rom0x2BHgO0#E5%G zCi%l~DrY`N1?56s)uW=iUOkmN4tP1TaT2;D@7VIIAtiCMwn;Yw3jvw7U0d+jZU&)7 zqc3k`mB8#VnamTO5xwR~CjKc_l?X!k#9wDxB+17VAus0b>KLG^x%X3UN)lpI>+0Cp z2{U%*el3q$%~F9HnJ1?m(y&GjMdMUBxf)6@#06L$XOADjEAy8?>zv$2+sO!2I16e{jErCf+JJk=j-tmD$$;kK>2+p-%jH->#o^eX*&VIhB_Nd zpG!+F$0#%*hpNrzkJ{YT9LCuV=&pVrMW^_|66Xk_+nBD1_SQ}my>^-c$?BHClacd? z(lpo17H+(fp%N!WF~^Q4F2LS<_;7#7LR6Qq7`wlHU$BTE@_tPkb1VkYs>uNN^V))> z0vA&Z87RHo5K19?8P7wR+yJqle1pL(8S8NIzT30xGwwgYCT4aCjteOdMyYPwmotK* zTX*!ZYjJ0HZ=DE$3vB9T$+@HSA@-;2?m$Q;r`pPGpx5qOyd0r&?$1r#su*&7Y!0_; zF1Uj-?C?Vh4G6tq4~@~kdIz1{GkOZ{lbUkD(s63bi>|+`@JBKJEJn(utV%G2-95Tl zM@86L3wfuf;)GNb_@|8s=sr7T&ST^{xxO$Uxmjp&DLE*$Ve4FtUXlo$a;5PH+-Fmb zcfj`+f1pOvqO9Hjp@h|Zz|PNRZnvar=kCn-vv9yK0Z8GqaxRu82RVvf%IOx}9s)NC z#ehT68sl%(UbSx6E$sW}?Y&s2;)8>*Bk)y^f0t3HJxi-5?hr4@S(s0N0LW$Lt5*@ih|~Iu5c^PIk`G=_`CU|K7B30+Dnk=9 z$9It!aThvad1qmDnJ9bStbM6I|Mn($Kf#NSB4XjO-?Ji2@lk7h8GF%V_sn!*EJ^R` z*nzRBrk6R2SI*3il}7`-`;eFaIZiVuq&zCs6HZV@tQ-}vpH`Rp^%q0mW?10_O)o1i(vc@(r!6l$@ zc$Wsdrx@JGzU_-?9 z+9$3v>VDu)j9go1vJX?W#7+PIQZcWON)RJN0$dXHv(mKlW~8U(@$CTCx*_p_#i+)h z+^AQA5_Lcxta`V<1XJf;ZjL!~8NUM-IxzIOFaJU&Z5yuIzq7B4%xzYRLusHgdtTHms$iO*S45ZC490fKMyjig> zPK$#@M*;4-2U^+6@iZi@IqCHZ`A%cj&;?(K+@6Rt%YG_!<>)*yg4fX{SvM$cA)ulMFDNLZwm~c-{bOV!S-ir(M|CTDz<#{6WHrbJr;x2 ze-&k<^pPk;OE%Ac70x?zjfczBnFcakC%tZj#C4SVQiqtN3Z%btal4Oamir|Tpe>RArlS>>J?h+Pm|;l0b=Uwf82N$b~IkGA>Em2RDU z|M?1ExOQXT?nFuWEA#J}v$@%4vH^ETWg!NFsJ&J^WxY`ZF>F&HD6^BTuRu?jMz_^f z;ptQf?bx-iEpbQ~RPHPPmjQS&)u)dBHv^APxxmO^qzp{8;(vX#350nvx5 ziEC%Mx0OoXL)GdHZqZ?&3yfFBH5axacVI+(6wxJYlGd((xl1=`{?!pJ@va35#e5mWuOA2hmz8GPU^`)){Wk(}K#IQLj;`POo*Pt2KRl z1^OS5IUrJn&tLseNipPj4_JNp3wVq{uoxTwi&f7bR%rULDKY!G)2->Jip%BoRK|BB zB6Bn}KcNO7%ytm2%lk>ElJk};W{x0wgztBIvDciD&LXnFa(T8(qXs0;Ltz0hC3l%Z z0!lZsv}yQm%ZXA<5f=guj~&TP5snM_kJ@#(D>iawCLch*Jw_CLx3-DDuR~SyDgB8Z1NAj1Bl`-y4IQ?SfKuAlvyf3$mlev>~5Fbe#R;wR9x$AdfUUlQGNf7!gd z+{45k=CSmw68m+S$DjQ+-PtgYrwCB`xV1d#BLNvUYr1n}?(%rE_i|%V^O3Q1wLQo? zxvND??Uq#{s6f!;$Z`;?|6GXD5yFf?aDaL0<&B(IQsg`{U^<^P-2gbowN)>zEQLji z+b`?}J@n4`!!-6EFZ_4!E~CpmdV7X6*WvMlx;S;0@oWWN^@0@u2A zMm1GyEj6o->OU-xrMDeYQvH!~OWrdD0LsVdDME#Fo*f@* zB@VsSYx{rAORdkKEwxiyGfRRlaeF_(A$#A19c@TNQM2qJOOUU6w;oNF?|fE_OEZyw z<8IoIKi1j9CcrO1(ZabE09WhCnaj{I5!qu>JaMJJ0o{2=Lp^X>;IIJmd*75?DaDEA z*avAeVO)*chp&AVz?=%vO9=QgLvjxeV#vra6zDes{_okc=0ES%BD#3)-^;psv6VT| z=V1@C$dQZuZZ4FWPE;4ZBxT2bz@D*$Op1n_V}s<8_92aO!~NN@u8imX;kjUVuGRj3 z^f^p|{g~ej7rStRsxvF=PDkDl#6&12YfHFL8J{4vd0JWi-+FeNvtn+Xj_D1{${Iborvu`U#>J{b;BgggxkzNqmQ;na`ieM2FV#% z#gbTmyD^kt8>V3fHu7BH`(h#>7h!ht3!mgfOyr<0=MjNDxW_~g{N2Pp^n11zp;`M? z(B&}AlculIb`6+hsjywaT#9UvKTBBg+H&GjaX;kMl|l{c2s#HZo1}#SE@*R*^L&q< z8NaOCh&*3|bex>5Zs6)CIiEO5?9EI&raWNTcFs87-y&oMevQ-N$YGDHM66z=Qwskn zf=)*6bYzEa9UD_#mvRD?+IYmLJ!~oQn5{b?3k+YO56zA}xA)B%x)9Ke^GLmT8CQsN z1aY8*z4V)P1A?G{GNXj>SPcoUyu=Xt;Dej>#RfuFr{*fyHqAgnYid5n9G`k{8!m*u z^jBhpUwLih*;O&x&OJAqI867I2 z-ynkUpl9f3(YRB*vM@6Z$)KSfB7F1IiJ@m-RLORp>d9M!OUd79n&vr-R;uZ>G9CEb zYoz2~yhtjYi8_RuzbrE}jZv{llDGJ2=De2uOPXfe!CcBm% z$ImQQJya&)Zy%j99}V`Y|NYOAF^UMDefFeJ_BAIb1wrS=pg^euFkr_Ica^ssTL(hw zB539*=G8tCwSpLW!nwRts$i1_a2qONFM)xE(j2R*?lJiZR9pOSW{fL-{DHrYp9H3^ zPA;E~8A*?eUhj9iBqK~2o;Ut*m$)FAy!{QToVBfpc^Y!q{5F#6{TRGl8EiuB6QT(9 z!EOWUlMA^?6PR@C%jhM_UejIp(HPBsxIyD{E#S61mbON{igQQa!ezS!*t+FAZLq0| zL+11j7}5%HGpZzYjyrcK!I3UK}$e z71zSbx3>MoHIq@IxmO4B=`B*bX=&ZB90pF-biH=pHvGnedt@7EbRCnAz1N97($S^~ z2}(#UhO)%H0bv@3I>Z%-fVIkbhl*@wC?`ga z2EoK-Z4#APSD4H|<%_h;NI_kg0-`)90NPH%`6rcQRarK61j`@tJB`lE}fLs23GNj{v^lMT*li!OravCF9+4Ml?8ZgPJ`(ieoTZm1GDNSCJcbl?8QkoJoPTk_=9LadoEi zElLHM{`7c8+0vSi_zi^Pqm)gWdn-yxCVYUQ{3j3ljrT)TN~})i_E&t0zAkiT4_EDj zaYc~>1vWKb>)nE_C>)u-Ymh`tUYh|Ipr?#7x)MG@zhk~K@e8rVorG-t(N2|}-}QL} z(4rf! Date: Tue, 4 Apr 2023 13:20:43 -0400 Subject: [PATCH 71/83] metrics and limits docs iteration (#283) --- .../metrics-and-limits/configuring-metrics.md | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/guides/metrics-and-limits/configuring-metrics.md b/docs/guides/metrics-and-limits/configuring-metrics.md index cc25e3f0..a5d055d4 100644 --- a/docs/guides/metrics-and-limits/configuring-metrics.md +++ b/docs/guides/metrics-and-limits/configuring-metrics.md @@ -8,4 +8,43 @@ A fully configured, production-scale `zrok` service instance looks like this: ![zrok simplified metrics architecture](images/metrics-architecture-simple.png) -Environments that horizontally scale the `zrok` control plane with multiple controllers should use an AMQP-based queue to "fan out" the metrics workload across the entire control plane. Simpler installations that use a single `zrok` controller can collect `fabric.usage` events from the OpenZiti controller by "tailing" the events log file, or collecting them from the OpenZiti controller's websocket implementation. \ No newline at end of file +Environments that horizontally scale the `zrok` control plane with multiple controllers should use an AMQP-based queue to "fan out" the metrics workload across the entire control plane. Simpler installations that use a single `zrok` controller can collect `fabric.usage` events from the OpenZiti controller by "tailing" the events log file, or collecting them from the OpenZiti controller's websocket implementation. + +## Configuring the OpenZiti Controller + +Emitting `fabric.usage` events to a file is currently the most reliable mechanism to capture usage events into `zrok`. We're going to configure the OpenZiti controller to append `fabric.usage` events to a file, by adding this stanza to the OpenZiti controller configuration: + +```yaml +events: + jsonLogger: + subscriptions: + - type: fabric.usage + version: 3 + handler: + type: file + format: json + path: /tmp/fabric-usage.json +``` + +You'll want to adjust the `events/jsonLogger/handler/path` to wherever you would like to send these events for ingestion into `zrok`. There are additional OpenZiti options that control file rotation. Be sure to consult the OpenZiti docs to tune these settings to be appropriate for your environment. + +By default the OpenZiti events infrastructure reports and batches events in 1 minute buckets. 1 minute is too large of an interval to provide a snappy `zrok` metrics experience. So, let's increase the frequency to every 5 seconds. Add this to the `network` stanza of your OpenZiti controller: + +```yaml +network: + intervalAgeThreshold: 5s + metricsReportInterval: 5s +``` + +And you'll want to add this stanza to the router configuration for every router on your OpenZiti network: + +```yaml +metrics: + reportInterval: 5s + intervalAgeThreshold: 5s +``` + +Be sure to restart all of the components of your OpenZiti network after making these configuration changes. + +## Configuring the zrok Metrics Bridge + From c96db2cd3f674ec49b8c2888d6fcf54218c6587d Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 4 Apr 2023 13:57:03 -0400 Subject: [PATCH 72/83] metrics and limits docs iteration (#283) --- .../metrics-and-limits/configuring-metrics.md | 62 ++++++++++++++++++ .../images/zrok-console-activity.png | Bin 0 -> 84928 bytes 2 files changed, 62 insertions(+) create mode 100755 docs/guides/metrics-and-limits/images/zrok-console-activity.png diff --git a/docs/guides/metrics-and-limits/configuring-metrics.md b/docs/guides/metrics-and-limits/configuring-metrics.md index a5d055d4..664e175f 100644 --- a/docs/guides/metrics-and-limits/configuring-metrics.md +++ b/docs/guides/metrics-and-limits/configuring-metrics.md @@ -48,3 +48,65 @@ Be sure to restart all of the components of your OpenZiti network after making t ## Configuring the zrok Metrics Bridge +`zrok` currently uses a "metrics bridge" component (running as a separate process) to consume the `fabric.usage` events from the OpenZiti controller, and publish them onto an AMQP queue. Add a stanza like the following to your `zrok` controller configuration: + +```yaml +bridge: + source: + type: fileSource + path: /tmp/fabric-usage.json + sink: + type: amqpSink + url: amqp://guest:guest@localhost:5672 + queue_name: events +``` + +This configuration consumes the `fabric.usage` events from the file we previously specified in our OpenZiti controller configuration, and publishes them onto an AMQP queue. + +### RabbitMQ + +For this example, we're going to use RabbitMQ as our AMQP implementation. The stock, default RabbitMQ configuration, launched as a `docker` container will work just fine: + +``` +$ docker run -it --rm --name rabbitmq -p 5672:5672 -p 15672:15672 rabbitmq:3.11-management +``` + +Once RabbitMQ is running, you can start the `zrok` metrics bridge by pointing it at your `zrok` controller configuration, like this: + +``` +$ zrok ctrl metrics bridge +``` + +## Configuring zrok Metrics + +Configure the `metrics` section of your `zrok` controller. Here is an example: + +```yaml +metrics: + agent: + source: + type: amqpSource + url: amqp://guest:guest@localhost:5672 + queue_name: events + influx: + url: "http://127.0.0.1:8086" + bucket: zrok + org: zrok + token: "" +``` + +This configures the `zrok` controller to consume usage events from the AMQP queue, and configures the InfluxDB metrics store. + +## Testing Metrics + +With all of the components configured and running, either use `zrok test loop` or manually create share(s) to generate traffic on the `zrok` instance. If everything is working correctly, you should see log messages from the controller like the following, which indicate that that the controller is processing OpenZiti usage events, and generating `zrok` metrics: + +``` +[5339.658] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 736z80mr4syu, circuit: Ad1V-6y48 backend {rx: 4.5 kB, tx: 4.6 kB} frontend {rx: 4.6 kB, tx: 4.5 kB} +[5349.652] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 736z80mr4syu, circuit: Ad1V-6y48 backend {rx: 2.5 kB, tx: 2.6 kB} frontend {rx: 2.6 kB, tx: 2.5 kB} +[5354.657] INFO zrok/controller/metrics.(*influxWriter).Handle: share: 5a4u7lqxb7pa, circuit: iG1--6H4S backend {rx: 13.2 kB, tx: 13.3 kB} frontend {rx: 13.3 kB, tx: 13.2 kB} +``` + +The `zrok` web console should also be showing activity for your share(s) like the following: + +![zrok web console activity](/home/michael/Repos/nf/zrok/docs/guides/metrics-and-limits/images/zrok-console-activity.png) \ No newline at end of file diff --git a/docs/guides/metrics-and-limits/images/zrok-console-activity.png b/docs/guides/metrics-and-limits/images/zrok-console-activity.png new file mode 100755 index 0000000000000000000000000000000000000000..376811a29dcedc04386b52a4d07e6db6cb416e7a GIT binary patch literal 84928 zcmd42by!qw_b`fqpn?dfG{_(zHFT#ilG5GMJ@n9F0E3iBcXxM#NOyNh$I#t;8{>K2 zUwr3W*ZJ%03z2=VJJ(wG>g_8dC4&A4{}B=r61tcuL=FiF#S;nXelO}>pe5Bg_bKpF zWul;Bry}`|OV82*tfOzKYXEk%uma#nNId+GRyulS26iuX4U9~nyr7+$2GC0reO{0< zt0aS@m5_n4iKw%UfxNSnf}XRP9;ZHtpYIWmBNs4%g@K*UOGgWHs4bTxFX-32T)^MA z&Gev`zq;6&@q$z&WnKzd+8DfK1+#(~Kzxs0^4RDba>+r2{|pA+ctOT?c2->U^bQUV zUMrd;k&jw-9qcgmbS*0 zR{tE}zbx?&_>u(f*&0}$~CssCk9e;2S-aI!L> zmou=nw71bScndRt+L8Uj#4TR9#4L1-4Bi<)jqHqhL5xhy4F3(JY@+`Q!pZa>klV@3 z?Qf}}2sHtgXZm-L%D*sP5IYk+3*()E1Z(*va4Z{spW4 zC;0D1e}RQ8%`I)@0roZk2L4*_A9`^8f=OEH10d%A1`-nzlCiNgG%*KYwsInZFU8&p zu`zP8vC%Pse@_Dtj|&)UV(0|Xu>*QBF)%UHF|YtX3XIHLtgKv&48Y%948KwHt2b~b z^mXiX{@?oFa`sCeV1%S3m#B%You!S_ucqJZE^lD{`|bCuxydg+fBEtkwYhZkZZXdb zva+$%x7Rb!|8*>2y5CS+OG7&c9UFr;MgXCCL2nEVO#m7>y_C7-;FnA553itSrxsY`zO;C%QU#jim6e^C2V^U2NQ?R}3a}o~F-`<)%sG4)>U7xmX%rPv@ z)to?|(3t)5maNmxRd|nAmZ~K9Y5Zryn8lDLYJ;{5I^r=clo^fXt81dNgX+8w8ETFK z%z3rD@ZE8nSvx0*N>vHU6))+}+;mhjr>!F)Z{1;>ic8WFbgXQsvk!F0kNiR!R3y|@ zb(}aqm>Y^+WVN}8#l^=&KbxY+c^-o+3wrI()cjJ%c`btX-B+#ZB}mli_Ri_U7vxGI zGy2GW$a1M2LsT2z^v7E8lMzhJxs_KpQ_54QM=wgo-X?AgSZhr+Ry_1VZ#(0#iv2;V z!1LWH|5-yNhMQ%wWoYSl;?uF8hsmkr74zmqYTG@Z!%oAg^_|e~X$2|!9l1u?-C^b~DWShx?E5HF6snS{6s~UYxO5sSt~z5zr8+^k1F|U3NL6J& z*%*~B9Q-!?!r_f1=3ALN69(FbM#rlsZLcwlsXn2)&lb42^zigK^>X&@ep_ZXs`rUv zHG6UAm-hPgi(@2^h?~QW7v8F2q|8B6Tf~xAo81@~{4;Xh? z4*7*sFxQ!X!v&;K@83-s7#}M^`UA|oK>H2qw>`bUN6No*zyJ69?eBh?|2&%Kf$wLY z{|a_=1Qc8*DVo2Z{+y#bLm?~G1 zL!S=Sk#mo;+njxe>tPb?1j^Q@BwrV@+p1$BpeTNi>q_fI8(UlD0xeB-bwqQoOW1(^ zTO{;*L;iToSlHNUa_gO8G-+}sy1EO8>qBD&+8*}&-g>&aT5d;*BqRxJZNMIjY?j6I zjyD>d{!pBw1zVbDUggs=0^!H1(odvYq(b}RM7VGJkym8Xd}NPc$wWzRcW9V_t|~~* z&_L4PH|sGP%ftJy58vNcuxF0&;$;e+zklwak(pt!_VP{_R?|~VMl@?qNb}PoSokK4 z%c0S=9txsa_A+MII8wO#Hl$hOc*7E ziM|3b`0(LF+tpsTow}3Q2^V@>K_ogX-(01f5z{Rv<(cNiJx8AS0prcIYjUg(;QXxt z1US#0K_KMh?BH>}YE07HA;Z7St6) z;=mJfeokDq9KN!$l0z&(gP)c?L_WP*cj`XImO;xFt`bfz{9TXlG=F_LfK@I?cFCXh z^8zMgyfnXMdxwq3VTX6Ym^HcFNkg~UWckF-+)j7{)0~3B&(DltX4vNV($doO^z`uX zF!0ALZt}??i)VP(G9x47+qZ8G3=E{AWE@V<&b&T<);BOPVZqOE;4|n9<+NR4VQ0_C z$grHQ42FOF^vS3<7HI1(EiFAgb@YWS9KLB(4X0JTMIN1cd9HRtz5VOl5mTbB1Z-?< zc{w>?;Ue>i;%x`S)>a)rXP6#>RLC1ZvK{-UmDglDk(^bBM(G!N4)@q!9&d9Cnepks z;ipYTyjPKzGmEkqnv zfbc-pblI+2wd0zhOmH0(i=mQF=egJ{Iauv$Ff7?11(>R0s%B^qT5)0}M$?7G=s#f2 ztgcexwY(4SVo0+OPtzX5QB;SwOLRzH_#_ON>Z!J>hVg3%x3XAF<({!>mRXeCOeH*( zm0fTPOzZ#gbyw?AqRUjeWDJ+~lY%kCYHys2ecZ`wPgrUnK=CmqH;UxqVy38fFR-x} z57C@EL#cqR%$1r99dAvkKLO|rovU}-o~}~9qHZd~Bqrtq&a zw=XNX-uKd4n4d2)9mxhOvr9*_tWS2gQJJ$5h`ipEk_ePBE}3xV1Nk(eo}yt|b2Kk9 z!MI~Q8eOZ0-=QHp@1JTc*hIyHhc1}&A>t4)nJ!^fiKD(h*p?r2;;5H)DEjNypP4y} z?-_8_u4BT(S2!ZW!^5MaCl89gg@kZpBoZxZ#pPfmst>O6ouLXqp-_GOR9j|aUkDSx ztZeJHU0Lnzq8a{o>+-rt@{y5|A+JI@oKN*C3kw-&X}k9v&kxt{(R*&k&UEUbd7*er zXD~hHAzOgfkj3YyURmUSAQ#5ty$H_Y3q$dk)PSB4@E-;GkQC-!ovWn8;qLXX2hH+A zt}gmakcCgyxS<{lAJ{CWwGGIz?#~{#-(x(EovWWp3rA0IZF38vl&WJ0n`}7zRoa8Ua5R5~wCmt{mtk|R53LdUhQdQn_)=?6Nb4&s zZKYmdo?Rg{B_y9?uD_CN%Aysap0Nd_+XcYBB}N#ScxVsWV8MV2q;I$^CZmmKG7xul zt(CvKrZwZYPo1N4VsXCtH6}(~Ss4c#n=GO$Zg9Rkia91GhF&_s_cbHqfGkF|bNGo+ zL1$kdB}iL)t~UnY@YFPG=~omKt-!o3Er6WYv>?$4Vziw-w;7uc;iP<$R&XKh8d|$N zMcT%@QebI585|X7yD@%E!tegdU!s}htb*RXqrJ0(*A29oadcD54HfWW>|GUbH7V zMtxFtZwwYao2?(6%&zWJ-gLQ+a_i$C+CAcum%XD*O_tn9)9`hUaY&VHd!mGr{r0J_ z7tqkq*yLDiPcQILF1dT|ESf+RTl1tkrNR?#Jd&-Lqm;Lyb7O_BZ)7AQBH~cr_TJ9U z4hB1Bjgh0&NXE(6vH)bBgV9Bi3>EJ)?!&FA9KiMO2UVX477RCu-Mk@igGZhE(b^Nrv8Hc>$o{R;XC1xD| zNaLfUFU$Zj7w2@ys< zw~;oM_5N**ClaANAryW0uDiE)drA6dc6Rnaibz{a3nv@pXN0N^O<)G(aMopq(;;+n zax$D$?U6M1qrrTwy7RSxlwSPInPK*Wl%S2i*pAD!6iQ|$E)!VGJHGPrAcIHHDiA=Pnyod&jT`vh5SZ^em4~uL-1}<)%|B0FnDI zH$JB!9(-!tYvFP}T-zAQQK$dvP>slcVE67FT2R};(b1M`R@R8=OpU|Nh-6O7Qb&kW zZ9XL)Hulg#(Z`P;)%f!7UMVOlhUtBt~;HxJ^az!5mj7_tk3g>Ob>kyxn|O4Rq{p!u(Ceu+{em-kC>rH^p4xB(b0s8d!ebu zohy<5y2am-=1=>tati7<78gt0ug+mpJbTU9zP`RR%K3Jkl!r;|xPWY})~nN+zjp&GmJ$h|xUmD|Fw{{2KiJv$HcyOiV>Z#kdSd-szS> zQW~0^1Ek5|Pbampg6zmBDD;}u&uTEWDvC==aA;NfonqG3))2{B%=E4bshQiW{D=)e7 zN9&QDyjA-PMpIn*!?XELznqwfbTYiE%`LKIo=EwIytzY}PTRTEpt-Y|@E&zoTy}{a z;)P1n`XPFlO_lQaxVojsrR;2kHc@tUD0jtB?7C~dX!LK(CLdyIYO1I>0_b9WCRK59 z@t6`ZF)_fyQ!mzkn9Z-<2e@Nx0R*cm&ePxC$mbkJ!2v&HbsBU(k9loYG{HA>VrHi3 z@%-%45(>MeA+BMc$Mqrs_JO{zcgh1hi4x)`%}mXU4Cwd-O$-`=aRzuGUDa?MIBu({ zfl%!kT9X31s*{HImTH=`grwruD$V8GHN?TO0T8%7oT{>VZFTQ?QcH@|3D5uvQGKdv z)pfZXaANJ6+njgf93H)Wuw;P)bh$jJOI0=#(3V8fR8+_)i*2hlBW7LQ8M*~V(F%Z{V+QHrK=*%qugBdVcs`{g`_iaz|V zOT8q}I?DNhHm$kK;UR5aeus(cR_OG{p3Lklg%O4D=j?l{H>@!8{$P?`M5%K)eef6D z!6HaAs=gdw&d|DQjh0?v$UAC-%qVh@u09jvy1jEaE5cs5Ri2{$@S{~K6O|r#Eb6KF zZwYC)@cjdz@d%A8>)5*El8@UG($dmWQc{wV_DlZGb?$o$EqMfvPEMSiLfj71 z;b_i_tpRniBI7?dHc%4}1_$>C2f0jyd8a=28&Xz99w5YopixvzGR7=ZRfIOfPe$;~ z=Re)2N!a64*Ev5*eFI(@cdngFS{bv`T;Us=z)&~M_J50Pxjh|{#ky{Bj6NUV7iW2@ zowx1v=1V90&A^ZjcyE7ye|Ps~^6BYmZ7q+Hy~~#7GR0>gngWmoAQnKP-Z+^P;C{V@ zKsaas=V8v{W+qq)rVD84hMVgedGAlNi6f>%gM%-Cdj)mF1xzNu`$TWub|7I|Fel+l%2{;}3Jd^7_NuXLBi-0nePce`wwA38%e zl6p2;lviWGp;;Sb#1h#p9^YP+Y+n?lIW_${lm%d8L^b7C0TrQQ{h+o>2can&cJ%^6o3^ez(5#$YrV?IDWV~qa76c+amM?9y4nPY81 z%FneS5$VI8plV#erefOj7pgLFlrifZ=$ki>5MIUEU0v$GY56^&xw*NtrjweR+vRX= zV9cG*>Md~p4)*uCCXr-7!L(Yn9Q5>kR5Xd~7L(5i38(iPfbavU8xex2m2pDGsmaxw#;RO$`s6sH!Z}*q)84 zdAas3yFAxI7P&(2vE(0dt6u;JH?ge?I#(RI#F3`Lj2nzOV#~-&ce%Mb(RSG?8`E;> zdCyQfytz#t!)76lP>_Cvquuk!*eBe+pDw6?JR_qI5*i=R2e`|C{cbT=UpMoh#IOg; zuZ74$0d>sx2r(8!%zK_!F$)MBQzDiw{o;yQ=Nupgkus?=_EB?YSRW@K()k1~KV@s` z;C_4*i|$KOpCK*An{^bqdz$RHoE3V6f~IeIZ6dib%_{AF$>936Mbgi*QApz?{#GO;beRsn#wCL`hO zbd)SsJppjpmiuwE4TpVw-oa%}X%JaAX;U)+hxpK?(n z4~XO$kO;f7kALwv@wedST0|g#$b#EK^Fz=h3=9vz`Tu290(Xc~D%SC6MA^;Ft-#OX zYoZ){pz%5DZ=O#6WwWP zTWn7sJc#!V996GYzFR8K#!b~6yTfG1#$9)vDH53=0Md_-Pj}pqdTWw*eBl=o5)!BGnMO9I ziN5xN4zF~?)ZSI@=j87GzK!$wI|v}L009NW3yf5c_d}gVb5#LGTHhj~Y@`ALF8h0J zBpIoxwFkZ2f#*Nx=jRJ*_kl2^L^y5LmnZgIGeC@C{K+g8z?f=)4yfSLRjlDtce;o< z(9(1?-x)GNSM6Hm&2y$&pC7xqsFl%O7 zUiFB%Jor$zL#Nj3vL3v|+9%2xZfK$Rc9x#i5LCS6TE}B?*cC+Qboz5q49Kk25Z(qM`!e>eeQ0X#9kNfi+NVj{pLCg@An)9UTpX3-0%i zaf+<3Mll;vP*C*7aaW|Keue|a>?uRk?1IPEV5{y+!Wwk9b2 z(i}I(8LDbIDKmPg#1s@73v~j7_4-6~;N;%sv?*V$Du@Pf##FBPG|x^g_A|B>+^a;c z#Hu&!vQ(*-Rf*Iwi&yepHXmrN)O$K89n`wLB^A29>MV&;8xp}8Zpe;S_Hq`jl&1%LY|fEPv_G$s*z5=te%(i@jEh8P+eBBH+yn*#1Q zIhGiGSjz_*`^|AiFTnvPf~a@pKx7oK*#H3r$Rt2nZEkKFbcJhb?gd0fMv_Z}JG!`3 zxRj@-8%`7(sH&#*KbXBtE1euB-IZ}Y*Z-)mi031LrZF)Z-5tFu!@;4XK zj|t>G`lo=nY~wnD^UuD3|Gt0!jD-L1Yiy8#>VGCke!8MHz4Y|H_R)!~kN=aR5oqvf zD7f5B`GFJW zZ1@3y$>Rn@!*EZ3rC(@PHa76Fu;f%#vj&V+!nY9Xe>QYTiPi*%B!B(-6tGrrvrfO} z)V8s=ZwV6nyJ=4Zq4@9Wj9)GPJ$-fe4$I%3NJxi|THD$(3ReGZLAocYsY%pSY_2VR zFVS+E$xcyQ`nQfjLV}{A^zn|3$s?`Z;~~Z%=lFM2qVh}62mic!YJK=m96BT4HU0;N zwDuhJ@8N;Q|Gy2q_v!!Z&KnVkhe+R#e&{Rapf6MSdBfi+1vE~7*Ky@qltQp^`v&a*9t8Elrqsn}1ZgLT64+4hBgt z)O2tubc+;(^C2h0yfOkL*Dil}H2s$>Kb?Uj?0b+=bC2JD^^jbr%?j~m_UFO z9vHZ}^sUZ?ncebwuVx2EOw(zFS$SR)aPmz=v*h7#OA`r6prNUdc52KPX^o%)O><;S zevQCcUee3ivM$^8@>JI2VoJ|;wY4Ianu>~^u1)|*J3R+}AkUsHkvv-}kZD&l8ge-? zbiY0un{z!#5qeFgG0zrjzc!xMB2nipm!Tuh}q9jHfaiW|h)f>rp z79u^1`smTi zA>J85>$DSN=^px}_DAi`H`Ka|h4??1JqJ>rR(gU+=1vF_!gIY;5UQW~|lI)0aX%fjiSx{PYD@yFcIf`tI4T z@H~C?tkA)7B_#-)o1Oi5FM1)Tc71XdNMGMR0ymz(q28e;&U?dzko!kAat~N*LcU>z zgy3Kf>@S_+k<1QA` zCFh*mz0r>9D#oCHq)TGM?LjDj6A9H=-(6!4n`7Jf!fx*9u|0j7r|v*bUPlr3WzJDg zw##|7yE~WHBWL1kyw?8}4lf)?A=A|p5fS$0&Y!HZ zor`9TIoN-ML+5O>KDf#beJAlwBlqbA#z{?37v}Lu&!0VL_&gP83V3*jWx5~w73uul zaz#&hr@#Ns$B%bcx}D~INZj4#y(qPu*Qr30E57Iivq6D@ksN#OIQer{ z4JgiA<-Zq0I>)rI*!uW;NQdF3_DW?NhZ`f7hDNKB@t3=C4zv4+0&OJ)g&;iJ zoi6$YGlooGYX8<^8nFc5cB#~|DVlfb$==?b6|?8BxSU5P%VHX?U`LPXd4eM=9f#5_ zl?&WUWR*4YdN=Y2{@zAU_q#NS^R29f$^r;U)`>K2u=cfOnp}KjmCu3y0)&aEsdiJva5tjwv#`q6bWgJH1sDDu4gCyF#;_c`SA*HVdPvdfv`WNbV9u2 zrQ?&}Nn=|3)V@1+?sP@a^ShqzA_^L2Dco+&5Gx*3R8(Wm*AZP)1pUp$%RKSR0qlGo z?d{R*^z(=Uk4QF)Z=?Mx89iG81iupuNJyUUckZx!7tkw-$yR8)*$Zl@yE@Lmrhqi; zwLW&3Va=$ljLrP8rhDX{0gz-h!NYwm$>-UZCldO`Eck7o{Qdo>r~No>fPfTA&z6!| zT1)a|8NyRIr3R5-Nv96;C*jX7)v3U?{HM0$9Ow3Y#+~?2R4mFTC(joa_WSrf4g$G} z3ozDRMXGq0ySua2Wgl&N&o)k)fa*hdN zRQvj@?<^{c5}#m%q9m($y)XfBKVydY4`U-8k^p-6K{S`e>^+QkQ-wNR=SMrWXDgI> zL#vsYnf8d$GH}*_nyKjqVBkA!Pfu)504LfG(6+32JnMc@a&~aDKOlm*x~enVkp5Ts zS2B`%KERbRWzC>d0tyhE%r}jH)ryFuT8bqA0UK~ZRnmYl;KY@{w$#ehwn8~o@`BZ?fgc0}RmKLt5x3YmA%84`z0rJNjJmlWHOI2*4dzqOp$dXY>4 zdIXS_-EFqaY-~77p8s-J$SAE30?}uW0%wlkdkHG3PYE=Kvu`?R{bS?wTfRqT`R=L9 zFot|_0oZ1M$}we8OI8DMW99*Qd&%?M0poDs{x}`1%tpaG-pd#m#>0WV)z)kR34IbQ*xnP+}2xqCyQCBN@=O~(+hHf{kcNoSzVMT2C*WjaVPUabY^8&WOIsfhDZ7pRyM_1If}ZZV z#p{j=OZ^4c6|VbB<8Zjl2T$o{V-_xN(weJ<{nhdD@$h4`CR**H+T7gfyg-S%s7DpN0s7%*@P2O!G9rq{%CWWWiBz;8JIeZh4$j@t)Y_FBT0L z^SCaF9(RPa1(J-d+vz0f#09l1?pT}AhRdHK^| zwLA+`Qyao)-izMHWk&gjhJS6g#JM-O%%ei9^g+OC9mu3C?ek~TX;)g!*X-1rIjL!C zZe`Axm#G0^9Vec6xumsJTMP^b;vL-q;y9_**AqG{_!8pc@b;#elurn|M_&H$jg4rv zCsbiJN#BP1RJVdv`6DKbz}cKibk36ZW_%1Y{raG|At^4kc~{Rp8H`Zvx-;2j1;} zVp`_h6U`dfrW6y_cD1PO)}KJb?hyMD8*^lgy)+-27YJ48G2Y41+_*0^S1}ts61o4% zj+5{oY&ihfI%i~G|0-}Yo+2U*F5ho%gyT+zX}q* zba=RVZSfbG0{pkg^muI5!1?{kP|`MP(FEGU4j}|dP)1}9Z%C0Xf)$jH4T!<(JXfFVMWMk9o-YN>9}rUP%}~ki7Rz zT=)FB~4-NBSQ1*_GU<>1ox;5d^Sx7ppjlRV`XSZp0z8a-YKl|Ms>D_@$Qc zk8oUFQz5z_vAN#QT}Lo$YuVTOkX#7pV!zA5Y@~{$X^5Qy8z%4yce*xwf6w( zS9o~t!?>+E+~aNtaC{-y6z@NA1Iwzsyl3?!GIKi%eeBh(M>#U+OP!EKJtySva&>)90>^tAejMA5uQ}0mZw!N?6OK5qLE@xsw+%FLLiBbuL?>?BAzFxUrW#!36sq5>Im z-&f5;zFav@e9}b~i7q^-j?(7*FflS1O1!ZBs?Wl^V)(fN$(;Jv_-K!Xk7bBsWs!x7 z-KmL(jYJLIqpbxb5}IzwV-$anIfin}q4G{urtM#U0+y}o%@w$Yx8E=aWLk3vNJ#Oz z90SJ@lacwV$7%;eQM;((q(mS*#M0DZTu<-=gyW(K|+>?j$WeA5p+NwqGhC1xJYKsn=# z*x{soKUwH$;;V;ob5h~Jmy=nc>1IUq6HUxNW=8D%oq5lRPkp!W`JZZFhTWtGfHg~A zxt^X##C@EaN(g#*gfHUvOOnh(i0}$kL9FxS-$Jo_kzdM}^z(SN_4W7157J z#-4Z5(#!Lg_W;lQx?d=WPIZ44SU17r`UtSrtg5hL^1L(S$P1Km%77iqK2Xb2_yPgy zt5X%I?YZLg=1auWHXK#(f~P(|WK^%npQyGUGb>0#!&|sGY2YWu$KN<4AVjU)|LQ9g z@3Al+)B4+6StLJ@z{uefYk+mhzY#dij2R@KsEJ^pf>&lL_Ak>vO8NBbFz7sPVW7}@00H<<G=x;_l$m1(UagI8|i&qRF)nCQ=zjyeoOx^_-a~O^|~$I zWFj_%85;;hPIh*;;lbsZM^{IOn3yUFmW{34V2GHQbgoK~UCWm=AUd}_VZe;#dX0Vv05BEGbCJX}KWYLfNjOCEK0f zp)%Q_STTQmsATAD@(7RQvKUlWEN<4|`T$Lv!&qZD*@4o3Clxz>yeWXx4P&J*c=~LA ze69DGxHL}H_H|u^7U(Y@YyINm}(SgCX>)k|v)I81W*T?%EK-(K zOR_244o}AEc6F&Q$n|R(JkCxyu08FHDyE^fuwaj!wP9jC+H53rt6*C~*gYwymefR9 zKOi@^@ah#NLg{a2;H7GRg~#HsC(|e8T!?_xIq+mup$OfcYXzx)KCF08dx__nerGdC%zV`k=P-w>^7 z+O{urxr-i+gbzf&qag}+UG5CqsjJA&5+lLHe1XHRl@-dZ7Tsf*GT?aaY69?!$vSe< zS8Hn>5J+&=o%=0ySZ~Y;P)FnbJ~61EK)GvjmCcUpo-A(@Dgglj-3ti;0fC*FdS>h9 zMeX{FfM)O*-&AM)#Zk-6GX>4{dgsy39!tWxp}fs1vzaW@#$u^FH&Rp=vel|>cI6$; zTEhg)5%yaIC6X+D5$PqwT5PGhq$lz2Qhah;WB(D_Wl6nRiH>B&d24_8khzVnqcvPR z13ITb{iLOfp0aGp=JGTclB$I87rrRz<0Hbx4vC6#y1qPHY%9SaI!{YZp48Rxg?AbJ zX!-;>f=`ISkB_9go^SQE6#N@ zOh}$3Ku7`?_1OZFX}cYdR=IgkXIKYNnu~^kp;^3S&!@e%vO@Y4^~{y8$qKF#*ZaO= zs;+jrvoo86Fi^^f!~L>SJZ=B*5Dgvu(UWo%FOt(P=#@ z{jVf}MlQaNW%fjZ$uYq^=1(##l>h_Pv<&>_c{1KhH_c|AnC#4jpO$6m?eiGT9fZ%9 zN-6t7M^rYkmL(nTKRLj8$l>uUvt&fc{Aznaiq?O4YXwD9GgJ`gj*nhNwm3D$rJAOZ z;j^Ys6cBieZoMDi2H92;7MqqD9mnx&nd(#}_g8+6@fJ^Y_11x_6i?A+4{rF+D6+Pe zml){_GGYc-UD~;YDY*{~C(eKRe5n+oQ;Fsyy0+@l^-SACnRrK-q%b5uYO)z;c@y2(7gpVHy!^;$vUa+teIPoMk zCIVGhsHnJ?r;9hWi2NFra7oD-*PS{&;6sI&Z~Nlf4T5o4RWk>Ty!29GFb$e0I9x^V zxNW__T`ldctZ)7}%|4&1IwDn#?Do<#JhjrOqwK3Z{JewZdQkY2Uuu84>49|a(5A%& zl@Ro2^Vi@HwNl5zI`=`+JK-xlV_!XEM9kbd25D^~F3ujR?S38FDtnn1>rb2?(|03U+nScZ3g1;8y{A_v3S63eJD5@{CK9u z1Zj;O`2G-$kdyPaE{WULu%ZXHcTxs?w-G?V4sHAh!|#i`m~Q+u;mSwPeKDPpP)~&G zh+K1*jh&rdb>CZsHoHLi?A+O6?plvB0LYCJ;NuSt4K+1U=Bbt}i$m9=12t>B0Tcvj>?z zy-9@=QeZOp|7dCnRjrRfI7s9UwgfxdeBO@d@rIXe92_esMT7J2>qrXox)jxunDs9a zW)-lHw~y$#=H`6X3Ec)0GO@(diqh8}a`QDk_9tNKjL`w={#(!xR4a^-TuzU%iB0@) zS;rYzPyx43YCZ^TU;BxTZ2xcpfQs4co0&xU;<&mrkH0~~WYjIH@rSq1DkMY=7{AAp zExhRo2G30FPN@v9gum6Ay$ftBzW_Ps8iSKDXx>C`QhLI31Mmm;D(n$aqec2TU2 zmAIt5rOvyT9B9FLNkx@O29oG0hgb)Apiw2KT#kwrIhk%Xe7YaIZ1jBE_?#1c(xN;V zzTbw%ujpxnJk(USQKQw3`b87&JfOZ+J#G8T+C;q70dQPzJXTYM)qqc*u6)~#S+0kD3#+QCfPzjQp38lCijb_*((HPz=eTPrGIoe7w=0NAn_-o@QjjP#)Hg5DQg2dp|=R7F(#kb+JpID<-hSIX};jhRH zbn0!uMfQ|mp2QU~<+I%+VM%-AdgtQLc$tPA1(`+qx!gpdFC$ChsNKMvOg`a_X3})# zSQcWZJQbF*3S)qvF*XOIoJkSxtWcPiSaLNnptTK9k)bBA702&#)(01Fc_O%QdjYEH@$HYvz&PWB@&4S?xDs3%6@A zP>`;gCd+n!fX!1(pQet^&W&1Yg<%gwtlGf9e&#`lN2 zGGAIiRT{HB>p*eM5~4tP?Wo5vCqE`agMg6zrXjDU25~r%JbTo6(jl5qfZBI;?bTGf zm-5SXJ-^#Fa9D9g=$1s~b9TH@e>OO^j)@^8OVxp+3YX7dAAxmNMY#P`5Av(G} zPyK3qO|nQpCN0(8*SB^z+s@C&xL=#}lwSD2j(YmXE4Nxm_|6r5(rT5(ox3E1B-?Ko z?hypuUg_^;t(6iQoDj%l?T-YHu_^e&NFESuH(|(lG%_OIqdI9MT05sj&;r}6CJs&x z@@^_p)_<|u{?Qv9%gVop)ysZaqt6->T<1zNGjv_|G$)(BlA)dE>mt?$e=F$c(}@jy ziIK6ccs6+aod>tyOy9fHC10R=VTd5(A?`CQvBwhNRs#FAfIv(!H>DC9wzef|d4Yj} zd=A?hLPDQ{f}~^2h_+gE2Yxoz>lRh|RTXWX9jvOyyEoX|Hy5hTROieBA+Y`ZNp8dY z4rS|!^{|5Z$B-q~_a`obG`@ts)uV7d9esKd_#(_|i8(~roHJU{bnACP zjes)PFzv4O_ovGFUh@Z4cQK{k53MGwk1nGHQHG*=iDwlgD-NKJ4$x?CQ#@rY;kl#y zdcn8ZRl*=()XGFu_}SBC18T)EQ^^M)JTf$5?#y-nB)*Kz_fitl&$$D$azpWi_>0oDXpn|^2=aJ+}4v`wv8ASc4_a_`8Oq#;lhTj z=)&Axt4tak3;hf2}$N2{(Gc;mWUZH&h(x)ZI9NZ@@;Z8@(Yx+ zMyf~e*a4v}0Rhhu(-C8qM_$QEN%Q+=c}ix$?RKmC0*8PG!lX`jJUmk0Xgj7Ub4GL( z23ta_^STZvW>xmqL-;{%+aD#57QakCj?2X`(x9NnB^sEo`Gr`dINdyQpD%q%QFOFL zv}TOz=xTa;0p-P=#_Nad_AYvotsh#2$-0VjRgGSAtHRbo>zDgpS!k`#+j<)k6wyxP zvV}C}ny?IO$&8t7(iWjpehG}XgqDBn(FJg>&m8-XoRoF zd{Vx(`Kr-K?6{`W8@-Yu{fY@RY9Wi1qo9Yy>HmklHxH+J{r`nkijuOEA+rX`SST|Y zN@YssxiV*}j0?+>!bXyW%pqim%(E3j$UIL==Gh9dOv`%SAA5hFb6wAKuID<>xz0I% z{noYjuFYrNpL=+}-}ifX-LKne_g;IyzCGWdy^gBOWRvvtY!Pegs7``LoM+9m7T55c zvYZBUhhE)SIH(~Ah3L++bkVHXg5fOY77_c#?wJU-vVkME1p_fIT-sIDnJVc;L(M`# z>QS*jU7wsww6HH<7MW2bIY46@s z!u{#PaF0y&)!)8Y95eAg-4*S&;*rbJ8PBRMX2iWx-t}h2OrP2HG;x-d)nV!fHS0C} zejEd~Gx_y##L9*=>y#1+eT(=1q_;UpZ8T~YXDkGetnh7nK%ZIm&XQg z-B+Y}c7);k2G0|bri(EjEQGFwkMOO>u8PW9IauCZX=N35t8^1Gi;Qimx)uoM?_<7y z`2OsO+*3L8uv>2}0~}&-gv^DUG4|=g_+v~x0>Noxc-gf~k2a@-b?M$*>+Sh;>})K> zmdoIRQXg7xSa56pL+20@aGn3KtnYP+*cC_FdKKuv5nG;c}{8>Lw zM}<-c_;TFYWOJ{|lRif-xAuD>WXzPPv)pLJ+4rYQFxEt3hZWn(&U=;T_{T@)i_43F z8`pov5-2e5HeRwz4Ga%^ofkrD#8bm*<}PMIS&!5I%<8V&ESsL3?Vi4st&YuFIaQq}p$E6A;^V$E-%$ zeL?@`6|6oZ7$dbz&ah&S8yz`cAaK*hcwJyNy#6pnErJ^s2)!v{`gWA0ttbT?f% zt2%LUN49*gR`{4~UcTv6Wjmi3DbLE(4&%mfn9Qe>)6;JHR&Vh)z4YP(h;&WD$P0#1Kr~Wq#;^1nSh6|i( z2L=x)qQN0xK;XBDO}?I(D0wivAIZwywa0n(gle4FvyNZO?D{)R>bMBlK`z`iwX!$H z-{;>|2ov77UX>+wR+K8L?@N-hRo4bS8*giL5nce^VJ8469(-Id(`or!4Qm<%k7e7_&|G3Y5CF#EP9Bapw zRrm8-b*vpksAd7=u+o0G_|CN*LuXUX2MfNBt@g)a@?{HS(O!@=;S-{Ap4?s?=4F`!seHQ;o6yyxen? zR%U0g>DGA!6wSzrjMs17V767#w&S6uX7`(A7xBSq?))R=rKKFAqO+ehTtM3aasP3K z`ASt)6&+n_eEg^!4Q#rs1fINv?=!8rr0(te^PF?+T6(xUHt`&c21-|IwgLWPy5WTT~D ztpY_yMn_pmN$%IHg7m?x*Iy=xI+~DPo}hq*g&t(|11X326N=Cmna$j?tKWkzG&Y9j zvh67DY^)4zMF@Tw9g@io=JGVz-E<@^)_tiPd039dpT-uHnEib`tXX!a_8i>}Imi=! zyZknO{;-eR%6($e1quS*_VUk+B`Fg!VtAsImg!heZ*O>16fJp8-xtH+;9$th6x*Rv zQYJ6Q#!#JDOH)r2fwKb{n&}P$Uv1hGhVF(ppaHt75~F2oU?2hKggA3tzB&-LQM0V# z-Zj44!zNN~(p|ZzjZ8sY-&VS*vGaNv3h%w>KGE?)X@7hWzh9i;$#{?XqW*3R13PiR zh8_4!?ZmW0zB-MROraJ7dsP`gT~Sd8$OUL{atrjHtfzH z@2Ly@#1*%!V!I3=bTTTiB$i-_!A;@Zf7AibV{2nGzw%~!c9#C^*}~#t%%7D0(dR8) z3IR;HRhturj#X>6rn|TVX^PYAGE&ZD##&-&Rnj`Tx}tDa;l(7{X5op{`|AGc{gjsK zgw3qVXJhJuSjVmY<&okI46b-Gogu-y)kSlrJu6Nq@8c&97Mad3-Z*Fd0tEVfHdrQ~M^q9g~Q&Uqx zLHXjD>XmY36O@H54LMYT2qR7;;dUv;6PMULm&Vj((al|{r1?ctQWU$_I=9vAD-*G| zTR&%I_x1^6dx7dBMGtF6I4Fx9MmOeC+C@YPr)Fm@txj*KBg{j|wOx@jZSdBbL(X^& zHt!$LE>2RI3_K@M8LqTDANYZdmwfy+?-?5#i^MRK%ZYfZTz38$p9g2Jg)LQ#%GP=% zSGJzhkb__v7Wcr(pD$mp&c+OrK27eWCE)!@N`{8S@RrlXShhk663bcQUY?aK6dQtr zscfHYwk67X5>{Sd99TH_a>`kZB6NTsUu7i zncD5c)?&RqJxf%+?{bPO>3DVc3<~EyI|1;*p?W*>lCpqG6}6xImoL(CG$6s(*`6%# z-BwjwEY;5HYklLA#>IE495XV&5?t@gVeN>ooKGY53=hY880zVX*>rv4T+q*ZtgYkq zeELK8&@hFpT<(`IU;d1a9%Qb#3(RCVPcQ~Pv9;aZ@#UZlT30jO7-`_NG{F!z;FL~N zOG{QxjyGv{j6LM`OO6i!9=zq$-V`rc^mu&>gsa%I`TYzBFQqT$RFGi#LNdS@(%Ry$ zKC0ct5vqd$=hvky2*{+^^I<>oa`J&z1y34Q(5`2={e0)m$&b52o#aSa*N5<$73`L_ zjuxx9p0V*}Lp6u2fpGx}pXfb9HMTuQ^5}885ccCzl#!72Q^2GWMt4n3y-~~a>FUN` zAJ)K`QR44@ONX6S&GlT<=XL3_o$NO$p2shp7_HT2sprGl#NF+ z*XpC>XbKB&u#2ac^q)e^w2x*qGTsfY9TRWVXs@nmpX`~C+XWXKiI1;zN|^PA+vZlU zrIEbb$Ko*i+X{TwVHs1E1;z1|i=fn9X053GZC!`T)vhw9SvX$-pllKtMh9XHvHRYB zCFZK6WL9=I-$r>(jsT@^70Ba2k;}+A&tv%{Ch9>N7uZNav$;s7>x?yZKm!eyuuwFgK{rt8u zs|@VeiJ+EaHzpXT;C#DD-s5^8t#HFj=&_WeekILBHtgat*7ZT!5fP3J9gBVT=QJN| z7q2iJbmP){l^t65+ZoE1xTyIvxG#Nrt(!xQphi1NkMrD}oQ9KySC~ZB?=Mm*J225U z-?JMjUG-;nC>{4_ZWEu4dz8T$cBgV{BA8bv+x>S91=KAuyUQhnjScjwuxV~Hn)J=a zr7HCMgz?5kh_Nv!&-;i7VBzxf;A#I{ZKKzWbRvAPxnDMM{tU;F{DKRvH|r>MSmJLu z7}YPAm`XnV{eqpi8z!6;Bxq2uFc>;kwzX1$_XY?SX(4wX&QVw7=Rd~*mUgyX7U-~G zLs9>o93LMKo9pL{j6uK<=UcSEPUAN2$6(^rdL2;Y>_7xCRAh@UT~^jEV`p;Kadq9= z3>|T=IqDP}gY#k<&-E@^O0|;2?W-bHud9y{r;b_EV6$G(JB_rwv$4p&T_5zz8Z+>g zxnd%gw5|&S!)!;np;w#|A%Sg(^+xC~c0wGzQc_dlD1_zS*6L(qXri!{ny05H6el2j zwic0hz z)G<5V{YEhUexq;+uwO2}4oC$6!CP2V1lV79DYno?&~D&)1B(|F91IW^E;xSxoH}wX z#U&-AP1x?%qOg^Yrn`HA80okY#MStAhs(!54Em4mxPwM3pz1rhOI(+m1Q`ikG1CN~ zZ2>|xIHusd_@%_eYZc)<0yY3-h(HDwXu1Q#2V4efDojb|i^0Y`^bxeMYA-+Z*vl4_ zriM4l*tqUnm_>`pkHFXBt1aN%kzJiY7GrT<4FV(^T;*U!NxLjofQ6+X)+xDUHeXrb zGFXIREO8jS9g9VQlmIYhj5L@?9303qLxi(W-%9nv)khP6x93*-3E9i0rg_;s7(!JQ z@%MNg0*fzmvV!2n5e`VXOL>pF^uq}y`!Ui!L9@Gd{JxU9v0R&mtGSYgC5BKT%g@79 z;J$iGjs}jt)a6#@&o4boWNB+hc3z)wmCMmm1_gSIM+Kr$rfJHI#|F!cD`GK*8rw5E z6ADH4lj}KDR?{plYdHT|(iBzHe(M<8p<@desQPn^cL8pW2T~?Ds|mGDU_)`Rw{K~^ zlFcTHBGuS~T)_v1zyg8}D0wSD&qCM1A31F>OPjbAT-5$5c%6%NW1%Sy9Wu)NAU7qf za&Ms4tQU#+(?3MyVr056-8^P)rL+V~$ln<+%1I!y1}u8)dOSaWiI(b~f6Pd~)BN-- zo4AZw1Biv?NtUw7H|?^-si7?*Ds3D{pnvbz`uJf2?q)fG)O!`P}xJtYO{^oRhp z-OvZV9E+uk2TPWJ{NuRR)1Dj#r67us)YL}1j%yt;*K?=+)LTy6^1c7*ODdrwNdZV3 znL^kI-zz8FJ6z7|LNCdF^;l7zf6bNTjwzYrTmrhC#Xe(kIhO@{H|6=2QHq6yg*ToP z6ZH)Bk0%aLHMomD{u-aE3g#8lh)Y(XN9FG8Y_Y3~mp3lEPyWJHd8$v1tb8ALGhjs; zPiHzL@|wO1EgjCSCLR{uE=}l~I&QUzZEX@5tVmx~biJt_{e0-zEb{DC50JXTxw(v3 zA(x5ObQfoUBlGjhIQ)9JjMD?b*X(|F_c?Bi>MshhRz}whBh>N?z{MW;0 zxK9i^ju{yl;w$q}M$hGJ`xD$u4tRFbX^{R0RwqOkq_MaSt+!*}F~~IEo>x>agvV1- z%;vi5#ycc=5!9HECY;f`9elrVy7csMK&IhAw63(qdru|3tf;MUV;6H^cmpfg!n^D> ztBuI+(-VK)th4?r*bU|&E`Ua6{b$h5PQ$ysyGm?4f>P*4=77zWQOQ3iG}Gef&FjZd z5y&{=-}Cn%J$@|#i$n*DlxshZDo z^*GOw-4=}ZY|f}>V?55ON9N$n&1)Xy`!SQu!4p;CTp`Sl)qoI@%QL`a7EIf!VpC7X zWa#yz{Cp8g-y_Q{T=b^9ZTArT@2~fqF!D!xTS1GFZC^5ZJXh;Ol-T$$|1iv-s;#Zv zBkp25yGP%J8$Zu>0foe_hEyyroWgERG*nmDC>0bGw8>&N2lD9HNdsf29Ek@*QA{!y zik@fU5dO#5xZ^L@eHmV4&ww*izEo}CLNYrS78KZar`h*TW}c|EI_oz>ou}rFLTh?0 z0|OGoT}B25UQ@4%Y)xauo_X2y`6Uh9ulNEQ{7_4OduLYI_^|m+SJ&^SV8jctn?`w4 zwQ#IqeZ*;&&84>7B{rvQC%n^I?!H)$g#AXnp{<+OqM>QkS~bPFi;5=+bgJi5VpGA z;k4(XlWQVvcO5ZdPZ!SItowYwfcSe7Yv49w!%wYW@28-8xtY0Qt&Bf6M_es<_FQ*p zH}`=b-O2G@g-h7@h$VSJ18L5lVB*R_pyt8(%mK1)k~v;A{m{Up}c*X!Rd)a4dr zfk%jR`rB{i%}YoBdE^tI@R6VS6#h%``b%h^2U7(e_8gu;Pxhi_T zxDWIHXj>wsZ>I`b>zvDWZ}*>v$YWiZpD)g?Wu;q`pZLKD4Gi&@wEpKkDqG(!bNg6o zRV-`H4v{0B`*c_1Zkv6gglM<+O)hA=Hz>$yf8A7|*_jT0`z>Q$mNcE5{Mc}ev6sB# zwX3r1(?0%^T|;1wCVTi{^!aXFD!UjjeAoZ@N6xKaY{gzCv!!`Ca<9R!o@c$(!uu0= zfp%xQx2S(a|JSFdc-=_ZonH+qy2zBgb}t!`QQA2vM+3VYdB^X_F{#S!Skji4p|W}! z74pRGPs#7%NPAJi&N!d5heS^H^$o3ulhzkfWCt(%BV|=JBNzdUu?)uKvB2>MQk>WL zG5MCQnN^0Oeapk$=U*d_z0uNo=QQ-`oxW3pfS%au!3430&Hra*f|r)==%m@ggGkRW z`WzH&IryRAhN59>DvK@ZiR%aPaEK=Ad6c^rb;|`A&KpaQ0=vVS` z&ums+1zr+v&oTJ=bYfapy~X*~)2`!w@=2eJsE^HAhZ48O`t}!+`VC^rg^iKJn^zyd zf8A8ItzdRzWhkrJJARE-`et;whzw_EGrJgj<_VmmzdO;S^ci%{UkZ?yBGK**Rf z?kutpp4j}oP6|=WgSCA4y<&P(9JA^cWd%|tbx5Rg9#7f}OlKQQnCF2ed^Epni~ii& zC86xovV2a$=|27FoI^r*TrOXf3}O-L<}w;f#&#p7qjL@msgt!@6Ji5I7b}qkK3So} zKC%V>i!^-|1lB;wb7`1V$kVM2za>jgb0l?VzS-1+hE>ko6wXA(SuElLPs4A2_4IW0 z0CA#W>^u)gI7sLdtjAoDN)+DP2!Q+3@R@Nk}aoY+RA2okX z7Sm}9LJ}>2^g?VSkb|)m8ZgPlx?b-7XZx1e8rLR6*o8U$zjZB?`ZV=bFrYxkv%~;$^@(mg5}RM z!O8M5Tcv^5t4Z{x(^1HMbs3hJU%#7`RpGQ)>>@{F-9=6IjRx7Um-{}a1;@RACWgu{oWDo2?-Z@MH$V^jhCa5 z+L=p%6z;@VBSL0YPWeJ+mM=#sV#*j3p3%R|UzOlK<2K>oD-jS1) zCE51?uzMI10mDvS16;)3#6?qfmo^6MRy|~+`uqC!*Maj@gL?#C19YHJf!-bJuig}J zfJJ)Kj-*K4)d9h|@l}iu(RO0ZaPMEprXhOFjCg#06d2!0!~YTQ$OMVi&l% z(=kO+rrz6-Y*MYN{?nxlM6gC$T9BB8h>}o{NL+&G4Q8ZbbI6v8iV9%CF!s9fr}p;C zKz_OoJjBh-&8NM8zeJ9G2Gq&X{Mp>>gIf36j|&jxGHU?}&W46OBqRg|3JE;DLTe4+ zf}$bMaAg^Px>eY%w!1egiAS z7aL<>^*24s5juK$9F%!sJ*lawVeuU`G&+H4g)UGLY6cJTqS3dqi01kekbR08*d+&VH~V#9O-2;Ezo@0l4^ zUNCfm($^`kVH|G%JFWT%2+8#HG=tI88pwWL2)iQ`y*Dk0xq=2|8Ks3l@F@?%dja$+ zQqLy7_oV#-{u39-VgbQX^-PSdFUMs;LCmig6iBKWY8YbIvDof_gb-qS%%?)- zr>6&KE-dT(sWHTUOuR(l^XXK5fn;bf^({O7U04P4oVZ$bcAt*D=SHXg3MXg~psDiMSm-52xkEt-Zj2FQsa`G~Zj;$v>M3GF|LKGpCVXBzqTdm@_~VYqm?RQV9uw~oCG7*9$k+|2_j%M{v38>k4^@_{IVZ5R_Tkn`(a%_=gcHPr#5BY?T{--8&KI}8=|LREHCtS@7GHSM%7piWNLz9mP2Cu z`g6QItdN|^uTNZvw{x**Z9bkiG3N?TaFniFgtpr(Lp?Y}jzKqiW^S~GW?a5OIXA9PSnM@-+K*Z4+GR%JQ& zyYAtBh9?>A(!X!Znb~`1dw&0(TR)v=F4WVx0Xn)g|3{UL%;&Ff{W@f1LVQ>;6X;WRc||E7m^Uj!a}u*(ND<0D zB#eU*RpxZ>@_RmkP}TtrvDdb)MHab+IxcM%8%VdlCTC`~Mxnr=bgeEpO!oFr(*69> z^E%L&mzSG^_xb3PMQm?YM)W+kZmwkHm6UuHNr;HBvuE_*XFjswYzgdR*Jt1Dltr-* zl_n>rSbXS*#T<9x@hYPrd-&Ix|X{+W!eR{Yd6uuPS*!D42?J& zK|UKNjTpN52Vz{v0Ab=kUSuPO`y+m=@Ylm^#eWG?iHdYhrIQXg{plLRis~y9Lw!_V zJ++nb$k&b3YXOAWbi4t0cyA6oJT-m&rgxbMS}QEA%gc+@*V&t;^)!2)a&WUzS(TR) z?wC9}FFDlT_W>8ughnT+W;I)xbcikbHZ13IExOWu+IS8vokCJ?P;cw3&VOcX(K~9E zSzb5Tl#r9%FjU#_64e^;d}l{KI%lOyM=SCPS^Oq9Iahen7s1GOu@akxGF9=iD$ip2 z^xYuD?L?Y#pMAbgJ55{OoK+Fr>>;1@G38P|DWn!!+e|C8%H=M2x2AVKvbT4i4Lyk4 zU-+P>-uFq^u)t~ONncd8D8o@QA6h`Sll@vg?&q7FpOC+>0{z;)6=-hydHQKv8+9r* z#kxoNZRPNgB)TYq2zwh!Ft^mx>29T@l)ZzKNV?;qvYGG0o@EINvR_k30`b?YR|Vmz z+M^b|#ujE37sSzldOBwID+&DdtpRIio;{m2%uhk1ZUqhzDle?Rh^fUX_Pt(bYA-K! zYzd}oswQkfe*EcKB#iPu)4I%?yTZfVul=AmJR)N0NxMv8$;Z`)4z>zV&X}K2yH`J* zajXMBe`@Z1x%75bV8>4?CM&`0vC8t~+DK;j3psE7a)F+~ z2c?+x9^Chg4*Bv@slGl&RW9M;-O}y*D-CoYuf6S6dma9C+N*5mKu*t1wp=EAD-brK zw{gz%Q+wRsd+#=bnu?-x9y7j@Z>qhP*XMuIan8Xu$iY2_=Uum`<7!fz6p5XjjKd84 zo}1-LdHNSmGbw&E0vz1)M?21 zxoc=CLwov}hYBiT`bFihv@b}ap7d>#G0PEu8z8uShpXGalYU{W%=Ik z{k?Ou*WV=eB^->G4jJ!NsUY6R_9103r`DiW5v-rD#N8-CH7`Y~FhG=;AqXjHGO|kQ zV0wOu**GU(jx7k-#zxUU8#Bf`5f*Ye z0NW+s3I=-N=T2iMSirM@ay=kk07-~jErMcTy39rf2jvaGE`WPpqD^=1J?nByt2)bo z$%lFQH@TK*oAdMuB3YN-4Ob8&@>QacE~$Vu{9WuS`1icI zmCwOz1qa&c=(|Iy5ugs8Dh$u%H|?%0%s0n8bI=r%+}=^PXfnUtO?Qaw=@Vo(z0d@) zO(g^(2T`q%-fcKh!GqrK&GkmW7yGMFQa5tx)gBxiglG($8NA)99^YmPJMr?yW4+Aj zDh(rC_}>8%2Ir#%rWAXZdFDa_ER8$fSABi7zOYs;gNSmSI`rCn0Gu|6<7PN06VRK3 zci~DT;Ekm8RgJyT+uxrEKcb_f1AOoLOe^9(ZuFbV0*iN#eg-`YGPlSba%*clx&UEB z0f1*$cmH>QtdX7`W1u3qVNe6B0(f>@TpS#U1>CG45N=CKO0t|vKearNcLA<3XnJ=E z-Wa5a`$;oAG-TqvJ>74LLEmrn^K&fL{|v`D0ibY;!T5c1O5M0sR^IZ_HRqm~|Kn~!ps?3ub zby=dbs%q=U&`WHg_|XbNv{gc2{b>TW9)yYd zThl}kZ_XSBjYG{+OF097{|bN*I`%+twJ4^{n%a?%dKExcO5s7*<)Mi9cxlLo?2AJH z05XxG0J_lzT$nF8=H}<2uMo$=kCD_xSkx&$T=RTR*-Xs7l*dPit*9$IB#W&(lfk8? zjTVPidMM6>HhOw^Jco~y)9laa0>`mxaHk>a>IQ5l+)f1{y=)^~(zpAJYK5e?bL)H< zB%Q9`xKRz0mJx=?5v&3BwN_;UtW_Db-3Yi)`;m1@$dvFgg%4C+!f$D+3xyPw#p-A$ zpgPB$1hU3T^Uwda{vIr!e>j*HF?~CDK}#e*fDWr*N?&peK#>*TSa9QvTg_1ha=H5t z9;`vOfX}G(_fkQ-TB{(h=lZw}tkoZgfgj`^gbe^#Fsb%J@zGa66EG6UB|%;$HYTQe zcND+b4p6{x0PR4E0n#jxdwKF3Qay5xj==cq1i=jBsQ}H2*ifD`fuFi&;o?C)fH|*C zH6c>E&;jTHz-wAtyZ`A_+TGa^DE;LN9FeR91u;PWp2lp|g`$_th!-G{Ax9%} z0ODj)@CzjM0I=6i>p!6ENNi7Pq$8J;Vlyqa(+5iyp=+Z5-~k+gE&zQ11T7@}GU4j8 zPoJhrMP<6^biey@3=X{wpO?o4&g{g2h51bcg9KyM)jv+lRw~u<8*%U7ek?4T_#TR7|)6 zXy+q~p^cMZ(R-Ubh6-)c6TLlX$l*c_;&20s5Y${g9Gi@;x*+xh|IeoW%C{`|^fcY2 zgIzP7(vTStoSCvmL3jVYodPBtE~bb7ZftBQ#6>KmcXo9_B4Y2Nq~tDSBNKo_?Bh{( zkGUjZ_c#@h0_h7Td)UJqV}L8V6Rrk8GFnJF3k)rBf&n{ciUEx|qOgqIX?p&Ms1l(C z3`JWTyo_;+r9K5`3AqNi{0EK@l&WEBGBS6@AZ<}l;kpdEeb2b$%NPCw>)#qJ4BTWt z<0VBYeL)P|TwNpIzyD`NUitM3^AeykuGoyIrsEg`$*ga)NW{six#>dF7)= zqXi~M7v_JOV6zwES|03v zNjyuBUkav19da=>J~##-7s5bRV4(xS_JM(cOYw?|ieO7JMsM`K>~plU^8$|D2U(Ef zjEsuHENCW5;stcAVM#j~e> z6$xX#TYUY^4{)}=L#Py~-C8#Fe}_u@TBTPY4!lEk3WhB4^w+n4vJjw0Qf&C8B|B?t z$X>ASWCPaikEe?B24A=z_Wm1Ni*a=Ev6F2Zig#Kd%pO^T7c2KaYH z9dRrN1{zfBOyKv)P}yq8cfbeQahJk@gbocEzxauRZN$HT3oaTTL(Dt~S@*%g=o9gpZfD7B00I9% z5SXvw{S?NRwV%R3&WFvdtT@=%AiM3{+(79wGuR}*HFN|jK871w7yEPBcT(7-gPI{P zFJ_THq{xTFfUNvsgH($x4NxgZyu;$R*m(1va`PF87!UnAt9xz!Vf|FOite-g(z)c| zHptp?P_n^3|AKSvSt^tBpFe+qfExTO$P+ow{EU{P*#n=0W$S5u5;$O$m6gY7x%l++ zFH-t050}Vp^k_PgxVjt`gHX`|JZ=|S7%=tdJ`SJaw%L-}djc9J&*SdK~96o!0>adWXka-mSiNcOs!xvkhyc15`Rk{}=P`>u(Q`0VSZ9_;}d~*2gK{LzL5EaSS6h_ym;6MD7zjDWjy;8t{4OPU&@T;?lp5j6 zHV;Ah4?w9p?`&l?izN~9sym)`RkjlF1Nc3lGNEiR$?RU5@wrV`i%v2o;KzQ*e)8pO zyDu7yH*E(k#-{1C>!U1tBR$TOkrhiH79soA2e+Rga8A31Be1>|o7FR_8*o0r7+g`q zt&7>3#%4EOb>NIl7UU*`W@V|%5_&c(ic(vE-CQQWXOfKUYsg!( zK@r$l8ND&M($iCA*uooBz|KM^pWFkNz<+JM6X^j3t^xnv>y|^Y0Yiu9iugT}EZZ0$@!xlw$=t(*Vn5X${Usv0tA$!Y+%~#J|L*5*o+)b+FOBdBFE2sRG|E# z%$r#5rlPItkXsiSseM*i8Umk}@&7(}ZMgFR{jIviWUuAcY4s85;>!H5?SCorW}Zl8 zgPe<;Dg@g;tzmZ*i>L=TXM!G_esI`$*&=jQ{6^U8O_|3w(|>zqG{SuRKJ)RAPwCkc z^AAG56<B$bxR|$0Ujd&JjI@(>9yuxw;-Gyfm9DX8)&{kks znJLImDv$;KI>rm9nh29i962VZ=B@I^nb zh5AOf1-l%<4UUtp+bXE;;K@iw+e76J&c#_5;@hcnVU~VaybB$>UR+UN*RBv1!ZNIR z(17z^y&f6I?P$7QssvnKkLFFj1!m#a;LLl{x8|pvnz-@jgqJ_2NuP4?o~iNYG6lw? zoo9!U+1J-rl`%G9NzmmmGR%AX0X6nvTA9aH@*`?w)A*kmQ$%J=Rj2>Man8c)j%EyM zuhq3qs*jBg6n^!HA{ub1Ec0!^H6thKK^n>I^2;0SWMakN)d0};A1uI<{-YFQOyJ<@ z8+_5Cj~8%XBvovAH?G1rPT;OVsK%9L|D*8wAU5xm6B}#9yqZ1M(-U>;SzAiVJ!EAs zaY5|$E$wa;LzeBjV0+e;4UU&R$_0x;(iz5^#tA%vGV2c&&R)8QTvU9?3^bR_E@r25 zkVZb>XvboSIqq+G@#}<+g2tNGCUY%hvM!%HkULdlvot*wXFfq*Xwjg%b|D?_n9V*a zOnhsRQFx4+?5fIfkif4tPnd;RTcUpSB|1P5WPzrNC&y@ zaHR}sl-<}By`i?49|RMwsS!W6`#W_y!tX&SJkWPA4zADkKe*0lC@EWHPz+ka%Q1U7 zdGl)%I3>O>pWC85!1@;R9CT04UV-bCqEawzM=T^4hy(Fv3Y3VfW$L^1cq#Y zd?RF#&VJsj?eLsx`20Ot@7V!)nKHt-vQm%LLEj7m57go#m?@l*#=@22yMqZmW^Qj< zol^-CcKXIC$T;~B1NAizbag)e^C%sqsIE+`*Tl*--l66+_WbNe6$!`yZY~|XQqLs> zWZAQTm&T!5US_qY%4{f>Cylj7%Hy#vqBtBT2V#)D(_oh-89%=%;1PT_I53)U{RV&L z$npmNWLoxRL|nfAFa(VE05qq+D|$@v>?KRtj?$^r1z1xVRLSPS8;!bB5m;_9H5k31 z1+U$>m6FChbb^T~5h)?yK32U^%}?jJ~zfA5VyiW;@Y z=Squ2qhy~E2G*KuWf0Ned4C`p?+<#fBP!89XaEP=$XF|KHuD=-SYlOfd9vpfK3A+h zFzL4rmg?i@oNL{kcBBTKu7dN#t$4GP$Bz>)MC$ZUA1GF=Ix;}&&*!3*xi0LA7N`C9 zEYpC>LFaEC>>@)`Nw%Fs#LGgm$6Ffa{|b!+X>;5z_=bkbc+9tY}D=luE|k7nAEI{Y-AH0=JOGV|IC~utr1D|tV95u_=N}+o_?HHwFtbT1gU9Fi-`!CR)Ow_xFg~cXpyPb8VX5P< zcYaZ`A&j4Eit|4+CMRR4j*$8P<%ou=XK&AS%k@EX-2k_?*ehXwKcJJIUprJ9^ef-* z(O_*eP0(~NtKq&~>KVH5?S#MlDly4qK96cnfJJh@?m`{y+L%~!vpnDP=EC~husifE znUZIkyfP>bkqt2I^+0j0kJp%9yMH>eQp{Lcp6@4b~F`i<OZ7Eoc^bq)C^)c& z4lW^sIdAg8;vn|tKsd4ujm;Vm8q|R^2d`7mwuc?X`6!@o2G0DuL>k6sT;PRVyy+;7 zts0M`seN6MNa}QR6x`sm-Knj29JSxsqF6zXuQ`rwwh^jVUb!s~4|TchM`Pho>u#mP z+B3&7&yx8`1xn1`#!FG-UA>HeqhySV2SW)c_g9gLU7BC>2g~%~D$1PlQRm(?=c~YQ zC4UhWxt++k@Q*6-=vkNyzqo~VIfPNl8S-vG^ff3 zDV|qft-Wn2$CEJa_69?%9cU+INlx$xr*FdUl6}S4Bs;T7&^sxu=jBP;N$xjNvy%bp zgD4e=sjxcWQ(^W$S$|%fk@sU%5 zHvQ7Yo;$0zN=WPTIA+gyw{eYG%Lw8Ih{L7jk|kZ>+G#)6`2T?$cS`#N@?jb6?K8)F=@bHaU%9JW`U`o z{&BKdnOjS$hcDtpn%q}V6uhAxr}0>5V}$vBUV zD6x6vk$__Mu`JCpde0bhJV*3(_YyT;s)Dq@hz7o1bjBc0D4( zqDd>cI7jH!eZHK{S5+-ByRug+*S?wV{mOxkUyHb>qaD@Xqb(x1I5L0Tte@>lBJa$) zpR)v={0|GNK*c*zsq&QDMw~$avM}1^gxl@75dnCzz$DChaVVC2?u$L>Zn&$pdu!pc z0CD@uZ2bOkJ(Yn{XhyVD>S}s_T-nelec|S^&;D=j(36Qq8yWE1T^Tm<%$-cE*GJM3 zhBi|iVYN{f*8#-pz>D$-tHKk6$BC%0P6$*-q zndyyZl{WnC2}VWW9dk8QvQKh@foSV<&a*!Br_XgdbVa}NdwvHTx5fKYap7NB-^vIf%9mJifLl7@S%{@W9a40k> z9tijYv{_mmvZXMJAuoSLBVW$Pq> z8?eC{h9d9de09J7ZD2Au&}7nYlWUv(hBjO-blbEPDcrKqwy@BqOwzkQb;#!tr||-8 zfe-J0w*85O0`&1}tatppB(yIpr#+8=`&G5IQ`xL&u9#;kOCJU&{ulW{f0Cis5MNyT z;SzFrCJw#AeQ&B<>FnAma2)G=MZrYSJh?snuIY1C(|*VchFgJlZmQ?Q)zg9k&69d2 z-|K1JVF=FnJa>_X?5fehV8H%yzACol{y2GtuG{Z%MgVjvYyG7g85t$`8Y^wiGKmHV7IsWvgkuZoue7r+Q z=y7!fg7!7U-=T6wW}g-|b;gw^%mwzUo4}ax?c2c*`noJMT;=fkR(Wz)7QVC_8}qLD$!9d4Q|!vna@hd3Gx&M4b6*7FQ!^e!4%hBo3=TBs1Rtdn zH`r;i@a$N;k$|qLtWX%=#doLxWUf5;Vvj(G-J}~SGn+ryr6wdcy}pe*{SNm5Cig5V z5o`twlMfjtd>$hk8+>S}5SI?zG(BXY;qzS{mIOW=2k|oeCGSp@2mGNo|F2VE5akw+ zX~5r;Zi!Kou)xx)59U^V*-gRkF0*;6@4>EjM!xj-h-`J`SNmO7=CR$HmuX8P<(cI` z|8q-Ycx{CwbrrwBi*L7}N@aub-Mfp30CntDlosL__0iORZ%RRI@T2*OO`2LI86s@& z9+%DphEexAO1K&*cLy==?^y@V0LVd3?%(fb$h`XcbbpPe_Epmb1xi7|C%p72F+)MP zoxQyRjC9Wt(RitK>()-3I!j)^Ew`>t#0IXb zJKS@j^QbIwOwzO5DHlyVmG1rPW8QEt*O@>X`LnlFl_hhxr(XxG#ClF#*^PH@_BQ6t z?>AomT@!#8dbX?4r1*G-mMXXO&l}imud{tp+)DT7U; zdhC#9bMu4Q#O(_3ld9$iY>)xs8)19df<4h_tv=QRvt3)zVq!8tFK=yQ&^a2B#ik*X zoc5U4y9iY9>hr(Ai#?W+GA2RXaQrZ&o2WnuJK!#R;UqYuyzy!Yo;A#kA-RhoKR910 zp1)>kv>YIVQbcVx3)A;rXhzm1eMKlO`Mj}C!jKhdIdqxu?eVoI@S;wfv4%!*`~EN% zJg;4o=}(%QySXLH;+97Ccfj1KtLHdIIB|uyo3B-|TM2I`Bp98&wMe9{6arkqNkcV6 zpYN!C5XmRJRF6liDjBM~sBWjq%EHxJZxThGYfb)*yNYh)FYv}pzow~eO2@9uO{VK= zy$@kC>?WL)-Dz7^moDarP&0Pz&B)MtUU7T_eR{8PLw#>v^t3;Xw@2%_-U~d%v3g^K z&ru9T#EsBUK9i1^)+q%_E9_!aV>LQgvt8nAIyWJSu=k`fzR_#C!tpex*QAqZ8`@#8 zbk)TW&x-^ROo$nf;|ViD`MM9jdvh=_y}uu1yj9_Pm+SQerL3#2F7WDWWtnY=k{=DY zMP&sSXyxb756}O#!v>r;E9>xYw_Ce8R;kr9XMW7>=e%rFTg4k{&Y$rG*?O;C2knh0 zIb)l!A|g=v2PDG_-=E)YR=4u_~APL^4~$cM^-*OE2`oZdzYbz z9}R+jYbm%`Sm;we(lT?w^P|Cm_|)>wVmc#csJWVA^ZIy&19}EDQ2pW=?OuxuPk443 z=k`zdJ(kh^Z{CkSis#kac827~m#2bxC4{R8BdC%*TJF$h^EFzokZd*$Hu*VeFWATG zq7V9%XYIWR>JJ1YyECC`)q9d9{01FEp1ZK-siaYQL2F}NWpcWtuD-VG)U^tB)Xx=k zVzc>kP^DdO)aq$m+0Z)jO#b=UrYPM{q{Ua#p%M-Tj{2uK_jb|=xJ^QOKifPKY;0-< z%We*e!Wr?^7h?f50;JYmc}hfJPLIjv;%&H9n99KEuHH@w-<5T3Te=jxVFAW1>4N^j zHkyhh&WGu%3p86$E!&Suk8T?}#%x3U!H`E^2G{v594L^gG32ULxsoYzMOrxuRtaX`Qa@aGH&jI7G`EJI7w@yGLUgZ*HOi&23; zEG7K26kMw+!EO20@qe-Ro>5J8VVf{EETD)8hzJM@C{_AHSLt19q$@}-k=_#hsRbH*03zZ>{+;KW6jiaI()nd*8d<^;~zN>MEeN zKfTo7uaVr>fa9U61I}6uuoEd*3AlQwQB#@6M$@mA{JcCz$2&|CFx0+}=|L``?}B+qDFMso0matjLa=@4<9VGc*5l2AIizc_47{|ART4>fXWH z_)Z|(BGZ@%Tt_Y{y0PhUN>R`qYIK*%!Pf3|1;a}NHW9^C(H;y)u0{XcU4SK;vg z(c=GBS|~J({Fd+^f8xLlqaKOJsw=uk`IINwCYuY8${YU4xD!P7`#5^T>aZ)bmF4y8 znl{_lRV|LpK}q+I*M)7Lb8-VFae!G=+~%-ZC`528gamabdz+uFF+3IU0Ac1;N0pCd zy!<8B`!oVAR~#$<2WIm>T^-SU`=?ElJ-ZF)uu6Aaf0ARykVv=HW*j)s=f$s6JsNoN z*9a->+&wCB+i#g2rI6b6TunPe6u!OB(Ab9Smg*l(CaCm67@fIihyDq8(kho}RZ6y& z#4AfU80XNq!~eL=y6oI<{l>s|wowZ|@eu``BOWLCUcbfm=kZIdA5>^Magd)s9|3W4 z6;*AmKTka!z%iU8{F3laA-R<;VgklPJY8Dp02l4)D8@V|?YhP>wTLsvnByHP*r;D`_T@!c;(R5C z<^-AvEIGh;?O#hY8Cj0->QOz2w1y7~(k30YK!NYbH1D*SV#I&B4M^>?ki3+Dd^7nM zKR6|wnneI6=Ox4p!AAI@DM|rFzj>z?+T;w7i#?s9#^(=<`G6|C248gQ3c5ZZzBf`( zplSnq*?VeY`)#G5pyP$_WgPqY3W|bTAD927R?5zHRwN=VN2cxZzjM*h*yxoZvHWqC zI7N;*YS*o-`7ZTGoCP|e>owyBbyM5O4p}rman_O&=^efFa_^5yNolUtm$u)oo}0?b z_a8j&5A}n!*|gU$e|A4_8&rzdj?ZEKO)&hrqUv=@^RvQjU($Bu zzaDr!&|+uwnz-6^N}JO?w{!#u zRQp+6B$>tcfFPoJFP+crY|6{64_;dQzZ-0Q#v${l@A6Mdhurj&DZDnH{Hdnho(a)g zwzC1!-|9Yoc>3c#`yRvz_SE7!Ze}A^ul=46< z&WaZUPlsIod7bZYcV4O4D)DcBm`<-L;QrqExo?Kl6n`B)jThG??6s zUyUSeMQU}QE{@jDLKc0W?~S=I&jSZjkL&o6@o|cuWzJqm{P#9oKXfHOhhnw1y7u6Z z<@AL=&Dp5e6%91gOj~N!jMa^EV0=V<+eQYboQ#%M^pDMel2HZN7V2# zGrmU~L+W3!n7z8JWVSn}rIj~5Bw%@i_qgpM)ydGwB%D!4GHvf8qiiG2XqGw5sSWBKuQsVmm#ekjV*lG*SNd12**8#Ec3@M zYBl0(`0teE8=fwv7oQchUtPG^8nl$rxJ%gMpZ$ezN+}U?Fzigl83-@{Uc zPV1fq3A_Q(iy#T_S)^weoY0G5YR;r%x#YKLw)?rzqd=cjS->Vm_So@8$Om9zUSj@P z+nPOE=?G|X1Y%hceuyOL>j* zp@e~s#qJe6_M29hNm+n5`MQhV?{pD?0a6vc5uLZXjbHWp$3_>T@P zFC~qOVWkaUqW1V##*gYm^fe{tvtUzxqc7Lwk!Cbv9+9Vurd1J0eS22eaiV(sMC8NI z@y~#YK-_LL2P4UwR|0qKBT3&rsYR-aAR1N^jER_%i_AXwqodx2%!24eFVt!{XNCv5 zyOc?nZg*>s89!cJi#nOh5?bvuof(SO_+&c+iF_aFM^YM)^6e@BjMyDtbI}mnCmdJ4 zyzF9PG-x)lS9cSOoEBeko(+lvnvo3-}1oMD^^jo|wV4Z3IK88u1{c zmVk5O1u$l(w`LqlrJ;Ck$$U`XVW*1?5dvwEO8M|}1nTWlGh57UWadjSZ}BCR7^5YQKQURG?dAHJZyNDxdh+=Z-?p4= ze}f{Ay9*%@6l#?9W(ItPVzpTRk=z5x%}hy%kE-A} zU{tGEPusu8Ogz?Nf$yPnA^xK_R0ZY?cPYZ0kLEiP5kA$=riW{6SJGQxJ{Tq{GwcRQ z+GVH*{3_W7dJ;_qSS0&8Wv$FwUL~$98haT?+IWH}Nzj;qTnI7uhgJ z1J;89vc!uAJ=T=~!fR*4K|_n#;UofgQc?Q^Q!`z{&hT}6$O%Mr*~iadV#6?X3#ly) zHsSag&UJ6>M|1q>X86X}`s9|w8LHYm8Sq4?AqcV-`cceM#TBvx_lD;5^~N{M^-oE)5a40S6!OKDKh_V%lt3%f`y(t@2>hUe%E zT*BOcjenN3mk)wX?f3SQ>WfJ9#TJonGC1_ZgJ*zz+U8YK`;3fFoLQH`Vc6qrcQ6Y)LhoVzt@rlT8yFgj01jVxNlndHYk3cSD+NQtin2{Ki^+DgUy6Z3-%O@LZ|KWc z1$8e_gi#J&=vEhz5R1c@O&g0*q`N`9(2Xs))&hA`lLqg}pqLH(vbn$WJ~t^|OLo-@ z1bGFULP=J!%yTL-;dl7ctK7IgT)z=Njj5Oi?*(>{;P{Imc-1#DX=3v+ z3{HJU;=}q0;sg`xO#B6-!%17=+FA~~Swgp3e7=%ov)A%-0}hIi7hTVtO0kdfvQ0l+ z7j64&`?GsXlQ2-);ySYwA5gtYdLoUKyVA{(8{D~%hxh|QI zaXD1eSZBh_cPM15k8vyyk?*1eSt-E4YTOj+wpTd$DLd#(rM7;7rQrwHRtJY8|MBz% z9R%?gA4tScv%JO%T*Q)uQvC5S>kQ_n#~v&ao5Nm3e&F6#)e#naJ_XACIIvcg=VvOLspzAN2D@eb|LzMkaNzPu7n^ z(S7_VDQNX^f`-IFZwvlFMVEUFm*@1#dAJWuuiz{@MySB&xCYL#%qy^m!_8mWU zY`*!z1LME+vn<&FVMiIaFl5}~-@{p&-Y7D#=KOYPVypA(_ZWTSe*_~*hwUfOriWUC z(#JTM%*KXSQ`6M9z0iD4Q$k@Wi=^G9%^YPXQU)U31KBO>ZDBg@TO@s$1rvAvL^uVX zlgl8xn!1=xqjiX`k07ITL9*`y-w@;SP3Gx3DXCBJ(u`8n>B0oo3XQbjm8~bEiK6t0RPrV zORjioK3LB4q4}&_*4Rb<(_Bip?-C2ms#`x+jE1MHyBCbRqaG7cM@`6ko zBhnXvSF}bgo)>n!y)UlWkq{VWP*n4<4qe7?cCt9^SpZr7Xw}gHf_aW8P&u~0TgKO# zLuNZZHurFa$?xhPJi{fbV@cYkdOSA)QFie4(l~S+Y{XEd<2DSr4H&JK&cF_Lv}BvKkk6)6qM^ibbJDWzhIk+9 zXfBe!fv6$tw|ZXo0`M{Gn=cw~f;7s$;R)X>wMuuQ@=KFz;)A9ot{go_6tB1y<}SZl ztZMekL(@MDo7|kz_Da7J_#j>rJbDyJ&#s)=**L6*Ph@G!F4LcvB!$9nIc2s^+ zyM*_#S}x=-dC=Fm(hK9hQ!Zx(#12>4qAImycd%i-<30rOXw^jar_c4x02!;dlq5Z0 zHW&gli9}XsR#s*;H&+)|dmQbqBqXfVY|ZZVQ?Z!PKE0GLBIvQRywS3gTp^~(6kRfn zYsWFmyj-s`cfsAqWzCH?#Ds~};3x(0dMbzCM>CCO)b^Lh-;z&~?wZJc8)2MO;t&~?Slm}~V`RHl9% za^ud2wUuob&Dydt27UQLt$Q z{Kb7d%@``QItE8zu+jHWB2i}~M4Z3Z=_So#E&zHR=_3ru^7M$Y6TbREx}a^>xW#YH zsrMMhRw}vK6bavJN7tmHo0p>Dd$S!1(Bm;uMi~iot$?9y5}?VOC!KhWiL!YkV82`W zG3~8%8}A&;FKoSR2JnIp#hssCl2Hr;>Zf=2yy$K zu_pJIu8sF1QH8F}um}m_$@?&{(G=-Q*z)i|-S*aE>{>qN1l#DfDQiF9;Q-&MRBE(j z1mKIv9*CT|Jyu*}J(}OQQ9_y?8CTypM#vnq)WPvHB3G(=3*zvxQ zXEDmY6y)|Rw*?=OPrGsC=sS`DmIOf4#wU}98=-T(tLSFKWbbxHnf-V#^?>-wPAf^+ zejAHFc0b%$XwSK6w;4g~Jih1{h*eb9@Ilb?{bMVgD%8N-OvrRt#m(N#NZTHuK-Kyg z9zqY$f>NP{;cbVNDIe}DpFMvy>bw^(8f|9unb#m&@r%M0aC24M*&N%j#zykmx!W_y zr%!`zJWy0;LV2#MdWyM{_+;XQY^*aTtp$6bh>Uw|&Q^X{aNN-vlLoyt>CwriS_0Q8 zTLoYhm~~&1)im#E2KF1*aLk{fpy!MGpyYN(Kd9inHSa@FtC_Uc02}bfu4IMc(Y5^& z=e42wMnB)Lh$vP|j#70qRB|PNE>`J2UrS8S?9n-H?ATLPPpf*cJr_7lwaCj=Y~1Mx z2~=y+E!JFA6j59K$$TvRda~PLj2`FCKkBve?x`**qudjd&-hZnI$p7G(0;!Jc*zVB zDJ^xHB0#`0=#yEpke{E;&O@Wl{mIw?UQy5Q?^@r}V>&`n{r&gktYK*iyEeXp(XJhk z$}wS7oi}lP>sZsIf#EiT4P77PNT< z&hXNpv7z42+p0SO$Pyi<3r+S}Tk~MkXRwFh?UuV(R`Z%}`|r2}7PlRc8l}Fl9>QK$ zGHPzDbnX52C%(~B&UpY9v*M0YMK<{qs_YqP4-Trh3TshCRu?9$_QVURMEtf`k&)ep z0{KQC{zEnm;26|fdq&K=)Kes}6lI4x1wr2n{n!T%q%$Qprk^?z#o_gy@N)c8|8fuu@* zFaN*v{Cj)z-6^uh@A88F|MI47(@n|d)D^QHO0jW?qUJWArg5Yfs(q!*@NL$V2eSIQ zyQdA>x_WAR0@Ra|nyE4>&dw>+%+57GHO&D>d2P1N&5lVCvVoVvDb7fW{ed{tuynd= zj;p7*=H~~hdg_kPyItEG4!Hfa#c0(b*iTn9VyfQ1DWGC`s~L5-QzL-FW@l$- zz;JIob|qVOa^t~#uPj&8>m%{mqqX3*7MAX^np$&f78Z=jz7kC9oUG^{QfL5xL^m`- zCM5&R#7CL`0um!88U%O&wU55T?s9=%i$-SW_l4bMuS?uHnaPq#pA*?G;X){8=?Itj3? z8-Qai#5yG6leWBkdiB%kF^?BH*9wXuIBZUZql*AA^wZh-H3t zn6PD6M^v3so7E_u5^j|WnQFA7*4ju%3MmfE_=t{dW0frv3r^_vOXud?$3|NW8n209 z+1Q4@uP%&5(n;&}f%wcs!oXO#z{y388DAM5RAz9nYVls5=@+CJ6f%1_H7B^WI#U}O zytV`uaU*=?7PTGjEZu_kTn90bpg9ASYBNH-^ka$aEeci7mt z5M#rspn}DTF#n+a0lbx;B`tM)VQLsFAIjBXEA(U*lt0hG#wL(!P4x@cwH}MJ=ikkvdaQE^^eOFjp`1z*uqPFtR3 zcqUx$%QJ~Hw&LVG_&sZP!!7EVqHMW}6?xe*c?$6 z4U?u`qgC)cYzCI?qs@DYyp6V@6GA=Cj6r`@JezitH%2qf!X2R7lfAX*k4hc4`{=^vztwxR;WaO5Y6=-V2NxUH z%W6^&(h)vuS@Ks0@1ug^pOmw)S%}Nllf)<@k>N*T0gza^SwrDGQeS{rrn=UFK~d+& z#M+1xLRYwUq+jy8T!{1`W?u;`CH$P$t*>8N7Y}Vge_Nh>4>!Ewk6FY1an!Ze-#T;k zeo7}NNVM9dl1o6T(|^XUFUz1BHJtmpS`_C0Tx5Ka%3Y5^ttz0@m-xxKr{9UxCwB*;r2FoyhMq+!|U91%f#ZwotL%XuCU-=XH%}C4OkbmEmy9S zFWf}%sCjX-zJMg$LE;5c&VdlG<`w2is+=8-R6<8+3;Rr+5Old`DbLi8W*^Q?j z``K#Mn7JXkeZ6hX(ncdRn8PL(3|)^R4gRo<7xnADF-6ZO9tqp8eJGeY4)a@J>E{e! zJ#a#;{1MHiqoJUd@B(MsygnWL$zf^I`MOJ7xI7+#Phji|GKWvZIHZ7)61mu4ZU|Ei=@mXW3TyL4F-$ZlvX0x7BTt7#(SrG#2X>_0WQ z`6Fkc`11tRp>`}+z~g-h{Sy(3emC$kLGGmib%#Ce;#M1X*MO|I$Pl*g%+11G%risN zK<8Pi{B<@qww1PA9yYeIG7Ttp8r#Pgx`1B$LRTKTwy@4}u@s{B+3!G**B1*mz8@f% z95rR3c_}RX7-Ii?J~vnNdGUg2ozdFnZEEW3Txd?i)hiD2@|R_sEk;0#HKq}dj9`%H zn;n_E4-IQ|TDM=lwo2f3Ft{Vc#g>xhar-8&$Gjz`tKh2k*u_)>$4JMY!p)-=+Bc~O zmr6#Ov;$^s;MNw4F*&YVLzBW^+ z-x_sJ3wIwzg8Z^BE{tXRXrse@4s(Vk@;)1n zfsKQN_v_T$prHbjp~gZJ)S9%vQLGxMjq>t2XN}xMQUf+^5*y?ky2HjEHF7K4MGuHPp#Qdu=&zqSNx6w9x)%KC7CUNc?fSZgCSTv+E}z1V z>og1yS^QZ2qL*3ayU(iWWxrmIY;CYo4R3^(3utRc&r@a1gpHK0L44X~PQEa&33)g8 zWxKg~41V~iB;oMn`=Q!29e-UhwKk=fr=~gmm*BiApL4tR`l~lwU-zc(*mN2MXQl+8o+WH0_N694AI@=oP1^n`=}Ov+IH@(0 zaR;MwZjU+Z^OmAeATA4;YWwA*M1T9w@)(oy9f)P^2iUpZaB1#wJic%t zGmGsWb5Pfk0@drsD3ZB zxf*`tUwzD%6#vsSwgnZRv}fnKG*A+MDfWijs{?+e3<;e{E&GulKDeJ#@qX(3o7o3f zek696x50|38@mpsPaJ7c`x0V^;{NA%TWWnVOoc}?*bI$a*~xe z9J#|Gs^ub@b};_P+W%-X#T3N5;->2-oQ_W!wm)};-d#Je)9~Ikbq0R|-u!C8%II`J zknM1Jd?7!ukYfvbGXq-mJ!q(@*cOh|L7-;b&s5J|kXV*#@P$VDdo zjA9C`GxPR$b}C-o<3*d&0&iBaQRFR$!+~g|r%d&e^m1F5SrD<)z(TV$9lMI6?+hz+ zzo2^4BcQPNj{lbx&D0J9SPP13JKDljzuL6Uy%kT>lE z)2d)7I?YhX>a~~0V8JL?JV&}#{eiUPO&8QfHLmIbX@|G#rVyAIpx9Q|>63i1^dh_7 z>m&Bz1EbP%D~|9pfol3iX4S!oM)HfKL9lE!S4xL|p;jVF(qTYKTqyC_AKU6nWN^0+ zclg?5l?vTCMlkI*Z`zN*nV?LAU(`Ll(q&8)VSO35FBWOpN>%vx>sJQb`lr_B&mAmP zR0`Ybw&}8Gwr?53y9>uz8U9}5EBtVg$3|*IapJ$qkrmzQ$i1JNt)Vu7h-@y&Ppnr%{)<#LOmmwMBYllu25K+ZERUM@ZXvf1p0mA>MQe{4lj6?WAU zfRW+P@%(>GA^O|gZ!^UI*mM+DkBj!dF%bOkweP%}3ND^k-?#yS{6G?_?Q}d;_p-uw z6m39nm)JXBy!QOPDWiOUkCvTG^YbuZ@%UG#SJBEH7COZP`CVH1{+suG@$mdx%PxNK z!Iv*LZm@Zwe=Ky|VC3!y4bkLO%$jNDR!dNp|1grrI$06&?e0ga-f+f9PaZQ9N^)@^ zee5~>VQ^TONedCIR&2De&h_Ps>-Jn0lf%e$5NL2?x&89x%hCA#4uqDVO#+YMeFuZ? zG()WKM-!c~~#HoLLtWvawuoYk_b4Lwd`ZAhg%CPt*t zr0y;ivvR7aPJYa*e*A;%fjE^B|3c>0jq*#Ze}*sS7j@YFBp@!;_QhuL3!Q6{c)_)Sc0Uz{Ri1v1bRlEw3R{)4Y&)+lOc7qwcS)p^c z+Guwv`Xklyyqw*zqJ&4x8?x?HP*ssSc{b!r4{>BX!{&+5x2KR*pHsH;*nfzTo2GvO5%WTjdd zolbUeo}MN*`QxEe(BI!33(EXaZ0veiwmI`~u}gB{6C57H9`@nGk8nnyJDf1(orM^2 zz@ExxSgD%8)9vp1>iv_aa_slzpUPQ{^G#MNxvtU`8=z+s!x=m5J>;)?tSi^M>U{ka z$IU738l`mZA&|Z7_luNA7ZzS2m>HyvM8P$f5IwG=O`*$|q5$f~Dpgq1Qbp=9rgDY0 zw<{|Ln>2IT4E)%h2NpA1=5R>&t3)B|UJhiVT8^qteMMdUP=Q0O@G zz__-jJNCxS`{Z&%uuK=rd~=faOqyh^t*{C-H13B*G^5zkfO6R)&O<<+W>pT<1Yy9fl*-5x`NFVfBdxb&W|7Y&=lSsP8-VEu{fvL1UB%Jpc*ZcX2>($yY0VwGty`FNbm?>l!$;t;o!PY@fgTabHa_%W;wo z1|=F4js_fl@!yX>fBsRQc>J@Qz!>TH6Da6k;haN4@%$G700z5p&GixLW|{nrbK6pX zjl@xDx56vzs{4}sXgrK7i9eFip)u~RDxb5!eBlL z)1mkaa}0m?@Lh(o=dZ^3j%GzemZc_RY&zVtn>q_20gH*=G@{6q)cqduTf@$MYr0lES1hMIIZyi^>$xh=ol(v z9o}CE@965?oCHp_JCX<1vL5xW-Az6^U)~|P&gsbhrQ6(fx-j*myXMdnYrke(MwJ7n z$400b7*-LLaEr3Sr_vvno+fof?q`MIv+PpMQlxCSNIduV*%vlnz6d&{5Dv4F`oU%6 zFldhwOGsfMvghe-I(p79`rfJP3cJ|$P7Zr|b8}$Q-~=5(n;JKl#|8hIGB@S4Urh?G zjcF__(ln~|CJfaS3){EPG`@}k)@7}xgcqc;*>fvqcX>6(QV>(!n&apC{2v-G4}$-*j% z;W1@p<6dKH*>V>FYW5^lL#-Y!0Ef5i$$duA`!&#`-HMmzEDzo${(CN}u!@3mhjLH_l=b<)RdHQq}0D;@y(UEcC%tP|O2ZA^7SuG_CpDZaYZ-B-B zZkCde3ZG8X#B;+cWe$xP%PzB7pYP|_W(?gWjuS83clh^Z8cF?v;0#Je4p1rlt4(jh z?s6iqMDp|^9#FofRER!zFpda1;af;uGRUxO03ZbDH^*Q#; ze?B;H;f$uoVHxF}q`W{erHDAQ<|N!5yI9R!kYwYZ!!K&n3B{9wb{ObhEL4K#reRxe=4zttiSzr^sTKQje*Cmk22|8 zk9rEQL?($J?=P0F>lP)cr*pBZNfdCj+Wp(Z3|d-tU>o7FK1hklrerLGx7q%>>qf#S z=92;$PA%0ORlyf;_1cIWM%-Sehu>-&YZ3GH|9%|~<1Wr0n=F++w5&HNF= zzPmr}ibEGRTp(8sza3F0ggZI8Uw`MpwH2vpAKo z^Rcd8yx?xD*57+NY28nO6E^$BLQsufFL5R;GZZSwpppWwMacP>$P+Cy!$W*RX&0 zSoRqkglfJ{>H2Fz-A$ffdG=&d8w(2gdMcTjj$mFRy_JFZr-65jX`;>pdgbXG-Uz=y zn*a+i$5_%mD)`cnO>6Smvr8))Yr|a!W4gK7dJm6|Y`VtI-@ghd?d4x<_?K`+1P_R$ zy>Hk(BRO3Ei9=rfP zA9{35YcIK+B67J$z!?n^oVQhZ{iwBEQ=OFU5}0{`aKbXHeZg3x!BJp~^u-9$a@He= zMU3<0VU~fv$piw?@}g}vl7YFl15L`;JOh;_vB8`kaV%UX$<(vO!e!=Uxi)c4?|3;( zs&*W8(1?vbsz*Avvb2HOcShbH8(xM7qUYg}jf)4Zr3K36On30un0@xy!NC~uy+Vo2Y0jfv_AT-FpBiDJfrbk0?$(n-i_+wYgVSi+ z`g)I^BmqwU1_QL-y^lun%i4KQx&{JAI|9rFpKD@3m6ofHs*fR{kQo_VWJ}Fj3O#zP z71nVhx}nO6`)F4so{N9$SwY+#sRwBEwW7=3+~XCCU4SmP|SZcJQ>e$Ux#QTs|ch0%;*zPaD zEoBDW2$5RkegDM0W}~K7`96Q;jhqxbRb2@gQ3dmtmftz)e8(V|LNg@v7dW+p+yq34IzcIOb{{w?Y836cbHLr^T&f6!}YU}JElOO3T@38wIP z*K;;IxWuG(zNJc&q*u5Gc)6Abh7cP(SE(*X(gJa;ncZp81NOO zYLMfMug)|P29qW+JaWP9+g4`gjo>LE)yF6MHn~svXV_HxzJ=Ty7b6vx`-}p>=aJBg zyx`@rDUviKEkrfPqgNZgvF&1RpmQ+c>&k#YU`Ug}8i{P53>0#Pe3l9hcYY+p;wm@7 z$R~ucfV|^x(GG8=i7sKE1*F;PeG4gBQW$w8LNvgbtm+bp0|#>+yb-gvrF} zBdZpj`RRT)*}i;}w280X!8Cj#jU(bBmdgQ!b$Y$#La%;_8D9;(WpKb%RdUGrewUYE zHQLrBO<&CNT-6V%U2F8-D@n~nqERZJ|LW_w3M+O&ogm8Sa!V)2W`D!mA~Cew?Y+J8 z@n=XtK*qLW-ztc35Kv|&KeE;f-BtnY`=7o!fDC(SOHrR+jGY@+Vs5Y;Fg$tBp)i!p z=bYBKeE1zY=6@Iq%qnU*)yiJXmj1P4$L_sp&H?G4U*+Df`$RaxIqDre9r2pWJmVl@ z6;?i}M0wb(+LckWIHUUH=cZ4iFVxCmHlV*3k?c7aA#Srjpb=WfI8c&WdD0+cPCM7) z?Jo9EV6|o@PTfk@7G%V~(iDGNY`2cZQm2TTbF5Un#%W>ZJ=5&fl(GZ=Y`-0A7QOB- zwA9sF$?64rmoI9wc}5aE=RMdR_E^uUvf$VE;@1goq+2wFJ8e{;dPa*Y&mt=C^Xp{yNyN8rF(9k` zKG!Pd4E>UFi*rI|xjlsx4q5T(WL16pEb0KU+>l z-Qy5cl;;5%5Gw%!N_;jOZ-6xXqH$QcxRomCwk#Op!^J2G?A(ym$Jz*3bIb5}q~vo<$oXvVtOc{!yz{;t z9z8vq|t9mWhSu?kJWfQr`pgMY~y4=h(T6_qCYC{R~`J+`X+@83EG(DtOuY z=l97cY;5){IcBAR%R5CXy^ELOxeoW(v=9n(Grr{2M=-hQ2&0!$5z6=I3+upLd%)2F zS`#638BI+KX%@-sw%R_*<)1Eq&tg(bRp8_>$bL!f@{+)NHr36w(O6AciR3X0QEm%M z^DI_AnR$6Rpt(R^2_YES?T3gKuh`(8{ei*7aY}JaNs^RLXR`9f;-lwQg16%j>%0|& z>({Z&hd$qw%u9h3j@PLdqhD6{0uCz6bmIHl-i$U3Hx&fTCxv|oUq3iYRCZ_9g#`an zg=}x|7Y}!pz{~Ids=0L)2mm2haKDg>W1JZ5syafyYuzJ1qw`s-r3h*{bfBhGi}aOB zf+9N^8P1-1-Gs(}?QgZlxVny@TZa?X73B`+J%GdaGAU68M=CGz<0m2!PUn8>)*c=R4Zgtweg;h1QP$gYTD)fM6Kj2)_tX+# zV)=q`rBUPev~u1xWN`x4R~2(DK(R%9fr#M&ZiSnzph(}ypjXCDD;TcaOEe!H z4-)~c?g~LCT3&Y7@WqSW38q0b6f;h)ojvo>9%V9+0SKCoj)KYUv*z#KWjh*vOuW#< zaMpG%yl#ZK9K5Tm``S@;VX?5;9>;P3`Sw9lcgu7_w^}#^L3W=$62rl6jlhM=rZq2@ z8cIrkUxkI}tvcTy%q@MheLN1?V)^6X(vI`tDE(+@fYiFId0=O4N3C4yvR)YAlbbXfXHurC@^NS} z#}34J*sz#L(Ocwmur)=?30Bxf`)_|!19C!g`AS}En{D>q*9ZgLj{d{ma!&a&IA0s))r?@m{nc5JNAIfdm%OTeZ;Tgl@Ugj7VLdYFpj$ zW2cg{%8I)y@xhk>a}j!`yu4bKcCrzqn&|0m*vbe*=fyX`o^n%%B3e_k#-nD4$^~5+rl|R+-K^3c{EiUb9SM2UT90dw1;oA z>dj%Iin)LJl=sHc%h*c}wX8Wlf?vYt*GsM!1{(m0I!4%3)qms+EAgjEdy26>RXg;l zotcqXtDWE(t4sQo^r>2wmxC>{Tf1_#Y>FtBLM15-ZYNxWHmvJqCSfvUJ*P(5`2?jl z-|Bp|5t+91)ALO?V0YnMg$#SaB$oQVJf|Kjc@iqT0e!D2%dC+v51XdR4IUGE-^kUY zXdd5hyy>(4q46G8XM3$EnNpjI<%1%*dx2560pgAmgQJMFZp60`+4s#)Z(?Gq zXHj2oHMD)Wxn3rh*w40xWae-DvICw0e`q=G@Zw~GNx6E2@wUiKG@>dfb}LO zGD*sx#9Sc8{KyRdVxWPxet%K8uY^t@Jx$tu2tKTqm{Ei-(GA)EhDvUOPIK+J3<^xS zN?)WM+R(FTZgC=beQlD_ZwSyioL&5-R_kPHJN9cXr&r#jr^?CnaPshJ&iIBNo6AKT z0;W+qS+yAlWJd;~!Oews4v8-YmmG&{8gflbZ@lKArw4o(aJ(vqdRcijB*3sj-hSk@ zx8Y-pN!+({*}Pn(zNQk1JA=qc5eFbG?r@C>EoNb>L2M_$uw+@g*exGs?eE`LZW8G4MSZSlGygaDjq-+1I)S^~NhPWHC zVqfxFT0QMPrufUgA88{3GF8_WtH{E0GJkC4kFBlMWRQHhK0}*&kWokciH%84H|6p| z8;W=}B<($s)KXk6(2=3v5z@(XdtxkBWK{0HvjC)i;x&V^eZS|GQv$2mu0GOxoo}&0 z=l2Da6CusDs!9(B4@~UDuJV|+Y)mXaYEwhscPi;|nuWG_1a4{}pQQNrH&pYivnPsa zqzbQL#GiQWPxg9k>2O>w{ zX(*yDw(zTZ3KkL5MuprP*&HqZuDhscs)ju*@DbHNOc0=vbMVsGWE5{LF(`=(y6ObR zqa`Qnhf!;l!JD6SgzFjoC+bzQ6vJ;T4%r%zNOKh(!J0TQB3YFovw)}Q#mz2km=N|^yh&1r`Pr(WxwuX4dvBq zuN9dz;eWNMD;Sd=7168-=~U+HE=S5bf{mJuf+$@aMILAVwV%C-+dRfB*UmbE@q1cp zqrK&$WMwV=sW}0DA=M&YlW`A+@?+a5&YG#_4xc7W21=C0f1&Ix!`kecwozIN6xu?u z;!pwrK%`c~fG^ z3Ghz?TD4iQ@D%?19xcWGTtF%OhL~k>wLYjLiFJ404sGclC!1XQhY#Izb63UZFq0N+ z2|KsFq$DlN>VK0g28qac2W9*pkdqbZUFDCyp=z+P zDrYX*(C{j(`j7pYK^rsu%h!k!^$0=tLVxe2%=LAtA~y$KwC{w{34!;Y)}<&ii&YuJ z!s>QB{QhyztB26Z7P?td94^$MVw*oE1lnGs4a6+49xe{tT=MZ@U?~Pa#^reW4~G1; z^iSRHl@%F(kUGlk@8k2I*6JShhtdjAU)ctytwd&l9F-Dl-2Z~|zqex(Iy7>asN6hX zGohw-L$<|U-QPNnjXs*E`3K-Gx5{pkg1W=8A0e)QwjUI$b|+j@Tswq#X*!B*VXpq5rJ3(@GaZb!VaW_eV{Ee}i!YxXWB(ZFl=Vfl|ZE-m;Y^nb7iGDL&X?y$V(HMWRDo0rK{(^H8-*V~3` zgeY`p_pNNaLX{!vGh6)s`N7+$fV0ThD`*fS5c9-O(5>J9SQJW1%#a&6C`IW){|~0% z8|z3m%Sy|FO?)(A@*rwrdd01YhbM1|2t!R z?3HAAp)Oar4*Kh!7C)`Ql;pHD*b;}yb}Nw!TJhT7iA6&NfYZg|sg+UVIkGvzB7Xm= zpxwH96#lqeYn4&!_5s1)hQBc9(6Itgp^>VIj|~Sag6>iya@?<_2QAJu82Wzd zGZ+a`9Qwl*b|-PGF>U`%Y4~=Big^ori$Q(Xzif^j$Jn721_*n}g<_8JH#l?DSqT=9 z7t$(q{s%lD!R@{kb+CFLV6i({IEIF7CFd=)pMz15uzfGr$^Q6!lJwuv$htUWNqi#; zyBA`9A?6iCVSd;d3@8r^)hE9)(BW_S#4#qWileajVg7|5Mo$rT>7WL&cXbBGCC3 z@Tbv(znTc~1*I!xG1b)kT?7B300z1$^!UibpS$`{p`pt2G3GjBKTBMw%T~jg#QE1b zO~#p7q|4RpcZs;V_F*JKe@(nUEuX^b3*i#duh+NM4AqR2MiB<1M2=d!;%9;w7KUIx z0UBs2)nw7;U}^4;=j%$-zr@KX8Teejf6m5hKN$96lnFz^E|oTv^!XV78SvTsqko40 zv&yh$V!orJYai_!Uks^P!Re7v_+0S2e;n*+)&>p3G8QW>D0L+k$J<+Ddn*m7jIk!4 z{IjF84zEvP24|c2$%AA?XmaS&b`*(X&WoEd>fdg-u>RW*6-%%}j)>a>k|#EkPabTC zngbUx$u>cu!BXvVk;#2eq1ezdryAWCr&c5+l`@%reNIQKjvMx`YHEI{w5L9|HvMOZ zXB~D&!ltb8@K9kL`5Cm)5>$dj;h!_a&uDAO0=igSvDO~*7kJUpv0PupBX~%MQ6Fq* zmYNp0-g?QV2EY3^AYVexvyDi};FT_Mmw79}&22dhsodv4WkMj2AhfHLgpLOjPHIZ^ z6AraP*9UP0b4xaq;;X2Ez5FLy{tL7{;twfOp|YNSL{EZZ#LjL#nm+aKly&m=Ssau< zd@HtkH&U{H1dUzj*w5f2R_&y1&=zX<7$Ea}ovb+HjsI;=*E#!s>)z4e1JaR;TI|FU z9?D2{|CRpltjW{!&B|D``=Lf$&UiGZEtes)53Kk{rl6;@BuBtpb=HA8+doN^f^vPm z%=2V{%5JkCW`LT-dqr%YXHUi@aDFW761GWh7}1|6zMAUnVXU@yE7jSPMDv2|6?gyjQMHj5-44P@V@zY2XH5|X9)LY?hz z6d-1akKeW)UNN$TWzO;ZZM$hN0ygztW4s{+>_yGXpoXzM9$QY6VuSzf@HDnuq^-rQ zwZUpAt16zHRrvP72)x$R7xV^KKe@@BLeknhZP8tQhXqTWViulpJ>A)-#l~&@;ivDz z_}du*q`ueGQ6p)@{rA+lZTUn!O!HX%<Wd7{^ z3rU%F{1jfItNy(ct@vN*2&sHyJjLk*r%po)U>u{Jiw9!y+k zuYZkB0nM20e0P8TSc2KK83ke`5Y>?pU$R>kYzjcd2v8CPLK+y~DO0=qpok2RLi=dB z!L*)rhFRl3@}Ptmd+`DZrP3uo@SfXi*6iQyR|n*z$KzFN{5rP%8xZla@S7BQ=?GnH zP$fs?jy-N&Y(pSY_E6U~x4$dZ?GrTC+_ILKE%|TYgbH z6tBuW8Zel^${eGus)D{pAf<<$T@M*i?+ib?Ad~dd&)Vhtq3t zi_L>~UqIS>HuGkeQt!rFgqzJ!5Y%$$0p$mFZWMFAKWMzCzU@gYZdVC;~kCL&^%z6Po ztd#^VJp2Q)5UbgxsN(}N2d*FZG{LZ!+g?pVV;vXC7k6(tR}xxoHGE-{q;tx>5KrBd zEUI%uFDT8e)mWvJi&PrAIvfQ(0qHCbxqfcUmpZ;Sb1mm4+@CWP^0P>imijUJf?2G} zZub{eO}<7(Y_-0N;mrCzUtg3hEalDD-kCi=<_oVA>z-aMHi0Uuz72Q+we=?G-CkT}Lie@>3w!SzlkVG%4NUv#tK>#^UJlZe zgy>B7lI!j1VOb*rAhD+O-jS1_feBAc%KS3vh9QX=GM7J)xo z9zs%zP_F74kLeq^=>8#~YaMeC6EpKe zw>q&0zMuY}qG%^(z$~5ks10Hz_6zNPuVZHByQe~y1o-wRKhtNP#g*%Os29L-O@MYT zNvG@SW{xE@YvS9ACH+ytV)6R#15DP3H@nk(0+c7sX>vPJK3v-^|Sl@;Q!%6l*H%!ghA2Jek&Crrvh4>GanO-0Q_LA2Mwq;Lm(I>kDxk$~BPt zfc9=d1H%!uIDX17CKMfDzI>zAoUwuhoUL*_9Na1g-+GPkq9PWyQ1}e<14sXLp=vgi zI*6)|9OD0}ProRd=a zH1k%c@Z$tdtY6G(AirQh@*^L`s2TH_9YZ7P331$+uFHZn9c^uXg| zUp-u^0=j0~7~e?52=l;)-VjF-v8xr0tW^soDUrre1(_=iO^(Q5p;B`$j|Rnv%Yn^* zkOhn_u?DvcjKGECSRJFc2SYYVe@(Uuy(#Quw1I(y`^MT&=O+WGS|v(Xim(p_fETKK zAg!{QXcdmEK-M>lr7A7QI5yxtHv6&EKt`QE>w2CyvJb=T`jfDJNT$4J)Tc6SHj3Ed z!W(A(*C=DN4S#>+TR-^=66ymnP;20j_w7}8gwLyMgv)W|?Vh%BYV%NSjzOruv*_|q zFQEyoO0Y!H=un<1Gwn|~0bfP9MrsOai94*AYF!@7%Ist#`SRCQqDwJ(7 zOPM2PEJWdLHGTQQKy$`mTLs?K>2`S+rlCjvZs2;lq&=z!z!i=gxL~T#&OIfz=YMcf zaBNNvk6ETntGY-tYUjt8W%FgWDN*{3mN*}#_$u?qt+bz(Rlfqcw5)+p85lW+haynDrWD3QG<4LiQVz(?|E*M|Zx^1i7wG zHQ`VkO=Jr&?2F>hRh-6mwrwX8jq<={ufPuL+b*5l>FuHGNfxlYLCe|lMtdVBeARFB zK5%9VG2O4Tr!@_8+PC%_YmN z>UO#V!V=oFtSEPycgEU!qq(Yalh~}|?@l!22L5EvQXX(`1J9Q^uBdOs{ozz7693*u zcLQ6P8ZXtBp8`Uj;zSV=lN`10msJ+bmVv>Z(?*T62o3uc?fumzeByc>96)IVQwTr+ zZNY_Z;219s{Zzv7kZR*R5YdOAhmRy~d^gMqI9GP1{1_7CYT7%N${ZB5X4O9uO30Fh zaJ{OTdz|nsz^N0CmG)=p{%Bc@w{i7wLBm^Y;C{0?O)0fUmi^jjMfA1f`GF&zrW8G% zA+M|Bhj{U-Cs-z{^IU!do62U*;yIddnslYR^NX)T;k`pw$|nAjDiw!6W3hs+N+;|Q zky9mj_w&mw&@zi46&ohnI9Lw>AIsTtTxcVqv^ke6^K@>$&$=f~XyGM2=Nf(!U78sb-NWbXqN%FRX_Kcw1iDeOzK`YMhnp7(RH~Xh! z@G%qey|%Rgm+|HvPeQsn_urVK8>un8*Y=~^{(2nkK}J`3BXNFi zP?r0-n@%=|;)xaie$(v_eO9qMztY@K^)&U>y{5~q+FY;Hr;l}BjeHpil4)i4ODd)3 z9y3Yq<<**ZPF0~+a`tMTnbeL$1N7=ktGF+k8&x;X8PAa)z=1JeD{Re36Ql+eipsDk z*d0`>I!qJVwT!uW_2*K#rL0CA4Rk@@37)7kghNRTZw02`a`08C7i6X`OtoSoIB=e& zb9j&9B*!|5PlHMgei5lA8qd}rE_drPbGeWo?A+yZQIXcjDVA8V-%GP`QOP?50Lj({>RAqIaV-MZQ*X({Wxq=> zzZGd>OM8oO59Z3qVJmCZ@yvpZOt-ev%0kz>L#-$YDZ>B{!y1y7C8>lFva%B75rMi2lv%x#YnHY6#%$!k~_8l34g*$3_ z?v|@}`zK4dCN?;I`XF=azO=@@6-Zcqcm2`Ydp5V+tM}@C6N(+YtNi;*G^w&4x&n2T zUyG9g{D<)WY@~VAM&jUBhmxTY$^N^EE*32CD$6pikUx+K?Pg!wvp24%KOyRN8q>)J zhhyURhAGH$zU36#EDfkI4++yXf-DkJ032Ig5`=0a85neGYB&Ap3T!ctUYuZ)Y8OJp z^}0DP5>nN`U^?|&P_&d($jP4%uX9~5eJZpX-{5AL7oE-c9>?@Tm$ZFg2hx*P0}*_C zb2BX-oZ`sW^76#G>*gv(CUuv3Sy{ROUv=_1m6s7s5kD(QMr5-=_6&fVYr^MS6vyb; z>?WRNr@_GuPl6fw;DIPdMz_t;22r{XZr#G!mZ;Y!D4#Y$TO#T9l>7M*jv+Fso=tP9 zuN0^p-Q7pkTEEuTgXh1J9drJbcR;SPL)=VRs*p$^XQBdsh`ZY!>4oAB6JFvV5+JTwQ;cx|#Zo|jx1_SOLWF`#MC=uh*@ zIHlp*y~=GuR}tCuWo@ZsazCw`3qG_cs+x`Kd*p_@@!amp5O8-fLbpm$l#lJ8`fKz< z+oe8(xm>?O|2tCFr&3X$#b>{O(}1gWcZ@4HZ1l!=`xOm+U#N}htQa|o= z@);U{;k*cF_c?DdyNy!{hjDjtrDMMS6-CGuF9wxdfb-qGkmp4Lmc;3I9yE{9{hJpo zFdEF8y>U%xU}CLBFMzU;QUvgyK?;JF;P4FWw<4-vm<^4@A&VTrY6Y3*aFhI*^|T@N?9b#{+mui%uPZ2xI#<~V40Za5=1_%g_OcXf9_ zN~-YVK%E8u#d1S>M8uFvqSufwItzho+fA_p@mPXfR<%Ng@5X8~Q6^1$jF<rRy8v(8D}E*vHaj%Zp+!Werba!rPEb{Hke514I^pIh1 z@4bEcAKIT+tJ9~4{vz<%u?#P zYjd)_%D84!w>dt?98tZmvBsmhC|;0SD0^0HjIEP%G9~cTFKoQ9bXuB~&db%LLy-qp z30G@<%KC%n63*FtA86~aOkq_dGI6?0%a~u<)loNPD~WkTb{I6&!bAdu`=hp$P=I0({<>|sr_QM#M4-wxwF_^WgkLYGHv0<&>XYH-&Es!^q7nqiBI4AAajdRC}8^yx@LR?RH3zAFs{GjVMzoX-`nOZ25hw`5ecp!flm&M=p z%JqW`O$gSJVV0*nG_Y==QH`QhxYbYu*v$^!tc`v%N%8dQz^KvnI9o%HBg8#;y36$w z*~dCP1{)|tk(tEvJ8JRlSUJ4qkXV{0ijk7qp~GVWyYRg@5&UV69(iZ!&|ou|_^uqVpjJMah`bc3o4E4xW#t|Bes z1zLg>Z!a6;`-nvCuB|yK9gn5~Eqhm8J7n7BU|ZG3S(oxPRnBi|34DSH<<>BQ;d;DC!-V<-QEi4r@lMLBmL^wHw5xP_Qd`df@h!V0k5x&LaIDq$ka{fLC6_3Nk zRE)Mp_<7Js)8%WpFEHSDwBp?4CbRVPX*eywJ(KwdfTV28g4IFU{nI_rQ?6h{GPT#X z#r&LunmwId&%&=S_zLj#&H3+$5h~P8nh}sg@4irNK6X>M`0nSh$)5G|N*Q*${-etk z{4DSAx!Y&Si@KVDuRPf~B|*b+O)t-t?)zTNc!2s>Vw#AGGz;J42wr_ychE9YlIngd z_%%zT^a+7a(Hr^R(ysd1qZRR(u*jq?Q@wN1FPf>wPRVy0@E4F+hO@pefekIiI zc&Y@hYG(*s%a~%+#w%q9xMqjNIgj4^Zp*hQobGK`(d8#o2z{4xw=k3@>x4l)UA*mJp5fGni1-tgJs|^gp{%~2M-0!ck&|}9`?Mn3GtijR{i^L~OnY=Wn}1R!KeJ)>tKE__QEhP^`oshV zT|fH;~5?g-~NQcUfzb()JWfj|xZC@xS;wY~vm+nk7bIIk*jz*BftsoNep z8FB>4jX_bM;?3+_jjlmjgiIcL_A2zyVLmTjByaiyp3Ud6_uB0b0@2MHjg4b3rBHVU zl}3;6FYzO@vt{Z{JK>s=eBu(Jf3X0)M=hkCdJP`AT$AeV zV!w*4yTw(B+tu3KCF{>dzG{G~EDo7%_iCJTbIudNx9w#W667&zCCyYhBkf{UJqyo4 zAd9(K9@!M@$o{e;esPE{vg_z5q&_4&wj>+m7fAs6s^vtbYFoOi%vSbC zP_JC*e|;5uBS5xUm>am!JK2Rvchx-oJ;KJL^*uy3wW6}w*H4Qpp;Vor{8k5YhmZ~l zwuChs=WI)JpsX$3`HbxV5%U_QH&&LuMfx-CI~1p&C>rnE%{vo^%fob}i7v@xWIWf& zUC~sX?_A(X+xIuTmC&y9RD6tE9L4PA;@G%!0pUEv`}nZlMUuf;`|FK){-HNVLag8e zkN^8MfO@a$2KeZE*GP;K3J_9g9TENeLJAICnS#> zu(CZZ>B|t0ttcxP5h*u&5q@a_8>3ov>ogecZ|V7Fdv5s5i3P#b6$&_5&MGW?jvc7E zG^2LUOQ^NmvC{~1Pr423em|GK-(i1}f7}sJ8$i8VEU^9M2!S(=y&7 z1=Q~FSzB&$L}N9nj&WD6_;qKbQ@N{_bmQXI`xo}5Kb21AgR&tO)y@sRfp^6%yTN*) z%RxgQvCT_pRH$hmV1W z2v&0QyA$7K{q#Ev@r8nK4|%P%mn`PXb`O;!`vZ44EF@vL&dtgWR$J3`l|aGER3j^G zP;L56<-)cvmS0zjcons%|LPoDYVlwhKUq_NtN6$i`{W&*TJ^Z;r2S7WnDV9l^9)hZ z%@Px#meHSAH;~j&`lN`6AmG(pbKqq$&A!%GVl8Cg`)UYO1v(pW%#X`T46(I0DQ&oy zF|)^c5T(ftNhs60{A&gf1f2U)R4*dv1iK^}k4*hd=T?r3LE+m!%gpUGN`9vo{4u%} zdSzp_0(kH;INoR2Ke{-i%rc>DEu`juj(R~BNRxtkInZEcE)UCFk#b}u*AnWrzX zV4jPUi4^+%To(6P9JKlLYSTpj{^q=gEgaKk9b!!v>IBhk z02v?~Wy2ViNc}zy{RtMNDuO%W`JS@+?)G|b4{(U_c}m=__>bX^q(1=nUm@naD-APb z|M*;gBZoA4EiBx~VJOf>N<}S2K8k8pikw;ddc=0dK8vHL%|~c0xUim;AEN=$ojZkl zv1cyw3;TTBdKXw~xlktd7ab(w+5_?_`abj^RI*Rex7GgY$$QYIK-`#rr8@L!#shB# zbEqVQc%R+-J@=T@O!UXCAPG2^ zUc0pZYUz3TJJBoq5T{&>Fd?yg=8&Q$qw)6o8WRJ9gwH+l)6MEUoew#4fcDNmS%_$P zjK}cLfr#(JBj6Ez?oWj*wcN>4^gJ&WiyZXJvLbYA`nxqVGFVBT1e=k%KMUO~Jjj}} z+Oo6-MP*IqfYOx`Mh3Q+0o>dvPao%lxaX&H#6!#r5*f$r*pjy|YX*it4(IyOuCHeD z3j58}){S>-F$a!3?yv(7L#`LuN8pHR5L2M(HW$W>~vGKu*jSQtC|Gh;Up4e-EO!w2~F`Kc6 z3{t?z&tUvIdYc=Lk^8N!jK^2hSy#Jx9PcE-;QqUksoSUA% ziP70D#cZB_p^3rq6f?{mlr%E@u}rf~>hF58vSLb7FlK#4mMOQjP#sNL|vTv9Xg1<*_Ti9e-@twq0%sqLiGkdeZ?jCCSI&PM;V|n|nZCrZxmNo44}SsLc&<7><~y_`k>mJB%1jbn)^rJS;AO*q zCIRh6?RWSZ+UM_#JQ`3kiMQeZ6s$y(s0mhhiMFHb=hrE>(fc3&(f?}O-Bn;p&!$Y{ zizY#Kg^VTX{XZKNe)t-a&?Erde1v8>ztW09ColhhJfcv$G9*FR`{ElWT9T#z6;VuE z0O&vK{^dmi4q6hIzBjMWW7+^Gb08yH9zPcs*Nw`5m(rk6 zvDHY%F~-K-T@u)CSBwnp`dHV%U|;wvTuR0cTP3IBTqG|^@)7x9)q2zZe~D#YmyuJe zI4`usUP2?_8MdQplLKpOAg0d9Bm zDvlz4hRI<(Ez6Ad(Q3I3#Gu;&89o0ziT6Jv*iI1Hzuac(()|?G9W=|{B1QcWKoZWM z^RC9f^~6vj0nOm&AeC2Tj7i5vd%%C6m&o`GbFOkqmigmN{=bB{dCpVCw%F@xU(ily z1pQN}MVLqr7Bo5h2Q4Qbz z-x}ftA@w-jP}ip?J9NZoOf&VbG8mO7eW^cc=luiBLX+UiQ%|Vl5e4dV=44mKJD}xV z*e!ey3Bkdfnv!RomF4?5nEyX*PhY>9+8oedKTa)tgodmsWomIRG5zUjubKO&BlO=j zB5}%T{v~B?5iB%@j;2rzwvU88!;GhyL+PoV_phEGMM%li>Z~wATfNjigoxEX`mD}K z8_eg-PMRf}8547n@FI!$f0h&xMK~Tw%qPodZDOzvz-~z3ATUBAWgG?Mr5B(rce~p8gNu2V!$w_y(^q; z&OsMyZjOVA_5Ag-XQ-LCe2Vig-EmLktStW3e#yBAn!@LwYuDiDJCmN0)3&^0!>IOH z3h)!_;c?VZZvU@NS@ZKY*v73g2?;`~7!0++(&fzgGNbj73`PBxG{L>AHTxW-ZqhS z4!G04w=5!^`y+#)K=Q%lP?g(J7z@2@ZY$vGz-UCGy)CUr`2nWJe{51-oa)3BDHnK^ zsf<|9+|GiXHQt(zAam%BdjmLCEu)TqY)ED}op~(W^yti0^_JP=8wOm^Ko9nJTR_0X z6~gT=;sbX&_5(eu4+9(3p21NB?xwIQ+!rf%_sHP~PBSqc)1}K;mftkQ_a9%(?2j1n zZQdm>#rb|Sj1ljAkK6Dc@w~mG?g9E-J!F0J@G*=h&Sxedz>PaVi80CwH=|*>Eq@%g^?|szH`_Rg#NbjJed6t<<#$VNY=UotR z*Z8Ag0NP5+-y|T`+BX6u z`)w>gB2`QpcjGW&+^~GoTJ=%^})a)5&>g|taTU}QTHqG95Q1Rbe)kK3g(Xr{X z&rFW2g{&UtU{#kCdbD;6XNQ#aK+y6(1A zKW(QvoSrvUT0kvjA>JywH|r+3h~_c=J~l}9>KqPKnrW+$y2Ci=B{j8|sRvu7_;ckp zZR6H+c!rbJ{HwhB#CjugQq<#6QtPi9ivzaTd704fHw_s-LCH^+%nq;WIpltzN=9hK zH^0M1#2f{O?s&mq_-I)YZD#R(&b)25s>?V+%W-{nWSP7Q0FWNfkT1^x>%0C@*|{_- z_$13XCWPFkG1+GUn_%*?QyN|>OboK-d+{h7AWc-r37bN|LYy&^5w)|T$+yTG=T>&c=| zmPO|dc`>MP#+t^{T^$(B-bTZ~(0TvfGh0ghB55UWWN!FAqC6ejz~AAd^i~`X#v_(q zc+omN@+Zi#qb2txiBJvlL1Yg(=DxF{BZ7~*S^hfSC^o*&A6rGTHQ?RzF}-7`MjU+J zrz3YpxM{UD*jJZ*+la4Qns#)`p;()$%8Dpot*gr3e%h~pm9;ck-XO(nQRFb#Lx(+z zzOQYKvQ`MR2N`lRefyq>+AOce1lQi(a4<9i5LDCKNkq)r??NM{xiI zlZ16oTFI3i3Ng0^`Ca*S7%VMuqzotT!)0>14(!q+3bkjjF2H_-H#cmmhMME^SByHN zf?V}O>Ee9156k;w*c8TA`1o)yZ|I1|E6icipU2S+t&l8AH*w{(C)hY@DL8rah%ZsXC4#5%e9sE6xW4$jgk3%;Jty?@=0#)U^8IE?qh*X|E zWH~(LegWv`fp6hKTw7dt(mYXYW*`M^ z%{F=1l-(0DaXk-#=YiUP6p1(UGgX5oK54Y}wlasl_a+Tb@@m!qk0op%25|jPCc_V6>AZN*)u6c)~l_T>vTVUewW|S>?lP&w;C1~9K zJK#6;;kLnm$GXoib`j6OWzTk$aZGe&xiPTgtvdvj-rFLtQSI6RRZ;V1RD8B1?D)B) zwY=QFmDW8@di##|OE#|*HCRk`b&2>8MUei zUk(N>%u#$uD&)RQ3;Oi1?9_l++Wg+N_#}IA-u>bt61}Ct{?Dr2FUsSrR=p&C7}$>M zw&eql>TOYTf zicYZo%k=x}>t(Y;iaB=|yGhI8Jr<6MB?5&q5kB+P$#Q;B?l47f|yQAS2X(ZWh6d-J_$?uQ+*n5?R zE8@F=A4shhzZ!Q&BS-}U@^TJk0S&UFOfj)XG`B)vWf*#+%!8P|pY72|2Tz33Aqi$I zynUgwtKV&U!WELg0SpytbZ!78`%LC{%^3zA-_%Xkhf3m^ z?9H&T`Sir`XPkQ_B$ySI6>GmDU^B2xK9wEEllp;4&)YR11z5%s_JIx;F~} zUXC^JBOmDaDWucnZhW!7EH*YYEh24C)X1RkNwv_+hn?eZ$*nxCF(3_jDJc}CL51$+ zML>l2$6`4EK;_<)NKA5Dto^pIxB7UHoK;_)@qVWud~b@B{UUM#9`pjSA27x)A3W4h zJ~e>b+Wx7P(?+eJC19^>Nq#^CU#)k`AoED`?(~j75H90<)607H`+QC$l6h`-K_>82 z&fyjQDOn(+1qIO}6-S6GF6TSg0ClDF{STe*y zmB!hE#>02ws1d`C{p@F6k?_~x=;iFyRvq zBSN87fB8ypzh}?VNm6bx=IokzT$Vd|!9?*qQS40ac%iU|o0iU6v(-kK%KHys@3YvY z2dBcM2h4}{Uop8HT1e!tk#Ct;jM@U{8wVEE7yw$ta>y|rXIJCxN;A;TseP%gL5dGTu57u#l`o5%-?RRd0PK}Kx+6} zjx(*EGwYMm6nRX`L#f1&#lXe6hl7Atd-v4r`(M|>WhbzdsW^ncaW#=u#xLpnuJ z4Y-dA=NVo{tEB|u4(rWK=tv@L1tfg#@+-Ro9>XJw*l0_P+=;+q$f+^fKN_) z{qtAw9(x7qH3qbMv6a_I(L0K{)%P@8+5>-oP+>C)oUuCK^nQ!)N-1kzv!nbALPhdp zvqE>+U9jWF7gm;PX#>W&HlL2Y;mW!UQIP;+Q7htHqN|31!2)acY?HZlWVJh9sjSq7 ziSJmZcim~0?+Ob9=)uCur|}vv)78#$Rf>7oanH#;>@AIjAkO{i0MIMwytjKoc@{WI zm;l)9-feWe@viNjnO%=Iv$^$RJIxI$ZW740Mn7428j`0O7^XOnvY>={HA(W8JVEC> zotd}ZCT`1pxHk1pr(km45&QWxtd)plxq7HL0Zr!3nY3M<_H2%E1Cm5h|B(=%wm%#e zju+Vpq4B^%T@+M*X;#jFV?jyD^B$E({K5=;YOl?^)wP)|^Z?p`ltEpPCOrUL9frW`|O5(5x_V%@y5DeQ|+ zR8>U%R{QfEc^&(n)7~yZ$>@pR}uv zY|3U>P~2U9bh)ZEwp4cnJ*qcaqVq0h{%-de(Fg7dFaXOEmj<&eyU^K8>1R}mYB-p4 z*nlsWtnK(Jmt^&AKQ{p={|nbUqRfl|DNF)T$dHjngDZzW3wFYETeFVV8wMk3DYDZg z>oBqyd-3kzt=z91)Pz2=(o*!_BN@(OV5;8DChL2U%O}%UAKOF-s<2X&cU2 zRqls!$@K>`yK9#^c--czNN@e{$+)O3c7(m@RgVmwGL?nl&5V%!e*oozgg^bi0pRg&FwZ?%3Ef^HgrlRL9z0ukxQ_=?o!V${CBXF~#hO|pUa6{oU) zGT6G3}k1t6*5h$YnIY) z_6@4muZhngTIfW*G}P0T-r#RR`=0LN{mxnNqS)3OnjRm9z%FJKF$LF17?J>kC!x30 z@5q-+v{z3h7QOU7ZUQ?-+9w~<>Ms<8@J}bO<|xgCl#H8UArBzSN@J&t+TY@^#VR>K z^1!D9#bZ65fCHFg+B|2;`De5He4cOxynw0lV?66&*}Lg6F(|LfRsl~YAJtQ(VQ(9& zn3z~8Zgfe)I@Ez39IQmEBTf~{H}yd5xd-F4VHIU3r@d| zYTTQ#oh}XOoYvf%!$WtK!0rKE7k;ToYFuz8tossl$~*fUt}ZaC}ux7n69fjUg-Nef`UeE9cX`z$XY48L5Lvw7WRT)f6LBJxej}w zS)Njr7+m|J*z6Ug=m+l&6j6A+35z@`NqQHNuI<(Q(c0&y868w4 zcl)DP%5euk%ljS@;wa?HC&sT+)Pm4;W%zqkatJlyp>Mob&ZS|ZCKoNQhNHyWJDS*C z{srv`urm%v&o8Sg4A*wsx4h&0(N-Brn*)oC*bwZ2D*AiwjOR;+$Dz-KQ;^kW#KsWA z>bo>|5KHMZKxQtfZqa!8GAkoBDZ_)hBs*R9U?i3z53V$hiJ{Vur#MeO_1T$+ZWzBQ-7HwYtdl zeL1#HhU2q9O0u6-Iq-F-eo7%S^l=8kfTnzS^YAS#YQZxqJlCqo4|G zdoEyxbue23mU94J`kb=7v3fPxwlfEKO9I2gT&}$Z1z+B(g8ydCR4}{&lRu zM+b*xIcjQ*Q17GhLX;C8a}7E;tXW(#Fvorc6A%;L&lUe5hr$Emel?t_vCbZX(R4(q zviw-6P}*85tPl_C06rZ3t)=qOX>Rv=UbFHX3!}|uvFr{0!gLKMcY<^nq{hj)N z?tFi>*gfxt0jeLIY+msC&lHh+10i`w%(VT2md4o00xG7Ym&yXWjp$a@gM018j;(_k z3{1yaQQU{CKc%(sElgMc>qWB^czKF%9RktkkrNPwdwm?O?S5s1rRC@I&IOu&VRi4oHfNjVF+}d=I(0bEdknpVIRcfl!C5^G59|{||a15#Z^??ju z((tHn6gxxSye48!mX5pNvTxGMH9C>OZK!Agg&3SlGR% zQ}=1dZM@4%d=Npd3nctHc!zg8*6<++*jjp1g4@rChc~mzhU%u(C+@u+(?Y{ z?EgCS$9Vz+PXU|u^e)G}Hb+uqx_$Th`kWmEWlPPPV(26$NhOB~U3E6#fmw zX%=DGU9R_vrQo%jF5P-=G9t-LY7s8R!%yAjvT>B{{%kLt%^*Mk_7sJvrdZYzX!Y-* z4Juy+x`mU!4d{5(gVr@p6Z9=Xt@ApCjflJ~{tFtCxJI|$Q~B3cUA!uF6C(lf@|+7Z zlFr|Miv}$NWwF%M)YQZBZkA>-qQD|9GHg7OzEk2>4_Dugi^1?b+r!y`VjJ+{-ublHRD!5$Q= za#2y9#s5rT+*qiwz=@{?^JW*2Nu~qtjvCmwT*(Y?He?yuS+bYedZY++>>D>DsDgSK zKQWnLJUJF(sXx%vQ|j*UY5HXMDR!3%YcZP0O(lO#FgQZDd{3q(l?JnqnRxeYFk^0e znYr;-26MJ7O77|dAZ21Fi*X;d*j3pN#iFzStG)AzYijxScodIVKtx34&=ds(Y0_&z zs`L_y^g|0$0@7=!f*=S|LWcm-LKOm`gB0n#7b((v3mpP?3+H(5%l+K9`}sfIyk_s2 zS+jm?_Uzf;S%a*yFkmEFacnVN%D==E;qtg2%6H#u(6f39KV_ncefTEKJ1mAY3|hkI zkhe*N-4gu>8&)s+0Y+zp?3o-?H#`k5?}*9b@YvyJaZx+Ttu44?R_wCW$J=S9`NTr& zMq&9E61zq5#t37U6SXx>q-y;2sG}2N4uAEh54nsFjyG;}t@iJpiKcTuKp-U&Z+9p$ zqMx!f3PR2cCAf4MZF!!uNHCD10!>4zKl5sg{DSYiE1?l+uF}fdyE>MI|DqTbtLE9Tl*b9&xRoZ_lKj2 z)+!Iphk)gAS4F-@@s0+1n+6uv%WE!TDrnn2p3tXyrfu`Kexj`;f$QY_bh<(_YMEX)14B?M>HL^)Hw+cmWRRDroi zg8}mPN{p2E_iA2cm|ljywGdm+3dhskB4 zfoY2wu*)ZNfEG-lch?4*C)+(^YJ@2ipS)M`3F12*mS1&m5Dw~)@4tHX2E9Ji&g7yp zGKu4n`|$(T*r@t;5~DXa$tAT(4^5Le$;VC9hHK#Cza%!fR*+bB>Tr%gk0Uh9@4RbN z&sF|J#1Cf_WDpQ|vNkfRB)u0G1j&n~GqJk;ahzN-RM%!%;(=mr$H?n?wm8|v>l?l&k+t@7)|DRBt683mq=i#n zGj&5z3*E!_HitgV%sM0yeyiPiygMj=9gT z!uO)1DSTsW{A_uw`w0^-Sa8k3VAB=L2*=T!-{cTv%99)UvFagooj;N9cp4XrgQu=k zs&jKIQ$LJz9(YdH{UkW(%fw6Q#M#b+SvVN4@0vqyE*Za#1V**lWy=dlXgY-BQq50v+Xk3 zb~i-A`u^rXd89Q&inhN<9 z_u7K^bW{g(oO|z4CY2-sF{IUC%s`7($tc|OKaNhYnPA6YyE_>($PfZ*E$Br9~?MQMYBI^+T(X)IvkRb+eMY!7#jI% zXeE>=J0ytnMh!Hjw5I0>yB*U)IWh*b?c?TuM!BhfkIb3TiiYb<^>S@CDcj64B>Hz$ z9z)_+my2u(k4uq5!*3u2rj3TNavg0uG9`@b7FsQXe>Hu>Mk{nL02>>zGI0aN9z@;x zl98-zUj75Ez?bltKr~()_3qoZx$qRrYJcGnINEY1zd@zFizGMidm*0Nuae z$U!=4paRieWrm>>PYa&vIa;b4NvLq%eLgI54?$k+G$PIuHZWvx{OZH9xVZb4kFYNm zKJ$M@BrXJp8BDr##bxmYNxF7OhpEE`Eb2b8S}c0(gjXnpr}heu7aVjsX*4Htw_KuhX)0wS!IiE0`w)L!hk zN^AjjRCQW>R_KejPPJ@9CkkvCc9N}e95{Vo3_DgNvw;mgKA2)x^%+_I5YgNmR7$W` zA*fnSNnkfBFINusymmHJGg&rhcW;4ZX`YIn2Q}zE-KOt0a|Noz4`ezi{Ft;xO z<| z9>GW($p<#%t#g>$d=tte!% zN3%tT4`QvoN5DHC~2pb-?nvN!%(mzHu5UE>l?wx77cS~NM z2-d}*+|K^Wl#JJaZ6azrsy(*3GX2#zOLcUzRL5AQV&nSuR*kI-uyX~GO|-0_-W zff#Z*r?>@GwdaUNAvzm16)+~%LyyIg1flP2*NnAQa=Oo>cD`4H)Rx+f4%Cd+`7&m1 zi!s~O|6+Kk#a>stE>SpaOkE#b49|pVk5y`%Jd>4WyUT*}Ebg7<+{NSatwit?-iOC?3v@#}w6UkmCgT zHPJrN-NN~*qs1ch76UvR8 z*)6DhadEBpM|HZUYbBfRy?V^4))uE#*`tq|%jh~7a5vx$LGt~SYoHsvdWsw!z{Cj3 z>QZ|A8Ow>Src1~e2?kAE+P-$l)nL-)Q*@}IT{ZM*a;{FUkkY|Jxvk3S-@d+-0V#4 zE+QTrKx~WbO*-d#DIOQ?*VQn$+5ag+Mhc=xMVmi4nq#X5k>0TLEMg}m+N_QRD9F`) z3<Ri6(Oy;z5Rq9h{U@fq z%geX$@V0a3Dcd#J+&w*LWE3N&s!401Kh?(hNp02H7};$W`tCSTFmq|%R7A#A4nk_V zIgG+1O1RUhPxgDWl(lA>nj&Meu$G{me?gqSEkQEUop*B)6Hsi(|3X~K{?Ejv zO22A4Bb~aM9MC{xO$Vf*3x<(d?TnMXH6z5YFmRyf`KFn4pf$UGQ+iE^;(1HP&aCPw zu%Mu^Z3ZxxT$-a}8f>L(oE$vPfOTzcs?f9-^**!Mf@HKd;;QCNC{)gzk=TY=yKwll zm^&T6Ikf*T7?G~Ari|=U;_K6Yz=1FG7xWk?`I?L(y{znd#3`; zH4+k3fJq;OwCeGrM<3HwxH-g7D%${!*77DUHbsfi1(?xU(jZK8iqL8_#72gUYN*2w zg7g4`;9y$u_Zc!K5cW|4Mb5S<)w42{R=7Fjh%`#=Y@2h31fVfVJA~ugnxkWR6xldT zA9NUzm*ae4b6B^1xs$`LjSc&sSINvbw+WLuezRJUas~R39QFGoUrhSGNU5STx=5~q z65aNLD;(3CLvJ@(4AOXdX1`IU=DSiQ2I!Fld}*G@O;yxAN?`>kREy5VRmHjN*L@SN zTV8AHC9254r9&H4WXhq)@7^%BwNJytFK(Qj$%4C8ILI#EyyBIAdPHAN%6`2ij%#i@ zdE658(dRdm7C65w?R(uUke*gKN4?lF4=CR2eDZq`Cp36p`uv0%j{RhXzH$iqO^?YG z=~ljNxw{^bA^&Bf(jlG@blQA}GL=fGSW&AYQZjT1#totdHthe?{8A%)+?haNapG(! zrL5;^3NXq}*A}{Pe0rq3l(tzr{OfW*;%l#__%81NeXDhxdB6XYH{fj-Mchl@N0OyC zuY6m4dSf8%rk|e~vci%~J}GznJqY{}fDwP&U0n9g+jV~c+}aXw?_Tji_;oMCkVTXQ zkP}_vWB@k)c1bC50dK4K**%6kf`3<(?ic-#QYag)0;MA(3lmKN%AD4RJYUKwDAqtS zGcPw00$m#ekd~L%C*NNlL)6!2LFLyW#$G~=LwC}^v9&i42x{um~&hb;9A%GQLE?%EcS0m0umrk6@v zz0b_dt;D!Q`T4fq{!D(k8z693f93!NmNEz=>y(ib-y@Ux7Xy@GxP4njKAp9e$=)B@ z2?%Lo9uGua?8wZ=uk!BeK1$z1`U6o!VWAaZj|SVdzmG5!F)MXqg`Y6D&W~jdp~Sqs zjo)UC1NP|J@^QTt>FxFF`s&UnAD33?4g%u?n+SdjfoyB*376%Y0fK`~$og0Xz-La9 zl1eoAla6i222@p}#PEOFN}sZXhC z&j9bt0sFizd(5lIcnS>UfNg8(0Xy_O)~ZR!&9#pw5z!p9;w6AM`g;+TTAi zQ9kGaDSrdnbnfRX;ZqR?27bBQ$;ECbQEiAX514P|f#YL(HYVm~eOEXl|NOm$LCIf2 zvhS;b{ehJ-Hke9&b(o}KiILu`@AciKErUP7r+Daz$^&oZlq)j#|21oY;eN3zS7)O3 zfFKlVkKsF`^2x2hz?}@6Q@@=p7{P8=ifzypd#Zb}_snE}@OPR>G zE2`{lJ!%RdT76gk5;L)bO`taxyMSUxM}P)^Ch8vjGf5B;7iVFKL`aR(eOI49aa?;WCblw^m$hOy+jRe~h>cAaSWMIF_oOD7u)7}v<21cf4L|Yb zBce#wtF{j(5qar|uCDL0aSE)pATnY>%;fnRoP|U1$6K2yM z$8cIxalx`95rfSECbF;`sShuxz4%dX>(7gu|gRVq}tOIfnWGZ3Z5 z55ly?#{yV7f{os%At*AUa_ab;6XpqMZt}jnlMrNe+PyGz+J02G*}z9Z6>9ULVG(;& z(_FUqBf9&5e5xf^a~|Kv+h81{19w2GTm6U$^{m?BWScI#aKn-5}SdCHU3;mkaF_(H-w;jfTO?&cRj+ZaVL3mml_ zVbQJ*(9X8cPv)C_IMOreLxrL$H@l|J9CyB33_}OA2`IHZm>wnTTm+u#Mq}U)M{RD) zrT&@K{C0mrGQOfMIi-=_FQdf_r9W(U5Sy+V>UnK#2r};6(>H9%H-9PZ8;ajUkeqvN zRlhr@Ajy5(R5-rdoeq)_NqsvzFP)YM97BEj7@r9E2ajkj0x1WB9TiS9J-!IR%*<+` zsS`c*f{p$evwqvvJk6qKi${#4czPNd9=l)MS0Hd*&rYt1Xq7!a*v0n^YN`%cJ zCX-cyd#d`w1)33v7aSh$t+CUpl)RreJ%v|?=I5goYszZCPLr{Up8QWGBasCE+$jWu zxop+BW8?YWP0t0bt`xmhwkoIX&x}SA94{0$+cprA=IU2+w?~@}WEtYsjmlr_)yQ5S zDYP7mBz?g+vXjm_zdz0al`D*$W~Sr^jxw}eOmxBR)mW7uEx0cVOuG7)(p}j0{d708 z{A8oJc-H}UdZby#Mtu@5_9BPYM=9t27oGxd2?vVe@EJxsR@fe&VJF4+-6U^4r&+Ts;4HpH@`ZZdR8s zCf&jWQzZg^tM`i);Bj#HPB)v-+Z$#L&nfbJFTK#!`H{4IA#PwGlI-?%j>}uo`)jQg z4w|Jbl#~eAk19U9Md-*^cB7ZhiM(M`XWXHJ;UB2;EobZffYRj>wV+JA^f!LFT*ULO zMf-3~)X7GUwcWNCW|kd8c`m!X9}n_%VinzK z&`^_>&sQVr%3ohNZ8K`7o&>6WKddxy~oKn&m6m9kCH)96ufiwE%Wg zpRv^*O6a*US30KW80JOzb| zPUyNP6$)*SV$yZ3SnNnjT<+(9yFxo!=33Pr1+T0ux5~24pPgV=^)o94@QD^$!XSd? zOFabSQR%YSc~;ZirEGFgXuXSU2v9(8rZgdJ4T_h*=# zod#2A&M}BwxfsLVbuBHo<`iV07=+JVs5}nk1h-RzRP>R$mEm(c?RNe0eCMYF!gXl5%+)rZ30+XVMjw zXj4VU^|oKqXrn&snZi@c%AnKL#R-Y~<=29-i;KS0zv9Zy|CRKGXPr3`6OUH)v8Px* zXEvY@%vwnlcWu%>*@HmWP`_A!13A-jK;AV6dmiOBeQA@W81Gk?!}k(;f4>;BkRILl zBOxlvax%g53|(vb=V!@JZMshL)tl?E&XN+G(A^&4Yrh_#Uj6;OVM9a3#cAby+NDV^ zSY>KwDNzO8L9BE@hX3-P{+yh~J9Af#H6@af7b615#NKIfZy0T@zrWGR(QUjoLLm?Z zKiN6O9`z%S-O5Tit#tee+iT#`wxcGRUmx%7xtxpx&#dsOa~v5@7ot-{|FRt1;^8(0 zZx1=D)T;An(X=DR-ahTiQ$=vBX;N#M4KS*6aV;;kEOb^^vXnpjBDL5Owl;Cnr2IuP zJ)h Date: Tue, 4 Apr 2023 13:57:42 -0400 Subject: [PATCH 73/83] oops (#283) --- docs/guides/metrics-and-limits/configuring-metrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/metrics-and-limits/configuring-metrics.md b/docs/guides/metrics-and-limits/configuring-metrics.md index 664e175f..c695bc8c 100644 --- a/docs/guides/metrics-and-limits/configuring-metrics.md +++ b/docs/guides/metrics-and-limits/configuring-metrics.md @@ -109,4 +109,4 @@ With all of the components configured and running, either use `zrok test loop` o The `zrok` web console should also be showing activity for your share(s) like the following: -![zrok web console activity](/home/michael/Repos/nf/zrok/docs/guides/metrics-and-limits/images/zrok-console-activity.png) \ No newline at end of file +![zrok web console activity](images/zrok-console-activity.png) \ No newline at end of file From 728c9fab9fc2b1fba6d2cc1f265517d62add7f33 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 4 Apr 2023 14:02:06 -0400 Subject: [PATCH 74/83] note (#283) --- docs/guides/metrics-and-limits/configuring-metrics.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/guides/metrics-and-limits/configuring-metrics.md b/docs/guides/metrics-and-limits/configuring-metrics.md index c695bc8c..de9531bb 100644 --- a/docs/guides/metrics-and-limits/configuring-metrics.md +++ b/docs/guides/metrics-and-limits/configuring-metrics.md @@ -12,6 +12,8 @@ Environments that horizontally scale the `zrok` control plane with multiple cont ## Configuring the OpenZiti Controller +> This requires a version of OpenZiti with a `fabric` dependency of `v0.22.52` or newer. + Emitting `fabric.usage` events to a file is currently the most reliable mechanism to capture usage events into `zrok`. We're going to configure the OpenZiti controller to append `fabric.usage` events to a file, by adding this stanza to the OpenZiti controller configuration: ```yaml From 9166e2d0aa0b9332ad38b991a575f79ee316d3ad Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 4 Apr 2023 14:02:23 -0400 Subject: [PATCH 75/83] lint (#283) --- docs/guides/v0.4_metrics.md | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 docs/guides/v0.4_metrics.md diff --git a/docs/guides/v0.4_metrics.md b/docs/guides/v0.4_metrics.md deleted file mode 100644 index 632a487c..00000000 --- a/docs/guides/v0.4_metrics.md +++ /dev/null @@ -1,31 +0,0 @@ -`v0.4` includes a new metrics infrastructure based on OpenZiti usage, which provides `zrok` with telemetry used to power end-user intelligence about shares, and also to power usage-based limits. - -# Configuration - -This requires a version of OpenZiti with a `fabric` dependency of `v0.22.52` or newer. - -## controller configuration: - -``` -network: - intervalAgeThreshold: 5s - metricsReportInterval: 5s - -events: - jsonLogger: - subscriptions: - - type: fabric.usage - version: 3 - handler: - type: file - format: json - path: /tmp/fabric-usage.log -``` - -## router configuration: - -``` -metrics: - reportInterval: 5s - intervalAgeThreshold: 5s -``` \ No newline at end of file From 3ca87532e60c9201231a5122ee80124cf96d2324 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 4 Apr 2023 15:25:12 -0400 Subject: [PATCH 76/83] configuring limits (#283) --- .../metrics-and-limits/configuring-limits.md | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 docs/guides/metrics-and-limits/configuring-limits.md diff --git a/docs/guides/metrics-and-limits/configuring-limits.md b/docs/guides/metrics-and-limits/configuring-limits.md new file mode 100644 index 00000000..3d472fc5 --- /dev/null +++ b/docs/guides/metrics-and-limits/configuring-limits.md @@ -0,0 +1,54 @@ +# Configuring Limits + +The limits facility in `zrok` is responsible for controlling the number of resources in use (environments, shares) and also for ensuring that any single account, environment, or share is held below the configured thresholds. + +Take this `zrok` controller configuration stanza as an example: + +```yaml +limits: + enforcing: true + cycle: 1m + environments: -1 + shares: -1 + bandwidth: + per_account: + period: 5m + warning: + rx: -1 + tx: -1 + total: 7242880 + limit: + rx: -1 + tx: -1 + total: 10485760 + per_environment: + period: 5m + warning: + rx: -1 + tx: -1 + total: -1 + limit: + rx: -1 + tx: -1 + total: -1 + per_share: + period: 5m + warning: + rx: -1 + tx: -1 + total: -1 + limit: + rx: -1 + tx: -1 + total: -1 +``` + +## The Global Controls + +The `enforcing` boolean will globally enable or disable limits for the controller. + +The `cycle` value controls how frequently the limits system will look for limited resources to re-enable. + +## Resource Counts + +The `environments` and `shares` values control the number of environments and shares that are allowed per-account. Any limit value can be set to `-1`, which means _unlimited_. \ No newline at end of file From fd5401b877c976a2d1d448259f82d88887e16ab6 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 5 Apr 2023 11:11:36 -0400 Subject: [PATCH 77/83] limits (#283) --- .../metrics-and-limits/configuring-limits.md | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/guides/metrics-and-limits/configuring-limits.md b/docs/guides/metrics-and-limits/configuring-limits.md index 3d472fc5..2f1a4be6 100644 --- a/docs/guides/metrics-and-limits/configuring-limits.md +++ b/docs/guides/metrics-and-limits/configuring-limits.md @@ -49,6 +49,34 @@ The `enforcing` boolean will globally enable or disable limits for the controlle The `cycle` value controls how frequently the limits system will look for limited resources to re-enable. -## Resource Counts +## Resource Limits -The `environments` and `shares` values control the number of environments and shares that are allowed per-account. Any limit value can be set to `-1`, which means _unlimited_. \ No newline at end of file +The `environments` and `shares` values control the number of environments and shares that are allowed per-account. Any limit value can be set to `-1`, which means _unlimited_. + +## Bandwidth Limits + +The `bandwidth` section is designed to provide a configurable system for controlling the amount of data transfer that can be performed by users of the `zrok` service instance. The bandwidth limits are configurable for each share, environment, and account. + +`per_account`, `per_environment`, and `per_share` are all configured the same way: + +The `period` specifies the time window for the bandwidth limit. See the documentation for [`time.Duration.ParseDuration`](https://pkg.go.dev/time#ParseDuration) for details about the format used for these durations. If the `period` is set to 5 minutes, then the limits implementation will monitor the send and receive traffic for the resource (share, environment, or account) for the last 5 minutes, and if the amount of data is greater than either the `warning` or the `limit` threshold, action will be taken. + +The `rx` value is the number of bytes _received_ by the resource. The `tx` value is the number of bytes _transmitted_ by the resource. And `total` is the combined `rx`+`tx` value. + +If the traffic quantity is greater than the `warning` threshold, the user will receive an email notification letting them know that their data transfer size is rising and will eventually be limited (the email details the limit threshold). + +If the traffic quantity is greater than the `limit` threshold, the resources will be limited until the traffic in the window (the last 5 minutes in our example) falls back below the `limit` threshold. + +### Limit Actions + +When a resource is limited, the actions taken differ depending on what kind of resource is being limited. + +When a share is limited, the dial service policies for that share are removed. No other action is taken. This means that public frontends will simply return a `404` as if the share is no longer there. Private frontends will also return `404` errors. When the limit is relaxed, the dial policies are put back in place and the share will continue operating normally. + +When an environment is limited, all of the shares in that environment become limited, and the user is not able to create new shares in that environment. When the limit is relaxed, all of the share limits are relaxed and the user is again able to add shares to the environment. + +When an account is limited, all of the environments in that account become limited (limiting all of the shares), and the user is not able to create new environments or shares. When the limit is relaxed, all of the environments and shares will return to normal operation. + +## Unlimited Accounts + +The `accounts` table in the database includes a `limitless` column. When this column is set to `true` the account is not subject to any of the limits in the system. \ No newline at end of file From f5ca92139b1fd7f695533f031f63175dd132a1ad Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 5 Apr 2023 11:12:29 -0400 Subject: [PATCH 78/83] link (#283) --- docs/guides/metrics-and-limits/configuring-metrics.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/guides/metrics-and-limits/configuring-metrics.md b/docs/guides/metrics-and-limits/configuring-metrics.md index de9531bb..5d58cfc7 100644 --- a/docs/guides/metrics-and-limits/configuring-metrics.md +++ b/docs/guides/metrics-and-limits/configuring-metrics.md @@ -111,4 +111,6 @@ With all of the components configured and running, either use `zrok test loop` o The `zrok` web console should also be showing activity for your share(s) like the following: -![zrok web console activity](images/zrok-console-activity.png) \ No newline at end of file +![zrok web console activity](images/zrok-console-activity.png) + +With metrics configured, you might be interested in [configuring limits](configuring-limits.md). \ No newline at end of file From 0284e41b55e15c6c86f4543d7aa3dcdeeaefab24 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 5 Apr 2023 11:45:40 -0400 Subject: [PATCH 79/83] update config version from 2 to 3 and update reference configuration (#288) --- controller/config/config.go | 2 +- etc/ctrl.yml | 90 ++++++++++++++++++++++++++++++------- 2 files changed, 75 insertions(+), 17 deletions(-) diff --git a/controller/config/config.go b/controller/config/config.go index 24422afd..9f54f241 100644 --- a/controller/config/config.go +++ b/controller/config/config.go @@ -13,7 +13,7 @@ import ( "github.com/pkg/errors" ) -const ConfigVersion = 2 +const ConfigVersion = 3 type Config struct { V int diff --git a/etc/ctrl.yml b/etc/ctrl.yml index 0bcf4250..e7d52977 100644 --- a/etc/ctrl.yml +++ b/etc/ctrl.yml @@ -9,7 +9,7 @@ # configuration, the software will expect this field to be incremented. This protects you against invalid configuration # versions. # -v: 2 +v: 3 admin: # The `secrets` array contains a list of strings that represent valid `ZROK_ADMIN_TOKEN` values to be used for @@ -23,6 +23,20 @@ admin: # tou_link: 'Terms and Conditions' +# The `bridge` section configures the `zrok controller metrics bridge`, specifying the source and sink where OpenZiti +# `fabric.usage` events are consumed and then sent into `zrok`. For production environments, we recommend that you use +# the `fileSource`, tailing the events from a JSON file written to by the OpenZiti controller. The `amqpSink` will then +# forward the events to an AMQP queue for consumption by multiple `zrok` controllers. +# +bridge: + source: + type: fileSource + path: /tmp/fabric-usage.log + sink: + type: amqpSink + url: amqp://guest:guest@localhost:5672 + queue_name: events + # The `endpoint` section determines where the HTTP listener that serves the API and web console will be bound. # endpoint: @@ -38,20 +52,46 @@ email: password: "" from: ziggy@zrok.io -# InfluxDB configuration. InfluxDB is used to support sparkline displays in the web console. +# Service instance limits configuration. # -influx: - url: http://127.0.0.1:8086 - bucket: zrok - org: zrok - token: "" - -# Instance-wide limits for per-user limits. `-1` represents unlimited. Each user can have the `limitless` flag set on -# their record in the `accounts` table in the database, to allow the user to ignore the instance-wide limits. +# See `docs/guides/metrics-and-limits/configuring-limits.md` for details. # limits: - environments: -1 - shares: -1 + environments: -1 + shares: -1 + bandwidth: + per_account: + period: 5m + warning: + rx: -1 + tx: -1 + total: 7242880 + limit: + rx: -1 + tx: -1 + total: 10485760 + per_environment: + period: 5m + warning: + rx: -1 + tx: -1 + total: -1 + limit: + rx: -1 + tx: -1 + total: -1 + per_share: + period: 5m + warning: + rx: -1 + tx: -1 + total: -1 + limit: + rx: -1 + tx: -1 + total: -1 + enforcing: false + cycle: 5m # Background maintenance job configuration. The `registration` job purges registration requests created through the # `zrok invite` tool. The `reset_password` job purges password reset requests. @@ -66,17 +106,35 @@ maintenance: check_frequency: 15m batch_limit: 500 -# The name of the service used to report metrics from the frontends (`zrok access public`) to the zrok controller -# fleet. +# Metrics configuration. # metrics: - service_name: metrics + agent: + # The `source` controls where the `zrok controller` looks to consume OpenZiti `fabric.usage` events. This works in + # concert with the `bridge` section above to consume events from an AMQP queue. This can also be configured to work + # with a `fileSource` (see the `bridge` section above for details), and also with a `websocketSource`. + # + source: + type: amqpSource + url: amqp://guest:guest@localhost:5672 + queue_name: events + # + # The `influx` section configures access to the InfluxDB instance used to store `zrok` metrics. + # + influx: + url: "http://127.0.0.1:8086" + bucket: zrok + org: zrok + token: "" # Configure the generated URL for the registration email. The registration token will be appended to this URL. # registration: registration_url_template: https://zrok.server.com/register - token_strategy: store + # + # Set `token_strategy` to `store` to require an invite token. + # + #token_strategy: store # Configure the generated URL for password resets. The reset token will be appended to this URL. # From e3338f5981b6d7d7625dfe4f522b429e1bff4659 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 5 Apr 2023 13:29:37 -0400 Subject: [PATCH 80/83] changelog (#235) --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eb3bb61..0c46ea1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # v0.4.0 -FEATURE: New metrics infrastructure based on OpenZiti usage events (https://github.com/openziti/zrok/issues/128). See the [v0.4 Metrics Guide](docs/guides/v0.4_metrics.md) for more information. +FEATURE: New metrics infrastructure based on OpenZiti usage events (https://github.com/openziti/zrok/issues/128). See the [v0.4 Metrics Guide](docs/guides/metrics-and-limits/configuring-metrics.md) for more information. + +FEATURE: New limits implementation based on the new metrics infrastructure (https://github.com/openziti/zrok/issues/235). See the [v0.4 Limits Guide](docs/guides/metrics-and-limits/configuring-limits.md) for more information. + +CHANGE: The controller configuration version bumps from `v: 2` to `v: 3` to support all of the new `v0.4` functionality. See the [example ctrl.yml](etc/ctrl.yml) for details on the new configuration. CHANGE: The underlying database store now utilizes a `deleted` flag on all tables to implement "soft deletes". This was necessary for the new metrics infrastructure, where we need to account for metrics data that arrived after the lifetime of a share or environment; and also we're going to need this for limits, where we need to see historical information about activity in the past (https://github.com/openziti/zrok/issues/262) From c8313d12b78cb2ee4f31627abe49375240641218 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 5 Apr 2023 13:57:22 -0400 Subject: [PATCH 81/83] fixes from video (#235) --- controller/limits/agent.go | 14 +++++++++----- controller/metrics/amqpSink.go | 2 ++ .../010_v0_4_0_frontend_private_share.sql | 6 +++--- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/controller/limits/agent.go b/controller/limits/agent.go index f2ff9d4c..1f05fd18 100644 --- a/controller/limits/agent.go +++ b/controller/limits/agent.go @@ -70,13 +70,17 @@ func (a *Agent) Stop() { func (a *Agent) CanCreateEnvironment(acctId int, trx *sqlx.Tx) (bool, error) { if a.cfg.Enforcing { - alj, err := a.str.FindLatestAccountLimitJournal(acctId, trx) - if err != nil { + if empty, err := a.str.IsAccountLimitJournalEmpty(acctId, trx); err == nil && !empty { + alj, err := a.str.FindLatestAccountLimitJournal(acctId, trx) + if err != nil { + return false, err + } + if alj.Action == store.LimitAction { + return false, nil + } + } else if err != nil { return false, err } - if alj.Action == store.LimitAction { - return false, nil - } if a.cfg.Environments > Unlimited { envs, err := a.str.FindEnvironmentsForAccount(acctId, trx) diff --git a/controller/metrics/amqpSink.go b/controller/metrics/amqpSink.go index 451bdb7b..520e86b7 100644 --- a/controller/metrics/amqpSink.go +++ b/controller/metrics/amqpSink.go @@ -6,6 +6,7 @@ import ( "github.com/openziti/zrok/controller/env" "github.com/pkg/errors" amqp "github.com/rabbitmq/amqp091-go" + "github.com/sirupsen/logrus" "time" ) @@ -57,6 +58,7 @@ func newAmqpSink(cfg *AmqpSinkConfig) (*amqpSink, error) { func (s *amqpSink) Handle(event ZitiEventJson) error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() + logrus.Infof("pushing '%v'", event) return s.ch.PublishWithContext(ctx, "", s.queue.Name, false, false, amqp.Publishing{ ContentType: "application/json", Body: []byte(event), diff --git a/controller/store/sql/postgresql/010_v0_4_0_frontend_private_share.sql b/controller/store/sql/postgresql/010_v0_4_0_frontend_private_share.sql index cabf651d..c08c193d 100644 --- a/controller/store/sql/postgresql/010_v0_4_0_frontend_private_share.sql +++ b/controller/store/sql/postgresql/010_v0_4_0_frontend_private_share.sql @@ -14,7 +14,7 @@ create table frontends ( reserved boolean not null default(false), created_at timestamptz not null default(current_timestamp), updated_at timestamptz not null default(current_timestamp), - deleted boolean not null default(false), + deleted boolean not null default(false) ); insert into frontends (id, environment_id, token, z_id, url_template, public_name, reserved, created_at, updated_at, deleted) @@ -25,7 +25,7 @@ select setval('frontends_id_seq', (select max(id) from frontends)); drop table frontends_old; alter index frontends_pkey1 rename to frontends_pkey; -alter index frontends_public_name_key1 to frontends_public_name_key; -alter index frontends_token_key1 to frontends_token_key; +alter index frontends_public_name_key1 rename to frontends_public_name_key; +alter index frontends_token_key1 rename to frontends_token_key; alter table frontends rename constraint frontends_environment_id_fkey1 to frontends_environment_id_fkey; From 35f991d42b588567e564386acb30016eeb105784 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 7 Apr 2023 10:17:19 -0400 Subject: [PATCH 82/83] pre-req in limits guide (#283) --- docs/guides/metrics-and-limits/configuring-limits.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/guides/metrics-and-limits/configuring-limits.md b/docs/guides/metrics-and-limits/configuring-limits.md index 2f1a4be6..79628927 100644 --- a/docs/guides/metrics-and-limits/configuring-limits.md +++ b/docs/guides/metrics-and-limits/configuring-limits.md @@ -1,5 +1,7 @@ # Configuring Limits +> If you have not yet configured [metrics](configuring-metrics.md), please visit the [metrics guide](configuring-metrics.md) first before working through the limits configuration. + The limits facility in `zrok` is responsible for controlling the number of resources in use (environments, shares) and also for ensuring that any single account, environment, or share is held below the configured thresholds. Take this `zrok` controller configuration stanza as an example: From e0e3e0286c07486c6a9a6a90ca8661d7a14177da Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 7 Apr 2023 13:15:34 -0400 Subject: [PATCH 83/83] preserve the form of the wss url for websocket_endpoint --- controller/metrics/websocketSource.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/controller/metrics/websocketSource.go b/controller/metrics/websocketSource.go index e6b66ac6..1eb7e4e5 100644 --- a/controller/metrics/websocketSource.go +++ b/controller/metrics/websocketSource.go @@ -27,8 +27,8 @@ func init() { } type WebsocketSourceConfig struct { - WebsocketEndpoint string - ApiEndpoint string + WebsocketEndpoint string // wss://127.0.0.1:1280/fabric/v1/ws-api + ApiEndpoint string // https://127.0.0.1:1280 Username string Password string `cf:"+secret"` } @@ -41,7 +41,7 @@ func loadWebsocketSourceConfig(v interface{}, _ *cf.Options) (interface{}, error } return &websocketSource{cfg: cfg}, nil } - return nil, errors.New("invalid config struture for 'websocketSource'") + return nil, errors.New("invalid config structure for 'websocketSource'") } type websocketSource struct {