From 9fdd2620d6f83252a67fe81e539aa9d23b796c6b Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 18 Apr 2025 11:57:18 -0400 Subject: [PATCH] enabler canary improvements (#771, #935) --- canary/disabler.go | 72 +++++++++++++++++++++++++++++++++++ canary/enabler.go | 10 ++++- cmd/zrok/testCanaryEnabler.go | 38 +++++++++++++----- sdk/golang/sdk/environment.go | 24 ++++++++++-- 4 files changed, 130 insertions(+), 14 deletions(-) create mode 100644 canary/disabler.go diff --git a/canary/disabler.go b/canary/disabler.go new file mode 100644 index 00000000..02533a6f --- /dev/null +++ b/canary/disabler.go @@ -0,0 +1,72 @@ +package canary + +import ( + "github.com/openziti/zrok/environment/env_core" + "github.com/openziti/zrok/sdk/golang/sdk" + "github.com/sirupsen/logrus" + "math/rand" + "time" +) + +type DisablerOptions struct { + Environments chan *sdk.Environment + MinDwell time.Duration + MaxDwell time.Duration + MinPacing time.Duration + MaxPacing time.Duration +} + +type Disabler struct { + Id uint + Done chan struct{} + opt *DisablerOptions + root env_core.Root +} + +func NewDisabler(id uint, opt *DisablerOptions, root env_core.Root) *Disabler { + return &Disabler{ + Id: id, + Done: make(chan struct{}), + opt: opt, + root: root, + } +} + +func (d *Disabler) Run() { + defer logrus.Infof("#%d stopping", d.Id) + defer close(d.Done) + d.dwell() + d.iterate() +} + +func (d *Disabler) dwell() { + dwell := d.opt.MinDwell.Milliseconds() + dwelta := d.opt.MaxDwell.Milliseconds() - d.opt.MinDwell.Milliseconds() + if dwelta > 0 { + dwell = int64(rand.Intn(int(dwelta)) + int(d.opt.MinDwell.Milliseconds())) + } + time.Sleep(time.Duration(dwell) * time.Millisecond) +} + +func (d *Disabler) iterate() { + for { + select { + case env, ok := <-d.opt.Environments: + if !ok { + return + } + if err := sdk.DisableEnvironment(env, d.root); err == nil { + logrus.Infof("#%d disabled environment '%v'", d.Id, env.ZitiIdentity) + } else { + logrus.Errorf("error disabling canary (#%d) environment '%v': %v", d.Id, env.ZitiIdentity, err) + } + } + + pacingMs := d.opt.MaxPacing.Milliseconds() + pacingDelta := d.opt.MaxPacing.Milliseconds() - d.opt.MinPacing.Milliseconds() + if pacingDelta > 0 { + pacingMs = (rand.Int63() % pacingDelta) + d.opt.MinPacing.Milliseconds() + time.Sleep(time.Duration(pacingMs) * time.Millisecond) + } + } +} diff --git a/canary/enabler.go b/canary/enabler.go index 7224d069..259fe3e2 100644 --- a/canary/enabler.go +++ b/canary/enabler.go @@ -57,8 +57,16 @@ func (e *Enabler) iterate() { }) if err == nil { e.Environments <- env + logrus.Infof("#%d enabled environment '%v'", e.Id, env.ZitiIdentity) } else { - logrus.Errorf("error creating canary environment: %v", err) + logrus.Errorf("error creating canary (#%d) environment: %v", e.Id, err) + } + + pacingMs := e.opt.MaxPacing.Milliseconds() + pacingDelta := e.opt.MaxPacing.Milliseconds() - e.opt.MinPacing.Milliseconds() + if pacingDelta > 0 { + pacingMs = (rand.Int63() % pacingDelta) + e.opt.MinPacing.Milliseconds() + time.Sleep(time.Duration(pacingMs) * time.Millisecond) } } } diff --git a/cmd/zrok/testCanaryEnabler.go b/cmd/zrok/testCanaryEnabler.go index 8f2cf7f9..028d4d74 100644 --- a/cmd/zrok/testCanaryEnabler.go +++ b/cmd/zrok/testCanaryEnabler.go @@ -23,6 +23,7 @@ type testCanaryEnabler struct { maxDwell time.Duration minPacing time.Duration maxPacing time.Duration + disable bool } func newTestCanaryEnabler() *testCanaryEnabler { @@ -35,10 +36,11 @@ func newTestCanaryEnabler() *testCanaryEnabler { cmd.Run = command.run cmd.Flags().UintVarP(&command.enablers, "enablers", "e", 1, "Number of concurrent enablers to start") cmd.Flags().UintVarP(&command.iterations, "iterations", "i", 1, "Number of iterations") - cmd.Flags().DurationVar(&command.minDwell, "min-dwell", 1*time.Second, "Minimum dwell time") - cmd.Flags().DurationVar(&command.maxDwell, "max-dwell", 1*time.Second, "Maximum dwell time") + cmd.Flags().DurationVar(&command.minDwell, "min-dwell", 0, "Minimum dwell time") + cmd.Flags().DurationVar(&command.maxDwell, "max-dwell", 0, "Maximum dwell time") cmd.Flags().DurationVar(&command.minPacing, "min-pacing", 0, "Minimum pacing time") cmd.Flags().DurationVar(&command.maxPacing, "max-pacing", 0, "Maximum pacing time") + cmd.Flags().BoolVar(&command.disable, "disable", true, "Disable (clean up) enabled environments") return command } @@ -69,15 +71,31 @@ func (cmd *testCanaryEnabler) run(_ *cobra.Command, _ []string) { go enabler.Run() } - for _, enabler := range enablers { - enablerLoop: - for { - select { - case env, ok := <-enabler.Environments: - if !ok { - break enablerLoop + if cmd.disable { + var disablers []*canary.Disabler + for i := uint(0); i < cmd.enablers; i++ { + disablerOpts := &canary.DisablerOptions{ + Environments: enablers[i].Environments, + } + disabler := canary.NewDisabler(i, disablerOpts, root) + disablers = append(disablers, disabler) + go disabler.Run() + } + for _, disabler := range disablers { + <-disabler.Done + } + + } else { + for _, enabler := range enablers { + enablerLoop: + for { + select { + case env, ok := <-enabler.Environments: + if !ok { + break enablerLoop + } + logrus.Infof("enabler #%d: %v", enabler.Id, env.ZitiIdentity) } - logrus.Infof("enabler #%d: %v", enabler.Id, env.ZitiIdentity) } } } diff --git a/sdk/golang/sdk/environment.go b/sdk/golang/sdk/environment.go index cc6b1a8a..ab5390ae 100644 --- a/sdk/golang/sdk/environment.go +++ b/sdk/golang/sdk/environment.go @@ -7,12 +7,12 @@ import ( "github.com/pkg/errors" ) -func EnableEnvironment(env env_core.Root, request *EnableRequest) (*Environment, error) { - zrok, err := env.Client() +func EnableEnvironment(root env_core.Root, request *EnableRequest) (*Environment, error) { + zrok, err := root.Client() if err != nil { return nil, errors.Wrap(err, "could not create zrok client") } - auth := httptransport.APIKeyAuth("X-TOKEN", "header", env.Environment().AccountToken) + auth := httptransport.APIKeyAuth("X-TOKEN", "header", root.Environment().AccountToken) req := restEnvironment.NewEnableParams() req.Body.Description = request.Description @@ -30,3 +30,21 @@ func EnableEnvironment(env env_core.Root, request *EnableRequest) (*Environment, ZitiConfig: resp.Payload.Cfg, }, nil } + +func DisableEnvironment(env *Environment, root env_core.Root) error { + zrok, err := root.Client() + if err != nil { + return errors.Wrap(err, "could not create zrok client") + } + auth := httptransport.APIKeyAuth("X-TOKEN", "header", root.Environment().AccountToken) + + req := restEnvironment.NewDisableParams() + req.Body.Identity = env.ZitiIdentity + + _, err = zrok.Environment.Disable(req, auth) + if err != nil { + return err + } + + return nil +}