enabler canary improvements (#771, #935)

canary/disabler.go

@@ -0,0 +1,72 @@
+package canary
+
+import (
+	"github.com/openziti/zrok/environment/env_core"
+	"github.com/openziti/zrok/sdk/golang/sdk"
+	"github.com/sirupsen/logrus"
+	"math/rand"
+	"time"
+)
+
+type DisablerOptions struct {
+	Environments chan *sdk.Environment
+	MinDwell     time.Duration
+	MaxDwell     time.Duration
+	MinPacing    time.Duration
+	MaxPacing    time.Duration
+}
+
+type Disabler struct {
+	Id   uint
+	Done chan struct{}
+	opt  *DisablerOptions
+	root env_core.Root
+}
+
+func NewDisabler(id uint, opt *DisablerOptions, root env_core.Root) *Disabler {
+	return &Disabler{
+		Id:   id,
+		Done: make(chan struct{}),
+		opt:  opt,
+		root: root,
+	}
+}
+
+func (d *Disabler) Run() {
+	defer logrus.Infof("#%d stopping", d.Id)
+	defer close(d.Done)
+	d.dwell()
+	d.iterate()
+}
+
+func (d *Disabler) dwell() {
+	dwell := d.opt.MinDwell.Milliseconds()
+	dwelta := d.opt.MaxDwell.Milliseconds() - d.opt.MinDwell.Milliseconds()
+	if dwelta > 0 {
+		dwell = int64(rand.Intn(int(dwelta)) + int(d.opt.MinDwell.Milliseconds()))
+	}
+	time.Sleep(time.Duration(dwell) * time.Millisecond)
+}
+
+func (d *Disabler) iterate() {
+	for {
+		select {
+		case env, ok := <-d.opt.Environments:
+			if !ok {
+				return
+			}
+			if err := sdk.DisableEnvironment(env, d.root); err == nil {
+				logrus.Infof("#%d disabled environment '%v'", d.Id, env.ZitiIdentity)
+			} else {
+				logrus.Errorf("error disabling canary (#%d) environment '%v': %v", d.Id, env.ZitiIdentity, err)
+			}
+		}
+
+		pacingMs := d.opt.MaxPacing.Milliseconds()
+		pacingDelta := d.opt.MaxPacing.Milliseconds() - d.opt.MinPacing.Milliseconds()
+		if pacingDelta > 0 {
+			pacingMs = (rand.Int63() % pacingDelta) + d.opt.MinPacing.Milliseconds()
+			time.Sleep(time.Duration(pacingMs) * time.Millisecond)
+		}
+	}
+}

canary/enabler.go

@@ -57,8 +57,16 @@ func (e *Enabler) iterate() {
 		})
 		if err == nil {
 			e.Environments <- env
+			logrus.Infof("#%d enabled environment '%v'", e.Id, env.ZitiIdentity)
 		} else {
-			logrus.Errorf("error creating canary environment: %v", err)
+			logrus.Errorf("error creating canary (#%d) environment: %v", e.Id, err)
+		}
+
+		pacingMs := e.opt.MaxPacing.Milliseconds()
+		pacingDelta := e.opt.MaxPacing.Milliseconds() - e.opt.MinPacing.Milliseconds()
+		if pacingDelta > 0 {
+			pacingMs = (rand.Int63() % pacingDelta) + e.opt.MinPacing.Milliseconds()
+			time.Sleep(time.Duration(pacingMs) * time.Millisecond)
 		}
 	}
 }

cmd/zrok/testCanaryEnabler.go

@@ -23,6 +23,7 @@ type testCanaryEnabler struct {
 	maxDwell    time.Duration
 	minPacing   time.Duration
 	maxPacing   time.Duration
+	disable     bool
 }
 
 func newTestCanaryEnabler() *testCanaryEnabler {
@@ -35,10 +36,11 @@ func newTestCanaryEnabler() *testCanaryEnabler {
 	cmd.Run = command.run
 	cmd.Flags().UintVarP(&command.enablers, "enablers", "e", 1, "Number of concurrent enablers to start")
 	cmd.Flags().UintVarP(&command.iterations, "iterations", "i", 1, "Number of iterations")
-	cmd.Flags().DurationVar(&command.minDwell, "min-dwell", 1*time.Second, "Minimum dwell time")
-	cmd.Flags().DurationVar(&command.maxDwell, "max-dwell", 1*time.Second, "Maximum dwell time")
+	cmd.Flags().DurationVar(&command.minDwell, "min-dwell", 0, "Minimum dwell time")
+	cmd.Flags().DurationVar(&command.maxDwell, "max-dwell", 0, "Maximum dwell time")
 	cmd.Flags().DurationVar(&command.minPacing, "min-pacing", 0, "Minimum pacing time")
 	cmd.Flags().DurationVar(&command.maxPacing, "max-pacing", 0, "Maximum pacing time")
+	cmd.Flags().BoolVar(&command.disable, "disable", true, "Disable (clean up) enabled environments")
 	return command
 }
 
@@ -69,15 +71,31 @@ func (cmd *testCanaryEnabler) run(_ *cobra.Command, _ []string) {
 		go enabler.Run()
 	}
 
-	for _, enabler := range enablers {
-	enablerLoop:
-		for {
-			select {
-			case env, ok := <-enabler.Environments:
-				if !ok {
-					break enablerLoop
+	if cmd.disable {
+		var disablers []*canary.Disabler
+		for i := uint(0); i < cmd.enablers; i++ {
+			disablerOpts := &canary.DisablerOptions{
+				Environments: enablers[i].Environments,
+			}
+			disabler := canary.NewDisabler(i, disablerOpts, root)
+			disablers = append(disablers, disabler)
+			go disabler.Run()
+		}
+		for _, disabler := range disablers {
+			<-disabler.Done
+		}
+
+	} else {
+		for _, enabler := range enablers {
+		enablerLoop:
+			for {
+				select {
+				case env, ok := <-enabler.Environments:
+					if !ok {
+						break enablerLoop
+					}
+					logrus.Infof("enabler #%d: %v", enabler.Id, env.ZitiIdentity)
 				}
-				logrus.Infof("enabler #%d: %v", enabler.Id, env.ZitiIdentity)
 			}
 		}
 	}

sdk/golang/sdk/environment.go

@@ -7,12 +7,12 @@ import (
 	"github.com/pkg/errors"
 )
 
-func EnableEnvironment(env env_core.Root, request *EnableRequest) (*Environment, error) {
-	zrok, err := env.Client()
+func EnableEnvironment(root env_core.Root, request *EnableRequest) (*Environment, error) {
+	zrok, err := root.Client()
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create zrok client")
 	}
-	auth := httptransport.APIKeyAuth("X-TOKEN", "header", env.Environment().AccountToken)
+	auth := httptransport.APIKeyAuth("X-TOKEN", "header", root.Environment().AccountToken)
 
 	req := restEnvironment.NewEnableParams()
 	req.Body.Description = request.Description
@@ -30,3 +30,21 @@ func EnableEnvironment(env env_core.Root, request *EnableRequest) (*Environment,
 		ZitiConfig:   resp.Payload.Cfg,
 	}, nil
 }
+
+func DisableEnvironment(env *Environment, root env_core.Root) error {
+	zrok, err := root.Client()
+	if err != nil {
+		return errors.Wrap(err, "could not create zrok client")
+	}
+	auth := httptransport.APIKeyAuth("X-TOKEN", "header", root.Environment().AccountToken)
+
+	req := restEnvironment.NewDisableParams()
+	req.Body.Identity = env.ZitiIdentity
+
+	_, err = zrok.Environment.Disable(req, auth)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}