Merge pull request #946 from openziti/lifecycle_2

Unbootstrapper; Canary Additions

.gitignore

@@ -4,6 +4,7 @@
 *.db
 /automated-release-build/
 etc/dev.yml
+etc/dev-canary.yml
 etc/dev-frontend.yml
 
 # Dependencies

CHANGELOG.md

@@ -1,5 +1,15 @@
 # CHANGELOG
 
+## v1.0.3
+
+FEATURE: New `zrok admin unbootstrap` to remove zrok resources from the underlying OpenZiti instance (https://github.com/openziti/zrok/issues/935)
+
+FEATURE: New InfluxDB metrics capture infrastructure for `zrok test canary` framework (https://github.com/openziti/zrok/issues/948)
+
+FEATURE: New `zrok test canary enabler` to validate `enable`/`disable` operations and gather performance metrics around how those paths are operating (https://github.com/openziti/zrok/issues/771)
+
+CHANGE: New _guard_ to prevent users from running potentially dangerous `zrok test canary` commands inadvertently without understanding what they do (https://github.com/openziti/zrok/issues/947)
+
 ## v1.0.2
 
 FEATURE: "Auto-rebase" for enabled environments where the `apiEndpoint` is set to `https://api.zrok.io`. This will automatically migrate existing environments to the new `apiEndpoint` for the `v1.0.x` series (https://github.com/openziti/zrok/issues/936)

canary/config.go

@@ -0,0 +1,31 @@
+package canary
+
+import (
+	"github.com/michaelquigley/cf"
+	"github.com/openziti/zrok/controller/metrics"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const ConfigVersion = 1
+
+type Config struct {
+	V      int
+	Influx *metrics.InfluxConfig
+}
+
+func DefaultConfig() *Config {
+	return &Config{}
+}
+
+func LoadConfig(path string) (*Config, error) {
+	cfg := DefaultConfig()
+	if err := cf.BindYaml(cfg, path, cf.DefaultOptions()); err != nil {
+		return nil, errors.Wrapf(err, "error loading canary configuration '%v'", path)
+	}
+	if cfg.V != ConfigVersion {
+		return nil, errors.Errorf("expecting canary configuration version '%v', got '%v'", ConfigVersion, cfg.V)
+	}
+	logrus.Info(cf.Dump(cfg, cf.DefaultOptions()))
+	return cfg, nil
+}

canary/dangerous.go

@@ -0,0 +1,13 @@
+package canary
+
+import (
+	"fmt"
+	"os"
+)
+
+func AcknowledgeDangerousCanary() error {
+	if _, ok := os.LookupEnv("ZROK_DANGEROUS_CANARY"); !ok {
+		return fmt.Errorf("this is a dangerous canary; see canary docs for details on enabling")
+	}
+	return nil
+}

canary/disabler.go

@@ -0,0 +1,92 @@
+package canary
+
+import (
+	"github.com/openziti/zrok/environment/env_core"
+	"github.com/openziti/zrok/sdk/golang/sdk"
+	"github.com/sirupsen/logrus"
+	"math/rand"
+	"time"
+)
+
+type DisablerOptions struct {
+	Environments  chan *sdk.Environment
+	MinDwell      time.Duration
+	MaxDwell      time.Duration
+	MinPacing     time.Duration
+	MaxPacing     time.Duration
+	SnapshotQueue chan *Snapshot
+}
+
+type Disabler struct {
+	Id   uint
+	Done chan struct{}
+	opt  *DisablerOptions
+	root env_core.Root
+}
+
+func NewDisabler(id uint, opt *DisablerOptions, root env_core.Root) *Disabler {
+	return &Disabler{
+		Id:   id,
+		Done: make(chan struct{}),
+		opt:  opt,
+		root: root,
+	}
+}
+
+func (d *Disabler) Run() {
+	defer logrus.Infof("#%d stopping", d.Id)
+	defer close(d.Done)
+	d.dwell()
+	d.iterate()
+}
+
+func (d *Disabler) dwell() {
+	dwell := d.opt.MinDwell.Milliseconds()
+	dwelta := d.opt.MaxDwell.Milliseconds() - d.opt.MinDwell.Milliseconds()
+	if dwelta > 0 {
+		dwell = int64(rand.Intn(int(dwelta)) + int(d.opt.MinDwell.Milliseconds()))
+	}
+	time.Sleep(time.Duration(dwell) * time.Millisecond)
+}
+
+func (d *Disabler) iterate() {
+	iteration := uint64(0)
+	for {
+		select {
+		case env, ok := <-d.opt.Environments:
+			if !ok {
+				return
+			}
+
+			snapshot := NewSnapshot("disable", d.Id, iteration)
+			iteration++
+
+			if err := sdk.DisableEnvironment(env, d.root); err == nil {
+				snapshot.Completed = time.Now()
+				snapshot.Ok = true
+
+				logrus.Infof("#%d disabled environment '%v'", d.Id, env.ZitiIdentity)
+
+			} else {
+				snapshot.Completed = time.Now()
+				snapshot.Ok = false
+				snapshot.Error = err
+
+				logrus.Errorf("error disabling canary (#%d) environment '%v': %v", d.Id, env.ZitiIdentity, err)
+			}
+
+			if d.opt.SnapshotQueue != nil {
+				d.opt.SnapshotQueue <- snapshot
+			} else {
+				logrus.Info(snapshot)
+			}
+		}
+
+		pacingMs := d.opt.MaxPacing.Milliseconds()
+		pacingDelta := d.opt.MaxPacing.Milliseconds() - d.opt.MinPacing.Milliseconds()
+		if pacingDelta > 0 {
+			pacingMs = (rand.Int63() % pacingDelta) + d.opt.MinPacing.Milliseconds()
+			time.Sleep(time.Duration(pacingMs) * time.Millisecond)
+		}
+	}
+}

canary/enabler.go

@@ -0,0 +1,93 @@
+package canary
+
+import (
+	"fmt"
+	"github.com/openziti/zrok/environment/env_core"
+	"github.com/openziti/zrok/sdk/golang/sdk"
+	"github.com/sirupsen/logrus"
+	"math/rand"
+	"time"
+)
+
+type EnablerOptions struct {
+	Iterations    uint
+	MinDwell      time.Duration
+	MaxDwell      time.Duration
+	MinPacing     time.Duration
+	MaxPacing     time.Duration
+	SnapshotQueue chan *Snapshot
+}
+
+type Enabler struct {
+	Id           uint
+	Done         chan struct{}
+	opt          *EnablerOptions
+	root         env_core.Root
+	Environments chan *sdk.Environment
+}
+
+func NewEnabler(id uint, opt *EnablerOptions, root env_core.Root) *Enabler {
+	return &Enabler{
+		Id:           id,
+		Done:         make(chan struct{}),
+		opt:          opt,
+		root:         root,
+		Environments: make(chan *sdk.Environment, opt.Iterations),
+	}
+}
+
+func (e *Enabler) Run() {
+	defer close(e.Environments)
+	defer close(e.Done)
+	defer logrus.Infof("#%d stopping", e.Id)
+	e.dwell()
+	e.iterate()
+}
+
+func (e *Enabler) dwell() {
+	dwell := e.opt.MinDwell.Milliseconds()
+	dwelta := e.opt.MaxDwell.Milliseconds() - e.opt.MinDwell.Milliseconds()
+	if dwelta > 0 {
+		dwell = int64(rand.Intn(int(dwelta)) + int(e.opt.MinDwell.Milliseconds()))
+	}
+	time.Sleep(time.Duration(dwell) * time.Millisecond)
+}
+
+func (e *Enabler) iterate() {
+	defer logrus.Info("done")
+	for i := uint(0); i < e.opt.Iterations; i++ {
+		snapshot := NewSnapshot("enable", e.Id, uint64(i))
+
+		env, err := sdk.EnableEnvironment(e.root, &sdk.EnableRequest{
+			Host:        fmt.Sprintf("canary_%d_%d", e.Id, i),
+			Description: "canary.Enabler",
+		})
+		if err == nil {
+			snapshot.Completed = time.Now()
+			snapshot.Ok = true
+
+			e.Environments <- env
+			logrus.Infof("#%d enabled environment '%v'", e.Id, env.ZitiIdentity)
+
+		} else {
+			snapshot.Completed = time.Now()
+			snapshot.Ok = false
+			snapshot.Error = err
+
+			logrus.Errorf("error creating canary (#%d) environment: %v", e.Id, err)
+		}
+
+		if e.opt.SnapshotQueue != nil {
+			e.opt.SnapshotQueue <- snapshot
+		} else {
+			logrus.Info(snapshot)
+		}
+
+		pacingMs := e.opt.MaxPacing.Milliseconds()
+		pacingDelta := e.opt.MaxPacing.Milliseconds() - e.opt.MinPacing.Milliseconds()
+		if pacingDelta > 0 {
+			pacingMs = (rand.Int63() % pacingDelta) + e.opt.MinPacing.Milliseconds()
+			time.Sleep(time.Duration(pacingMs) * time.Millisecond)
+		}
+	}
+}

canary/metrics.go

@@ -0,0 +1,102 @@
+package canary
+
+import (
+	"context"
+	"fmt"
+	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
+	"github.com/openziti/zrok/util"
+	"github.com/sirupsen/logrus"
+	"slices"
+	"sort"
+	"time"
+)
+
+type Snapshot struct {
+	Operation string
+	Instance  uint
+	Iteration uint64
+	Started   time.Time
+	Completed time.Time
+	Ok        bool
+	Error     error
+	Size      uint64
+}
+
+func NewSnapshot(operation string, instance uint, iteration uint64) *Snapshot {
+	return &Snapshot{Operation: operation, Instance: instance, Iteration: iteration, Started: time.Now()}
+}
+
+func (s *Snapshot) String() string {
+	if s.Ok {
+		return fmt.Sprintf("[%v, %d, %d] (ok) %v, %v", s.Operation, s.Instance, s.Iteration, s.Completed.Sub(s.Started), util.BytesToSize(int64(s.Size)))
+	} else {
+		return fmt.Sprintf("[%v, %d, %d] (err) %v, %v, (%v)", s.Operation, s.Instance, s.Iteration, s.Completed.Sub(s.Started), util.BytesToSize(int64(s.Size)), s.Error)
+	}
+}
+
+type SnapshotCollector struct {
+	InputQueue chan *Snapshot
+	Closed     chan struct{}
+	ctx        context.Context
+	cfg        *Config
+	snapshots  map[string][]*Snapshot
+}
+
+func NewSnapshotCollector(ctx context.Context, cfg *Config) *SnapshotCollector {
+	return &SnapshotCollector{
+		InputQueue: make(chan *Snapshot),
+		Closed:     make(chan struct{}),
+		ctx:        ctx,
+		cfg:        cfg,
+		snapshots:  make(map[string][]*Snapshot),
+	}
+}
+
+func (sc *SnapshotCollector) Run() {
+	defer close(sc.Closed)
+	defer logrus.Info("stopping")
+	logrus.Info("starting")
+	for {
+		select {
+		case <-sc.ctx.Done():
+			return
+
+		case snapshot := <-sc.InputQueue:
+			var snapshots []*Snapshot
+			if v, ok := sc.snapshots[snapshot.Operation]; ok {
+				snapshots = v
+			}
+			i := sort.Search(len(snapshots), func(i int) bool { return snapshots[i].Completed.After(snapshot.Started) })
+			snapshots = slices.Insert(snapshots, i, snapshot)
+			sc.snapshots[snapshot.Operation] = snapshots
+		}
+	}
+}
+
+func (sc *SnapshotCollector) Store() error {
+	idb := influxdb2.NewClient(sc.cfg.Influx.Url, sc.cfg.Influx.Token)
+	writeApi := idb.WriteAPIBlocking(sc.cfg.Influx.Org, sc.cfg.Influx.Bucket)
+	for key, arr := range sc.snapshots {
+		for _, snapshot := range arr {
+			tags := map[string]string{
+				"instance":  fmt.Sprintf("%d", snapshot.Instance),
+				"iteration": fmt.Sprintf("%d", snapshot.Iteration),
+				"ok":        fmt.Sprintf("%t", snapshot.Ok),
+			}
+			if snapshot.Error != nil {
+				tags["error"] = snapshot.Error.Error()
+			}
+			pt := influxdb2.NewPoint(snapshot.Operation, tags, map[string]interface{}{
+				"duration": snapshot.Completed.Sub(snapshot.Started).Milliseconds(),
+				"size":     snapshot.Size,
+			}, snapshot.Started)
+			if err := writeApi.WritePoint(context.Background(), pt); err != nil {
+				return err
+			}
+		}
+		logrus.Infof("wrote '%v' points for '%v'", len(arr), key)
+	}
+	idb.Close()
+	logrus.Infof("complete")
+	return nil
+}

cmd/zrok/adminUnbootstrap.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/michaelquigley/cf"
+	"github.com/openziti/zrok/controller"
+	"github.com/openziti/zrok/controller/config"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	adminCmd.AddCommand(newAdminUnbootstrap().cmd)
+}
+
+type adminUnbootstrap struct {
+	cmd *cobra.Command
+}
+
+func newAdminUnbootstrap() *adminUnbootstrap {
+	cmd := &cobra.Command{
+		Use:   "unbootstrap <configPath>",
+		Short: "Unbootstrap the underlying Ziti network from zrok",
+		Args:  cobra.ExactArgs(1),
+	}
+	command := &adminUnbootstrap{cmd: cmd}
+	cmd.Run = command.run
+	return command
+}
+
+func (cmd *adminUnbootstrap) run(_ *cobra.Command, args []string) {
+	cfg, err := config.LoadConfig(args[0])
+	if err != nil {
+		panic(err)
+	}
+	logrus.Infof(cf.Dump(cfg, cf.DefaultOptions()))
+	if err := controller.Unbootstrap(cfg); err != nil {
+		panic(err)
+	}
+	logrus.Infof("unbootstrap complete!")
+}

cmd/zrok/testCanaryEnabler.go

@@ -0,0 +1,143 @@
+package main
+
+import (
+	"context"
+	"github.com/openziti/zrok/canary"
+	"github.com/openziti/zrok/environment"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"math/rand"
+	"time"
+)
+
+func init() {
+	testCanaryCmd.AddCommand(newTestCanaryEnabler().cmd)
+}
+
+type testCanaryEnabler struct {
+	cmd          *cobra.Command
+	enablers     uint
+	iterations   uint
+	minPreDelay  time.Duration
+	maxPreDelay  time.Duration
+	minDwell     time.Duration
+	maxDwell     time.Duration
+	minPacing    time.Duration
+	maxPacing    time.Duration
+	skipDisable  bool
+	canaryConfig string
+}
+
+func newTestCanaryEnabler() *testCanaryEnabler {
+	cmd := &cobra.Command{
+		Use:   "enabler",
+		Short: "Enable a canary enabling environments",
+		Args:  cobra.NoArgs,
+	}
+	command := &testCanaryEnabler{cmd: cmd}
+	cmd.Run = command.run
+	cmd.Flags().UintVarP(&command.enablers, "enablers", "e", 1, "Number of concurrent enablers to start")
+	cmd.Flags().UintVarP(&command.iterations, "iterations", "i", 1, "Number of iterations")
+	cmd.Flags().DurationVar(&command.minDwell, "min-dwell", 0, "Minimum dwell time")
+	cmd.Flags().DurationVar(&command.maxDwell, "max-dwell", 0, "Maximum dwell time")
+	cmd.Flags().DurationVar(&command.minPacing, "min-pacing", 0, "Minimum pacing time")
+	cmd.Flags().DurationVar(&command.maxPacing, "max-pacing", 0, "Maximum pacing time")
+	cmd.Flags().BoolVar(&command.skipDisable, "skip-disable", false, "Disable (clean up) enabled environments")
+	cmd.Flags().StringVar(&command.canaryConfig, "canary-config", "", "Path to canary configuration file")
+	return command
+}
+
+func (cmd *testCanaryEnabler) run(_ *cobra.Command, _ []string) {
+	if err := canary.AcknowledgeDangerousCanary(); err != nil {
+		logrus.Fatal(err)
+	}
+
+	root, err := environment.LoadRoot()
+	if err != nil {
+		panic(err)
+	}
+
+	var sc *canary.SnapshotCollector
+	var scCtx context.Context
+	var scCancel context.CancelFunc
+	if cmd.canaryConfig != "" {
+		cfg, err := canary.LoadConfig(cmd.canaryConfig)
+		if err != nil {
+			panic(err)
+		}
+		scCtx, scCancel = context.WithCancel(context.Background())
+		sc = canary.NewSnapshotCollector(scCtx, cfg)
+		go sc.Run()
+	}
+
+	var enablers []*canary.Enabler
+	for i := uint(0); i < cmd.enablers; i++ {
+		preDelay := cmd.maxPreDelay.Milliseconds()
+		preDelayDelta := cmd.maxPreDelay.Milliseconds() - cmd.minPreDelay.Milliseconds()
+		if preDelayDelta > 0 {
+			preDelay = int64(rand.Intn(int(preDelayDelta))) + cmd.minPreDelay.Milliseconds()
+			time.Sleep(time.Duration(preDelay) * time.Millisecond)
+		}
+
+		enablerOpts := &canary.EnablerOptions{
+			Iterations: cmd.iterations,
+			MinDwell:   cmd.minDwell,
+			MaxDwell:   cmd.maxDwell,
+			MinPacing:  cmd.minPacing,
+			MaxPacing:  cmd.maxPacing,
+		}
+		if sc != nil {
+			enablerOpts.SnapshotQueue = sc.InputQueue
+		}
+		enabler := canary.NewEnabler(i, enablerOpts, root)
+		enablers = append(enablers, enabler)
+		go enabler.Run()
+	}
+
+	if !cmd.skipDisable {
+		var disablers []*canary.Disabler
+		for i := uint(0); i < cmd.enablers; i++ {
+			disablerOpts := &canary.DisablerOptions{
+				Environments: enablers[i].Environments,
+			}
+			if sc != nil {
+				disablerOpts.SnapshotQueue = sc.InputQueue
+			}
+			disabler := canary.NewDisabler(i, disablerOpts, root)
+			disablers = append(disablers, disabler)
+			go disabler.Run()
+		}
+		for _, disabler := range disablers {
+			logrus.Infof("waiting for disabler #%d", disabler.Id)
+			<-disabler.Done
+		}
+
+	} else {
+		for _, enabler := range enablers {
+		enablerLoop:
+			for {
+				select {
+				case env, ok := <-enabler.Environments:
+					if !ok {
+						break enablerLoop
+					}
+					logrus.Infof("enabler #%d: %v", enabler.Id, env.ZitiIdentity)
+				}
+			}
+		}
+	}
+
+	for _, enabler := range enablers {
+		<-enabler.Done
+	}
+
+	if sc != nil {
+		scCancel()
+		<-sc.Closed
+		if err := sc.Store(); err != nil {
+			panic(err)
+		}
+	}
+
+	logrus.Infof("complete")
+}

controller/unbootstrap.go

@@ -0,0 +1,284 @@
+package controller
+
+import (
+	"context"
+	"fmt"
+	"github.com/openziti/edge-api/rest_management_api_client"
+	apiConfig "github.com/openziti/edge-api/rest_management_api_client/config"
+	"github.com/openziti/edge-api/rest_management_api_client/edge_router_policy"
+	"github.com/openziti/edge-api/rest_management_api_client/identity"
+	"github.com/openziti/edge-api/rest_management_api_client/service"
+	"github.com/openziti/edge-api/rest_management_api_client/service_edge_router_policy"
+	"github.com/openziti/edge-api/rest_management_api_client/service_policy"
+	"github.com/openziti/zrok/controller/config"
+	"github.com/openziti/zrok/controller/zrokEdgeSdk"
+	"github.com/openziti/zrok/sdk/golang/sdk"
+	"github.com/sirupsen/logrus"
+)
+
+func Unbootstrap(cfg *config.Config) error {
+	edge, err := zrokEdgeSdk.Client(cfg.Ziti)
+	if err != nil {
+		return err
+	}
+	if err := unbootstrapServiceEdgeRouterPolicies(edge); err != nil {
+		logrus.Errorf("error unbootstrapping service edge router policies: %v", err)
+	}
+	if err := unbootstrapServicePolicies(edge); err != nil {
+		logrus.Errorf("error unbootstrapping service policies: %v", err)
+	}
+	if err := unbootstrapConfigs(edge); err != nil {
+		logrus.Errorf("error unbootrapping configs: %v", err)
+	}
+	if err := unbootstrapServices(edge); err != nil {
+		logrus.Errorf("error unbootstrapping services: %v", err)
+	}
+	if err := unbootstrapEdgeRouterPolicies(edge); err != nil {
+		logrus.Errorf("error unbootstrapping edge router policies: %v", err)
+	}
+	if err := unbootstrapIdentities(edge); err != nil {
+		logrus.Errorf("error unbootstrapping identities: %v", err)
+	}
+	if err := unbootstrapConfigType(edge); err != nil {
+		logrus.Errorf("error unbootstrapping config type: %v", err)
+	}
+	return nil
+}
+
+func unbootstrapServiceEdgeRouterPolicies(edge *rest_management_api_client.ZitiEdgeManagement) error {
+	for {
+		filter := "tags.zrok != null"
+		limit := int64(100)
+		offset := int64(0)
+		listReq := &service_edge_router_policy.ListServiceEdgeRouterPoliciesParams{
+			Filter:  &filter,
+			Limit:   &limit,
+			Offset:  &offset,
+			Context: context.Background(),
+		}
+		listResp, err := edge.ServiceEdgeRouterPolicy.ListServiceEdgeRouterPolicies(listReq, nil)
+		if err != nil {
+			return err
+		}
+		if len(listResp.Payload.Data) < 1 {
+			break
+		}
+		for _, serp := range listResp.Payload.Data {
+			delReq := &service_edge_router_policy.DeleteServiceEdgeRouterPolicyParams{
+				ID:      *serp.ID,
+				Context: context.Background(),
+			}
+			_, err := edge.ServiceEdgeRouterPolicy.DeleteServiceEdgeRouterPolicy(delReq, nil)
+			if err == nil {
+				logrus.Infof("deleted service edge router policy '%v'", *serp.ID)
+			} else {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func unbootstrapServicePolicies(edge *rest_management_api_client.ZitiEdgeManagement) error {
+	for {
+		filter := "tags.zrok != null"
+		limit := int64(100)
+		offset := int64(0)
+		listReq := &service_policy.ListServicePoliciesParams{
+			Filter:  &filter,
+			Limit:   &limit,
+			Offset:  &offset,
+			Context: context.Background(),
+		}
+		listResp, err := edge.ServicePolicy.ListServicePolicies(listReq, nil)
+		if err != nil {
+			return err
+		}
+		if len(listResp.Payload.Data) < 1 {
+			break
+		}
+		for _, sp := range listResp.Payload.Data {
+			delReq := &service_policy.DeleteServicePolicyParams{
+				ID:      *sp.ID,
+				Context: context.Background(),
+			}
+			_, err := edge.ServicePolicy.DeleteServicePolicy(delReq, nil)
+			if err == nil {
+				logrus.Infof("deleted service policy '%v'", *sp.ID)
+			} else {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func unbootstrapServices(edge *rest_management_api_client.ZitiEdgeManagement) error {
+	for {
+		filter := "tags.zrok != null"
+		limit := int64(100)
+		offset := int64(0)
+		listReq := &service.ListServicesParams{
+			Filter:  &filter,
+			Limit:   &limit,
+			Offset:  &offset,
+			Context: context.Background(),
+		}
+		listResp, err := edge.Service.ListServices(listReq, nil)
+		if err != nil {
+			return err
+		}
+		if len(listResp.Payload.Data) < 1 {
+			break
+		}
+		for _, svc := range listResp.Payload.Data {
+			delReq := &service.DeleteServiceParams{
+				ID:      *svc.ID,
+				Context: context.Background(),
+			}
+			_, err := edge.Service.DeleteService(delReq, nil)
+			if err == nil {
+				logrus.Infof("deleted service '%v' (%v)", *svc.ID, *svc.Name)
+			} else {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func unbootstrapEdgeRouterPolicies(edge *rest_management_api_client.ZitiEdgeManagement) error {
+	for {
+		filter := "tags.zrok != null"
+		limit := int64(100)
+		offset := int64(0)
+		listReq := &edge_router_policy.ListEdgeRouterPoliciesParams{
+			Filter:  &filter,
+			Limit:   &limit,
+			Offset:  &offset,
+			Context: context.Background(),
+		}
+		listResp, err := edge.EdgeRouterPolicy.ListEdgeRouterPolicies(listReq, nil)
+		if err != nil {
+			return err
+		}
+		if len(listResp.Payload.Data) < 1 {
+			break
+		}
+		for _, erp := range listResp.Payload.Data {
+			delReq := &edge_router_policy.DeleteEdgeRouterPolicyParams{
+				ID:      *erp.ID,
+				Context: context.Background(),
+			}
+			_, err := edge.EdgeRouterPolicy.DeleteEdgeRouterPolicy(delReq, nil)
+			if err == nil {
+				logrus.Infof("deleted edge router policy '%v'", *erp.ID)
+			} else {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func unbootstrapIdentities(edge *rest_management_api_client.ZitiEdgeManagement) error {
+	for {
+		filter := "tags.zrok != null"
+		limit := int64(100)
+		offset := int64(0)
+		listReq := &identity.ListIdentitiesParams{
+			Filter:  &filter,
+			Limit:   &limit,
+			Offset:  &offset,
+			Context: context.Background(),
+		}
+		listResp, err := edge.Identity.ListIdentities(listReq, nil)
+		if err != nil {
+			return err
+		}
+		if len(listResp.Payload.Data) < 1 {
+			break
+		}
+		for _, i := range listResp.Payload.Data {
+			delReq := &identity.DeleteIdentityParams{
+				ID:      *i.ID,
+				Context: context.Background(),
+			}
+			_, err := edge.Identity.DeleteIdentity(delReq, nil)
+			if err == nil {
+				logrus.Infof("deleted identity '%v' (%v)", *i.ID, *i.Name)
+			} else {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func unbootstrapConfigs(edge *rest_management_api_client.ZitiEdgeManagement) error {
+	for {
+		filter := "tags.zrok != null"
+		limit := int64(100)
+		offset := int64(0)
+		listReq := &apiConfig.ListConfigsParams{
+			Filter:  &filter,
+			Limit:   &limit,
+			Offset:  &offset,
+			Context: context.Background(),
+		}
+		listResp, err := edge.Config.ListConfigs(listReq, nil)
+		if err != nil {
+			return err
+		}
+		if len(listResp.Payload.Data) < 1 {
+			break
+		}
+		for _, listCfg := range listResp.Payload.Data {
+			delReq := &apiConfig.DeleteConfigParams{
+				ID:      *listCfg.ID,
+				Context: context.Background(),
+			}
+			_, err := edge.Config.DeleteConfig(delReq, nil)
+			if err == nil {
+				logrus.Infof("deleted config '%v'", *listCfg.ID)
+			} else {
+				return nil
+			}
+		}
+	}
+	return nil
+}
+
+func unbootstrapConfigType(edge *rest_management_api_client.ZitiEdgeManagement) error {
+	for {
+		filter := fmt.Sprintf("name = \"%v\"", sdk.ZrokProxyConfig)
+		limit := int64(100)
+		offset := int64(0)
+		listReq := &apiConfig.ListConfigTypesParams{
+			Filter:  &filter,
+			Limit:   &limit,
+			Offset:  &offset,
+			Context: context.Background(),
+		}
+		listResp, err := edge.Config.ListConfigTypes(listReq, nil)
+		if err != nil {
+			return err
+		}
+		if len(listResp.Payload.Data) < 1 {
+			break
+		}
+		for _, listCfgType := range listResp.Payload.Data {
+			delReq := &apiConfig.DeleteConfigTypeParams{
+				ID:      *listCfgType.ID,
+				Context: context.Background(),
+			}
+			_, err := edge.Config.DeleteConfigType(delReq, nil)
+			if err == nil {
+				logrus.Infof("deleted config type '%v' (%v)", *listCfgType.ID, *listCfgType.Name)
+			} else {
+				return err
+			}
+		}
+	}
+	return nil
+}

sdk/golang/sdk/environment.go

@@ -0,0 +1,50 @@
+package sdk
+
+import (
+	httptransport "github.com/go-openapi/runtime/client"
+	"github.com/openziti/zrok/environment/env_core"
+	restEnvironment "github.com/openziti/zrok/rest_client_zrok/environment"
+	"github.com/pkg/errors"
+)
+
+func EnableEnvironment(root env_core.Root, request *EnableRequest) (*Environment, error) {
+	zrok, err := root.Client()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create zrok client")
+	}
+	auth := httptransport.APIKeyAuth("X-TOKEN", "header", root.Environment().AccountToken)
+
+	req := restEnvironment.NewEnableParams()
+	req.Body.Description = request.Description
+	req.Body.Host = request.Host
+
+	resp, err := zrok.Environment.Enable(req, auth)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Environment{
+		Host:         request.Host,
+		Description:  request.Description,
+		ZitiIdentity: resp.Payload.Identity,
+		ZitiConfig:   resp.Payload.Cfg,
+	}, nil
+}
+
+func DisableEnvironment(env *Environment, root env_core.Root) error {
+	zrok, err := root.Client()
+	if err != nil {
+		return errors.Wrap(err, "could not create zrok client")
+	}
+	auth := httptransport.APIKeyAuth("X-TOKEN", "header", root.Environment().AccountToken)
+
+	req := restEnvironment.NewDisableParams()
+	req.Body.Identity = env.ZitiIdentity
+
+	_, err = zrok.Environment.Disable(req, auth)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

sdk/golang/sdk/model.go

@@ -2,6 +2,18 @@ package sdk
 
 import "time"
 
+type EnableRequest struct {
+	Host        string
+	Description string
+}
+
+type Environment struct {
+	Host         string
+	Description  string
+	ZitiIdentity string
+	ZitiConfig   string
+}
+
 type BackendMode string
 
 const (