[client] Set up signal to generate debug bundles (#3683)

This commit is contained in:
Viktor Liu 2025-04-16 11:06:22 +02:00 committed by GitHub
parent 7cb366bc7d
commit a675531b5c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 1316 additions and 1009 deletions

View File

@ -11,9 +11,12 @@ import (
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
"github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal"
"github.com/netbirdio/netbird/client/internal/debug"
"github.com/netbirdio/netbird/client/internal/peer"
"github.com/netbirdio/netbird/client/proto" "github.com/netbirdio/netbird/client/proto"
"github.com/netbirdio/netbird/client/server" "github.com/netbirdio/netbird/client/server"
nbstatus "github.com/netbirdio/netbird/client/status" nbstatus "github.com/netbirdio/netbird/client/status"
mgmProto "github.com/netbirdio/netbird/management/proto"
) )
const errCloseConnection = "Failed to close connection: %v" const errCloseConnection = "Failed to close connection: %v"
@ -326,3 +329,34 @@ func formatDuration(d time.Duration) string {
s := d / time.Second s := d / time.Second
return fmt.Sprintf("%02d:%02d:%02d", h, m, s) return fmt.Sprintf("%02d:%02d:%02d", h, m, s)
} }
func generateDebugBundle(config *internal.Config, recorder *peer.Status, connectClient *internal.ConnectClient, logFilePath string) {
var networkMap *mgmProto.NetworkMap
var err error
if connectClient != nil {
networkMap, err = connectClient.GetLatestNetworkMap()
if err != nil {
log.Warnf("Failed to get latest network map: %v", err)
}
}
bundleGenerator := debug.NewBundleGenerator(
debug.GeneratorDependencies{
InternalConfig: config,
StatusRecorder: recorder,
NetworkMap: networkMap,
LogFile: logFilePath,
},
debug.BundleConfig{
IncludeSystemInfo: true,
},
)
path, err := bundleGenerator.Generate()
if err != nil {
log.Errorf("Failed to generate debug bundle: %v", err)
return
}
log.Infof("Generated debug bundle from SIGUSR1 at: %s", path)
}

39
client/cmd/debug_unix.go Normal file
View File

@ -0,0 +1,39 @@
//go:build unix
package cmd
import (
"context"
"os"
"os/signal"
"syscall"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/internal"
"github.com/netbirdio/netbird/client/internal/peer"
)
func SetupDebugHandler(
ctx context.Context,
config *internal.Config,
recorder *peer.Status,
connectClient *internal.ConnectClient,
logFilePath string,
) {
usr1Ch := make(chan os.Signal, 1)
signal.Notify(usr1Ch, syscall.SIGUSR1)
go func() {
for {
select {
case <-ctx.Done():
return
case <-usr1Ch:
log.Info("Received SIGUSR1. Triggering debug bundle generation.")
go generateDebugBundle(config, recorder, connectClient, logFilePath)
}
}
}()
}

126
client/cmd/debug_windows.go Normal file
View File

@ -0,0 +1,126 @@
package cmd
import (
"context"
"errors"
"os"
"strconv"
"time"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/windows"
"github.com/netbirdio/netbird/client/internal"
"github.com/netbirdio/netbird/client/internal/peer"
)
const (
envListenEvent = "NB_LISTEN_DEBUG_EVENT"
debugTriggerEventName = `Global\NetbirdDebugTriggerEvent`
waitTimeout = 5 * time.Second
)
// SetupDebugHandler sets up a Windows event to listen for a signal to generate a debug bundle.
// Example usage with PowerShell:
// $evt = [System.Threading.EventWaitHandle]::OpenExisting("Global\NetbirdDebugTriggerEvent")
// $evt.Set()
// $evt.Close()
func SetupDebugHandler(
ctx context.Context,
config *internal.Config,
recorder *peer.Status,
connectClient *internal.ConnectClient,
logFilePath string,
) {
env := os.Getenv(envListenEvent)
if env == "" {
return
}
listenEvent, err := strconv.ParseBool(env)
if err != nil {
log.Errorf("Failed to parse %s: %v", envListenEvent, err)
return
}
if !listenEvent {
return
}
eventNamePtr, err := windows.UTF16PtrFromString(debugTriggerEventName)
if err != nil {
log.Errorf("Failed to convert event name '%s' to UTF16: %v", debugTriggerEventName, err)
return
}
// TODO: restrict access by ACL
eventHandle, err := windows.CreateEvent(nil, 1, 0, eventNamePtr)
if err != nil {
if errors.Is(err, windows.ERROR_ALREADY_EXISTS) {
log.Warnf("Debug trigger event '%s' already exists. Attempting to open.", debugTriggerEventName)
// SYNCHRONIZE is needed for WaitForSingleObject, EVENT_MODIFY_STATE for ResetEvent.
eventHandle, err = windows.OpenEvent(windows.SYNCHRONIZE|windows.EVENT_MODIFY_STATE, false, eventNamePtr)
if err != nil {
log.Errorf("Failed to open existing debug trigger event '%s': %v", debugTriggerEventName, err)
return
}
log.Infof("Successfully opened existing debug trigger event '%s'.", debugTriggerEventName)
} else {
log.Errorf("Failed to create debug trigger event '%s': %v", debugTriggerEventName, err)
return
}
}
if eventHandle == windows.InvalidHandle {
log.Errorf("Obtained an invalid handle for debug trigger event '%s'", debugTriggerEventName)
return
}
log.Infof("Debug handler waiting for signal on event: %s", debugTriggerEventName)
go waitForEvent(ctx, config, recorder, connectClient, logFilePath, eventHandle)
}
func waitForEvent(
ctx context.Context,
config *internal.Config,
recorder *peer.Status,
connectClient *internal.ConnectClient,
logFilePath string,
eventHandle windows.Handle,
) {
defer func() {
if err := windows.CloseHandle(eventHandle); err != nil {
log.Errorf("Failed to close debug event handle '%s': %v", debugTriggerEventName, err)
}
}()
for {
if ctx.Err() != nil {
return
}
status, err := windows.WaitForSingleObject(eventHandle, uint32(waitTimeout.Milliseconds()))
switch status {
case windows.WAIT_OBJECT_0:
log.Info("Received signal on debug event. Triggering debug bundle generation.")
// reset the event so it can be triggered again later (manual reset == 1)
if err := windows.ResetEvent(eventHandle); err != nil {
log.Errorf("Failed to reset debug event '%s': %v", debugTriggerEventName, err)
}
go generateDebugBundle(config, recorder, connectClient, logFilePath)
case uint32(windows.WAIT_TIMEOUT):
default:
log.Errorf("Unexpected status %d from WaitForSingleObject for debug event '%s': %v", status, debugTriggerEventName, err)
select {
case <-time.After(5 * time.Second):
case <-ctx.Done():
return
}
}
}
}

View File

@ -115,6 +115,7 @@ var runCmd = &cobra.Command{
ctx, cancel := context.WithCancel(cmd.Context()) ctx, cancel := context.WithCancel(cmd.Context())
SetupCloseHandler(ctx, cancel) SetupCloseHandler(ctx, cancel)
SetupDebugHandler(ctx, nil, nil, nil, logFile)
s, err := newSVC(newProgram(ctx, cancel), newSVCConfig()) s, err := newSVC(newProgram(ctx, cancel), newSVCConfig())
if err != nil { if err != nil {

View File

@ -219,6 +219,8 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error {
r.GetFullStatus() r.GetFullStatus()
connectClient := internal.NewConnectClient(ctx, config, r) connectClient := internal.NewConnectClient(ctx, config, r)
SetupDebugHandler(ctx, config, r, connectClient, "")
return connectClient.Run(nil) return connectClient.Run(nil)
} }

View File

@ -349,6 +349,25 @@ func (c *ConnectClient) Engine() *Engine {
return e return e
} }
// GetLatestNetworkMap returns the latest network map from the engine.
func (c *ConnectClient) GetLatestNetworkMap() (*mgmProto.NetworkMap, error) {
engine := c.Engine()
if engine == nil {
return nil, errors.New("engine is not initialized")
}
networkMap, err := engine.GetLatestNetworkMap()
if err != nil {
return nil, fmt.Errorf("get latest network map: %w", err)
}
if networkMap == nil {
return nil, errors.New("network map is not available")
}
return networkMap, nil
}
// Status returns the current client status // Status returns the current client status
func (c *ConnectClient) Status() StatusType { func (c *ConnectClient) Status() StatusType {
if c == nil { if c == nil {

File diff suppressed because it is too large Load Diff

View File

@ -1,9 +1,8 @@
//go:build linux && !android //go:build linux && !android
package server package debug
import ( import (
"archive/zip"
"bytes" "bytes"
"encoding/binary" "encoding/binary"
"fmt" "fmt"
@ -14,36 +13,31 @@ import (
"github.com/google/nftables" "github.com/google/nftables"
"github.com/google/nftables/expr" "github.com/google/nftables/expr"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/anonymize"
"github.com/netbirdio/netbird/client/proto"
) )
// addFirewallRules collects and adds firewall rules to the archive // addFirewallRules collects and adds firewall rules to the archive
func (s *Server) addFirewallRules(req *proto.DebugBundleRequest, anonymizer *anonymize.Anonymizer, archive *zip.Writer) error { func (g *BundleGenerator) addFirewallRules() error {
log.Info("Collecting firewall rules") log.Info("Collecting firewall rules")
// Collect and add iptables rules
iptablesRules, err := collectIPTablesRules() iptablesRules, err := collectIPTablesRules()
if err != nil { if err != nil {
log.Warnf("Failed to collect iptables rules: %v", err) log.Warnf("Failed to collect iptables rules: %v", err)
} else { } else {
if req.GetAnonymize() { if g.anonymize {
iptablesRules = anonymizer.AnonymizeString(iptablesRules) iptablesRules = g.anonymizer.AnonymizeString(iptablesRules)
} }
if err := addFileToZip(archive, strings.NewReader(iptablesRules), "iptables.txt"); err != nil { if err := g.addFileToZip(strings.NewReader(iptablesRules), "iptables.txt"); err != nil {
log.Warnf("Failed to add iptables rules to bundle: %v", err) log.Warnf("Failed to add iptables rules to bundle: %v", err)
} }
} }
// Collect and add nftables rules
nftablesRules, err := collectNFTablesRules() nftablesRules, err := collectNFTablesRules()
if err != nil { if err != nil {
log.Warnf("Failed to collect nftables rules: %v", err) log.Warnf("Failed to collect nftables rules: %v", err)
} else { } else {
if req.GetAnonymize() { if g.anonymize {
nftablesRules = anonymizer.AnonymizeString(nftablesRules) nftablesRules = g.anonymizer.AnonymizeString(nftablesRules)
} }
if err := addFileToZip(archive, strings.NewReader(nftablesRules), "nftables.txt"); err != nil { if err := g.addFileToZip(strings.NewReader(nftablesRules), "nftables.txt"); err != nil {
log.Warnf("Failed to add nftables rules to bundle: %v", err) log.Warnf("Failed to add nftables rules to bundle: %v", err)
} }
} }
@ -65,16 +59,13 @@ func collectIPTablesRules() (string, error) {
builder.WriteString("\n") builder.WriteString("\n")
} }
// Then get verbose statistics for each table
builder.WriteString("=== iptables -v -n -L output ===\n") builder.WriteString("=== iptables -v -n -L output ===\n")
// Get list of tables
tables := []string{"filter", "nat", "mangle", "raw", "security"} tables := []string{"filter", "nat", "mangle", "raw", "security"}
for _, table := range tables { for _, table := range tables {
builder.WriteString(fmt.Sprintf("*%s\n", table)) builder.WriteString(fmt.Sprintf("*%s\n", table))
// Get verbose statistics for the entire table
stats, err := getTableStatistics(table) stats, err := getTableStatistics(table)
if err != nil { if err != nil {
log.Warnf("Failed to get statistics for table %s: %v", table, err) log.Warnf("Failed to get statistics for table %s: %v", table, err)
@ -182,12 +173,10 @@ func formatTables(conn *nftables.Conn, tables []*nftables.Table) string {
continue continue
} }
// Format chains
for _, chain := range chains { for _, chain := range chains {
formatChain(conn, table, chain, &builder) formatChain(conn, table, chain, &builder)
} }
// Format sets
if sets, err := conn.GetSets(table); err != nil { if sets, err := conn.GetSets(table); err != nil {
log.Warnf("Failed to get sets for table %s: %v", table.Name, err) log.Warnf("Failed to get sets for table %s: %v", table.Name, err)
} else if len(sets) > 0 { } else if len(sets) > 0 {

View File

@ -0,0 +1,7 @@
//go:build ios || android
package debug
func (g *BundleGenerator) addRoutes() error {
return nil
}

View File

@ -0,0 +1,8 @@
//go:build !linux || android
package debug
// collectFirewallRules returns nothing on non-linux systems
func (g *BundleGenerator) addFirewallRules() error {
return nil
}

View File

@ -0,0 +1,25 @@
//go:build !ios && !android
package debug
import (
"fmt"
"strings"
"github.com/netbirdio/netbird/client/internal/routemanager/systemops"
)
func (g *BundleGenerator) addRoutes() error {
routes, err := systemops.GetRoutesFromTable()
if err != nil {
return fmt.Errorf("get routes: %w", err)
}
// TODO: get routes including nexthop
routesContent := formatRoutes(routes, g.anonymize, g.anonymizer)
routesReader := strings.NewReader(routesContent)
if err := g.addFileToZip(routesReader, "routes.txt"); err != nil {
return fmt.Errorf("add routes file to zip: %w", err)
}
return nil
}

View File

@ -1,4 +1,4 @@
package server package debug
import ( import (
"encoding/json" "encoding/json"

File diff suppressed because it is too large Load Diff

View File

@ -1,15 +0,0 @@
//go:build !linux || android
package server
import (
"archive/zip"
"github.com/netbirdio/netbird/client/anonymize"
"github.com/netbirdio/netbird/client/proto"
)
// collectFirewallRules returns nothing on non-linux systems
func (s *Server) addFirewallRules(req *proto.DebugBundleRequest, anonymizer *anonymize.Anonymizer, archive *zip.Writer) error {
return nil
}