From 3cf87b68467c3ebe45e86d6cb5d3abe1c6209759 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Fri, 25 Apr 2025 18:50:44 +0200 Subject: [PATCH] [client] Run container tests more generically (#3737) --- .github/workflows/golang-test-linux.yml | 196 ++++++------------ client/Dockerfile | 5 +- client/firewall/uspfilter/tracer_test.go | 16 +- .../systemops/systemops_bsd_test.go | 1 - .../systemops/systemops_linux_test.go | 55 ++--- .../systemops/systemops_unix_test.go | 12 +- 6 files changed, 101 insertions(+), 184 deletions(-) diff --git a/.github/workflows/golang-test-linux.yml b/.github/workflows/golang-test-linux.yml index 4e690ff1b..2f1df9b1a 100644 --- a/.github/workflows/golang-test-linux.yml +++ b/.github/workflows/golang-test-linux.yml @@ -146,6 +146,64 @@ jobs: - name: Test run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} CI=true go test -tags devcert -exec 'sudo' -timeout 10m -p 1 $(go list ./... | grep -v -e /management -e /signal -e /relay) + test_client_on_docker: + name: "Client (Docker) / Unit" + needs: [build-cache] + runs-on: ubuntu-22.04 + steps: + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: "1.23.x" + cache: false + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Get Go environment + id: go-env + run: | + echo "cache_dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT + echo "modcache_dir=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT + + - name: Cache Go modules + uses: actions/cache/restore@v4 + id: cache-restore + with: + path: | + ${{ steps.go-env.outputs.cache_dir }} + ${{ steps.go-env.outputs.modcache_dir }} + key: ${{ runner.os }}-gotest-cache-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-gotest-cache- + + - name: Run tests in container + env: + HOST_GOCACHE: ${{ steps.go-env.outputs.cache_dir }} + HOST_GOMODCACHE: ${{ steps.go-env.outputs.modcache_dir }} + run: | + CONTAINER_GOCACHE="/root/.cache/go-build" + CONTAINER_GOMODCACHE="/go/pkg/mod" + + docker run --rm \ + --cap-add=NET_ADMIN \ + --privileged \ + -v $PWD:/app \ + -w /app \ + -v "${HOST_GOCACHE}:${CONTAINER_GOCACHE}" \ + -v "${HOST_GOMODCACHE}:${CONTAINER_GOMODCACHE}" \ + -e CGO_ENABLED=1 \ + -e CI=true \ + -e GOARCH=${GOARCH_TARGET} \ + -e GOCACHE=${CONTAINER_GOCACHE} \ + -e GOMODCACHE=${CONTAINER_GOMODCACHE} \ + golang:1.23-alpine \ + sh -c ' \ + apk update; apk add --no-cache \ + ca-certificates iptables ip6tables dbus dbus-dev libpcap-dev build-base; \ + go test -buildvcs=false -tags devcert -v -timeout 10m -p 1 $(go list -buildvcs=false ./... | grep -v -e /management -e /signal -e /relay -e /client/ui) + ' + test_relay: name: "Relay / Unit" needs: [build-cache] @@ -179,13 +237,6 @@ jobs: restore-keys: | ${{ runner.os }}-gotest-cache- - - name: Install dependencies - run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev - - - name: Install 32-bit libpcap - if: matrix.arch == '386' - run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386 - - name: Install modules run: go mod tidy @@ -232,13 +283,6 @@ jobs: restore-keys: | ${{ runner.os }}-gotest-cache- - - name: Install dependencies - run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev - - - name: Install 32-bit libpcap - if: matrix.arch == '386' - run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386 - - name: Install modules run: go mod tidy @@ -286,13 +330,6 @@ jobs: restore-keys: | ${{ runner.os }}-gotest-cache- - - name: Install dependencies - run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev - - - name: Install 32-bit libpcap - if: matrix.arch == '386' - run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386 - - name: Install modules run: go mod tidy @@ -354,13 +391,6 @@ jobs: restore-keys: | ${{ runner.os }}-gotest-cache- - - name: Install dependencies - run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev - - - name: Install 32-bit libpcap - if: matrix.arch == '386' - run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386 - - name: Install modules run: go mod tidy @@ -449,13 +479,6 @@ jobs: restore-keys: | ${{ runner.os }}-gotest-cache- - - name: Install dependencies - run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev - - - name: Install 32-bit libpcap - if: matrix.arch == '386' - run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386 - - name: Install modules run: go mod tidy @@ -520,13 +543,6 @@ jobs: restore-keys: | ${{ runner.os }}-gotest-cache- - - name: Install dependencies - run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev - - - name: Install 32-bit libpcap - if: matrix.arch == '386' - run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386 - - name: Install modules run: go mod tidy @@ -541,99 +557,3 @@ jobs: go test -tags=integration \ -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' \ -timeout 20m ./management/... - - test_client_on_docker: - name: "Client (Docker) / Unit" - needs: [ build-cache ] - runs-on: ubuntu-22.04 - steps: - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version: "1.23.x" - cache: false - - - name: Checkout code - uses: actions/checkout@v4 - - - name: Get Go environment - run: | - echo "cache=$(go env GOCACHE)" >> $GITHUB_ENV - echo "modcache=$(go env.GOMODCACHE)" >> $GITHUB_ENV - - - name: Cache Go modules - uses: actions/cache/restore@v4 - with: - path: | - ${{ env.cache }} - ${{ env.modcache }} - key: ${{ runner.os }}-gotest-cache-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-gotest-cache- - - - name: Install dependencies - run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev - - - name: Install modules - run: go mod tidy - - - name: Check git status - run: git --no-pager diff --exit-code - - - name: Generate Shared Sock Test bin - run: CGO_ENABLED=0 go test -c -o sharedsock-testing.bin ./sharedsock - - - name: Generate RouteManager Test bin - run: CGO_ENABLED=0 go test -c -o routemanager-testing.bin ./client/internal/routemanager - - - name: Generate SystemOps Test bin (static via Alpine) - run: | - docker run --rm -v $PWD:/app -w /app \ - alpine:latest \ - sh -c " - apk add --no-cache go gcc musl-dev libpcap-dev dbus-dev && \ - adduser -D -u $(id -u) builder && \ - su builder -c '\ - cd /app && \ - CGO_ENABLED=1 GOOS=linux GOARCH=amd64 \ - go test -c -o /app/systemops-testing.bin \ - -tags netgo \ - -ldflags=\"-w -extldflags \\\"-static -ldbus-1 -lpcap\\\"\" \ - ./client/internal/routemanager/systemops \ - ' - " - - - name: Generate nftables Manager Test bin - run: CGO_ENABLED=0 go test -c -o nftablesmanager-testing.bin ./client/firewall/nftables/... - - - name: Generate Engine Test bin - run: CGO_ENABLED=1 go test -c -o engine-testing.bin ./client/internal - - - name: Generate Peer Test bin - run: CGO_ENABLED=0 go test -c -o peer-testing.bin ./client/internal/peer/ - - - run: chmod +x *testing.bin - - - name: Run Shared Sock tests in docker - run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/sharedsock --entrypoint /ci/sharedsock-testing.bin gcr.io/distroless/base:debug -test.timeout 5m -test.parallel 1 - - - name: Run Iface tests in docker - run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/netbird -v /tmp/cache:/tmp/cache -v /tmp/modcache:/tmp/modcache -w /netbird -e GOCACHE=/tmp/cache -e GOMODCACHE=/tmp/modcache -e CGO_ENABLED=0 golang:1.23-alpine go test -test.timeout 5m -test.parallel 1 ./client/iface/... - - - name: Run RouteManager tests in docker - run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/routemanager --entrypoint /ci/routemanager-testing.bin gcr.io/distroless/base:debug -test.timeout 5m -test.parallel 1 - - - name: Run SystemOps tests in docker - run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/routemanager/systemops --entrypoint /ci/systemops-testing.bin gcr.io/distroless/base:debug -test.timeout 5m -test.parallel 1 - - - name: Run nftables Manager tests in docker - run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/firewall --entrypoint /ci/nftablesmanager-testing.bin gcr.io/distroless/base:debug -test.timeout 5m -test.parallel 1 - - - name: Run Engine tests in docker with file store - run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal -e NETBIRD_STORE_ENGINE="jsonfile" --entrypoint /ci/engine-testing.bin gcr.io/distroless/base:debug -test.timeout 5m -test.parallel 1 - - - name: Run Engine tests in docker with sqlite store - run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal -e NETBIRD_STORE_ENGINE="sqlite" --entrypoint /ci/engine-testing.bin gcr.io/distroless/base:debug -test.timeout 5m -test.parallel 1 - - - name: Run Peer tests in docker - run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/peer --entrypoint /ci/peer-testing.bin gcr.io/distroless/base:debug -test.timeout 5m -test.parallel 1 diff --git a/client/Dockerfile b/client/Dockerfile index 35c1d04c2..16b2916c7 100644 --- a/client/Dockerfile +++ b/client/Dockerfile @@ -1,5 +1,6 @@ FROM alpine:3.21.3 -RUN apk add --no-cache ca-certificates iptables ip6tables +# iproute2: busybox doesn't display ip rules properly +RUN apk add --no-cache ca-certificates ip6tables iproute2 iptables ENV NB_FOREGROUND_MODE=true ENTRYPOINT [ "/usr/local/bin/netbird","up"] -COPY netbird /usr/local/bin/netbird \ No newline at end of file +COPY netbird /usr/local/bin/netbird diff --git a/client/firewall/uspfilter/tracer_test.go b/client/firewall/uspfilter/tracer_test.go index 53ee6c886..bd87879a5 100644 --- a/client/firewall/uspfilter/tracer_test.go +++ b/client/firewall/uspfilter/tracer_test.go @@ -198,12 +198,12 @@ func TestTracePacket(t *testing.T) { m.forwarder.Store(&forwarder.Forwarder{}) src := netip.PrefixFrom(netip.AddrFrom4([4]byte{1, 1, 1, 1}), 32) - dst := netip.PrefixFrom(netip.AddrFrom4([4]byte{172, 17, 0, 2}), 32) + dst := netip.PrefixFrom(netip.AddrFrom4([4]byte{192, 168, 17, 2}), 32) _, err := m.AddRouteFiltering(nil, []netip.Prefix{src}, fw.Network{Prefix: dst}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { - return createPacketBuilder("1.1.1.1", "172.17.0.2", "tcp", 12345, 80, fw.RuleDirectionIN) + return createPacketBuilder("1.1.1.1", "192.168.17.2", "tcp", 12345, 80, fw.RuleDirectionIN) }, expectedStages: []PacketStage{ StageReceived, @@ -222,12 +222,12 @@ func TestTracePacket(t *testing.T) { m.nativeRouter.Store(false) src := netip.PrefixFrom(netip.AddrFrom4([4]byte{1, 1, 1, 1}), 32) - dst := netip.PrefixFrom(netip.AddrFrom4([4]byte{172, 17, 0, 2}), 32) + dst := netip.PrefixFrom(netip.AddrFrom4([4]byte{192, 168, 17, 2}), 32) _, err := m.AddRouteFiltering(nil, []netip.Prefix{src}, fw.Network{Prefix: dst}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionDrop) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { - return createPacketBuilder("1.1.1.1", "172.17.0.2", "tcp", 12345, 80, fw.RuleDirectionIN) + return createPacketBuilder("1.1.1.1", "192.168.17.2", "tcp", 12345, 80, fw.RuleDirectionIN) }, expectedStages: []PacketStage{ StageReceived, @@ -245,7 +245,7 @@ func TestTracePacket(t *testing.T) { m.nativeRouter.Store(true) }, packetBuilder: func() *PacketBuilder { - return createPacketBuilder("1.1.1.1", "172.17.0.2", "tcp", 12345, 80, fw.RuleDirectionIN) + return createPacketBuilder("1.1.1.1", "192.168.17.2", "tcp", 12345, 80, fw.RuleDirectionIN) }, expectedStages: []PacketStage{ StageReceived, @@ -263,7 +263,7 @@ func TestTracePacket(t *testing.T) { m.routingEnabled.Store(false) }, packetBuilder: func() *PacketBuilder { - return createPacketBuilder("1.1.1.1", "172.17.0.2", "tcp", 12345, 80, fw.RuleDirectionIN) + return createPacketBuilder("1.1.1.1", "192.168.17.2", "tcp", 12345, 80, fw.RuleDirectionIN) }, expectedStages: []PacketStage{ StageReceived, @@ -425,8 +425,8 @@ func TestTracePacket(t *testing.T) { require.True(t, m.localipmanager.IsLocalIP(netip.MustParseAddr("100.10.0.100")), "100.10.0.100 should be recognized as a local IP") - require.False(t, m.localipmanager.IsLocalIP(netip.MustParseAddr("172.17.0.2")), - "172.17.0.2 should not be recognized as a local IP") + require.False(t, m.localipmanager.IsLocalIP(netip.MustParseAddr("192.168.17.2")), + "192.168.17.2 should not be recognized as a local IP") pb := tc.packetBuilder() diff --git a/client/internal/routemanager/systemops/systemops_bsd_test.go b/client/internal/routemanager/systemops/systemops_bsd_test.go index 84b84483e..a83d7f1de 100644 --- a/client/internal/routemanager/systemops/systemops_bsd_test.go +++ b/client/internal/routemanager/systemops/systemops_bsd_test.go @@ -24,7 +24,6 @@ func init() { testCases = append(testCases, []testCase{ { name: "To more specific route without custom dialer via vpn", - destination: "10.10.0.2:53", expectedInterface: expectedVPNint, dialer: &net.Dialer{}, expectedPacket: createPacketExpectation("100.64.0.1", 12345, "10.10.0.2", 53), diff --git a/client/internal/routemanager/systemops/systemops_linux_test.go b/client/internal/routemanager/systemops/systemops_linux_test.go index 8f12740d0..f0d7472dc 100644 --- a/client/internal/routemanager/systemops/systemops_linux_test.go +++ b/client/internal/routemanager/systemops/systemops_linux_test.go @@ -27,14 +27,12 @@ func init() { testCases = append(testCases, []testCase{ { name: "To more specific route without custom dialer via physical interface", - destination: "10.10.0.2:53", expectedInterface: expectedInternalInt, dialer: &net.Dialer{}, expectedPacket: createPacketExpectation("192.168.1.1", 12345, "10.10.0.2", 53), }, { name: "To more specific route (local) without custom dialer via physical interface", - destination: "127.0.10.1:53", expectedInterface: expectedLoopbackInt, dialer: &net.Dialer{}, expectedPacket: createPacketExpectation("127.0.0.1", 12345, "127.0.10.1", 53), @@ -134,6 +132,16 @@ func addDummyRoute(t *testing.T, dstCIDR string, gw net.IP, intf string) { _, dstIPNet, err := net.ParseCIDR(dstCIDR) require.NoError(t, err) + link, err := netlink.LinkByName(intf) + require.NoError(t, err) + linkIndex := link.Attrs().Index + + route := &netlink.Route{ + Dst: dstIPNet, + Gw: gw, + LinkIndex: linkIndex, + } + // Handle existing routes with metric 0 var originalNexthop net.IP var originalLinkIndex int @@ -145,32 +153,24 @@ func addDummyRoute(t *testing.T, dstCIDR string, gw net.IP, intf string) { } if originalNexthop != nil { + // remove original route err = netlink.RouteDel(&netlink.Route{Dst: dstIPNet, Priority: 0}) - switch { - case err != nil && !errors.Is(err, syscall.ESRCH): - t.Logf("Failed to delete route: %v", err) - case err == nil: - t.Cleanup(func() { - err := netlink.RouteAdd(&netlink.Route{Dst: dstIPNet, Gw: originalNexthop, LinkIndex: originalLinkIndex, Priority: 0}) - if err != nil && !errors.Is(err, syscall.EEXIST) { - t.Fatalf("Failed to add route: %v", err) - } - }) - default: - t.Logf("Failed to delete route: %v", err) - } + assert.NoError(t, err) + + // add new route + assert.NoError(t, netlink.RouteAdd(route)) + + t.Cleanup(func() { + // restore original route + assert.NoError(t, netlink.RouteDel(route)) + err := netlink.RouteAdd(&netlink.Route{Dst: dstIPNet, Gw: originalNexthop, LinkIndex: originalLinkIndex, Priority: 0}) + assert.NoError(t, err) + }) + + return } } - link, err := netlink.LinkByName(intf) - require.NoError(t, err) - linkIndex := link.Attrs().Index - - route := &netlink.Route{ - Dst: dstIPNet, - Gw: gw, - LinkIndex: linkIndex, - } err = netlink.RouteDel(route) if err != nil && !errors.Is(err, syscall.ESRCH) { t.Logf("Failed to delete route: %v", err) @@ -180,7 +180,6 @@ func addDummyRoute(t *testing.T, dstCIDR string, gw net.IP, intf string) { if err != nil && !errors.Is(err, syscall.EEXIST) { t.Fatalf("Failed to add route: %v", err) } - require.NoError(t, err) } func fetchOriginalGateway(family int) (net.IP, int, error) { @@ -190,7 +189,11 @@ func fetchOriginalGateway(family int) (net.IP, int, error) { } for _, route := range routes { - if route.Dst == nil && route.Priority == 0 { + ones := -1 + if route.Dst != nil { + ones, _ = route.Dst.Mask.Size() + } + if route.Dst == nil || ones == 0 && route.Priority == 0 { return route.Gw, route.LinkIndex, nil } } diff --git a/client/internal/routemanager/systemops/systemops_unix_test.go b/client/internal/routemanager/systemops/systemops_unix_test.go index d88c1ab6b..ad37f611f 100644 --- a/client/internal/routemanager/systemops/systemops_unix_test.go +++ b/client/internal/routemanager/systemops/systemops_unix_test.go @@ -31,7 +31,6 @@ type PacketExpectation struct { type testCase struct { name string - destination string expectedInterface string dialer dialer expectedPacket PacketExpectation @@ -40,14 +39,12 @@ type testCase struct { var testCases = []testCase{ { name: "To external host without custom dialer via vpn", - destination: "192.0.2.1:53", expectedInterface: expectedVPNint, dialer: &net.Dialer{}, expectedPacket: createPacketExpectation("100.64.0.1", 12345, "192.0.2.1", 53), }, { name: "To external host with custom dialer via physical interface", - destination: "192.0.2.1:53", expectedInterface: expectedExternalInt, dialer: nbnet.NewDialer(), expectedPacket: createPacketExpectation("192.168.0.1", 12345, "192.0.2.1", 53), @@ -55,14 +52,12 @@ var testCases = []testCase{ { name: "To duplicate internal route with custom dialer via physical interface", - destination: "10.0.0.2:53", expectedInterface: expectedInternalInt, dialer: nbnet.NewDialer(), expectedPacket: createPacketExpectation("192.168.1.1", 12345, "10.0.0.2", 53), }, { name: "To duplicate internal route without custom dialer via physical interface", // local route takes precedence - destination: "10.0.0.2:53", expectedInterface: expectedInternalInt, dialer: &net.Dialer{}, expectedPacket: createPacketExpectation("192.168.1.1", 12345, "10.0.0.2", 53), @@ -70,14 +65,12 @@ var testCases = []testCase{ { name: "To unique vpn route with custom dialer via physical interface", - destination: "172.16.0.2:53", expectedInterface: expectedExternalInt, dialer: nbnet.NewDialer(), expectedPacket: createPacketExpectation("192.168.0.1", 12345, "172.16.0.2", 53), }, { name: "To unique vpn route without custom dialer via vpn", - destination: "172.16.0.2:53", expectedInterface: expectedVPNint, dialer: &net.Dialer{}, expectedPacket: createPacketExpectation("100.64.0.1", 12345, "172.16.0.2", 53), @@ -94,10 +87,11 @@ func TestRouting(t *testing.T) { t.Run(tc.name, func(t *testing.T) { setupTestEnv(t) - filter := createBPFFilter(tc.destination) + dst := fmt.Sprintf("%s:%d", tc.expectedPacket.DstIP, tc.expectedPacket.DstPort) + filter := createBPFFilter(dst) handle := startPacketCapture(t, tc.expectedInterface, filter) - sendTestPacket(t, tc.destination, tc.expectedPacket.SrcPort, tc.dialer) + sendTestPacket(t, dst, tc.expectedPacket.SrcPort, tc.dialer) packetSource := gopacket.NewPacketSource(handle, handle.LinkType()) packet, err := packetSource.NextPacket()