[client] Run container tests more generically (#3737)

.github/workflows/golang-test-linux.yml

@@ -146,6 +146,64 @@ jobs:
       - name: Test
         run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} CI=true go test -tags devcert -exec 'sudo' -timeout 10m -p 1 $(go list ./... | grep -v -e /management -e /signal -e /relay)
 
+  test_client_on_docker:
+    name: "Client (Docker) / Unit"
+    needs: [build-cache]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.23.x"
+          cache: false
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get Go environment
+        id: go-env
+        run: |
+          echo "cache_dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT
+          echo "modcache_dir=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
+
+      - name: Cache Go modules
+        uses: actions/cache/restore@v4
+        id: cache-restore
+        with:
+          path: |
+            ${{ steps.go-env.outputs.cache_dir }}
+            ${{ steps.go-env.outputs.modcache_dir }}
+          key: ${{ runner.os }}-gotest-cache-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-gotest-cache-
+
+      - name: Run tests in container
+        env:
+          HOST_GOCACHE: ${{ steps.go-env.outputs.cache_dir }}
+          HOST_GOMODCACHE: ${{ steps.go-env.outputs.modcache_dir }}
+        run: |
+          CONTAINER_GOCACHE="/root/.cache/go-build"
+          CONTAINER_GOMODCACHE="/go/pkg/mod"
+
+          docker run --rm \
+            --cap-add=NET_ADMIN \
+            --privileged \
+            -v $PWD:/app \
+            -w /app \
+            -v "${HOST_GOCACHE}:${CONTAINER_GOCACHE}" \
+            -v "${HOST_GOMODCACHE}:${CONTAINER_GOMODCACHE}" \
+            -e CGO_ENABLED=1 \
+            -e CI=true \
+            -e GOARCH=${GOARCH_TARGET} \
+            -e GOCACHE=${CONTAINER_GOCACHE} \
+            -e GOMODCACHE=${CONTAINER_GOMODCACHE} \
+            golang:1.23-alpine \
+            sh -c ' \
+              apk update; apk add --no-cache \
+                ca-certificates iptables ip6tables dbus dbus-dev libpcap-dev build-base; \
+              go test -buildvcs=false -tags devcert -v -timeout 10m -p 1 $(go list -buildvcs=false ./... | grep -v -e /management -e /signal -e /relay -e /client/ui)
+            '
+
   test_relay:
     name: "Relay / Unit"
     needs: [build-cache]
@@ -179,13 +237,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-gotest-cache-
 
-      - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
-
-      - name: Install 32-bit libpcap
-        if: matrix.arch == '386'
-        run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386
-
       - name: Install modules
         run: go mod tidy
 
@@ -232,13 +283,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-gotest-cache-
 
-      - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
-
-      - name: Install 32-bit libpcap
-        if: matrix.arch == '386'
-        run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386
-
       - name: Install modules
         run: go mod tidy
 
@@ -286,13 +330,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-gotest-cache-
 
-      - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
-
-      - name: Install 32-bit libpcap
-        if: matrix.arch == '386'
-        run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386
-
       - name: Install modules
         run: go mod tidy
 
@@ -354,13 +391,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-gotest-cache-
 
-      - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
-
-      - name: Install 32-bit libpcap
-        if: matrix.arch == '386'
-        run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386
-
       - name: Install modules
         run: go mod tidy
 
@@ -449,13 +479,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-gotest-cache-
 
-      - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
-
-      - name: Install 32-bit libpcap
-        if: matrix.arch == '386'
-        run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386
-
       - name: Install modules
         run: go mod tidy
 
@@ -520,13 +543,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-gotest-cache-
 
-      - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
-
-      - name: Install 32-bit libpcap
-        if: matrix.arch == '386'
-        run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386
-
       - name: Install modules
         run: go mod tidy
 
@@ -541,99 +557,3 @@ jobs:
           go test -tags=integration \
           -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' \
           -timeout 20m ./management/...
-
-  test_client_on_docker:
-    name: "Client (Docker) / Unit"
-    needs: [ build-cache ]
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Install Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: "1.23.x"
-          cache: false
-
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Get Go environment
-        run: |
-          echo "cache=$(go env GOCACHE)" >> $GITHUB_ENV
-          echo "modcache=$(go env.GOMODCACHE)" >> $GITHUB_ENV     
-
-      - name: Cache Go modules
-        uses: actions/cache/restore@v4
-        with:
-          path: |
-            ${{ env.cache }}
-            ${{ env.modcache }}
-          key: ${{ runner.os }}-gotest-cache-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-gotest-cache-
-
-      - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
-
-      - name: Install modules
-        run: go mod tidy
-
-      - name: Check git status
-        run: git --no-pager diff --exit-code
-
-      - name: Generate Shared Sock Test bin
-        run: CGO_ENABLED=0 go test -c -o sharedsock-testing.bin ./sharedsock
-
-      - name: Generate RouteManager Test bin
-        run: CGO_ENABLED=0 go test -c -o routemanager-testing.bin ./client/internal/routemanager
-
-      - name: Generate SystemOps Test bin (static via Alpine)
-        run: |
-          docker run --rm -v $PWD:/app -w /app \
-            alpine:latest \
-            sh -c "
-              apk add --no-cache go gcc musl-dev libpcap-dev dbus-dev && \
-              adduser -D -u $(id -u) builder && \
-              su builder -c '\
-                cd /app && \
-                CGO_ENABLED=1 GOOS=linux GOARCH=amd64 \
-                go test -c -o /app/systemops-testing.bin \
-                  -tags netgo \
-                  -ldflags=\"-w -extldflags \\\"-static -ldbus-1 -lpcap\\\"\" \
-                  ./client/internal/routemanager/systemops \
-              '
-            "
-
-      - name: Generate nftables Manager Test bin
-        run: CGO_ENABLED=0 go test -c -o nftablesmanager-testing.bin ./client/firewall/nftables/...
-
-      - name: Generate Engine Test bin
-        run: CGO_ENABLED=1 go test -c -o engine-testing.bin ./client/internal
-
-      - name: Generate Peer Test bin
-        run: CGO_ENABLED=0 go test -c -o peer-testing.bin ./client/internal/peer/
-
-      - run: chmod +x *testing.bin
-
-      - name: Run Shared Sock tests in docker
-        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/sharedsock --entrypoint /ci/sharedsock-testing.bin gcr.io/distroless/base:debug -test.timeout 5m -test.parallel 1
-
-      - name: Run Iface tests in docker
-        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/netbird -v /tmp/cache:/tmp/cache -v /tmp/modcache:/tmp/modcache -w /netbird -e GOCACHE=/tmp/cache -e GOMODCACHE=/tmp/modcache -e CGO_ENABLED=0 golang:1.23-alpine go test -test.timeout 5m -test.parallel 1 ./client/iface/...
-
-      - name: Run RouteManager tests in docker
-        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/routemanager --entrypoint /ci/routemanager-testing.bin gcr.io/distroless/base:debug -test.timeout 5m -test.parallel 1
-
-      - name: Run SystemOps tests in docker
-        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/routemanager/systemops --entrypoint /ci/systemops-testing.bin gcr.io/distroless/base:debug -test.timeout 5m -test.parallel 1
-
-      - name: Run nftables Manager tests in docker
-        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/firewall --entrypoint /ci/nftablesmanager-testing.bin gcr.io/distroless/base:debug -test.timeout 5m -test.parallel 1
-
-      - name: Run Engine tests in docker with file store
-        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal -e NETBIRD_STORE_ENGINE="jsonfile" --entrypoint /ci/engine-testing.bin gcr.io/distroless/base:debug -test.timeout 5m -test.parallel 1
-
-      - name: Run Engine tests in docker with sqlite store
-        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal -e NETBIRD_STORE_ENGINE="sqlite" --entrypoint /ci/engine-testing.bin gcr.io/distroless/base:debug -test.timeout 5m -test.parallel 1
-
-      - name: Run Peer tests in docker
-        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/peer --entrypoint /ci/peer-testing.bin gcr.io/distroless/base:debug -test.timeout 5m -test.parallel 1

client/Dockerfile

@@ -1,5 +1,6 @@
 FROM alpine:3.21.3
-RUN apk add --no-cache ca-certificates iptables ip6tables
+# iproute2: busybox doesn't display ip rules properly
+RUN apk add --no-cache ca-certificates ip6tables iproute2 iptables
 ENV NB_FOREGROUND_MODE=true
 ENTRYPOINT [ "/usr/local/bin/netbird","up"]
 COPY netbird /usr/local/bin/netbird

client/firewall/uspfilter/tracer_test.go

@@ -198,12 +198,12 @@ func TestTracePacket(t *testing.T) {
 				m.forwarder.Store(&forwarder.Forwarder{})
 
 				src := netip.PrefixFrom(netip.AddrFrom4([4]byte{1, 1, 1, 1}), 32)
-				dst := netip.PrefixFrom(netip.AddrFrom4([4]byte{172, 17, 0, 2}), 32)
+				dst := netip.PrefixFrom(netip.AddrFrom4([4]byte{192, 168, 17, 2}), 32)
 				_, err := m.AddRouteFiltering(nil, []netip.Prefix{src}, fw.Network{Prefix: dst}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept)
 				require.NoError(t, err)
 			},
 			packetBuilder: func() *PacketBuilder {
-				return createPacketBuilder("1.1.1.1", "172.17.0.2", "tcp", 12345, 80, fw.RuleDirectionIN)
+				return createPacketBuilder("1.1.1.1", "192.168.17.2", "tcp", 12345, 80, fw.RuleDirectionIN)
 			},
 			expectedStages: []PacketStage{
 				StageReceived,
@@ -222,12 +222,12 @@ func TestTracePacket(t *testing.T) {
 				m.nativeRouter.Store(false)
 
 				src := netip.PrefixFrom(netip.AddrFrom4([4]byte{1, 1, 1, 1}), 32)
-				dst := netip.PrefixFrom(netip.AddrFrom4([4]byte{172, 17, 0, 2}), 32)
+				dst := netip.PrefixFrom(netip.AddrFrom4([4]byte{192, 168, 17, 2}), 32)
 				_, err := m.AddRouteFiltering(nil, []netip.Prefix{src}, fw.Network{Prefix: dst}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionDrop)
 				require.NoError(t, err)
 			},
 			packetBuilder: func() *PacketBuilder {
-				return createPacketBuilder("1.1.1.1", "172.17.0.2", "tcp", 12345, 80, fw.RuleDirectionIN)
+				return createPacketBuilder("1.1.1.1", "192.168.17.2", "tcp", 12345, 80, fw.RuleDirectionIN)
 			},
 			expectedStages: []PacketStage{
 				StageReceived,
@@ -245,7 +245,7 @@ func TestTracePacket(t *testing.T) {
 				m.nativeRouter.Store(true)
 			},
 			packetBuilder: func() *PacketBuilder {
-				return createPacketBuilder("1.1.1.1", "172.17.0.2", "tcp", 12345, 80, fw.RuleDirectionIN)
+				return createPacketBuilder("1.1.1.1", "192.168.17.2", "tcp", 12345, 80, fw.RuleDirectionIN)
 			},
 			expectedStages: []PacketStage{
 				StageReceived,
@@ -263,7 +263,7 @@ func TestTracePacket(t *testing.T) {
 				m.routingEnabled.Store(false)
 			},
 			packetBuilder: func() *PacketBuilder {
-				return createPacketBuilder("1.1.1.1", "172.17.0.2", "tcp", 12345, 80, fw.RuleDirectionIN)
+				return createPacketBuilder("1.1.1.1", "192.168.17.2", "tcp", 12345, 80, fw.RuleDirectionIN)
 			},
 			expectedStages: []PacketStage{
 				StageReceived,
@@ -425,8 +425,8 @@ func TestTracePacket(t *testing.T) {
 
 			require.True(t, m.localipmanager.IsLocalIP(netip.MustParseAddr("100.10.0.100")),
 				"100.10.0.100 should be recognized as a local IP")
-			require.False(t, m.localipmanager.IsLocalIP(netip.MustParseAddr("172.17.0.2")),
+			require.False(t, m.localipmanager.IsLocalIP(netip.MustParseAddr("192.168.17.2")),
-				"172.17.0.2 should not be recognized as a local IP")
+				"192.168.17.2 should not be recognized as a local IP")
 
 			pb := tc.packetBuilder()
 

client/internal/routemanager/systemops/systemops_bsd_test.go

@@ -24,7 +24,6 @@ func init() {
 	testCases = append(testCases, []testCase{
 		{
 			name:              "To more specific route without custom dialer via vpn",
-			destination:       "10.10.0.2:53",
 			expectedInterface: expectedVPNint,
 			dialer:            &net.Dialer{},
 			expectedPacket:    createPacketExpectation("100.64.0.1", 12345, "10.10.0.2", 53),

client/internal/routemanager/systemops/systemops_linux_test.go

@@ -27,14 +27,12 @@ func init() {
 	testCases = append(testCases, []testCase{
 		{
 			name:              "To more specific route without custom dialer via physical interface",
-			destination:       "10.10.0.2:53",
 			expectedInterface: expectedInternalInt,
 			dialer:            &net.Dialer{},
 			expectedPacket:    createPacketExpectation("192.168.1.1", 12345, "10.10.0.2", 53),
 		},
 		{
 			name:              "To more specific route (local) without custom dialer via physical interface",
-			destination:       "127.0.10.1:53",
 			expectedInterface: expectedLoopbackInt,
 			dialer:            &net.Dialer{},
 			expectedPacket:    createPacketExpectation("127.0.0.1", 12345, "127.0.10.1", 53),
@@ -134,6 +132,16 @@ func addDummyRoute(t *testing.T, dstCIDR string, gw net.IP, intf string) {
 	_, dstIPNet, err := net.ParseCIDR(dstCIDR)
 	require.NoError(t, err)
 
+	link, err := netlink.LinkByName(intf)
+	require.NoError(t, err)
+	linkIndex := link.Attrs().Index
+
+	route := &netlink.Route{
+		Dst:       dstIPNet,
+		Gw:        gw,
+		LinkIndex: linkIndex,
+	}
+
 	// Handle existing routes with metric 0
 	var originalNexthop net.IP
 	var originalLinkIndex int
@@ -145,32 +153,24 @@ func addDummyRoute(t *testing.T, dstCIDR string, gw net.IP, intf string) {
 		}
 
 		if originalNexthop != nil {
+			// remove original route
 			err = netlink.RouteDel(&netlink.Route{Dst: dstIPNet, Priority: 0})
-			switch {
+			assert.NoError(t, err)
-			case err != nil && !errors.Is(err, syscall.ESRCH):
+
-				t.Logf("Failed to delete route: %v", err)
+			// add new route
-			case err == nil:
+			assert.NoError(t, netlink.RouteAdd(route))
+
 			t.Cleanup(func() {
+				// restore original route
+				assert.NoError(t, netlink.RouteDel(route))
 				err := netlink.RouteAdd(&netlink.Route{Dst: dstIPNet, Gw: originalNexthop, LinkIndex: originalLinkIndex, Priority: 0})
-					if err != nil && !errors.Is(err, syscall.EEXIST) {
+				assert.NoError(t, err)
-						t.Fatalf("Failed to add route: %v", err)
-					}
 			})
-			default:
+
-				t.Logf("Failed to delete route: %v", err)
+			return
-			}
 		}
 	}
 
-	link, err := netlink.LinkByName(intf)
-	require.NoError(t, err)
-	linkIndex := link.Attrs().Index
-
-	route := &netlink.Route{
-		Dst:       dstIPNet,
-		Gw:        gw,
-		LinkIndex: linkIndex,
-	}
 	err = netlink.RouteDel(route)
 	if err != nil && !errors.Is(err, syscall.ESRCH) {
 		t.Logf("Failed to delete route: %v", err)
@@ -180,7 +180,6 @@ func addDummyRoute(t *testing.T, dstCIDR string, gw net.IP, intf string) {
 	if err != nil && !errors.Is(err, syscall.EEXIST) {
 		t.Fatalf("Failed to add route: %v", err)
 	}
-	require.NoError(t, err)
 }
 
 func fetchOriginalGateway(family int) (net.IP, int, error) {
@@ -190,7 +189,11 @@ func fetchOriginalGateway(family int) (net.IP, int, error) {
 	}
 
 	for _, route := range routes {
-		if route.Dst == nil && route.Priority == 0 {
+		ones := -1
+		if route.Dst != nil {
+			ones, _ = route.Dst.Mask.Size()
+		}
+		if route.Dst == nil || ones == 0 && route.Priority == 0 {
 			return route.Gw, route.LinkIndex, nil
 		}
 	}

client/internal/routemanager/systemops/systemops_unix_test.go

@@ -31,7 +31,6 @@ type PacketExpectation struct {
 
 type testCase struct {
 	name              string
-	destination       string
 	expectedInterface string
 	dialer            dialer
 	expectedPacket    PacketExpectation
@@ -40,14 +39,12 @@ type testCase struct {
 var testCases = []testCase{
 	{
 		name:              "To external host without custom dialer via vpn",
-		destination:       "192.0.2.1:53",
 		expectedInterface: expectedVPNint,
 		dialer:            &net.Dialer{},
 		expectedPacket:    createPacketExpectation("100.64.0.1", 12345, "192.0.2.1", 53),
 	},
 	{
 		name:              "To external host with custom dialer via physical interface",
-		destination:       "192.0.2.1:53",
 		expectedInterface: expectedExternalInt,
 		dialer:            nbnet.NewDialer(),
 		expectedPacket:    createPacketExpectation("192.168.0.1", 12345, "192.0.2.1", 53),
@@ -55,14 +52,12 @@ var testCases = []testCase{
 
 	{
 		name:              "To duplicate internal route with custom dialer via physical interface",
-		destination:       "10.0.0.2:53",
 		expectedInterface: expectedInternalInt,
 		dialer:            nbnet.NewDialer(),
 		expectedPacket:    createPacketExpectation("192.168.1.1", 12345, "10.0.0.2", 53),
 	},
 	{
 		name:              "To duplicate internal route without custom dialer via physical interface", // local route takes precedence
-		destination:       "10.0.0.2:53",
 		expectedInterface: expectedInternalInt,
 		dialer:            &net.Dialer{},
 		expectedPacket:    createPacketExpectation("192.168.1.1", 12345, "10.0.0.2", 53),
@@ -70,14 +65,12 @@ var testCases = []testCase{
 
 	{
 		name:              "To unique vpn route with custom dialer via physical interface",
-		destination:       "172.16.0.2:53",
 		expectedInterface: expectedExternalInt,
 		dialer:            nbnet.NewDialer(),
 		expectedPacket:    createPacketExpectation("192.168.0.1", 12345, "172.16.0.2", 53),
 	},
 	{
 		name:              "To unique vpn route without custom dialer via vpn",
-		destination:       "172.16.0.2:53",
 		expectedInterface: expectedVPNint,
 		dialer:            &net.Dialer{},
 		expectedPacket:    createPacketExpectation("100.64.0.1", 12345, "172.16.0.2", 53),
@@ -94,10 +87,11 @@ func TestRouting(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			setupTestEnv(t)
 
-			filter := createBPFFilter(tc.destination)
+			dst := fmt.Sprintf("%s:%d", tc.expectedPacket.DstIP, tc.expectedPacket.DstPort)
+			filter := createBPFFilter(dst)
 			handle := startPacketCapture(t, tc.expectedInterface, filter)
 
-			sendTestPacket(t, tc.destination, tc.expectedPacket.SrcPort, tc.dialer)
+			sendTestPacket(t, dst, tc.expectedPacket.SrcPort, tc.dialer)
 
 			packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
 			packet, err := packetSource.NextPacket()