mirror of
https://github.com/netbirdio/netbird.git
synced 2025-08-13 17:07:30 +02:00
Support client default routes for Linux (#1667)
All routes are now installed in a custom netbird routing table. Management and wireguard traffic is now marked with a custom fwmark. When the mark is present the traffic is routed via the main routing table, bypassing the VPN. When the mark is absent the traffic is routed via the netbird routing table, if: - there's no match in the main routing table - it would match the default route in the routing table IPv6 traffic is blocked when a default route IPv4 route is configured to avoid leakage.
This commit is contained in:
Viktor Liu
GitHub
@ -3,142 +3,298 @@
|
||||
package routemanager
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
|
||||
"github.com/hashicorp/go-multierror"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vishvananda/netlink"
|
||||
|
||||
nbnet "github.com/netbirdio/netbird/util/net"
|
||||
)
|
||||
|
||||
// Pulled from http://man7.org/linux/man-pages/man7/rtnetlink.7.html
|
||||
// See the section on RTM_NEWROUTE, specifically 'struct rtmsg'.
|
||||
type routeInfoInMemory struct {
|
||||
Family byte
|
||||
DstLen byte
|
||||
SrcLen byte
|
||||
TOS byte
|
||||
const (
|
||||
// NetbirdVPNTableID is the ID of the custom routing table used by Netbird.
|
||||
NetbirdVPNTableID = 0x1BD0
|
||||
// NetbirdVPNTableName is the name of the custom routing table used by Netbird.
|
||||
NetbirdVPNTableName = "netbird"
|
||||
|
||||
Table byte
|
||||
Protocol byte
|
||||
Scope byte
|
||||
Type byte
|
||||
// rtTablesPath is the path to the file containing the routing table names.
|
||||
rtTablesPath = "/etc/iproute2/rt_tables"
|
||||
|
||||
Flags uint32
|
||||
// ipv4ForwardingPath is the path to the file containing the IP forwarding setting.
|
||||
ipv4ForwardingPath = "/proc/sys/net/ipv4/ip_forward"
|
||||
)
|
||||
|
||||
var ErrTableIDExists = errors.New("ID exists with different name")
|
||||
|
||||
type ruleParams struct {
|
||||
fwmark int
|
||||
tableID int
|
||||
family int
|
||||
priority int
|
||||
invert bool
|
||||
suppressPrefix int
|
||||
description string
|
||||
}
|
||||
|
||||
const ipv4ForwardingPath = "/proc/sys/net/ipv4/ip_forward"
|
||||
func getSetupRules() []ruleParams {
|
||||
return []ruleParams{
|
||||
{nbnet.NetbirdFwmark, NetbirdVPNTableID, netlink.FAMILY_V4, -1, true, -1, "add rule v4 netbird"},
|
||||
{nbnet.NetbirdFwmark, NetbirdVPNTableID, netlink.FAMILY_V6, -1, true, -1, "add rule v6 netbird"},
|
||||
{-1, syscall.RT_TABLE_MAIN, netlink.FAMILY_V4, -1, false, 0, "add rule with suppress prefixlen v4"},
|
||||
{-1, syscall.RT_TABLE_MAIN, netlink.FAMILY_V6, -1, false, 0, "add rule with suppress prefixlen v6"},
|
||||
}
|
||||
}
|
||||
|
||||
func addToRouteTable(prefix netip.Prefix, addr string) error {
|
||||
_, ipNet, err := net.ParseCIDR(prefix.String())
|
||||
if err != nil {
|
||||
return err
|
||||
// setupRouting establishes the routing configuration for the VPN, including essential rules
|
||||
// to ensure proper traffic flow for management, locally configured routes, and VPN traffic.
|
||||
//
|
||||
// Rule 1 (Main Route Precedence): Safeguards locally installed routes by giving them precedence over
|
||||
// potential routes received and configured for the VPN. This rule is skipped for the default route and routes
|
||||
// that are not in the main table.
|
||||
//
|
||||
// Rule 2 (VPN Traffic Routing): Directs all remaining traffic to the 'NetbirdVPNTableID' custom routing table.
|
||||
// This table is where a default route or other specific routes received from the management server are configured,
|
||||
// enabling VPN connectivity.
|
||||
//
|
||||
// The rules are inserted in reverse order, as rules are added from the bottom up in the rule list.
|
||||
func setupRouting() (err error) {
|
||||
if err = addRoutingTableName(); err != nil {
|
||||
log.Errorf("Error adding routing table name: %v", err)
|
||||
}
|
||||
|
||||
addrMask := "/32"
|
||||
if prefix.Addr().Unmap().Is6() {
|
||||
addrMask = "/128"
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
if cleanErr := cleanupRouting(); cleanErr != nil {
|
||||
log.Errorf("Error cleaning up routing: %v", cleanErr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
ip, _, err := net.ParseCIDR(addr + addrMask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
route := &netlink.Route{
|
||||
Scope: netlink.SCOPE_UNIVERSE,
|
||||
Dst: ipNet,
|
||||
Gw: ip,
|
||||
}
|
||||
|
||||
err = netlink.RouteAdd(route)
|
||||
if err != nil {
|
||||
return err
|
||||
rules := getSetupRules()
|
||||
for _, rule := range rules {
|
||||
if err := addRule(rule); err != nil {
|
||||
return fmt.Errorf("%s: %w", rule.description, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeFromRouteTable(prefix netip.Prefix, addr string) error {
|
||||
_, ipNet, err := net.ParseCIDR(prefix.String())
|
||||
if err != nil {
|
||||
return err
|
||||
// cleanupRouting performs a thorough cleanup of the routing configuration established by 'setupRouting'.
|
||||
// It systematically removes the three rules and any associated routing table entries to ensure a clean state.
|
||||
// The function uses error aggregation to report any errors encountered during the cleanup process.
|
||||
func cleanupRouting() error {
|
||||
var result *multierror.Error
|
||||
|
||||
if err := flushRoutes(NetbirdVPNTableID, netlink.FAMILY_V4); err != nil {
|
||||
result = multierror.Append(result, fmt.Errorf("flush routes v4: %w", err))
|
||||
}
|
||||
if err := flushRoutes(NetbirdVPNTableID, netlink.FAMILY_V6); err != nil {
|
||||
result = multierror.Append(result, fmt.Errorf("flush routes v6: %w", err))
|
||||
}
|
||||
|
||||
addrMask := "/32"
|
||||
if prefix.Addr().Unmap().Is6() {
|
||||
addrMask = "/128"
|
||||
rules := getSetupRules()
|
||||
for _, rule := range rules {
|
||||
if err := removeAllRules(rule); err != nil {
|
||||
result = multierror.Append(result, fmt.Errorf("%s: %w", rule.description, err))
|
||||
}
|
||||
}
|
||||
|
||||
ip, _, err := net.ParseCIDR(addr + addrMask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return result.ErrorOrNil()
|
||||
}
|
||||
|
||||
route := &netlink.Route{
|
||||
Scope: netlink.SCOPE_UNIVERSE,
|
||||
Dst: ipNet,
|
||||
Gw: ip,
|
||||
}
|
||||
func addToRouteTableIfNoExists(prefix netip.Prefix, _ string, intf string) error {
|
||||
// No need to check if routes exist as main table takes precedence over the VPN table via Rule 2
|
||||
|
||||
err = netlink.RouteDel(route)
|
||||
if err != nil {
|
||||
return err
|
||||
// TODO remove this once we have ipv6 support
|
||||
if prefix == defaultv4 {
|
||||
if err := addUnreachableRoute(&defaultv6, NetbirdVPNTableID, netlink.FAMILY_V6); err != nil {
|
||||
return fmt.Errorf("add blackhole: %w", err)
|
||||
}
|
||||
}
|
||||
if err := addRoute(&prefix, nil, &intf, NetbirdVPNTableID, netlink.FAMILY_V4); err != nil {
|
||||
return fmt.Errorf("add route: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeFromRouteTableIfNonSystem(prefix netip.Prefix, _ string, intf string) error {
|
||||
// TODO remove this once we have ipv6 support
|
||||
if prefix == defaultv4 {
|
||||
if err := removeUnreachableRoute(&defaultv6, NetbirdVPNTableID, netlink.FAMILY_V6); err != nil {
|
||||
return fmt.Errorf("remove unreachable route: %w", err)
|
||||
}
|
||||
}
|
||||
if err := removeRoute(&prefix, nil, &intf, NetbirdVPNTableID, netlink.FAMILY_V4); err != nil {
|
||||
return fmt.Errorf("remove route: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getRoutesFromTable() ([]netip.Prefix, error) {
|
||||
tab, err := syscall.NetlinkRIB(syscall.RTM_GETROUTE, syscall.AF_UNSPEC)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return getRoutes(NetbirdVPNTableID, netlink.FAMILY_V4)
|
||||
}
|
||||
|
||||
// addRoute adds a route to a specific routing table identified by tableID.
|
||||
func addRoute(prefix *netip.Prefix, addr, intf *string, tableID, family int) error {
|
||||
route := &netlink.Route{
|
||||
Scope: netlink.SCOPE_UNIVERSE,
|
||||
Table: tableID,
|
||||
Family: family,
|
||||
}
|
||||
msgs, err := syscall.ParseNetlinkMessage(tab)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
if prefix != nil {
|
||||
_, ipNet, err := net.ParseCIDR(prefix.String())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse prefix %s: %w", prefix, err)
|
||||
}
|
||||
route.Dst = ipNet
|
||||
}
|
||||
var prefixList []netip.Prefix
|
||||
loop:
|
||||
for _, m := range msgs {
|
||||
switch m.Header.Type {
|
||||
case syscall.NLMSG_DONE:
|
||||
break loop
|
||||
case syscall.RTM_NEWROUTE:
|
||||
rt := (*routeInfoInMemory)(unsafe.Pointer(&m.Data[0]))
|
||||
msg := m
|
||||
attrs, err := syscall.ParseNetlinkRouteAttr(&msg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
if err := addNextHop(addr, intf, route); err != nil {
|
||||
return fmt.Errorf("add gateway and device: %w", err)
|
||||
}
|
||||
|
||||
if err := netlink.RouteAdd(route); err != nil && !errors.Is(err, syscall.EEXIST) {
|
||||
return fmt.Errorf("netlink add route: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// addUnreachableRoute adds an unreachable route for the specified IP family and routing table.
|
||||
// ipFamily should be netlink.FAMILY_V4 for IPv4 or netlink.FAMILY_V6 for IPv6.
|
||||
// tableID specifies the routing table to which the unreachable route will be added.
|
||||
func addUnreachableRoute(prefix *netip.Prefix, tableID, ipFamily int) error {
|
||||
_, ipNet, err := net.ParseCIDR(prefix.String())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse prefix %s: %w", prefix, err)
|
||||
}
|
||||
|
||||
route := &netlink.Route{
|
||||
Type: syscall.RTN_UNREACHABLE,
|
||||
Table: tableID,
|
||||
Family: ipFamily,
|
||||
Dst: ipNet,
|
||||
}
|
||||
|
||||
if err := netlink.RouteAdd(route); err != nil && !errors.Is(err, syscall.EEXIST) {
|
||||
return fmt.Errorf("netlink add unreachable route: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeUnreachableRoute(prefix *netip.Prefix, tableID, ipFamily int) error {
|
||||
_, ipNet, err := net.ParseCIDR(prefix.String())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse prefix %s: %w", prefix, err)
|
||||
}
|
||||
|
||||
route := &netlink.Route{
|
||||
Type: syscall.RTN_UNREACHABLE,
|
||||
Table: tableID,
|
||||
Family: ipFamily,
|
||||
Dst: ipNet,
|
||||
}
|
||||
|
||||
if err := netlink.RouteDel(route); err != nil && !errors.Is(err, syscall.ESRCH) {
|
||||
return fmt.Errorf("netlink remove unreachable route: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
// removeRoute removes a route from a specific routing table identified by tableID.
|
||||
func removeRoute(prefix *netip.Prefix, addr, intf *string, tableID, family int) error {
|
||||
_, ipNet, err := net.ParseCIDR(prefix.String())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse prefix %s: %w", prefix, err)
|
||||
}
|
||||
|
||||
route := &netlink.Route{
|
||||
Scope: netlink.SCOPE_UNIVERSE,
|
||||
Table: tableID,
|
||||
Family: family,
|
||||
Dst: ipNet,
|
||||
}
|
||||
|
||||
if err := addNextHop(addr, intf, route); err != nil {
|
||||
return fmt.Errorf("add gateway and device: %w", err)
|
||||
}
|
||||
|
||||
if err := netlink.RouteDel(route); err != nil && !errors.Is(err, syscall.ESRCH) {
|
||||
return fmt.Errorf("netlink remove route: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func flushRoutes(tableID, family int) error {
|
||||
routes, err := netlink.RouteListFiltered(family, &netlink.Route{Table: tableID}, netlink.RT_FILTER_TABLE)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list routes from table %d: %w", tableID, err)
|
||||
}
|
||||
|
||||
var result *multierror.Error
|
||||
for i := range routes {
|
||||
route := routes[i]
|
||||
// unreachable default routes don't come back with Dst set
|
||||
if route.Gw == nil && route.Src == nil && route.Dst == nil {
|
||||
if family == netlink.FAMILY_V4 {
|
||||
routes[i].Dst = &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)}
|
||||
} else {
|
||||
routes[i].Dst = &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)}
|
||||
}
|
||||
if rt.Family != syscall.AF_INET {
|
||||
continue loop
|
||||
}
|
||||
if err := netlink.RouteDel(&routes[i]); err != nil {
|
||||
result = multierror.Append(result, fmt.Errorf("failed to delete route %v from table %d: %w", routes[i], tableID, err))
|
||||
}
|
||||
}
|
||||
|
||||
return result.ErrorOrNil()
|
||||
}
|
||||
|
||||
// getRoutes fetches routes from a specific routing table identified by tableID.
|
||||
func getRoutes(tableID, family int) ([]netip.Prefix, error) {
|
||||
var prefixList []netip.Prefix
|
||||
|
||||
routes, err := netlink.RouteListFiltered(family, &netlink.Route{Table: tableID}, netlink.RT_FILTER_TABLE)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list routes from table %d: %v", tableID, err)
|
||||
}
|
||||
|
||||
for _, route := range routes {
|
||||
if route.Dst != nil {
|
||||
addr, ok := netip.AddrFromSlice(route.Dst.IP)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("parse route destination IP: %v", route.Dst.IP)
|
||||
}
|
||||
|
||||
for _, attr := range attrs {
|
||||
if attr.Attr.Type == syscall.RTA_DST {
|
||||
addr, ok := netip.AddrFromSlice(attr.Value)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
mask := net.CIDRMask(int(rt.DstLen), len(attr.Value)*8)
|
||||
cidr, _ := mask.Size()
|
||||
routePrefix := netip.PrefixFrom(addr, cidr)
|
||||
if routePrefix.IsValid() && routePrefix.Addr().Is4() {
|
||||
prefixList = append(prefixList, routePrefix)
|
||||
}
|
||||
}
|
||||
ones, _ := route.Dst.Mask.Size()
|
||||
|
||||
prefix := netip.PrefixFrom(addr, ones)
|
||||
if prefix.IsValid() {
|
||||
prefixList = append(prefixList, prefix)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return prefixList, nil
|
||||
}
|
||||
|
||||
func enableIPForwarding() error {
|
||||
bytes, err := os.ReadFile(ipv4ForwardingPath)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("read file %s: %w", ipv4ForwardingPath, err)
|
||||
}
|
||||
|
||||
// check if it is already enabled
|
||||
@ -147,5 +303,142 @@ func enableIPForwarding() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
return os.WriteFile(ipv4ForwardingPath, []byte("1"), 0644) //nolint:gosec
|
||||
//nolint:gosec
|
||||
if err := os.WriteFile(ipv4ForwardingPath, []byte("1"), 0644); err != nil {
|
||||
return fmt.Errorf("write file %s: %w", ipv4ForwardingPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// entryExists checks if the specified ID or name already exists in the rt_tables file
|
||||
// and verifies if existing names start with "netbird_".
|
||||
func entryExists(file *os.File, id int) (bool, error) {
|
||||
if _, err := file.Seek(0, 0); err != nil {
|
||||
return false, fmt.Errorf("seek rt_tables: %w", err)
|
||||
}
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
var existingID int
|
||||
var existingName string
|
||||
if _, err := fmt.Sscanf(line, "%d %s\n", &existingID, &existingName); err == nil {
|
||||
if existingID == id {
|
||||
if existingName != NetbirdVPNTableName {
|
||||
return true, ErrTableIDExists
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return false, fmt.Errorf("scan rt_tables: %w", err)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// addRoutingTableName adds human-readable names for custom routing tables.
|
||||
func addRoutingTableName() error {
|
||||
file, err := os.Open(rtTablesPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("open rt_tables: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := file.Close(); err != nil {
|
||||
log.Errorf("Error closing rt_tables: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
exists, err := entryExists(file, NetbirdVPNTableID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("verify entry %d, %s: %w", NetbirdVPNTableID, NetbirdVPNTableName, err)
|
||||
}
|
||||
if exists {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reopen the file in append mode to add new entries
|
||||
if err := file.Close(); err != nil {
|
||||
log.Errorf("Error closing rt_tables before appending: %v", err)
|
||||
}
|
||||
file, err = os.OpenFile(rtTablesPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open rt_tables for appending: %w", err)
|
||||
}
|
||||
|
||||
if _, err := file.WriteString(fmt.Sprintf("\n%d\t%s\n", NetbirdVPNTableID, NetbirdVPNTableName)); err != nil {
|
||||
return fmt.Errorf("append entry to rt_tables: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// addRule adds a routing rule to a specific routing table identified by tableID.
|
||||
func addRule(params ruleParams) error {
|
||||
rule := netlink.NewRule()
|
||||
rule.Table = params.tableID
|
||||
rule.Mark = params.fwmark
|
||||
rule.Family = params.family
|
||||
rule.Priority = params.priority
|
||||
rule.Invert = params.invert
|
||||
rule.SuppressPrefixlen = params.suppressPrefix
|
||||
|
||||
if err := netlink.RuleAdd(rule); err != nil {
|
||||
return fmt.Errorf("add routing rule: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeRule removes a routing rule from a specific routing table identified by tableID.
|
||||
func removeRule(params ruleParams) error {
|
||||
rule := netlink.NewRule()
|
||||
rule.Table = params.tableID
|
||||
rule.Mark = params.fwmark
|
||||
rule.Family = params.family
|
||||
rule.Invert = params.invert
|
||||
rule.Priority = params.priority
|
||||
rule.SuppressPrefixlen = params.suppressPrefix
|
||||
|
||||
if err := netlink.RuleDel(rule); err != nil {
|
||||
return fmt.Errorf("remove routing rule: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeAllRules(params ruleParams) error {
|
||||
for {
|
||||
if err := removeRule(params); err != nil {
|
||||
if errors.Is(err, syscall.ENOENT) {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// addNextHop adds the gateway and device to the route.
|
||||
func addNextHop(addr *string, intf *string, route *netlink.Route) error {
|
||||
if addr != nil {
|
||||
ip := net.ParseIP(*addr)
|
||||
if ip == nil {
|
||||
return fmt.Errorf("parsing address %s failed", *addr)
|
||||
}
|
||||
|
||||
route.Gw = ip
|
||||
}
|
||||
|
||||
if intf != nil {
|
||||
link, err := netlink.LinkByName(*intf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("set interface %s: %w", *intf, err)
|
||||
}
|
||||
route.LinkIndex = link.Attrs().Index
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user