Migrate blob net ip fields to json serializer (#1906)

* serialize net.IP as json

* migrate net ip field from blob to json

* run net ip migration

* remove duplicate index

* Refactor

* Add tests

* fix tests

* migrate null blob values

management/server/migration/migration.go

@@ -1,10 +1,12 @@
 package migration
 
 import (
+	"database/sql"
 	"encoding/gob"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
@@ -99,3 +101,104 @@ func MigrateFieldFromGobToJSON[T any, S any](db *gorm.DB, fieldName string) erro
 
 	return nil
 }
+
+// MigrateNetIPFieldFromBlobToJSON migrates a Net IP column from Blob encoding to JSON encoding.
+// T is the type of the model that contains the field to be migrated.
+func MigrateNetIPFieldFromBlobToJSON[T any](db *gorm.DB, fieldName string, indexName string) error {
+	oldColumnName := fieldName
+	newColumnName := fieldName + "_tmp"
+
+	var model T
+
+	if !db.Migrator().HasTable(&model) {
+		log.Printf("Table for %T does not exist, no migration needed", model)
+		return nil
+	}
+
+	stmt := &gorm.Statement{DB: db}
+	err := stmt.Parse(&model)
+	if err != nil {
+		return fmt.Errorf("parse model: %w", err)
+	}
+	tableName := stmt.Schema.Table
+
+	var item sql.NullString
+	if err := db.Model(&model).Select(oldColumnName).First(&item).Error; err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			log.Printf("No records in table %s, no migration needed", tableName)
+			return nil
+		}
+		return fmt.Errorf("fetch first record: %w", err)
+	}
+
+	if item.Valid {
+		var js json.RawMessage
+		var syntaxError *json.SyntaxError
+		err = json.Unmarshal([]byte(item.String), &js)
+		if err == nil || !errors.As(err, &syntaxError) {
+			log.Debugf("No migration needed for %s, %s", tableName, fieldName)
+			return nil
+		}
+	}
+
+	if err := db.Transaction(func(tx *gorm.DB) error {
+		if err := tx.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s TEXT", tableName, newColumnName)).Error; err != nil {
+			return fmt.Errorf("add column %s: %w", newColumnName, err)
+		}
+
+		var rows []map[string]any
+		if err := tx.Table(tableName).Select("id", oldColumnName).Find(&rows).Error; err != nil {
+			if errors.Is(err, gorm.ErrRecordNotFound) {
+				log.Printf("No records in table %s, no migration needed", tableName)
+				return nil
+			}
+			return fmt.Errorf("find rows: %w", err)
+		}
+
+		for _, row := range rows {
+			var blobValue string
+			if columnValue := row[oldColumnName]; columnValue != nil {
+				value, ok := columnValue.(string)
+				if !ok {
+					return fmt.Errorf("type assertion failed")
+				}
+				blobValue = value
+			}
+
+			columnIpValue := net.IP(blobValue)
+			if net.ParseIP(columnIpValue.String()) == nil {
+				log.Debugf("failed to parse %s as ip, fallback to ipv6 loopback", oldColumnName)
+				columnIpValue = net.IPv6loopback
+			}
+
+			jsonValue, err := json.Marshal(columnIpValue)
+			if err != nil {
+				return fmt.Errorf("re-encode to JSON: %w", err)
+			}
+
+			if err := tx.Table(tableName).Where("id = ?", row["id"]).Update(newColumnName, jsonValue).Error; err != nil {
+				return fmt.Errorf("update row: %w", err)
+			}
+		}
+
+		if indexName != "" {
+			if err := tx.Migrator().DropIndex(&model, indexName); err != nil {
+				return fmt.Errorf("drop index %s: %w", indexName, err)
+			}
+		}
+
+		if err := tx.Exec(fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", tableName, oldColumnName)).Error; err != nil {
+			return fmt.Errorf("drop column %s: %w", oldColumnName, err)
+		}
+		if err := tx.Exec(fmt.Sprintf("ALTER TABLE %s RENAME COLUMN %s TO %s", tableName, newColumnName, oldColumnName)).Error; err != nil {
+			return fmt.Errorf("rename column %s to %s: %w", newColumnName, oldColumnName, err)
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	log.Printf("Migration of %s.%s from blob to json completed", tableName, fieldName)
+
+	return nil
+}

management/server/migration/migration_test.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/netbirdio/netbird/management/server"
 	"github.com/netbirdio/netbird/management/server/migration"
+	nbpeer "github.com/netbirdio/netbird/management/server/peer"
 	"github.com/netbirdio/netbird/route"
 )
 
@@ -89,3 +90,72 @@ func TestMigrateFieldFromGobToJSON_WithJSONData(t *testing.T) {
 	db.Model(&server.Account{}).Select("network_net").First(&jsonStr)
 	assert.JSONEq(t, `{"IP":"10.0.0.0","Mask":"////AA=="}`, jsonStr, "Data should be unchanged")
 }
+
+func TestMigrateNetIPFieldFromBlobToJSON_EmptyDB(t *testing.T) {
+	db := setupDatabase(t)
+	err := migration.MigrateNetIPFieldFromBlobToJSON[nbpeer.Peer](db, "ip", "idx_peers_account_id_ip")
+	require.NoError(t, err, "Migration should not fail for an empty database")
+}
+
+func TestMigrateNetIPFieldFromBlobToJSON_WithBlobData(t *testing.T) {
+	db := setupDatabase(t)
+
+	err := db.AutoMigrate(&server.Account{}, &nbpeer.Peer{})
+	require.NoError(t, err, "Failed to auto-migrate tables")
+
+	type location struct {
+		nbpeer.Location
+		ConnectionIP net.IP
+	}
+
+	type peer struct {
+		nbpeer.Peer
+		Location location `gorm:"embedded;embeddedPrefix:location_"`
+	}
+
+	type account struct {
+		server.Account
+		Peers []peer `gorm:"foreignKey:AccountID;references:id"`
+	}
+
+	err = db.Save(&account{
+		Account: server.Account{Id: "123"},
+		Peers: []peer{
+			{Location: location{ConnectionIP: net.IP{10, 0, 0, 1}}},
+		}},
+	).Error
+	require.NoError(t, err, "Failed to insert blob data")
+
+	var blobValue string
+	err = db.Model(&nbpeer.Peer{}).Select("location_connection_ip").First(&blobValue).Error
+	assert.NoError(t, err, "Failed to fetch blob data")
+
+	err = migration.MigrateNetIPFieldFromBlobToJSON[nbpeer.Peer](db, "location_connection_ip", "")
+	require.NoError(t, err, "Migration should not fail with net.IP blob data")
+
+	var jsonStr string
+	db.Model(&nbpeer.Peer{}).Select("location_connection_ip").First(&jsonStr)
+	assert.JSONEq(t, `"10.0.0.1"`, jsonStr, "Data should be migrated")
+}
+
+func TestMigrateNetIPFieldFromBlobToJSON_WithJSONData(t *testing.T) {
+	db := setupDatabase(t)
+
+	err := db.AutoMigrate(&server.Account{}, &nbpeer.Peer{})
+	require.NoError(t, err, "Failed to auto-migrate tables")
+
+	err = db.Save(&server.Account{
+		Id: "1234",
+		PeersG: []nbpeer.Peer{
+			{Location: nbpeer.Location{ConnectionIP: net.IP{10, 0, 0, 1}}},
+		}},
+	).Error
+	require.NoError(t, err, "Failed to insert JSON data")
+
+	err = migration.MigrateNetIPFieldFromBlobToJSON[nbpeer.Peer](db, "location_connection_ip", "")
+	require.NoError(t, err, "Migration should not fail with net.IP JSON data")
+
+	var jsonStr string
+	db.Model(&nbpeer.Peer{}).Select("location_connection_ip").First(&jsonStr)
+	assert.JSONEq(t, `"10.0.0.1"`, jsonStr, "Data should be unchanged")
+}

management/server/peer/peer.go

@@ -13,13 +13,13 @@ type Peer struct {
 	// ID is an internal ID of the peer
 	ID string `gorm:"primaryKey"`
 	// AccountID is a reference to Account that this object belongs
-	AccountID string `json:"-" gorm:"index;uniqueIndex:idx_peers_account_id_ip"`
+	AccountID string `json:"-" gorm:"index"`
 	// WireGuard public key
 	Key string `gorm:"index"`
 	// A setup key this peer was registered with
 	SetupKey string
 	// IP address of the Peer
-	IP net.IP `gorm:"uniqueIndex:idx_peers_account_id_ip"`
+	IP net.IP `gorm:"serializer:json"`
 	// Meta is a Peer system meta data
 	Meta PeerSystemMeta `gorm:"embedded;embeddedPrefix:meta_"`
 	// Name is peer's name (machine name)
@@ -61,7 +61,7 @@ type PeerStatus struct { //nolint:revive
 
 // Location is a geo location information of a Peer based on public connection IP
 type Location struct {
-	ConnectionIP net.IP // from grpc peer or reverse proxy headers depends on setup
+	ConnectionIP net.IP `gorm:"serializer:json"` // from grpc peer or reverse proxy headers depends on setup
 	CountryCode  string
 	CityName     string
 	GeoNameID    uint // city level geoname id

management/server/sqlite_store.go

@@ -571,13 +571,17 @@ func getMigrations() []migrationFunc {
 		func(db *gorm.DB) error {
 			return migration.MigrateFieldFromGobToJSON[Account, net.IPNet](db, "network_net")
 		},
-
 		func(db *gorm.DB) error {
 			return migration.MigrateFieldFromGobToJSON[route.Route, netip.Prefix](db, "network")
 		},
-
 		func(db *gorm.DB) error {
 			return migration.MigrateFieldFromGobToJSON[route.Route, []string](db, "peer_groups")
 		},
+		func(db *gorm.DB) error {
+			return migration.MigrateNetIPFieldFromBlobToJSON[nbpeer.Peer](db, "location_connection_ip", "")
+		},
+		func(db *gorm.DB) error {
+			return migration.MigrateNetIPFieldFromBlobToJSON[nbpeer.Peer](db, "ip", "idx_peers_account_id_ip")
+		},
 	}
 }

management/server/sqlite_store_test.go

@@ -519,15 +519,29 @@ func TestMigrate(t *testing.T) {
 		Net net.IPNet `gorm:"serializer:gob"`
 	}
 
+	type location struct {
+		nbpeer.Location
+		ConnectionIP net.IP
+	}
+
+	type peer struct {
+		nbpeer.Peer
+		Location location `gorm:"embedded;embeddedPrefix:location_"`
+	}
+
 	type account struct {
 		Account
 		Network *network `gorm:"embedded;embeddedPrefix:network_"`
+		Peers   []peer   `gorm:"foreignKey:AccountID;references:id"`
 	}
 
 	act := &account{
 		Network: &network{
 			Net: *ipnet,
 		},
+		Peers: []peer{
+			{Location: location{ConnectionIP: net.IP{10, 0, 0, 1}}},
+		},
 	}
 
 	err = store.db.Save(act).Error