extern/netbird
1
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2025-03-04 01:41:17 +01:00
Code Issues Packages Projects Releases Wiki Activity

[management] Improve group peer/resource counting (#3192)

Browse Source
This commit is contained in:
Pascal Fischer 2025-01-16 21:32:12 +01:00 committed by GitHub
parent d962f7fd27
commit 97ff9a3e5d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 46 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/golang-test-linux.yml vendored
View File

@ -314,7 +314,7 @@ jobs:
run: docker pull mlsmaycon/warmed-mysql:8
- name: Test
run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -run=^$ -tags=benchmark -bench=. -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 20m $(go list -tags=benchmark ./... | grep /management)
run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -run=^$ -tags=benchmark -bench=. -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 30m $(go list -tags=benchmark ./... | grep /management)
api_integration_test:
needs: [ build-cache ]
@ -363,7 +363,7 @@ jobs:
run: git --no-pager diff --exit-code
- name: Test
run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -tags=integration -p 1 -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 20m $(go list -tags=integration ./... | grep /management)
run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -tags=integration -p 1 -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 30m $(go list -tags=integration ./... | grep /management)
test_client_on_docker:
needs: [ build-cache ]

39
management/server/groups/manager.go
View File

@ -140,41 +140,36 @@ func (m *managerImpl) GetResourceGroupsInTransaction(ctx context.Context, transa
return transaction.GetResourceGroups(ctx, lockingStrength, accountID, resourceID)
}
func ToGroupsInfo(groups []*types.Group, id string) []api.GroupMinimum {
groupsInfo := []api.GroupMinimum{}
groupsChecked := make(map[string]struct{})
func ToGroupsInfoMap(groups []*types.Group, idCount int) map[string][]api.GroupMinimum {
groupsInfoMap := make(map[string][]api.GroupMinimum, idCount)
groupsChecked := make(map[string]struct{}, len(groups)) // not sure why this is needed (left over from old implementation)
for _, group := range groups {
_, ok := groupsChecked[group.ID]
if ok {
continue
}
groupsChecked[group.ID] = struct{}{}
for _, pk := range group.Peers {
if pk == id {
info := api.GroupMinimum{
Id: group.ID,
Name: group.Name,
PeersCount: len(group.Peers),
ResourcesCount: len(group.Resources),
}
groupsInfo = append(groupsInfo, info)
break
info := api.GroupMinimum{
Id: group.ID,
Name: group.Name,
PeersCount: len(group.Peers),
ResourcesCount: len(group.Resources),
}
groupsInfoMap[pk] = append(groupsInfoMap[pk], info)
}
for _, rk := range group.Resources {
if rk.ID == id {
info := api.GroupMinimum{
Id: group.ID,
Name: group.Name,
PeersCount: len(group.Peers),
ResourcesCount: len(group.Resources),
}
groupsInfo = append(groupsInfo, info)
break
info := api.GroupMinimum{
Id: group.ID,
Name: group.Name,
PeersCount: len(group.Peers),
ResourcesCount: len(group.Resources),
}
groupsInfoMap[rk.ID] = append(groupsInfoMap[rk.ID], info)
}
}
return groupsInfo
return groupsInfoMap
}
func (m *mockManager) GetAllGroups(ctx context.Context, accountID, userID string) ([]*types.Group, error) {

25
management/server/http/handlers/networks/resources_handler.go
View File

@ -66,10 +66,11 @@ func (h *resourceHandler) getAllResourcesInNetwork(w http.ResponseWriter, r *htt
return
}
grpsInfoMap := groups.ToGroupsInfoMap(grps, len(resources))
var resourcesResponse []*api.NetworkResource
for _, resource := range resources {
groupMinimumInfo := groups.ToGroupsInfo(grps, resource.ID)
resourcesResponse = append(resourcesResponse, resource.ToAPIResponse(groupMinimumInfo))
resourcesResponse = append(resourcesResponse, resource.ToAPIResponse(grpsInfoMap[resource.ID]))
}
util.WriteJSONObject(r.Context(), w, resourcesResponse)
@ -94,10 +95,11 @@ func (h *resourceHandler) getAllResourcesInAccount(w http.ResponseWriter, r *htt
return
}
grpsInfoMap := groups.ToGroupsInfoMap(grps, 0)
var resourcesResponse []*api.NetworkResource
for _, resource := range resources {
groupMinimumInfo := groups.ToGroupsInfo(grps, resource.ID)
resourcesResponse = append(resourcesResponse, resource.ToAPIResponse(groupMinimumInfo))
resourcesResponse = append(resourcesResponse, resource.ToAPIResponse(grpsInfoMap[resource.ID]))
}
util.WriteJSONObject(r.Context(), w, resourcesResponse)
@ -136,8 +138,9 @@ func (h *resourceHandler) createResource(w http.ResponseWriter, r *http.Request)
return
}
groupMinimumInfo := groups.ToGroupsInfo(grps, resource.ID)
util.WriteJSONObject(r.Context(), w, resource.ToAPIResponse(groupMinimumInfo))
grpsInfoMap := groups.ToGroupsInfoMap(grps, 0)
util.WriteJSONObject(r.Context(), w, resource.ToAPIResponse(grpsInfoMap[resource.ID]))
}
func (h *resourceHandler) getResource(w http.ResponseWriter, r *http.Request) {
@ -162,8 +165,9 @@ func (h *resourceHandler) getResource(w http.ResponseWriter, r *http.Request) {
return
}
groupMinimumInfo := groups.ToGroupsInfo(grps, resource.ID)
util.WriteJSONObject(r.Context(), w, resource.ToAPIResponse(groupMinimumInfo))
grpsInfoMap := groups.ToGroupsInfoMap(grps, 0)
util.WriteJSONObject(r.Context(), w, resource.ToAPIResponse(grpsInfoMap[resource.ID]))
}
func (h *resourceHandler) updateResource(w http.ResponseWriter, r *http.Request) {
@ -199,8 +203,9 @@ func (h *resourceHandler) updateResource(w http.ResponseWriter, r *http.Request)
return
}
groupMinimumInfo := groups.ToGroupsInfo(grps, resource.ID)
util.WriteJSONObject(r.Context(), w, resource.ToAPIResponse(groupMinimumInfo))
grpsInfoMap := groups.ToGroupsInfoMap(grps, 0)
util.WriteJSONObject(r.Context(), w, resource.ToAPIResponse(grpsInfoMap[resource.ID]))
}
func (h *resourceHandler) deleteResource(w http.ResponseWriter, r *http.Request) {

12
management/server/http/handlers/peers/peers_handler.go
View File

@ -73,7 +73,7 @@ func (h *Handler) getPeer(ctx context.Context, accountID, peerID, userID string,
dnsDomain := h.accountManager.GetDNSDomain()
grps, _ := h.accountManager.GetPeerGroups(ctx, accountID, peerID)
groupsInfo := groups.ToGroupsInfo(grps, peerID)
grpsInfoMap := groups.ToGroupsInfoMap(grps, 0)
validPeers, err := h.accountManager.GetValidatedPeers(ctx, accountID)
if err != nil {
@ -83,7 +83,7 @@ func (h *Handler) getPeer(ctx context.Context, accountID, peerID, userID string,
}
_, valid := validPeers[peer.ID]
util.WriteJSONObject(ctx, w, toSinglePeerResponse(peerToReturn, groupsInfo, dnsDomain, valid))
util.WriteJSONObject(ctx, w, toSinglePeerResponse(peerToReturn, grpsInfoMap[peerID], dnsDomain, valid))
}
func (h *Handler) updatePeer(ctx context.Context, accountID, userID, peerID string, w http.ResponseWriter, r *http.Request) {
@ -123,7 +123,7 @@ func (h *Handler) updatePeer(ctx context.Context, accountID, userID, peerID stri
return
}
groupMinimumInfo := groups.ToGroupsInfo(peerGroups, peer.ID)
grpsInfoMap := groups.ToGroupsInfoMap(peerGroups, 0)
validPeers, err := h.accountManager.GetValidatedPeers(ctx, accountID)
if err != nil {
@ -134,7 +134,7 @@ func (h *Handler) updatePeer(ctx context.Context, accountID, userID, peerID stri
_, valid := validPeers[peer.ID]
util.WriteJSONObject(r.Context(), w, toSinglePeerResponse(peer, groupMinimumInfo, dnsDomain, valid))
util.WriteJSONObject(r.Context(), w, toSinglePeerResponse(peer, grpsInfoMap[peerID], dnsDomain, valid))
}
func (h *Handler) deletePeer(ctx context.Context, accountID, userID string, peerID string, w http.ResponseWriter) {
@ -196,6 +196,7 @@ func (h *Handler) GetAllPeers(w http.ResponseWriter, r *http.Request) {
grps, _ := h.accountManager.GetAllGroups(r.Context(), accountID, userID)
grpsInfoMap := groups.ToGroupsInfoMap(grps, len(peers))
respBody := make([]*api.PeerBatch, 0, len(peers))
for _, peer := range peers {
peerToReturn, err := h.checkPeerStatus(peer)
@ -203,9 +204,8 @@ func (h *Handler) GetAllPeers(w http.ResponseWriter, r *http.Request) {
util.WriteError(r.Context(), err, w)
return
}
groupMinimumInfo := groups.ToGroupsInfo(grps, peer.ID)
respBody = append(respBody, toPeerListItemResponse(peerToReturn, groupMinimumInfo, dnsDomain, 0))
respBody = append(respBody, toPeerListItemResponse(peerToReturn, grpsInfoMap[peer.ID], dnsDomain, 0))
}
validPeersMap, err := h.accountManager.GetValidatedPeers(r.Context(), accountID)

12
management/server/http/testing/benchmarks/peers_handler_benchmark_test.go
View File

@ -40,8 +40,8 @@ func BenchmarkUpdatePeer(b *testing.B) {
"Peers - M": {MinMsPerOpLocal: 130, MaxMsPerOpLocal: 150, MinMsPerOpCICD: 100, MaxMsPerOpCICD: 300},
"Peers - L": {MinMsPerOpLocal: 230, MaxMsPerOpLocal: 270, MinMsPerOpCICD: 200, MaxMsPerOpCICD: 500},
"Groups - L": {MinMsPerOpLocal: 400, MaxMsPerOpLocal: 600, MinMsPerOpCICD: 650, MaxMsPerOpCICD: 3500},
"Users - L": {MinMsPerOpLocal: 200, MaxMsPerOpLocal: 400, MinMsPerOpCICD: 300, MaxMsPerOpCICD: 600},
"Setup Keys - L": {MinMsPerOpLocal: 200, MaxMsPerOpLocal: 400, MinMsPerOpCICD: 300, MaxMsPerOpCICD: 600},
"Users - L": {MinMsPerOpLocal: 200, MaxMsPerOpLocal: 400, MinMsPerOpCICD: 250, MaxMsPerOpCICD: 600},
"Setup Keys - L": {MinMsPerOpLocal: 200, MaxMsPerOpLocal: 400, MinMsPerOpCICD: 250, MaxMsPerOpCICD: 600},
"Peers - XL": {MinMsPerOpLocal: 600, MaxMsPerOpLocal: 1000, MinMsPerOpCICD: 600, MaxMsPerOpCICD: 2000},
}
@ -77,7 +77,7 @@ func BenchmarkUpdatePeer(b *testing.B) {
func BenchmarkGetOnePeer(b *testing.B) {
var expectedMetrics = map[string]testing_tools.PerformanceMetrics{
"Peers - XS": {MinMsPerOpLocal: 40, MaxMsPerOpLocal: 60, MinMsPerOpCICD: 30, MaxMsPerOpCICD: 70},
"Peers - XS": {MinMsPerOpLocal: 15, MaxMsPerOpLocal: 40, MinMsPerOpCICD: 30, MaxMsPerOpCICD: 70},
"Peers - S": {MinMsPerOpLocal: 1, MaxMsPerOpLocal: 5, MinMsPerOpCICD: 2, MaxMsPerOpCICD: 30},
"Peers - M": {MinMsPerOpLocal: 9, MaxMsPerOpLocal: 18, MinMsPerOpCICD: 15, MaxMsPerOpCICD: 50},
"Peers - L": {MinMsPerOpLocal: 40, MaxMsPerOpLocal: 90, MinMsPerOpCICD: 50, MaxMsPerOpCICD: 130},
@ -114,11 +114,11 @@ func BenchmarkGetAllPeers(b *testing.B) {
"Peers - XS": {MinMsPerOpLocal: 40, MaxMsPerOpLocal: 70, MinMsPerOpCICD: 50, MaxMsPerOpCICD: 150},
"Peers - S": {MinMsPerOpLocal: 2, MaxMsPerOpLocal: 10, MinMsPerOpCICD: 5, MaxMsPerOpCICD: 30},
"Peers - M": {MinMsPerOpLocal: 20, MaxMsPerOpLocal: 50, MinMsPerOpCICD: 20, MaxMsPerOpCICD: 70},
"Peers - L": {MinMsPerOpLocal: 130, MaxMsPerOpLocal: 170, MinMsPerOpCICD: 100, MaxMsPerOpCICD: 300},
"Groups - L": {MinMsPerOpLocal: 4800, MaxMsPerOpLocal: 5300, MinMsPerOpCICD: 5000, MaxMsPerOpCICD: 8000},
"Peers - L": {MinMsPerOpLocal: 110, MaxMsPerOpLocal: 150, MinMsPerOpCICD: 100, MaxMsPerOpCICD: 300},
"Groups - L": {MinMsPerOpLocal: 150, MaxMsPerOpLocal: 200, MinMsPerOpCICD: 130, MaxMsPerOpCICD: 500},
"Users - L": {MinMsPerOpLocal: 100, MaxMsPerOpLocal: 170, MinMsPerOpCICD: 100, MaxMsPerOpCICD: 400},
"Setup Keys - L": {MinMsPerOpLocal: 100, MaxMsPerOpLocal: 170, MinMsPerOpCICD: 100, MaxMsPerOpCICD: 400},
"Peers - XL": {MinMsPerOpLocal: 900, MaxMsPerOpLocal: 1300, MinMsPerOpCICD: 800, MaxMsPerOpCICD: 2300},
"Peers - XL": {MinMsPerOpLocal: 450, MaxMsPerOpLocal: 800, MinMsPerOpCICD: 500, MaxMsPerOpCICD: 1500},
}
log.SetOutput(io.Discard)