diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go
index ea6d892b9..baff1372a 100644
--- a/client/internal/peer/conn.go
+++ b/client/internal/peer/conn.go
@@ -586,13 +586,17 @@ func (conn *Conn) onWorkerRelayStateDisconnected() {
 		return
 	}
 
-	if conn.wgProxyRelay != nil {
-		log.Debugf("relayed connection is closed, clean up WireGuard config")
+	log.Debugf("relay connection is disconnected")
+
+	if conn.currentConnPriority == connPriorityRelay {
+		log.Debugf("clean up WireGuard config")
 		err := conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey)
 		if err != nil {
 			conn.log.Errorf("failed to remove wg endpoint: %v", err)
 		}
+	}
 
+	if conn.wgProxyRelay != nil {
 		conn.endpointRelay = nil
 		_ = conn.wgProxyRelay.CloseConn()
 		conn.wgProxyRelay = nil