From d655a8cc216e686a3f49206a0722593f23916a17 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Wed, 20 Dec 2023 23:29:24 +0800 Subject: [PATCH] WIP --- backend/agent-manager.ts | 151 ++++++++++++++++++ backend/dockge-instance-manager.ts | 121 -------------- backend/dockge-server.ts | 31 +--- .../migrations/2023-12-20-2117-agent-table.ts | 16 ++ backend/models/agent.ts | 33 ++++ .../manage-agent-socket-handler.ts | 13 ++ backend/util-server.ts | 4 +- extra/reset-password.ts | 1 - frontend/src/components/StackListItem.vue | 3 +- frontend/src/lang/en.json | 6 +- frontend/src/mixins/socket.ts | 63 ++++++-- frontend/src/pages/Compose.vue | 2 +- frontend/src/pages/DashboardHome.vue | 56 +++++-- 13 files changed, 322 insertions(+), 178 deletions(-) create mode 100644 backend/agent-manager.ts delete mode 100644 backend/dockge-instance-manager.ts create mode 100644 backend/migrations/2023-12-20-2117-agent-table.ts create mode 100644 backend/models/agent.ts create mode 100644 backend/socket-handlers/manage-agent-socket-handler.ts diff --git a/backend/agent-manager.ts b/backend/agent-manager.ts new file mode 100644 index 0000000..71c244e --- /dev/null +++ b/backend/agent-manager.ts @@ -0,0 +1,151 @@ +import { DockgeSocket } from "./util-server"; +import { io, Socket as SocketClient } from "socket.io-client"; +import { log } from "./log"; +import { Agent } from "./models/agent"; +import { LooseObject } from "../common/util-common"; + +/** + * Dockge Instance Manager + */ +export class AgentManager { + + protected socket : DockgeSocket; + protected instanceSocketList : Record = {}; + + constructor(socket: DockgeSocket) { + this.socket = socket; + } + + connect(url : string, username : string, password : string) { + let obj = new URL(url); + let endpoint = obj.host; + + this.socket.emit("agentStatus", { + endpoint: endpoint, + status: "connecting", + }); + + if (!endpoint) { + log.error("agent-manager", "Invalid endpoint: " + endpoint + " URL: " + url); + return; + } + + if (this.instanceSocketList[endpoint]) { + log.debug("agent-manager", "Already connected to the socket server: " + endpoint); + return; + } + + log.info("agent-manager", "Connecting to the socket server: " + endpoint); + let client = io(url, { + extraHeaders: { + endpoint, + } + }); + + client.on("connect", () => { + log.info("agent-manager", "Connected to the socket server: " + endpoint); + + client.emit("login", { + username: username, + password: password, + }, (res) => { + if (res.ok) { + log.info("agent-manager", "Logged in to the socket server: " + endpoint); + this.socket.emit("agentStatus", { + endpoint: endpoint, + status: "online", + }); + } else { + log.error("agent-manager", "Failed to login to the socket server: " + endpoint); + this.socket.emit("agentStatus", { + endpoint: endpoint, + status: "offline", + }); + } + }); + }); + + client.on("error", (err) => { + log.error("agent-manager", "Error from the socket server: " + endpoint); + log.error("agent-manager", err); + this.socket.emit("agentStatus", { + endpoint: endpoint, + status: "offline", + }); + }); + + client.on("disconnect", () => { + log.info("agent-manager", "Disconnected from the socket server: " + endpoint); + this.socket.emit("agentStatus", { + endpoint: endpoint, + status: "offline", + }); + }); + + client.on("agent", (...args : unknown[]) => { + log.debug("agent-manager", "Forward event"); + this.socket.emit("agent", ...args); + }); + + this.instanceSocketList[endpoint] = client; + } + + disconnect(endpoint : string) { + let client = this.instanceSocketList[endpoint]; + client?.disconnect(); + } + + async connectAll() { + if (this.socket.endpoint) { + log.info("agent-manager", "This connection is connected as an agent, skip connectAll()"); + return; + } + + let list : Record = await Agent.getAgentList(); + + if (Object.keys(list).length !== 0) { + log.info("agent-manager", "Connecting to all instance socket server(s)..."); + } + + for (let endpoint in list) { + let agent = list[endpoint]; + this.connect(agent.url, agent.username, agent.password); + } + } + + disconnectAll() { + for (let endpoint in this.instanceSocketList) { + this.disconnect(endpoint); + } + } + + emitToEndpoint(endpoint: string, eventName: string, ...args : unknown[]) { + log.debug("agent-manager", "Emitting event to endpoint: " + endpoint); + let client = this.instanceSocketList[endpoint]; + if (!client) { + log.error("agent-manager", "Socket client not found for endpoint: " + endpoint); + throw new Error("Socket client not found for endpoint: " + endpoint); + } + client?.emit("agent", endpoint, eventName, ...args); + } + + emitToAllEndpoints(eventName: string, ...args : unknown[]) { + log.debug("agent-manager", "Emitting event to all endpoints"); + for (let endpoint in this.instanceSocketList) { + this.emitToEndpoint(endpoint, eventName, ...args); + } + } + + async sendAgentList() { + let list = await Agent.getAgentList(); + let result : Record = {}; + for (let endpoint in list) { + let agent = list[endpoint]; + result[endpoint] = agent.toJSON(); + } + this.socket.emit("agentList", { + ok: true, + agentList: result, + }); + } +} diff --git a/backend/dockge-instance-manager.ts b/backend/dockge-instance-manager.ts deleted file mode 100644 index 88c5063..0000000 --- a/backend/dockge-instance-manager.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { DockgeSocket } from "./util-server"; -import { io, Socket as SocketClient } from "socket.io-client"; -import { log } from "./log"; - -/** - * Dockge Instance Manager - */ -export class DockgeInstanceManager { - - protected socket : DockgeSocket; - protected instanceSocketList : Record = {}; - - constructor(socket: DockgeSocket) { - this.socket = socket; - } - - connect(endpoint : string, tls : boolean, username : string, password : string) { - if (this.instanceSocketList[endpoint]) { - log.debug("INSTANCEMANAGER", "Already connected to the socket server: " + endpoint); - return; - } - - let url = ((tls) ? "wss://" : "ws://") + endpoint; - - log.info("INSTANCEMANAGER", "Connecting to the socket server: " + endpoint); - let client = io(url, { - transports: [ "websocket", "polling" ], - extraHeaders: { - endpoint, - } - }); - - client.on("connect", () => { - log.info("INSTANCEMANAGER", "Connected to the socket server: " + endpoint); - - client.emit("login", { - username: username, - password: password, - }, (res) => { - if (res.ok) { - log.info("INSTANCEMANAGER", "Logged in to the socket server: " + endpoint); - } else { - log.error("INSTANCEMANAGER", "Failed to login to the socket server: " + endpoint); - } - }); - }); - - client.on("error", (err) => { - log.error("INSTANCEMANAGER", "Error from the socket server: " + endpoint); - log.error("INSTANCEMANAGER", err); - }); - - client.on("disconnect", () => { - log.info("INSTANCEMANAGER", "Disconnected from the socket server: " + endpoint); - }); - - client.on("agent", (...args : unknown[]) => { - log.debug("INSTANCEMANAGER", "Forward event"); - this.socket.emit("agent", ...args); - }); - - this.instanceSocketList[endpoint] = client; - } - - disconnect(endpoint : string) { - let client = this.instanceSocketList[endpoint]; - client?.disconnect(); - } - - connectAll() { - if (this.socket.endpoint) { - log.info("INSTANCEMANAGER", "This connection is connected as an agent, skip connectAll()"); - return; - } - - let list : Record = { - - }; - - if (process.env.DOCKGE_TEST_REMOTE_HOST) { - list[process.env.DOCKGE_TEST_REMOTE_HOST] = { - tls: false, - username: "admin", - password: process.env.DOCKGE_TEST_REMOTE_PW || "", - }; - } - - if (Object.keys(list).length !== 0) { - log.info("INSTANCEMANAGER", "Connecting to all instance socket server(s)..."); - } - - for (let endpoint in list) { - let item = list[endpoint]; - this.connect(endpoint, item.tls, item.username, item.password); - } - } - - disconnectAll() { - for (let endpoint in this.instanceSocketList) { - this.disconnect(endpoint); - } - } - - emitToEndpoint(endpoint: string, eventName: string, ...args : unknown[]) { - log.debug("INSTANCEMANAGER", "Emitting event to endpoint: " + endpoint); - let client = this.instanceSocketList[endpoint]; - if (!client) { - log.error("INSTANCEMANAGER", "Socket client not found for endpoint: " + endpoint); - throw new Error("Socket client not found for endpoint: " + endpoint); - } - client?.emit("agent", endpoint, eventName, ...args); - } - - emitToAllEndpoints(eventName: string, ...args : unknown[]) { - log.debug("INSTANCEMANAGER", "Emitting event to all endpoints"); - for (let endpoint in this.instanceSocketList) { - this.emitToEndpoint(endpoint, eventName, ...args); - } - } - -} diff --git a/backend/dockge-server.ts b/backend/dockge-server.ts index 39272ae..14b9014 100644 --- a/backend/dockge-server.ts +++ b/backend/dockge-server.ts @@ -31,10 +31,11 @@ import { Cron } from "croner"; import gracefulShutdown from "http-graceful-shutdown"; import User from "./models/user"; import childProcessAsync from "promisify-child-process"; -import { DockgeInstanceManager } from "./dockge-instance-manager"; +import { AgentManager } from "./agent-manager"; import { AgentProxySocketHandler } from "./socket-handlers/agent-proxy-socket-handler"; import { AgentSocketHandler } from "./agent-socket-handler"; import { AgentSocket } from "../common/agent-socket"; +import { ManageAgentSocketHandler } from "./socket-handlers/manage-agent-socket-handler"; export class DockgeServer { app : Express; @@ -56,6 +57,7 @@ export class DockgeServer { */ socketHandlerList : SocketHandler[] = [ new MainSocketHandler(), + new ManageAgentSocketHandler(), ]; agentProxySocketHandler = new AgentProxySocketHandler(); @@ -206,7 +208,7 @@ export class DockgeServer { cors, allowRequest: (req, callback) => { let isOriginValid = true; - const bypass = isDev; + const bypass = isDev || process.env.UPTIME_KUMA_WS_ORIGIN_CHECK === "bypass"; if (!bypass) { let host = req.headers.host; @@ -241,14 +243,14 @@ export class DockgeServer { this.io.on("connection", async (socket: Socket) => { let dockgeSocket = socket as DockgeSocket; - dockgeSocket.instanceManager = new DockgeInstanceManager(dockgeSocket); + dockgeSocket.instanceManager = new AgentManager(dockgeSocket); dockgeSocket.emitAgent = (event : string, ...args : unknown[]) => { let obj = args[0]; if (typeof(obj) === "object") { let obj2 = obj as LooseObject; obj2.endpoint = dockgeSocket.endpoint; } - dockgeSocket.emit("agent", event, ...args); + this.io.to(dockgeSocket.userID + "").emit("agent", event, ...args); }; if (typeof(socket.request.headers.endpoint) === "string") { @@ -330,6 +332,8 @@ export class DockgeServer { log.error("server", e); } + socket.instanceManager.sendAgentList(); + // Also connect to other dockge instances socket.instanceManager.connectAll(); } @@ -605,25 +609,6 @@ export class DockgeServer { } } - async sendStackStatusList() { - let statusList = await Stack.getStatusList(); - - let roomList = this.io.sockets.adapter.rooms.keys(); - - for (let room of roomList) { - // Check if the room is a number (user id) - if (Number(room)) { - log.debug("server", "Send stack status list to room " + room); - this.io.to(room).emit("stackStatusList", { - ok: true, - stackStatusList: Object.fromEntries(statusList), - }); - } else { - log.debug("server", "Skip sending stack status list to room " + room); - } - } - } - async getDockerNetworkList() : Promise { let res = await childProcessAsync.spawn("docker", [ "network", "ls", "--format", "{{.Name}}" ], { encoding: "utf-8", diff --git a/backend/migrations/2023-12-20-2117-agent-table.ts b/backend/migrations/2023-12-20-2117-agent-table.ts new file mode 100644 index 0000000..ae18d1d --- /dev/null +++ b/backend/migrations/2023-12-20-2117-agent-table.ts @@ -0,0 +1,16 @@ +import { Knex } from "knex"; + +export async function up(knex: Knex): Promise { + // Create the user table + return knex.schema.createTable("agent", (table) => { + table.increments("id"); + table.string("url", 255).notNullable().unique(); + table.string("username", 255).notNullable(); + table.string("password", 255).notNullable(); + table.boolean("active").notNullable().defaultTo(true); + }); +} + +export async function down(knex: Knex): Promise { + return knex.schema.dropTable("agent"); +} diff --git a/backend/models/agent.ts b/backend/models/agent.ts new file mode 100644 index 0000000..7d97bc8 --- /dev/null +++ b/backend/models/agent.ts @@ -0,0 +1,33 @@ +import { BeanModel } from "redbean-node/dist/bean-model"; +import { R } from "redbean-node"; +import { LooseObject } from "../../common/util-common"; +import User from "./user"; + +export class Agent extends BeanModel { + + static async getAgentList() : Promise> { + let list = await R.findAll("agent") as Agent[]; + let result : Record = {}; + for (let agent of list) { + result[agent.endpoint] = agent; + } + return result; + } + + get endpoint() : string { + let obj = new URL(this.url); + return obj.host; + } + + toJSON() : LooseObject { + return { + url: this.url, + username: this.username, + password: this.password, + endpoint: this.endpoint, + }; + } + +} + +export default Agent; diff --git a/backend/socket-handlers/manage-agent-socket-handler.ts b/backend/socket-handlers/manage-agent-socket-handler.ts new file mode 100644 index 0000000..b96e812 --- /dev/null +++ b/backend/socket-handlers/manage-agent-socket-handler.ts @@ -0,0 +1,13 @@ +import { SocketHandler } from "../socket-handler.js"; +import { DockgeServer } from "../dockge-server"; +import { log } from "../log"; +import { checkLogin, DockgeSocket } from "../util-server"; +import { AgentSocket } from "../../common/agent-socket"; +import { ALL_ENDPOINTS } from "../../common/util-common"; + +export class ManageAgentSocketHandler extends SocketHandler { + + create(socket : DockgeSocket, server : DockgeServer) { + + } +} diff --git a/backend/util-server.ts b/backend/util-server.ts index f4f53ee..24c3ffe 100644 --- a/backend/util-server.ts +++ b/backend/util-server.ts @@ -6,7 +6,7 @@ import { ERROR_TYPE_VALIDATION } from "../common/util-common"; import { R } from "redbean-node"; import { verifyPassword } from "./password-hash"; import fs from "fs"; -import { DockgeInstanceManager } from "./dockge-instance-manager"; +import { AgentManager } from "./agent-manager"; export interface JWTDecoded { username : string; @@ -16,7 +16,7 @@ export interface JWTDecoded { export interface DockgeSocket extends Socket { userID: number; consoleTerminal? : Terminal; - instanceManager : DockgeInstanceManager; + instanceManager : AgentManager; endpoint : string; emitAgent : (eventName : string, ...args : unknown[]) => void; } diff --git a/extra/reset-password.ts b/extra/reset-password.ts index f6de5e5..696fb4c 100644 --- a/extra/reset-password.ts +++ b/extra/reset-password.ts @@ -92,7 +92,6 @@ function disconnectAllSocketClients(username : string, password : string) : Prom // Disconnect all socket connections const socket = io(url, { - transports: [ "websocket" ], reconnection: false, timeout: 5000, }); diff --git a/frontend/src/components/StackListItem.vue b/frontend/src/components/StackListItem.vue index 86ea803..83e525d 100644 --- a/frontend/src/components/StackListItem.vue +++ b/frontend/src/components/StackListItem.vue @@ -3,13 +3,12 @@
{{ stackName }} -
{{ endpointDisplay }}
+
{{ endpointDisplay }}