From d0067a0a12979a8eb99737b517cf3dcf935a21b7 Mon Sep 17 00:00:00 2001 From: vishalsabhaya <86877856+vishalsabhaya@users.noreply.github.com> Date: Sun, 6 Oct 2024 10:36:54 +0900 Subject: [PATCH] improve page load performance of large amount urls (#5025) Co-authored-by: vishal sabhaya Co-authored-by: Frank Elsinga --- server/model/monitor.js | 173 +++++++++++++++++++++++++++-------- server/server.js | 30 +++--- server/uptime-kuma-server.js | 47 ++++++++-- src/mixins/socket.js | 41 +++++++-- src/pages/EditMonitor.vue | 1 - src/util.js | 1 + 6 files changed, 225 insertions(+), 68 deletions(-) diff --git a/server/model/monitor.js b/server/model/monitor.js index 4beeb003..78485c4c 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -72,23 +72,12 @@ class Monitor extends BeanModel { /** * Return an object that ready to parse to JSON + * @param {object} preloadData to prevent n+1 problems, we query the data in a batch outside of this function * @param {boolean} includeSensitiveData Include sensitive data in * JSON - * @returns {Promise} Object ready to parse + * @returns {object} Object ready to parse */ - async toJSON(includeSensitiveData = true) { - - let notificationIDList = {}; - - let list = await R.find("monitor_notification", " monitor_id = ? ", [ - this.id, - ]); - - for (let bean of list) { - notificationIDList[bean.notification_id] = true; - } - - const tags = await this.getTags(); + toJSON(preloadData = {}, includeSensitiveData = true) { let screenshot = null; @@ -96,7 +85,7 @@ class Monitor extends BeanModel { screenshot = "/screenshots/" + jwt.sign(this.id, UptimeKumaServer.getInstance().jwtSecret) + ".png"; } - const path = await this.getPath(); + const path = preloadData.paths.get(this.id) || []; const pathName = path.join(" / "); let data = { @@ -106,15 +95,15 @@ class Monitor extends BeanModel { path, pathName, parent: this.parent, - childrenIDs: await Monitor.getAllChildrenIDs(this.id), + childrenIDs: preloadData.childrenIDs.get(this.id) || [], url: this.url, method: this.method, hostname: this.hostname, port: this.port, maxretries: this.maxretries, weight: this.weight, - active: await this.isActive(), - forceInactive: !await Monitor.isParentActive(this.id), + active: preloadData.activeStatus.get(this.id), + forceInactive: preloadData.forceInactive.get(this.id), type: this.type, timeout: this.timeout, interval: this.interval, @@ -134,9 +123,9 @@ class Monitor extends BeanModel { docker_container: this.docker_container, docker_host: this.docker_host, proxyId: this.proxy_id, - notificationIDList, - tags: tags, - maintenance: await Monitor.isUnderMaintenance(this.id), + notificationIDList: preloadData.notifications.get(this.id) || {}, + tags: preloadData.tags.get(this.id) || [], + maintenance: preloadData.maintenanceStatus.get(this.id), mqttTopic: this.mqttTopic, mqttSuccessMessage: this.mqttSuccessMessage, mqttCheckType: this.mqttCheckType, @@ -202,16 +191,6 @@ class Monitor extends BeanModel { return data; } - /** - * Checks if the monitor is active based on itself and its parents - * @returns {Promise} Is the monitor active? - */ - async isActive() { - const parentActive = await Monitor.isParentActive(this.id); - - return (this.active === 1) && parentActive; - } - /** * Get all tags applied to this monitor * @returns {Promise[]>} List of tags on the @@ -1197,6 +1176,18 @@ class Monitor extends BeanModel { return checkCertificateResult; } + /** + * Checks if the monitor is active based on itself and its parents + * @param {number} monitorID ID of monitor to send + * @param {boolean} active is active + * @returns {Promise} Is the monitor active? + */ + static async isActive(monitorID, active) { + const parentActive = await Monitor.isParentActive(monitorID); + + return (active === 1) && parentActive; + } + /** * Send statistics to clients * @param {Server} io Socket server instance @@ -1333,7 +1324,10 @@ class Monitor extends BeanModel { for (let notification of notificationList) { try { const heartbeatJSON = bean.toJSON(); - + const monitorData = [{ id: monitor.id, + active: monitor.active + }]; + const preloadData = await Monitor.preparePreloadData(monitorData); // Prevent if the msg is undefined, notifications such as Discord cannot send out. if (!heartbeatJSON["msg"]) { heartbeatJSON["msg"] = "N/A"; @@ -1344,7 +1338,7 @@ class Monitor extends BeanModel { heartbeatJSON["timezoneOffset"] = UptimeKumaServer.getInstance().getTimezoneOffset(); heartbeatJSON["localDateTime"] = dayjs.utc(heartbeatJSON["time"]).tz(heartbeatJSON["timezone"]).format(SQL_DATETIME_FORMAT); - await Notification.send(JSON.parse(notification.config), msg, await monitor.toJSON(false), heartbeatJSON); + await Notification.send(JSON.parse(notification.config), msg, monitor.toJSON(preloadData, false), heartbeatJSON); } catch (e) { log.error("monitor", "Cannot send notification to " + notification.name); log.error("monitor", e); @@ -1507,6 +1501,111 @@ class Monitor extends BeanModel { } } + /** + * Gets monitor notification of multiple monitor + * @param {Array} monitorIDs IDs of monitor to get + * @returns {Promise>} object + */ + static async getMonitorNotification(monitorIDs) { + return await R.getAll(` + SELECT monitor_notification.monitor_id, monitor_notification.notification_id + FROM monitor_notification + WHERE monitor_notification.monitor_id IN (?) + `, [ + monitorIDs, + ]); + } + + /** + * Gets monitor tags of multiple monitor + * @param {Array} monitorIDs IDs of monitor to get + * @returns {Promise>} object + */ + static async getMonitorTag(monitorIDs) { + return await R.getAll(` + SELECT monitor_tag.monitor_id, tag.name, tag.color + FROM monitor_tag + JOIN tag ON monitor_tag.tag_id = tag.id + WHERE monitor_tag.monitor_id IN (?) + `, [ + monitorIDs, + ]); + } + + /** + * prepare preloaded data for efficient access + * @param {Array} monitorData IDs & active field of monitor to get + * @returns {Promise>} object + */ + static async preparePreloadData(monitorData) { + + const notificationsMap = new Map(); + const tagsMap = new Map(); + const maintenanceStatusMap = new Map(); + const childrenIDsMap = new Map(); + const activeStatusMap = new Map(); + const forceInactiveMap = new Map(); + const pathsMap = new Map(); + + if (monitorData.length > 0) { + const monitorIDs = monitorData.map(monitor => monitor.id); + const notifications = await Monitor.getMonitorNotification(monitorIDs); + const tags = await Monitor.getMonitorTag(monitorIDs); + const maintenanceStatuses = await Promise.all(monitorData.map(monitor => Monitor.isUnderMaintenance(monitor.id))); + const childrenIDs = await Promise.all(monitorData.map(monitor => Monitor.getAllChildrenIDs(monitor.id))); + const activeStatuses = await Promise.all(monitorData.map(monitor => Monitor.isActive(monitor.id, monitor.active))); + const forceInactiveStatuses = await Promise.all(monitorData.map(monitor => Monitor.isParentActive(monitor.id))); + const paths = await Promise.all(monitorData.map(monitor => Monitor.getAllPath(monitor.id, monitor.name))); + + notifications.forEach(row => { + if (!notificationsMap.has(row.monitor_id)) { + notificationsMap.set(row.monitor_id, {}); + } + notificationsMap.get(row.monitor_id)[row.notification_id] = true; + }); + + tags.forEach(row => { + if (!tagsMap.has(row.monitor_id)) { + tagsMap.set(row.monitor_id, []); + } + tagsMap.get(row.monitor_id).push({ + name: row.name, + color: row.color + }); + }); + + monitorData.forEach((monitor, index) => { + maintenanceStatusMap.set(monitor.id, maintenanceStatuses[index]); + }); + + monitorData.forEach((monitor, index) => { + childrenIDsMap.set(monitor.id, childrenIDs[index]); + }); + + monitorData.forEach((monitor, index) => { + activeStatusMap.set(monitor.id, activeStatuses[index]); + }); + + monitorData.forEach((monitor, index) => { + forceInactiveMap.set(monitor.id, !forceInactiveStatuses[index]); + }); + + monitorData.forEach((monitor, index) => { + pathsMap.set(monitor.id, paths[index]); + }); + } + + return { + notifications: notificationsMap, + tags: tagsMap, + maintenanceStatus: maintenanceStatusMap, + childrenIDs: childrenIDsMap, + activeStatus: activeStatusMap, + forceInactive: forceInactiveMap, + paths: pathsMap, + }; + } + /** * Gets Parent of the monitor * @param {number} monitorID ID of monitor to get @@ -1539,16 +1638,18 @@ class Monitor extends BeanModel { /** * Gets the full path + * @param {number} monitorID ID of the monitor to get + * @param {string} name of the monitor to get * @returns {Promise} Full path (includes groups and the name) of the monitor */ - async getPath() { - const path = [ this.name ]; + static async getAllPath(monitorID, name) { + const path = [ name ]; if (this.parent === null) { return path; } - let parent = await Monitor.getParent(this.id); + let parent = await Monitor.getParent(monitorID); while (parent !== null) { path.unshift(parent.name); parent = await Monitor.getParent(parent.id); diff --git a/server/server.js b/server/server.js index d040d6e8..3579df5d 100644 --- a/server/server.js +++ b/server/server.js @@ -726,7 +726,7 @@ let needSetup = false; await updateMonitorNotification(bean.id, notificationIDList); - await server.sendMonitorList(socket); + await server.sendUpdateMonitorIntoList(socket, bean.id); if (monitor.active !== false) { await startMonitor(socket.userID, bean.id); @@ -879,11 +879,11 @@ let needSetup = false; await updateMonitorNotification(bean.id, monitor.notificationIDList); - if (await bean.isActive()) { + if (await Monitor.isActive(bean.id, bean.active)) { await restartMonitor(socket.userID, bean.id); } - await server.sendMonitorList(socket); + await server.sendUpdateMonitorIntoList(socket, bean.id); callback({ ok: true, @@ -923,14 +923,17 @@ let needSetup = false; log.info("monitor", `Get Monitor: ${monitorID} User ID: ${socket.userID}`); - let bean = await R.findOne("monitor", " id = ? AND user_id = ? ", [ + let monitor = await R.findOne("monitor", " id = ? AND user_id = ? ", [ monitorID, socket.userID, ]); - + const monitorData = [{ id: monitor.id, + active: monitor.active + }]; + const preloadData = await Monitor.preparePreloadData(monitorData); callback({ ok: true, - monitor: await bean.toJSON(), + monitor: monitor.toJSON(preloadData), }); } catch (e) { @@ -981,7 +984,7 @@ let needSetup = false; try { checkLogin(socket); await startMonitor(socket.userID, monitorID); - await server.sendMonitorList(socket); + await server.sendUpdateMonitorIntoList(socket, monitorID); callback({ ok: true, @@ -1001,7 +1004,7 @@ let needSetup = false; try { checkLogin(socket); await pauseMonitor(socket.userID, monitorID); - await server.sendMonitorList(socket); + await server.sendUpdateMonitorIntoList(socket, monitorID); callback({ ok: true, @@ -1047,8 +1050,7 @@ let needSetup = false; msg: "successDeleted", msgi18n: true, }); - - await server.sendMonitorList(socket); + await server.sendDeleteMonitorFromList(socket, monitorID); } catch (e) { callback({ @@ -1678,13 +1680,13 @@ async function afterLogin(socket, user) { await StatusPage.sendStatusPageList(io, socket); + const monitorPromises = []; for (let monitorID in monitorList) { - await sendHeartbeatList(socket, monitorID); + monitorPromises.push(sendHeartbeatList(socket, monitorID)); + monitorPromises.push(Monitor.sendStats(io, monitorID, user.id)); } - for (let monitorID in monitorList) { - await Monitor.sendStats(io, monitorID, user.id); - } + await Promise.all(monitorPromises); // Set server timezone from client browser if not set // It should be run once only diff --git a/server/uptime-kuma-server.js b/server/uptime-kuma-server.js index 573d791a..76bf4256 100644 --- a/server/uptime-kuma-server.js +++ b/server/uptime-kuma-server.js @@ -205,24 +205,56 @@ class UptimeKumaServer { return list; } + /** + * Update Monitor into list + * @param {Socket} socket Socket to send list on + * @param {number} monitorID update or deleted monitor id + * @returns {Promise} + */ + async sendUpdateMonitorIntoList(socket, monitorID) { + let list = await this.getMonitorJSONList(socket.userID, monitorID); + this.io.to(socket.userID).emit("updateMonitorIntoList", list); + } + + /** + * Delete Monitor from list + * @param {Socket} socket Socket to send list on + * @param {number} monitorID update or deleted monitor id + * @returns {Promise} + */ + async sendDeleteMonitorFromList(socket, monitorID) { + this.io.to(socket.userID).emit("deleteMonitorFromList", monitorID); + } + /** * Get a list of monitors for the given user. * @param {string} userID - The ID of the user to get monitors for. + * @param {number} monitorID - The ID of monitor for. * @returns {Promise} A promise that resolves to an object with monitor IDs as keys and monitor objects as values. * * Generated by Trelent */ - async getMonitorJSONList(userID) { - let result = {}; + async getMonitorJSONList(userID, monitorID = null) { - let monitorList = await R.find("monitor", " user_id = ? ORDER BY weight DESC, name", [ - userID, - ]); + let query = " user_id = ? "; + let queryParams = [ userID ]; - for (let monitor of monitorList) { - result[monitor.id] = await monitor.toJSON(); + if (monitorID) { + query += "AND id = ? "; + queryParams.push(monitorID); } + let monitorList = await R.find("monitor", query + "ORDER BY weight DESC, name", queryParams); + + const monitorData = monitorList.map(monitor => ({ + id: monitor.id, + active: monitor.active, + name: monitor.name, + })); + const preloadData = await Monitor.preparePreloadData(monitorData); + + const result = {}; + monitorList.forEach(monitor => result[monitor.id] = monitor.toJSON(preloadData)); return result; } @@ -520,3 +552,4 @@ const { DnsMonitorType } = require("./monitor-types/dns"); const { MqttMonitorType } = require("./monitor-types/mqtt"); const { SNMPMonitorType } = require("./monitor-types/snmp"); const { MongodbMonitorType } = require("./monitor-types/mongodb"); +const Monitor = require("./model/monitor"); diff --git a/src/mixins/socket.js b/src/mixins/socket.js index 4541161c..3272e042 100644 --- a/src/mixins/socket.js +++ b/src/mixins/socket.js @@ -141,19 +141,23 @@ export default { }); socket.on("monitorList", (data) => { - // Add Helper function - Object.entries(data).forEach(([ monitorID, monitor ]) => { - monitor.getUrl = () => { - try { - return new URL(monitor.url); - } catch (_) { - return null; - } - }; - }); + this.assignMonitorUrlParser(data); this.monitorList = data; }); + socket.on("updateMonitorIntoList", (data) => { + this.assignMonitorUrlParser(data); + Object.entries(data).forEach(([ monitorID, updatedMonitor ]) => { + this.monitorList[monitorID] = updatedMonitor; + }); + }); + + socket.on("deleteMonitorFromList", (monitorID) => { + if (this.monitorList[monitorID]) { + delete this.monitorList[monitorID]; + } + }); + socket.on("monitorTypeList", (data) => { this.monitorTypeList = data; }); @@ -289,6 +293,23 @@ export default { location.reload(); }); }, + /** + * parse all urls from list. + * @param {object} data Monitor data to modify + * @returns {object} list + */ + assignMonitorUrlParser(data) { + Object.entries(data).forEach(([ monitorID, monitor ]) => { + monitor.getUrl = () => { + try { + return new URL(monitor.url); + } catch (_) { + return null; + } + }; + }); + return data; + }, /** * The storage currently in use diff --git a/src/pages/EditMonitor.vue b/src/pages/EditMonitor.vue index e684dd70..5d999b59 100644 --- a/src/pages/EditMonitor.vue +++ b/src/pages/EditMonitor.vue @@ -1813,7 +1813,6 @@ message HealthCheckResponse { await this.startParentGroupMonitor(); } this.processing = false; - this.$root.getMonitorList(); this.$router.push("/dashboard/" + res.monitorID); } else { this.processing = false; diff --git a/src/util.js b/src/util.js index 8b2c2b9d..370004e0 100644 --- a/src/util.js +++ b/src/util.js @@ -83,6 +83,7 @@ exports.CONSOLE_STYLE_BgMagenta = "\x1b[45m"; exports.CONSOLE_STYLE_BgCyan = "\x1b[46m"; exports.CONSOLE_STYLE_BgWhite = "\x1b[47m"; exports.CONSOLE_STYLE_BgGray = "\x1b[100m"; + const consoleModuleColors = [ exports.CONSOLE_STYLE_FgCyan, exports.CONSOLE_STYLE_FgGreen,