Feat: Full server-side pagination for important events (#3515)

* Feat: Serverside pagination for importantBeats

* Chore: Remove unused state

* Apply suggestions from code review

Co-authored-by: Frank Elsinga <frank@elsinga.de>

* Fix: Add watch for monitor

* Fix: Fix compatibility with dynamic page length

* Chore: Fix lint

* Merge conflict

---------

Co-authored-by: Frank Elsinga <frank@elsinga.de>
Co-authored-by: Louis Lam <louislam@users.noreply.github.com>

package.json

@@ -113,6 +113,7 @@
         "knex": "^2.4.2",
         "limiter": "~2.1.0",
         "liquidjs": "^10.7.0",
+        "mitt": "~3.0.1",
         "mongodb": "~4.17.1",
         "mqtt": "~4.3.7",
         "mssql": "~8.1.4",

server/server.js

@@ -144,7 +144,7 @@ if (config.demoMode) {
 }
 
 // Must be after io instantiation
-const { sendNotificationList, sendHeartbeatList, sendImportantHeartbeatList, sendInfo, sendProxyList, sendDockerHostList, sendAPIKeyList } = require("./client");
+const { sendNotificationList, sendHeartbeatList, sendInfo, sendProxyList, sendDockerHostList, sendAPIKeyList } = require("./client");
 const { statusPageSocketHandler } = require("./socket-handlers/status-page-socket-handler");
 const databaseSocketHandler = require("./socket-handlers/database-socket-handler");
 const TwoFA = require("./2fa");
@@ -1003,8 +1003,6 @@ let needSetup = false;
                 });
 
                 await server.sendMonitorList(socket);
-                // Clear heartbeat list on client
-                await sendImportantHeartbeatList(socket, monitorID, true, true);
 
             } catch (e) {
                 callback({
@@ -1174,6 +1172,72 @@ let needSetup = false;
             }
         });
 
+        socket.on("monitorImportantHeartbeatListCount", async (monitorID, callback) => {
+            try {
+                checkLogin(socket);
+
+                let count;
+                if (monitorID == null) {
+                    count = await R.count("heartbeat", "important = 1");
+                } else {
+                    count = await R.count("heartbeat", "monitor_id = ? AND important = 1", [
+                        monitorID,
+                    ]);
+                }
+
+                callback({
+                    ok: true,
+                    count: count,
+                });
+            } catch (e) {
+                callback({
+                    ok: false,
+                    msg: e.message,
+                });
+            }
+        });
+
+        socket.on("monitorImportantHeartbeatListPaged", async (monitorID, offset, count, callback) => {
+            try {
+                checkLogin(socket);
+
+                let list;
+                if (monitorID == null) {
+                    list = await R.find("heartbeat", `
+                        important = 1
+                        ORDER BY time DESC
+                        LIMIT ?
+                        OFFSET ?
+                    `, [
+                        count,
+                        offset,
+                    ]);
+                } else {
+                    list = await R.find("heartbeat", `
+                        monitor_id = ?
+                        AND important = 1
+                        ORDER BY time DESC
+                        LIMIT ?
+                        OFFSET ?
+                    `, [
+                        monitorID,
+                        count,
+                        offset,
+                    ]);
+                }
+
+                callback({
+                    ok: true,
+                    data: list,
+                });
+            } catch (e) {
+                callback({
+                    ok: false,
+                    msg: e.message,
+                });
+            }
+        });
+
         socket.on("changePassword", async (password, callback) => {
             try {
                 checkLogin(socket);
@@ -1573,8 +1637,6 @@ let needSetup = false;
                     monitorID,
                 ]);
 
-                await sendImportantHeartbeatList(socket, monitorID, true, true);
-
                 callback({
                     ok: true,
                 });
@@ -1755,10 +1817,6 @@ async function afterLogin(socket, user) {
         await sendHeartbeatList(socket, monitorID);
     }
 
-    for (let monitorID in monitorList) {
-        await sendImportantHeartbeatList(socket, monitorID);
-    }
-
     for (let monitorID in monitorList) {
         await Monitor.sendStats(io, monitorID, user.id);
     }

src/mixins/socket.js

@@ -3,6 +3,8 @@ import { useToast } from "vue-toastification";
 import jwtDecode from "jwt-decode";
 import Favico from "favico.js";
 import dayjs from "dayjs";
+import mitt from "mitt";
+
 import { DOWN, MAINTENANCE, PENDING, UP } from "../util.ts";
 import { getDevContainerServerHostname, isDevContainer, getToastSuccessTimeout, getToastErrorTimeout } from "../util-frontend.js";
 const toast = useToast();
@@ -39,7 +41,6 @@ export default {
             maintenanceList: {},
             apiKeyList: {},
             heartbeatList: { },
-            importantHeartbeatList: { },
             avgPingList: { },
             uptimeList: { },
             tlsInfoList: {},
@@ -59,6 +60,7 @@ export default {
                 currentPassword: "",
             },
             faviconUpdateDebounce: null,
+            emitter: mitt(),
         };
     },
 
@@ -201,11 +203,7 @@ export default {
                         }
                     }
 
-                    if (! (data.monitorID in this.importantHeartbeatList)) {
+                    this.emitter.emit("newImportantHeartbeat", data);
-                        this.importantHeartbeatList[data.monitorID] = [];
-                    }
-
-                    this.importantHeartbeatList[data.monitorID].unshift(data);
                 }
             });
 
@@ -229,14 +227,6 @@ export default {
                 this.tlsInfoList[monitorID] = JSON.parse(data);
             });
 
-            socket.on("importantHeartbeatList", (monitorID, data, overwrite) => {
-                if (! (monitorID in this.importantHeartbeatList) || overwrite) {
-                    this.importantHeartbeatList[monitorID] = data;
-                } else {
-                    this.importantHeartbeatList[monitorID] = data.concat(this.importantHeartbeatList[monitorID]);
-                }
-            });
-
             socket.on("connect_error", (err) => {
                 console.error(`Failed to connect to the backend. Socket.io connect_error: ${err.message}`);
                 this.connectionErrorMsg = `${this.$t("Cannot connect to the socket server.")} [${err}] ${this.$t("Reconnecting...")}`;
@@ -630,7 +620,6 @@ export default {
         clearData() {
             console.log("reset heartbeat list");
             this.heartbeatList = {};
-            this.importantHeartbeatList = {};
         },
 
         /**

src/pages/DashboardHome.vue

@@ -42,13 +42,13 @@
                     </thead>
                     <tbody>
                         <tr v-for="(beat, index) in displayedRecords" :key="index" :class="{ 'shadow-box': $root.windowWidth <= 550}">
-                            <td><router-link :to="`/dashboard/${beat.monitorID}`">{{ beat.name }}</router-link></td>
+                            <td><router-link :to="`/dashboard/${beat.monitorID}`">{{ $root.monitorList[beat.monitorID]?.name }}</router-link></td>
                             <td><Status :status="beat.status" /></td>
                             <td :class="{ 'border-0':! beat.msg}"><Datetime :value="beat.time" /></td>
                             <td class="border-0">{{ beat.msg }}</td>
                         </tr>
 
-                        <tr v-if="importantHeartBeatList.length === 0">
+                        <tr v-if="importantHeartBeatListLength === 0">
                             <td colspan="4">
                                 {{ $t("No important events") }}
                             </td>
@@ -59,7 +59,7 @@
                 <div class="d-flex justify-content-center kuma_pagination">
                     <pagination
                         v-model="page"
-                        :records="importantHeartBeatList.length"
+                        :records="importantHeartBeatListLength"
                         :per-page="perPage"
                         :options="paginationConfig"
                     />
@@ -92,72 +92,89 @@ export default {
             page: 1,
             perPage: 25,
             initialPerPage: 25,
-            heartBeatList: [],
             paginationConfig: {
                 hideCount: true,
                 chunksNavigation: "scroll",
             },
+            importantHeartBeatListLength: 0,
+            displayedRecords: [],
         };
     },
-    computed: {
-
-        importantHeartBeatList() {
-            let result = [];
-
-            for (let monitorID in this.$root.importantHeartbeatList) {
-                let list = this.$root.importantHeartbeatList[monitorID];
-                result = result.concat(list);
-            }
-
-            for (let beat of result) {
-                let monitor = this.$root.monitorList[beat.monitorID];
-
-                if (monitor) {
-                    beat.name = monitor.name;
-                }
-            }
-
-            result.sort((a, b) => {
-                if (a.time > b.time) {
-                    return -1;
-                }
-
-                if (a.time < b.time) {
-                    return 1;
-                }
-
-                return 0;
-            });
-
-            // eslint-disable-next-line vue/no-side-effects-in-computed-properties
-            this.heartBeatList = result;
-
-            return result;
-        },
-
-        displayedRecords() {
-            const startIndex = this.perPage * (this.page - 1);
-            const endIndex = startIndex + this.perPage;
-            return this.heartBeatList.slice(startIndex, endIndex);
-        },
-    },
     watch: {
-        importantHeartBeatList() {
+        perPage() {
             this.$nextTick(() => {
-                this.updatePerPage();
+                this.getImportantHeartbeatListPaged();
             });
         },
+
+        page() {
+            this.getImportantHeartbeatListPaged();
+        },
     },
+
     mounted() {
+        this.getImportantHeartbeatListLength();
+
+        this.$root.emitter.on("newImportantHeartbeat", this.onNewImportantHeartbeat);
+
         this.initialPerPage = this.perPage;
 
         window.addEventListener("resize", this.updatePerPage);
         this.updatePerPage();
     },
+
     beforeUnmount() {
+        this.$root.emitter.off("newImportantHeartbeat", this.onNewImportantHeartbeat);
+
         window.removeEventListener("resize", this.updatePerPage);
     },
+
     methods: {
+        /**
+         * Updates the displayed records when a new important heartbeat arrives.
+         * @param {object} heartbeat - The heartbeat object received.
+         * @returns {void}
+         */
+        onNewImportantHeartbeat(heartbeat) {
+            if (this.page === 1) {
+                this.displayedRecords.unshift(heartbeat);
+                if (this.displayedRecords.length > this.perPage) {
+                    this.displayedRecords.pop();
+                }
+                this.importantHeartBeatListLength += 1;
+            }
+        },
+
+        /**
+         * Retrieves the length of the important heartbeat list for all monitors.
+         * @returns {void}
+         */
+        getImportantHeartbeatListLength() {
+            this.$root.getSocket().emit("monitorImportantHeartbeatListCount", null, (res) => {
+                if (res.ok) {
+                    this.importantHeartBeatListLength = res.count;
+                    this.getImportantHeartbeatListPaged();
+                }
+            });
+        },
+
+        /**
+         * Retrieves the important heartbeat list for the current page.
+         * @returns {void}
+         */
+        getImportantHeartbeatListPaged() {
+            const offset = (this.page - 1) * this.perPage;
+            this.$root.getSocket().emit("monitorImportantHeartbeatListPaged", null, offset, this.perPage, (res) => {
+                if (res.ok) {
+                    this.displayedRecords = res.data;
+                }
+            });
+        },
+
+        /**
+         * Updates the number of items shown per page based on the available height.
+         * @returns {void}
+         */
         updatePerPage() {
             const tableContainer = this.$refs.tableContainer;
             const tableContainerHeight = tableContainer.offsetHeight;

src/pages/Details.vue

@@ -195,7 +195,7 @@
                             <td class="border-0">{{ beat.msg }}</td>
                         </tr>
 
-                        <tr v-if="importantHeartBeatList.length === 0">
+                        <tr v-if="importantHeartBeatListLength === 0">
                             <td colspan="3">
                                 {{ $t("No important events") }}
                             </td>
@@ -206,7 +206,7 @@
                 <div class="d-flex justify-content-center kuma_pagination">
                     <pagination
                         v-model="page"
-                        :records="importantHeartBeatList.length"
+                        :records="importantHeartBeatListLength"
                         :per-page="perPage"
                         :options="paginationConfig"
                     />
@@ -275,6 +275,8 @@ export default {
                 chunksNavigation: "scroll",
             },
             cacheTime: Date.now(),
+            importantHeartBeatListLength: 0,
+            displayedRecords: [],
         };
     },
     computed: {
@@ -313,16 +315,6 @@ export default {
             return this.$t("notAvailableShort");
         },
 
-        importantHeartBeatList() {
-            if (this.$root.importantHeartbeatList[this.monitor.id]) {
-                // eslint-disable-next-line vue/no-side-effects-in-computed-properties
-                this.heartBeatList = this.$root.importantHeartbeatList[this.monitor.id];
-                return this.$root.importantHeartbeatList[this.monitor.id];
-            }
-
-            return [];
-        },
-
         status() {
             if (this.$root.statusList[this.monitor.id]) {
                 return this.$root.statusList[this.monitor.id];
@@ -346,12 +338,6 @@ export default {
             return this.tlsInfo != null && this.toggleCertInfoBox;
         },
 
-        displayedRecords() {
-            const startIndex = this.perPage * (this.page - 1);
-            const endIndex = startIndex + this.perPage;
-            return this.heartBeatList.slice(startIndex, endIndex);
-        },
-
         group() {
             if (!this.monitor.pathName.includes("/")) {
                 return "";
@@ -367,9 +353,27 @@ export default {
             return getResBaseURL() + this.monitor.screenshot + "?time=" + this.cacheTime;
         }
     },
+
+    watch: {
+        page(to) {
+            this.getImportantHeartbeatListPaged();
+        },
+
+        monitor(to) {
+            this.getImportantHeartbeatListLength();
+        }
+    },
+
     mounted() {
+        this.getImportantHeartbeatListLength();
 
+        this.$root.emitter.on("newImportantHeartbeat", this.onNewImportantHeartbeat);
     },
+
+    beforeUnmount() {
+        this.$root.emitter.off("newImportantHeartbeat", this.onNewImportantHeartbeat);
+    },
+
     methods: {
         getResBaseURL,
         /**
@@ -454,7 +458,9 @@ export default {
          */
         clearEvents() {
             this.$root.clearEvents(this.monitor.id, (res) => {
-                if (! res.ok) {
+                if (res.ok) {
+                    this.getImportantHeartbeatListLength();
+                } else {
                     toast.error(res.msg);
                 }
             });
@@ -515,7 +521,54 @@ export default {
                 // Handle SQL Server
                 return urlString.replaceAll(/Password=(.+);/ig, "Password=******;");
             }
-        }
+        },
+
+        /**
+         * Retrieves the length of the important heartbeat list for this monitor.
+         * @returns {void}
+         */
+        getImportantHeartbeatListLength() {
+            if (this.monitor) {
+                this.$root.getSocket().emit("monitorImportantHeartbeatListCount", this.monitor.id, (res) => {
+                    if (res.ok) {
+                        this.importantHeartBeatListLength = res.count;
+                        this.getImportantHeartbeatListPaged();
+                    }
+                });
+            }
+        },
+
+        /**
+         * Retrieves the important heartbeat list for the current page.
+         * @returns {void}
+         */
+        getImportantHeartbeatListPaged() {
+            if (this.monitor) {
+                const offset = (this.page - 1) * this.perPage;
+                this.$root.getSocket().emit("monitorImportantHeartbeatListPaged", this.monitor.id, offset, this.perPage, (res) => {
+                    if (res.ok) {
+                        this.displayedRecords = res.data;
+                    }
+                });
+            }
+        },
+
+        /**
+         * Updates the displayed records when a new important heartbeat arrives.
+         * @param {object} heartbeat - The heartbeat object received.
+         * @returns {void}
+         */
+        onNewImportantHeartbeat(heartbeat) {
+            if (heartbeat.monitorID === this.monitor?.id) {
+                if (this.page === 1) {
+                    this.displayedRecords.unshift(heartbeat);
+                    if (this.displayedRecords.length > this.perPage) {
+                        this.displayedRecords.pop();
+                    }
+                    this.importantHeartBeatListLength += 1;
+                }
+            }
+        },
     },
 };
 </script>