From daacc2842eb92f1701a4b061bd81677f61f5811b Mon Sep 17 00:00:00 2001 From: Stephen Papierski Date: Thu, 26 Oct 2023 22:40:44 -0600 Subject: [PATCH 01/44] Reimplement functionality from https://github.com/louislam/uptime-kuma/pull/1878 with modern knex_migration --- .../2023-10-26-slow-response-notification.js | 20 +++++ server/model/monitor.js | 82 +++++++++++++++++++ server/server.js | 7 +- src/lang/en.json | 12 +++ src/pages/EditMonitor.vue | 51 ++++++++++++ 5 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 db/knex_migrations/2023-10-26-slow-response-notification.js diff --git a/db/knex_migrations/2023-10-26-slow-response-notification.js b/db/knex_migrations/2023-10-26-slow-response-notification.js new file mode 100644 index 00000000..0c33ff99 --- /dev/null +++ b/db/knex_migrations/2023-10-26-slow-response-notification.js @@ -0,0 +1,20 @@ +exports.up = function (knex) { + // add various slow_response_notification parameters + return knex.schema + .alterTable("monitor", function(table) { + table.boolean("slow_response_notification").defaultTo(false); + table.integer("slow_response_notification_threshold").defaultTo(0); + table.integer("slow_response_notification_range").defaultTo(0); + table.string("slow_response_notification_method").defaultTo(""); + }); +} + +exports.down = function (knex) { + return knex.schema + .alterTable("monitor", function(table) { + table.boolean("slow_response_notification").defaultTo(false); + table.integer("slow_response_notification_threshold").defaultTo(0); + table.integer("slow_response_notification_range").defaultTo(0); + table.string("slow_response_notification_method").defaultTo(""); + }); +} diff --git a/server/model/monitor.js b/server/model/monitor.js index 5dcb7171..60f64115 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -132,6 +132,10 @@ class Monitor extends BeanModel { mqttSuccessMessage: this.mqttSuccessMessage, databaseQuery: this.databaseQuery, authMethod: this.authMethod, + slowResponseNotification: this.isEnabledSlowResponseNotification(), + slowResponseNotificationThreshold: this.slowResponseNotificationThreshold, + slowResponseNotificationRange: this.slowResponseNotificationRange, + slowResponseNotificationMethod: this.slowResponseNotificationMethod, grpcUrl: this.grpcUrl, grpcProtobuf: this.grpcProtobuf, grpcMethod: this.grpcMethod, @@ -298,6 +302,14 @@ class Monitor extends BeanModel { return Boolean(this.gamedigGivenPortOnly); } + /** + * Is the slow response notification enabled? + * @returns {boolean} + */ + isEnabledSlowResponseNotification() { + return Boolean(this.slowResponseNotification); + } + /** * Start monitor * @param {Server} io Socket server instance @@ -938,6 +950,11 @@ class Monitor extends BeanModel { log.debug("monitor", `[${this.name}] Store`); await R.store(bean); + if (this.isEnabledSlowResponseNotification()) { + log.debug("monitor", `[${this.name}] Check response is slow`); + await this.checkSlowResponseNotification(this); + } + log.debug("monitor", `[${this.name}] prometheus.update`); this.prometheus?.update(bean, tlsInfo); @@ -1375,6 +1392,71 @@ class Monitor extends BeanModel { } } + /** + * Check heartbeat response time is slower than threshold. + * @param {Monitor} monitor The monitor to send a notification about + * @returns {Promise} + */ + async checkSlowResponseNotification(monitor) { + + //Get recent heartbeat list with range of time + const afterThisDate = new Date(Date.now() - (1000 * monitor.slowResponseNotificationRange)); + const previousBeats = await R.getAll(` + SELECT * FROM heartbeat + WHERE monitor_id = ? AND time > datetime(?) AND status = ?`, + [ + monitor.id, + afterThisDate.toISOString(), + UP, + ]); + const method = monitor.slowResponseNotificationMethod; + const thresholdResponseTime = monitor.slowResponseNotificationThreshold; + let actualResponseTime = 0; + + switch (method) { + case "average": + previousBeats.forEach(beat => { + actualResponseTime = actualResponseTime + beat.ping; + }); + actualResponseTime = actualResponseTime / previousBeats.length; + break; + + case "max": + previousBeats.forEach(beat => { + actualResponseTime = Math.max(actualResponseTime, beat.ping); + }); + break; + + default: + log.error("monitor", `[${this.name}] Unknown slow response notification method ${method}`); + return; + } + + if (actualResponseTime < thresholdResponseTime) { + log.debug("monitor", `[${this.name}] No need to send slow notification. ${actualResponseTime} < ${thresholdResponseTime}`); + return; + } + + log.debug("monitor", `[${this.name}] Try to send slow response notification (${actualResponseTime} > ${thresholdResponseTime})`); + + const notificationList = await Monitor.getNotificationList(monitor); + + if (notificationList.length > 0) { + for (let notification of notificationList) { + try { + log.debug("monitor", `[${this.name}] Sending to ${notification.name}`); + await Notification.send(JSON.parse(notification.config), `[${this.name}] Responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms)`); + } catch (e) { + log.error("monitor", `[${this.name}] Cannot send slow response notification to ${notification.name}`); + log.error("monitor", e); + } + } + + } else { + log.debug("monitor", `[${this.name}] No notification, no need to send slow response notification`); + } + } + /** * Get the status of the previous heartbeat * @param {number} monitorID ID of monitor to check diff --git a/server/server.js b/server/server.js index f726790c..acb38b8a 100644 --- a/server/server.js +++ b/server/server.js @@ -43,7 +43,8 @@ log.debug("server", "Arguments"); log.debug("server", args); if (! process.env.NODE_ENV) { - process.env.NODE_ENV = "production"; + // process.env.NODE_ENV = "production"; + process.env.NODE_ENV = "development"; } log.info("server", "Env: " + process.env.NODE_ENV); @@ -803,6 +804,10 @@ let needSetup = false; bean.authMethod = monitor.authMethod; bean.authWorkstation = monitor.authWorkstation; bean.authDomain = monitor.authDomain; + bean.slowResponseNotification = monitor.slowResponseNotification; + bean.slowResponseNotificationThreshold = monitor.slowResponseNotificationThreshold; + bean.slowResponseNotificationRange = monitor.slowResponseNotificationRange; + bean.slowResponseNotificationMethod = monitor.slowResponseNotificationMethod; bean.grpcUrl = monitor.grpcUrl; bean.grpcProtobuf = monitor.grpcProtobuf; bean.grpcServiceName = monitor.grpcServiceName; diff --git a/src/lang/en.json b/src/lang/en.json index 057b2790..5be3b303 100644 --- a/src/lang/en.json +++ b/src/lang/en.json @@ -485,6 +485,18 @@ "uninstall": "Uninstall", "uninstalling": "Uninstalling", "confirmUninstallPlugin": "Are you sure want to uninstall this plugin?", + "slowResponseNotification": "Slow Response Notification", + "slowResponseNotificationUse": "Use Slow Response Notification", + "slowResponseNotificationUseDescription": "When response time is slow, notify.", + "slowResponseNotificationThreshold": "Threshold (ms)", + "slowResponseNotificationThresholdDescription": "If response time greater than {0} ms, notify.", + "slowResponseNotificationRange": "Time Range (seconds)", + "slowResponseNotificationRangeDescription": "Gets the heartbeat information for the last {0} seconds and calculates the condition.", + "slowResponseNotificationMethod": "Calculation Method", + "slowResponseNotificationMethodAverage": "Average", + "slowResponseNotificationMethodAverageDescription": "Get the average response time of the last {0} seconds.", + "slowResponseNotificationMethodMax": "Max", + "slowResponseNotificationMethodMaxDescription": "Get the max response time of the last {0} seconds.", "notificationRegional": "Regional", "Clone Monitor": "Clone Monitor", "Clone": "Clone", diff --git a/src/pages/EditMonitor.vue b/src/pages/EditMonitor.vue index aef96423..78e7417a 100644 --- a/src/pages/EditMonitor.vue +++ b/src/pages/EditMonitor.vue @@ -418,6 +418,53 @@ +

{{ $t("slowResponseNotification") }}

+ +
+ + +
+ {{ $t("slowResponseNotificationUseDescription") }} +
+
+ + +
+ + +
+ {{ $t("slowResponseNotificationMethodAverageDescription", [monitor.slowResponseNotificationRange]) }} +
+
+ {{ $t("slowResponseNotificationMethodMaxDescription", [monitor.slowResponseNotificationRange]) }} +
+
+ +
+ + +
+ {{ $t("slowResponseNotificationThresholdDescription", [monitor.slowResponseNotificationThreshold]) }} +
+
+ +
+ + +
+ {{ $t("slowResponseNotificationRangeDescription", [monitor.slowResponseNotificationRange]) }} +
+
+

{{ $t("Advanced") }}

@@ -888,6 +935,10 @@ const monitorDefaults = { }, kafkaProducerSsl: false, gamedigGivenPortOnly: true, + slowResponseNotification: false, + slowResponseNotificationThreshold: 5000, + slowResponseNotificationRange: 60, + slowResponseNotificationMethod: "average", }; export default { From 8669c5df500f04b0902a83b772058257dfd193bd Mon Sep 17 00:00:00 2001 From: Stephen Papierski Date: Sun, 29 Oct 2023 12:28:49 -0600 Subject: [PATCH 02/44] Add slow response notification resend interval to db/gui --- .../2023-10-26-slow-response-notification.js | 2 ++ src/pages/EditMonitor.vue | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/db/knex_migrations/2023-10-26-slow-response-notification.js b/db/knex_migrations/2023-10-26-slow-response-notification.js index 0c33ff99..89270396 100644 --- a/db/knex_migrations/2023-10-26-slow-response-notification.js +++ b/db/knex_migrations/2023-10-26-slow-response-notification.js @@ -6,6 +6,7 @@ exports.up = function (knex) { table.integer("slow_response_notification_threshold").defaultTo(0); table.integer("slow_response_notification_range").defaultTo(0); table.string("slow_response_notification_method").defaultTo(""); + table.integer("slow_response_notification_resend_interval").defaultTo(0); }); } @@ -16,5 +17,6 @@ exports.down = function (knex) { table.integer("slow_response_notification_threshold").defaultTo(0); table.integer("slow_response_notification_range").defaultTo(0); table.string("slow_response_notification_method").defaultTo(""); + table.integer("slow_response_notification_resend_interval").defaultTo(0); }); } diff --git a/src/pages/EditMonitor.vue b/src/pages/EditMonitor.vue index 78e7417a..eebb52dc 100644 --- a/src/pages/EditMonitor.vue +++ b/src/pages/EditMonitor.vue @@ -465,6 +465,15 @@
+
+ + +
+

{{ $t("Advanced") }}

@@ -939,6 +948,7 @@ const monitorDefaults = { slowResponseNotificationThreshold: 5000, slowResponseNotificationRange: 60, slowResponseNotificationMethod: "average", + slowResponseNotificationResendInterval: 0, }; export default { From 8fc0b6a0a180138cc4a30fbc1be5d20dc8dbf2b0 Mon Sep 17 00:00:00 2001 From: Stephen Papierski Date: Tue, 31 Oct 2023 23:17:35 -0600 Subject: [PATCH 03/44] Add slow response notification resend interval, also notify when response time returns to normal --- .../2023-10-26-slow-response-notification.js | 26 +++--- server/model/monitor.js | 88 ++++++++++++++----- server/server.js | 1 + src/lang/en.json | 8 +- src/pages/EditMonitor.vue | 7 +- 5 files changed, 89 insertions(+), 41 deletions(-) diff --git a/db/knex_migrations/2023-10-26-slow-response-notification.js b/db/knex_migrations/2023-10-26-slow-response-notification.js index 89270396..72ce236c 100644 --- a/db/knex_migrations/2023-10-26-slow-response-notification.js +++ b/db/knex_migrations/2023-10-26-slow-response-notification.js @@ -2,21 +2,27 @@ exports.up = function (knex) { // add various slow_response_notification parameters return knex.schema .alterTable("monitor", function(table) { - table.boolean("slow_response_notification").defaultTo(false); - table.integer("slow_response_notification_threshold").defaultTo(0); - table.integer("slow_response_notification_range").defaultTo(0); - table.string("slow_response_notification_method").defaultTo(""); - table.integer("slow_response_notification_resend_interval").defaultTo(0); + table.boolean("slow_response_notification").notNullable().defaultTo(false); + table.integer("slow_response_notification_threshold").notNullable().defaultTo(0); + table.integer("slow_response_notification_range").notNullable().defaultTo(0); + table.string("slow_response_notification_method").notNullable().defaultTo(""); + table.integer("slow_response_notification_resend_interval").notNullable().defaultTo(0); + }) + .alterTable("heartbeat", function(table) { + table.integer("slow_response_count").notNullable().defaultTo(0); }); } exports.down = function (knex) { return knex.schema .alterTable("monitor", function(table) { - table.boolean("slow_response_notification").defaultTo(false); - table.integer("slow_response_notification_threshold").defaultTo(0); - table.integer("slow_response_notification_range").defaultTo(0); - table.string("slow_response_notification_method").defaultTo(""); - table.integer("slow_response_notification_resend_interval").defaultTo(0); + table.boolean("slow_response_notification").notNullable().defaultTo(false); + table.integer("slow_response_notification_threshold").notNullable().defaultTo(0); + table.integer("slow_response_notification_range").notNullable().defaultTo(0); + table.string("slow_response_notification_method").notNullable().defaultTo(""); + table.integer("slow_response_notification_resend_interval").notNullable().defaultTo(0); + }) + .alterTable("heartbeat", function(table) { + table.integer("slow_response_count").notNullable().defaultTo(0); }); } diff --git a/server/model/monitor.js b/server/model/monitor.js index 60f64115..eb6041f0 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -136,6 +136,7 @@ class Monitor extends BeanModel { slowResponseNotificationThreshold: this.slowResponseNotificationThreshold, slowResponseNotificationRange: this.slowResponseNotificationRange, slowResponseNotificationMethod: this.slowResponseNotificationMethod, + slowResponseNotificationResendInterval: this.slowResponseNotificationResendInterval, grpcUrl: this.grpcUrl, grpcProtobuf: this.grpcProtobuf, grpcMethod: this.grpcMethod, @@ -353,6 +354,7 @@ class Monitor extends BeanModel { bean.time = R.isoDateTimeMillis(dayjs.utc()); bean.status = DOWN; bean.downCount = previousBeat?.downCount || 0; + bean.slowResponseCount = previousBeat?.slowResponseCount || 0; if (this.isUpsideDown()) { bean.status = flipStatus(bean.status); @@ -941,6 +943,12 @@ class Monitor extends BeanModel { let endTimeDayjs = await uptimeCalculator.update(bean.status, parseFloat(bean.ping)); bean.end_time = R.isoDateTimeMillis(endTimeDayjs); + // Check if response time is slow + if (this.isEnabledSlowResponseNotification()) { + log.debug("monitor", `[${this.name}] Check if response is slow`); + await this.checkSlowResponseNotification(this, bean); + } + // Send to frontend log.debug("monitor", `[${this.name}] Send to socket`); io.to(this.user_id).emit("heartbeat", bean.toJSON()); @@ -950,11 +958,6 @@ class Monitor extends BeanModel { log.debug("monitor", `[${this.name}] Store`); await R.store(bean); - if (this.isEnabledSlowResponseNotification()) { - log.debug("monitor", `[${this.name}] Check response is slow`); - await this.checkSlowResponseNotification(this); - } - log.debug("monitor", `[${this.name}] prometheus.update`); this.prometheus?.update(bean, tlsInfo); @@ -1392,12 +1395,40 @@ class Monitor extends BeanModel { } } + /** + * Send a slow response notification about a monitor + * @param {Monitor} monitor The monitor to send a notificaton about + * @param {Bean} bean Status information about monitor + * @param {string} message Notification text to be sent + * @returns {void} + */ + static async sendSlowResponseNotification(monitor, bean, message) { + // Send notification + const notificationList = await Monitor.getNotificationList(monitor); + + if (notificationList.length > 0) { + for (let notification of notificationList) { + try { + log.debug("monitor", `[${this.name}] Sending to ${notification.name}`); + await Notification.send(JSON.parse(notification.config), message); + } catch (e) { + log.error("monitor", `[${this.name}] Cannot send slow response notification to ${notification.name}`); + log.error("monitor", e); + } + } + + } else { + log.debug("monitor", `[${this.name}] No notification configured, no need to send slow response notification`); + } + } + /** * Check heartbeat response time is slower than threshold. * @param {Monitor} monitor The monitor to send a notification about + * @param {Bean} bean Status information about monitor * @returns {Promise} */ - async checkSlowResponseNotification(monitor) { + async checkSlowResponseNotification(monitor, bean) { //Get recent heartbeat list with range of time const afterThisDate = new Date(Date.now() - (1000 * monitor.slowResponseNotificationRange)); @@ -1432,28 +1463,39 @@ class Monitor extends BeanModel { return; } + // Responding normally if (actualResponseTime < thresholdResponseTime) { - log.debug("monitor", `[${this.name}] No need to send slow notification. ${actualResponseTime} < ${thresholdResponseTime}`); - return; - } - - log.debug("monitor", `[${this.name}] Try to send slow response notification (${actualResponseTime} > ${thresholdResponseTime})`); + if (bean.slowResponseCount == 0) { + log.debug("monitor", `[${this.name}] Responding normally. No need to send slow response notification (${actualResponseTime}ms < ${thresholdResponseTime}ms)`); + } else { + log.debug("monitor", `[${this.name}] Returned to normal response time (${actualResponseTime}ms < ${thresholdResponseTime}ms)`); + let message = `[${this.name}] Returned to normal response time (${actualResponseTime}ms < ${thresholdResponseTime}ms)`; + Monitor.sendSlowResponseNotification(monitor, bean, message); + } - const notificationList = await Monitor.getNotificationList(monitor); + // Reset slow response count + bean.slowResponseCount = 0; + return; - if (notificationList.length > 0) { - for (let notification of notificationList) { - try { - log.debug("monitor", `[${this.name}] Sending to ${notification.name}`); - await Notification.send(JSON.parse(notification.config), `[${this.name}] Responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms)`); - } catch (e) { - log.error("monitor", `[${this.name}] Cannot send slow response notification to ${notification.name}`); - log.error("monitor", e); + // Responding slowly + } else { + ++bean.slowResponseCount; + log.debug("monitor", `[${this.name}] Responded slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`); + + // Always send first notification + if (this.slowResponseNotificationResendInterval <= 0 || bean.slowResponseCount == 1) { + log.debug("monitor", `[${this.name}] Responded slowly, sending notification (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`); + let message = `[${this.name}] Started responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms)`; + Monitor.sendSlowResponseNotification(monitor, bean, message); + } else { + // Send notification every x times + if (((bean.slowResponseCount) % this.slowResponseNotificationResendInterval) == 0) { + // Send notification again, because we are still responding slow + log.debug("monitor", `[${this.name}] sendSlowResponseNotification again (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`); + let message = `[${this.name}] Still responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`; + Monitor.sendSlowResponseNotification(monitor, bean, message); } } - - } else { - log.debug("monitor", `[${this.name}] No notification, no need to send slow response notification`); } } diff --git a/server/server.js b/server/server.js index acb38b8a..c49ccd35 100644 --- a/server/server.js +++ b/server/server.js @@ -808,6 +808,7 @@ let needSetup = false; bean.slowResponseNotificationThreshold = monitor.slowResponseNotificationThreshold; bean.slowResponseNotificationRange = monitor.slowResponseNotificationRange; bean.slowResponseNotificationMethod = monitor.slowResponseNotificationMethod; + bean.slowResponseNotificationResendInterval = monitor.slowResponseNotificationResendInterval; bean.grpcUrl = monitor.grpcUrl; bean.grpcProtobuf = monitor.grpcProtobuf; bean.grpcServiceName = monitor.grpcServiceName; diff --git a/src/lang/en.json b/src/lang/en.json index 5be3b303..2a42f711 100644 --- a/src/lang/en.json +++ b/src/lang/en.json @@ -485,18 +485,18 @@ "uninstall": "Uninstall", "uninstalling": "Uninstalling", "confirmUninstallPlugin": "Are you sure want to uninstall this plugin?", - "slowResponseNotification": "Slow Response Notification", - "slowResponseNotificationUse": "Use Slow Response Notification", - "slowResponseNotificationUseDescription": "When response time is slow, notify.", + "slowResponseNotificationEnable": "Slow Response Notification", + "slowResponseNotificationUseDescription": "Send a notification when service response time is slow.", "slowResponseNotificationThreshold": "Threshold (ms)", "slowResponseNotificationThresholdDescription": "If response time greater than {0} ms, notify.", "slowResponseNotificationRange": "Time Range (seconds)", "slowResponseNotificationRangeDescription": "Gets the heartbeat information for the last {0} seconds and calculates the condition.", "slowResponseNotificationMethod": "Calculation Method", "slowResponseNotificationMethodAverage": "Average", - "slowResponseNotificationMethodAverageDescription": "Get the average response time of the last {0} seconds.", + "slowResponseNotificationMethodAverageDescription": "Get the average response time over the last {0} seconds.", "slowResponseNotificationMethodMax": "Max", "slowResponseNotificationMethodMaxDescription": "Get the max response time of the last {0} seconds.", + "slowResponseNotificationResendInterval": "Resend Notification if Slow Response X times consecutively", "notificationRegional": "Regional", "Clone Monitor": "Clone Monitor", "Clone": "Clone", diff --git a/src/pages/EditMonitor.vue b/src/pages/EditMonitor.vue index eebb52dc..c68f8cde 100644 --- a/src/pages/EditMonitor.vue +++ b/src/pages/EditMonitor.vue @@ -418,12 +418,11 @@
-

{{ $t("slowResponseNotification") }}

- +
{{ $t("slowResponseNotificationUseDescription") }} @@ -467,7 +466,7 @@
From fd745d1dcc3d278e628e074e1d47e0cb13b17155 Mon Sep 17 00:00:00 2001 From: Stephen Papierski Date: Tue, 31 Oct 2023 23:46:20 -0600 Subject: [PATCH 04/44] Prevent slowResponseNotificationRange from being less than one Heartbeat Interval --- server/model/monitor.js | 2 +- src/pages/EditMonitor.vue | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/server/model/monitor.js b/server/model/monitor.js index 661a7f5d..c4890339 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -1432,7 +1432,7 @@ class Monitor extends BeanModel { async checkSlowResponseNotification(monitor, bean) { //Get recent heartbeat list with range of time - const afterThisDate = new Date(Date.now() - (1000 * monitor.slowResponseNotificationRange)); + const afterThisDate = new Date(Date.now() - (1000 * (monitor.slowResponseNotificationRange + 1))); // add 1 second otherwise we grab 0 previous beats when Time Range == Heartbeat Interval const previousBeats = await R.getAll(` SELECT * FROM heartbeat WHERE monitor_id = ? AND time > datetime(?) AND status = ?`, diff --git a/src/pages/EditMonitor.vue b/src/pages/EditMonitor.vue index c68f8cde..820c17a8 100644 --- a/src/pages/EditMonitor.vue +++ b/src/pages/EditMonitor.vue @@ -458,7 +458,7 @@
- +
{{ $t("slowResponseNotificationRangeDescription", [monitor.slowResponseNotificationRange]) }}
@@ -1198,6 +1198,14 @@ message HealthCheckResponse { if (this.monitor.retryInterval === oldValue) { this.monitor.retryInterval = value; } + // Link interval and slowResponseNotificationRange if the are the same value + if (this.monitor.slowResponseNotificationRange === oldValue) { + this.monitor.slowResponseNotificationRange = value; + } + // But always keep slowResponseNotificationRange >= interval + if (this.monitor.slowResponseNotificationRange < value) { + this.monitor.slowResponseNotificationRange = value; + } }, "monitor.timeout"(value, oldValue) { From 6c3199438749c6a113ae7f4ca6493b8b34b0e673 Mon Sep 17 00:00:00 2001 From: Stephen Papierski Date: Wed, 1 Nov 2023 00:06:05 -0600 Subject: [PATCH 05/44] Fix behavior of slowResponseNotificationRetryInterval --- server/model/monitor.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/model/monitor.js b/server/model/monitor.js index c4890339..8fde44c1 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -1484,11 +1484,11 @@ class Monitor extends BeanModel { log.debug("monitor", `[${this.name}] Responded slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`); // Always send first notification - if (this.slowResponseNotificationResendInterval <= 0 || bean.slowResponseCount == 1) { + if (bean.slowResponseCount == 1) { log.debug("monitor", `[${this.name}] Responded slowly, sending notification (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`); let message = `[${this.name}] Started responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms)`; Monitor.sendSlowResponseNotification(monitor, bean, message); - } else { + } else if (this.slowResponseNotificationResendInterval > 0){ // Send notification every x times if (((bean.slowResponseCount) % this.slowResponseNotificationResendInterval) == 0) { // Send notification again, because we are still responding slow From da43e1b7072926e3b66bb17a377ec62558626728 Mon Sep 17 00:00:00 2001 From: Stephen Papierski Date: Wed, 1 Nov 2023 00:37:37 -0600 Subject: [PATCH 06/44] Add calculation method to just use last heartbeat --- server/model/monitor.js | 4 ++++ src/lang/en.json | 4 +++- src/pages/EditMonitor.vue | 8 +++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/server/model/monitor.js b/server/model/monitor.js index 8fde44c1..596ca305 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -1459,6 +1459,10 @@ class Monitor extends BeanModel { }); break; + case "last": + actualResponseTime = bean.ping + break; + default: log.error("monitor", `[${this.name}] Unknown slow response notification method ${method}`); return; diff --git a/src/lang/en.json b/src/lang/en.json index 2a42f711..20633154 100644 --- a/src/lang/en.json +++ b/src/lang/en.json @@ -488,7 +488,7 @@ "slowResponseNotificationEnable": "Slow Response Notification", "slowResponseNotificationUseDescription": "Send a notification when service response time is slow.", "slowResponseNotificationThreshold": "Threshold (ms)", - "slowResponseNotificationThresholdDescription": "If response time greater than {0} ms, notify.", + "slowResponseNotificationThresholdDescription": "Send a notification if response time is greater than {0} ms.", "slowResponseNotificationRange": "Time Range (seconds)", "slowResponseNotificationRangeDescription": "Gets the heartbeat information for the last {0} seconds and calculates the condition.", "slowResponseNotificationMethod": "Calculation Method", @@ -496,6 +496,8 @@ "slowResponseNotificationMethodAverageDescription": "Get the average response time over the last {0} seconds.", "slowResponseNotificationMethodMax": "Max", "slowResponseNotificationMethodMaxDescription": "Get the max response time of the last {0} seconds.", + "slowResponseNotificationMethodLast": "Last", + "slowResponseNotificationMethodLastDescription": "Get the response time of the last heartbeat.", "slowResponseNotificationResendInterval": "Resend Notification if Slow Response X times consecutively", "notificationRegional": "Regional", "Clone Monitor": "Clone Monitor", diff --git a/src/pages/EditMonitor.vue b/src/pages/EditMonitor.vue index 820c17a8..bc553b0e 100644 --- a/src/pages/EditMonitor.vue +++ b/src/pages/EditMonitor.vue @@ -439,6 +439,9 @@ +
{{ $t("slowResponseNotificationMethodAverageDescription", [monitor.slowResponseNotificationRange]) }} @@ -446,6 +449,9 @@
{{ $t("slowResponseNotificationMethodMaxDescription", [monitor.slowResponseNotificationRange]) }}
+
+ {{ $t("slowResponseNotificationMethodLastDescription", [monitor.slowResponseNotificationRange]) }} +
@@ -456,7 +462,7 @@
-
+
From f81c48d1cacd17f30fd47f169c825751dc1ad347 Mon Sep 17 00:00:00 2001 From: Stephen Papierski Date: Wed, 1 Nov 2023 00:41:19 -0600 Subject: [PATCH 07/44] Round response time to nearest ms when using average --- server/model/monitor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/model/monitor.js b/server/model/monitor.js index 596ca305..9b00c332 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -1450,7 +1450,7 @@ class Monitor extends BeanModel { previousBeats.forEach(beat => { actualResponseTime = actualResponseTime + beat.ping; }); - actualResponseTime = actualResponseTime / previousBeats.length; + actualResponseTime = Math.round(actualResponseTime / previousBeats.length); break; case "max": From 51e48e60548c0fd66457f6ff8daf33d3bb5848bb Mon Sep 17 00:00:00 2001 From: Stephen Papierski Date: Tue, 7 Nov 2023 15:51:59 -0700 Subject: [PATCH 08/44] Clean up notification logic and messages --- server/model/monitor.js | 93 +++++++++++++++++++++------------------ server/server.js | 3 +- src/lang/en.json | 6 +-- src/pages/EditMonitor.vue | 2 +- 4 files changed, 55 insertions(+), 49 deletions(-) diff --git a/server/model/monitor.js b/server/model/monitor.js index 727c3d00..c6ff7a83 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -1404,26 +1404,21 @@ class Monitor extends BeanModel { * Send a slow response notification about a monitor * @param {Monitor} monitor The monitor to send a notificaton about * @param {Bean} bean Status information about monitor - * @param {string} message Notification text to be sent + * @param {string} msg Notification text to be sent * @returns {void} */ - static async sendSlowResponseNotification(monitor, bean, message) { + static async sendSlowResponseNotification(monitor, bean, msg) { // Send notification const notificationList = await Monitor.getNotificationList(monitor); - if (notificationList.length > 0) { - for (let notification of notificationList) { - try { - log.debug("monitor", `[${this.name}] Sending to ${notification.name}`); - await Notification.send(JSON.parse(notification.config), message); - } catch (e) { - log.error("monitor", `[${this.name}] Cannot send slow response notification to ${notification.name}`); - log.error("monitor", e); - } + for (let notification of notificationList) { + try { + log.debug("monitor", `[${this.name}] Sending to ${notification.name}`); + await Notification.send(JSON.parse(notification.config), msg); + } catch (e) { + log.error("monitor", `[${this.name}] Cannot send slow response notification to ${notification.name}`); + log.error("monitor", e); } - - } else { - log.debug("monitor", `[${this.name}] No notification configured, no need to send slow response notification`); } } @@ -1447,6 +1442,7 @@ class Monitor extends BeanModel { ]); const method = monitor.slowResponseNotificationMethod; const thresholdResponseTime = monitor.slowResponseNotificationThreshold; + const windowDuration = monitor.slowResponseNotificationRange; let actualResponseTime = 0; switch (method) { @@ -1472,39 +1468,50 @@ class Monitor extends BeanModel { return; } - // Responding normally - if (actualResponseTime < thresholdResponseTime) { - if (bean.slowResponseCount == 0) { - log.debug("monitor", `[${this.name}] Responding normally. No need to send slow response notification (${actualResponseTime}ms < ${thresholdResponseTime}ms)`); - } else { - log.debug("monitor", `[${this.name}] Returned to normal response time (${actualResponseTime}ms < ${thresholdResponseTime}ms)`); - let message = `[${this.name}] Returned to normal response time (${actualResponseTime}ms < ${thresholdResponseTime}ms)`; - Monitor.sendSlowResponseNotification(monitor, bean, message); - } + // Create stats to append to messages/logs + let msgStats = `\nResponse: ${actualResponseTime}ms | Threshold: ${thresholdResponseTime}ms | Method: ${method}` + // Add window duration for methods that make sense + if (["average", "max"].includes(method)) { + msgStats += ` over ${windowDuration}s` + } - // Reset slow response count - bean.slowResponseCount = 0; - return; + // Verify something was actually calculated + if (actualResponseTime != 0 && Number.isInteger(actualResponseTime)) { + // Responding normally + if (actualResponseTime < thresholdResponseTime) { + if (bean.slowResponseCount == 0) { + log.debug("monitor", `[${this.name}] Responding normally. No need to send slow response notification ${msgStats}`); + } else { + log.debug("monitor", `[${this.name}] Returned to normal response time ${msgStats}`); + let msg = `[${this.name}] Returned to Normal Response Time ${msgStats}`; + Monitor.sendSlowResponseNotification(monitor, bean, msg); + } - // Responding slowly - } else { - ++bean.slowResponseCount; - log.debug("monitor", `[${this.name}] Responded slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`); - - // Always send first notification - if (bean.slowResponseCount == 1) { - log.debug("monitor", `[${this.name}] Responded slowly, sending notification (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`); - let message = `[${this.name}] Started responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms)`; - Monitor.sendSlowResponseNotification(monitor, bean, message); - } else if (this.slowResponseNotificationResendInterval > 0){ - // Send notification every x times - if (((bean.slowResponseCount) % this.slowResponseNotificationResendInterval) == 0) { - // Send notification again, because we are still responding slow - log.debug("monitor", `[${this.name}] sendSlowResponseNotification again (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`); - let message = `[${this.name}] Still responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`; - Monitor.sendSlowResponseNotification(monitor, bean, message); + // Reset slow response count + bean.slowResponseCount = 0; + return; + + // Responding slowly + } else { + ++bean.slowResponseCount; + + // Always send first notification + if (bean.slowResponseCount == 1) { + log.debug("monitor", `[${this.name}] Responded slowly, sending notification ${msgStats}`); + let msg = `[${this.name}] Responded Slowly ${msgStats}`; + Monitor.sendSlowResponseNotification(monitor, bean, msg); + } else if (this.slowResponseNotificationResendInterval > 0){ + // Send notification every x times + if (((bean.slowResponseCount) % this.slowResponseNotificationResendInterval) == 0) { + // Send notification again, because we are still responding slow + log.debug("monitor", `[${this.name}] sendSlowResponseNotification again ${msgStats}`); + let msg = `[${this.name}] Still Responding Slowly ${msgStats}`; + Monitor.sendSlowResponseNotification(monitor, bean, msg); + } } } + } else { + log.debug("monitor", `[${this.name}] Failed to calculate valid response time`); } } diff --git a/server/server.js b/server/server.js index c49ccd35..2fe30e60 100644 --- a/server/server.js +++ b/server/server.js @@ -43,8 +43,7 @@ log.debug("server", "Arguments"); log.debug("server", args); if (! process.env.NODE_ENV) { - // process.env.NODE_ENV = "production"; - process.env.NODE_ENV = "development"; + process.env.NODE_ENV = "production"; } log.info("server", "Env: " + process.env.NODE_ENV); diff --git a/src/lang/en.json b/src/lang/en.json index de8464a5..e0a0e6c7 100644 --- a/src/lang/en.json +++ b/src/lang/en.json @@ -490,9 +490,9 @@ "slowResponseNotificationEnable": "Slow Response Notification", "slowResponseNotificationUseDescription": "Send a notification when service response time is slow.", "slowResponseNotificationThreshold": "Threshold (ms)", - "slowResponseNotificationThresholdDescription": "Send a notification if response time is greater than {0} ms.", - "slowResponseNotificationRange": "Time Range (seconds)", - "slowResponseNotificationRangeDescription": "Gets the heartbeat information for the last {0} seconds and calculates the condition.", + "slowResponseNotificationThresholdDescription": "Send a notification if calculated response time is greater than {0} ms.", + "slowResponseNotificationRange": "Window Duration (seconds)", + "slowResponseNotificationRangeDescription": "Window duration for calculating the {0}.", "slowResponseNotificationMethod": "Calculation Method", "slowResponseNotificationMethodAverage": "Average", "slowResponseNotificationMethodAverageDescription": "Get the average response time over the last {0} seconds.", diff --git a/src/pages/EditMonitor.vue b/src/pages/EditMonitor.vue index 91a707a9..0ab30742 100644 --- a/src/pages/EditMonitor.vue +++ b/src/pages/EditMonitor.vue @@ -461,7 +461,7 @@
- {{ $t("slowResponseNotificationRangeDescription", [monitor.slowResponseNotificationRange]) }} + {{ $t("slowResponseNotificationRangeDescription", [monitor.slowResponseNotificationMethod]) }}
From d73a17fd8ab65bf7a82f3badc8bbc1aee6216b34 Mon Sep 17 00:00:00 2001 From: Stephen Papierski Date: Wed, 8 Nov 2023 14:50:03 -0700 Subject: [PATCH 09/44] Add threshold calculations methods --- .../2023-10-26-slow-response-notification.js | 21 ++- server/model/monitor.js | 149 ++++++++++++------ server/server.js | 6 +- src/lang/en.json | 9 +- src/pages/EditMonitor.vue | 55 +++++-- 5 files changed, 165 insertions(+), 75 deletions(-) diff --git a/db/knex_migrations/2023-10-26-slow-response-notification.js b/db/knex_migrations/2023-10-26-slow-response-notification.js index 72ce236c..8a2c1908 100644 --- a/db/knex_migrations/2023-10-26-slow-response-notification.js +++ b/db/knex_migrations/2023-10-26-slow-response-notification.js @@ -3,9 +3,11 @@ exports.up = function (knex) { return knex.schema .alterTable("monitor", function(table) { table.boolean("slow_response_notification").notNullable().defaultTo(false); - table.integer("slow_response_notification_threshold").notNullable().defaultTo(0); - table.integer("slow_response_notification_range").notNullable().defaultTo(0); table.string("slow_response_notification_method").notNullable().defaultTo(""); + table.integer("slow_response_notification_range").notNullable().defaultTo(0); + table.string("slow_response_notification_threshold_method").notNullable().defaultTo(""); + table.integer("slow_response_notification_threshold").notNullable().defaultTo(0); + table.float("slow_response_notification_threshold_multiplier").notNullable().defaultTo(0.0); table.integer("slow_response_notification_resend_interval").notNullable().defaultTo(0); }) .alterTable("heartbeat", function(table) { @@ -14,15 +16,18 @@ exports.up = function (knex) { } exports.down = function (knex) { + // remove various slow_response_notification parameters return knex.schema .alterTable("monitor", function(table) { - table.boolean("slow_response_notification").notNullable().defaultTo(false); - table.integer("slow_response_notification_threshold").notNullable().defaultTo(0); - table.integer("slow_response_notification_range").notNullable().defaultTo(0); - table.string("slow_response_notification_method").notNullable().defaultTo(""); - table.integer("slow_response_notification_resend_interval").notNullable().defaultTo(0); + table.dropColumn("slow_response_notification"); + table.dropColumn("slow_response_notification_method"); + table.dropColumn("slow_response_notification_range"); + table.dropColumn("slow_response_notification_threshold_method"); + table.dropColumn("slow_response_notification_threshold"); + table.dropColumn("slow_response_notification_threshold_multiplier"); + table.dropColumn("slow_response_notification_resend_interval"); }) .alterTable("heartbeat", function(table) { - table.integer("slow_response_count").notNullable().defaultTo(0); + table.dropColumn("slow_response_count"); }); } diff --git a/server/model/monitor.js b/server/model/monitor.js index c6ff7a83..6678a1c8 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -133,9 +133,11 @@ class Monitor extends BeanModel { databaseQuery: this.databaseQuery, authMethod: this.authMethod, slowResponseNotification: this.isEnabledSlowResponseNotification(), - slowResponseNotificationThreshold: this.slowResponseNotificationThreshold, - slowResponseNotificationRange: this.slowResponseNotificationRange, slowResponseNotificationMethod: this.slowResponseNotificationMethod, + slowResponseNotificationRange: this.slowResponseNotificationRange, + slowResponseNotificationThresholdMethod: this.slowResponseNotificationThresholdMethod, + slowResponseNotificationThreshold: this.slowResponseNotificationThreshold, + slowResponseNotificationThresholdMultiplier: this.slowResponseNotificationThresholdMultiplier, slowResponseNotificationResendInterval: this.slowResponseNotificationResendInterval, grpcUrl: this.grpcUrl, grpcProtobuf: this.grpcProtobuf, @@ -949,7 +951,7 @@ class Monitor extends BeanModel { bean.end_time = R.isoDateTimeMillis(endTimeDayjs); // Check if response time is slow - if (this.isEnabledSlowResponseNotification()) { + if (this.isEnabledSlowResponseNotification() && !isFirstBeat) { log.debug("monitor", `[${this.name}] Check if response is slow`); await this.checkSlowResponseNotification(this, bean); } @@ -1423,28 +1425,33 @@ class Monitor extends BeanModel { } /** - * Check heartbeat response time is slower than threshold. + * Check if heartbeat response time is slower than threshold. * @param {Monitor} monitor The monitor to send a notification about * @param {Bean} bean Status information about monitor * @returns {Promise} */ async checkSlowResponseNotification(monitor, bean) { - //Get recent heartbeat list with range of time - const afterThisDate = new Date(Date.now() - (1000 * (monitor.slowResponseNotificationRange + 1))); // add 1 second otherwise we grab 0 previous beats when Time Range == Heartbeat Interval - const previousBeats = await R.getAll(` - SELECT * FROM heartbeat - WHERE monitor_id = ? AND time > datetime(?) AND status = ?`, - [ - monitor.id, - afterThisDate.toISOString(), - UP, - ]); const method = monitor.slowResponseNotificationMethod; - const thresholdResponseTime = monitor.slowResponseNotificationThreshold; + const thresholdMethod = monitor.slowResponseNotificationThresholdMethod; + const thresholdMultipler = monitor.slowResponseNotificationThresholdMultiplier; const windowDuration = monitor.slowResponseNotificationRange; let actualResponseTime = 0; + let previousBeats; + if (method != "last") { + //Get recent heartbeat list with range of time + const afterThisDate = new Date(Date.now() - (1000 * (monitor.slowResponseNotificationRange + 1))); // add 1 second otherwise we grab 0 previous beats when Time Range == Heartbeat Interval + previousBeats = await R.getAll(` + SELECT * FROM heartbeat + WHERE monitor_id = ? AND time > datetime(?) AND status = ?`, + [ + monitor.id, + afterThisDate.toISOString(), + UP, + ]); + } + switch (method) { case "average": previousBeats.forEach(beat => { @@ -1460,58 +1467,98 @@ class Monitor extends BeanModel { break; case "last": - actualResponseTime = bean.ping + actualResponseTime = bean.ping; break; default: - log.error("monitor", `[${this.name}] Unknown slow response notification method ${method}`); + log.error("monitor", `[${this.name}] Unknown response time calculation method for slow response notification: ${method}`); + return; + } + + let threshold; + let thresholdDescription; + switch (thresholdMethod) { + case "threshold-static": + threshold = monitor.slowResponseNotificationThreshold; + thresholdDescription = "static"; + break; + + case "threshold-relative-24-hour": + //Get average response time over last 24 hours + const afterThisDate = new Date(Date.now() - (1000 * (24 * 60 * 60))); // 24 hours in milliseconds + const avgPing = parseInt(await R.getCell(` + SELECT AVG(ping) FROM heartbeat + WHERE time > datetime(?) + AND ping IS NOT NULL + AND monitor_id = ? + AND status = ? + `, + [ afterThisDate.toISOString(), monitor.id, UP ] + )); + //calculate threshold + threshold = Math.round(avgPing * thresholdMultipler); + thresholdDescription = `${thresholdMultipler}x 24H Avg`; + break; + + default: + log.error("monitor", `[${this.name}] Unknown threshold calculation method for slow response notification: ${thresholdMethod}`); return; } // Create stats to append to messages/logs - let msgStats = `\nResponse: ${actualResponseTime}ms | Threshold: ${thresholdResponseTime}ms | Method: ${method}` + const methodDescription = ["average", "max"].includes(method) ? `${method} of ${windowDuration}s` : method; + let msgStats = `Response: ${actualResponseTime}ms (${methodDescription}) | Threshold: ${threshold}ms (${thresholdDescription})` // Add window duration for methods that make sense - if (["average", "max"].includes(method)) { - msgStats += ` over ${windowDuration}s` - } - // Verify something was actually calculated - if (actualResponseTime != 0 && Number.isInteger(actualResponseTime)) { - // Responding normally - if (actualResponseTime < thresholdResponseTime) { - if (bean.slowResponseCount == 0) { - log.debug("monitor", `[${this.name}] Responding normally. No need to send slow response notification ${msgStats}`); - } else { - log.debug("monitor", `[${this.name}] Returned to normal response time ${msgStats}`); - let msg = `[${this.name}] Returned to Normal Response Time ${msgStats}`; - Monitor.sendSlowResponseNotification(monitor, bean, msg); - } + // Verify valid response time was calculated + if (actualResponseTime == 0 || !Number.isInteger(actualResponseTime)) { + log.debug("monitor", `[${this.name}] Failed to calculate valid response time`); + return; + } - // Reset slow response count - bean.slowResponseCount = 0; - return; + // Verify valid threshold was calculated + if (!Number.isInteger(threshold)) { + log.debug("monitor", `[${this.name}] Failed to calculate valid threshold`); + return; + } - // Responding slowly + // Responding normally + if (actualResponseTime < threshold) { + if (bean.slowResponseCount == 0) { + log.debug("monitor", `[${this.name}] Responding normally. No need to send slow response notification | ${msgStats}`); } else { - ++bean.slowResponseCount; + msgStats += ` | Slow for: ${bean.slowResponseCount * monitor.interval}s`; + log.debug("monitor", `[${this.name}] Returned to normal response time | ${msgStats}`); + let msg = `[${this.name}] Returned to Normal Response Time \n${msgStats}`; + Monitor.sendSlowResponseNotification(monitor, bean, msg); + } + + // Reset slow response count + bean.slowResponseCount = 0; - // Always send first notification - if (bean.slowResponseCount == 1) { - log.debug("monitor", `[${this.name}] Responded slowly, sending notification ${msgStats}`); - let msg = `[${this.name}] Responded Slowly ${msgStats}`; + // Responding slowly + } else { + ++bean.slowResponseCount; + + // Always send first notification + if (bean.slowResponseCount == 1) { + log.debug("monitor", `[${this.name}] Responded slowly, sending notification | ${msgStats}`); + let msg = `[${this.name}] Responded Slowly \n${msgStats}`; + Monitor.sendSlowResponseNotification(monitor, bean, msg); + // Send notification every x times + } else if (this.slowResponseNotificationResendInterval > 0){ + if (((bean.slowResponseCount) % this.slowResponseNotificationResendInterval) == 0) { + // Send notification again, because we are still responding slow + msgStats += ` | Slow for: ${bean.slowResponseCount * monitor.interval}s`; + log.debug("monitor", `[${this.name}] Still responding slowly, sendSlowResponseNotification again | ${msgStats}`); + let msg = `[${this.name}] Still Responding Slowly \n${msgStats}`; Monitor.sendSlowResponseNotification(monitor, bean, msg); - } else if (this.slowResponseNotificationResendInterval > 0){ - // Send notification every x times - if (((bean.slowResponseCount) % this.slowResponseNotificationResendInterval) == 0) { - // Send notification again, because we are still responding slow - log.debug("monitor", `[${this.name}] sendSlowResponseNotification again ${msgStats}`); - let msg = `[${this.name}] Still Responding Slowly ${msgStats}`; - Monitor.sendSlowResponseNotification(monitor, bean, msg); - } + } else { + log.debug("monitor", `[${this.name}] Still responding slowly, waiting for resend interal | ${msgStats}`); } + } else { + log.debug("monitor", `[${this.name}] Still responding slowly, but resend is disabled | ${msgStats}`); } - } else { - log.debug("monitor", `[${this.name}] Failed to calculate valid response time`); } } diff --git a/server/server.js b/server/server.js index 2fe30e60..6f55ef88 100644 --- a/server/server.js +++ b/server/server.js @@ -804,9 +804,11 @@ let needSetup = false; bean.authWorkstation = monitor.authWorkstation; bean.authDomain = monitor.authDomain; bean.slowResponseNotification = monitor.slowResponseNotification; - bean.slowResponseNotificationThreshold = monitor.slowResponseNotificationThreshold; - bean.slowResponseNotificationRange = monitor.slowResponseNotificationRange; bean.slowResponseNotificationMethod = monitor.slowResponseNotificationMethod; + bean.slowResponseNotificationRange = monitor.slowResponseNotificationRange; + bean.slowResponseNotificationThresholdMethod = monitor.slowResponseNotificationThresholdMethod; + bean.slowResponseNotificationThreshold = monitor.slowResponseNotificationThreshold; + bean.slowResponseNotificationThresholdMultiplier = monitor.slowResponseNotificationThresholdMultiplier; bean.slowResponseNotificationResendInterval = monitor.slowResponseNotificationResendInterval; bean.grpcUrl = monitor.grpcUrl; bean.grpcProtobuf = monitor.grpcProtobuf; diff --git a/src/lang/en.json b/src/lang/en.json index e0a0e6c7..a5d0cb11 100644 --- a/src/lang/en.json +++ b/src/lang/en.json @@ -491,9 +491,16 @@ "slowResponseNotificationUseDescription": "Send a notification when service response time is slow.", "slowResponseNotificationThreshold": "Threshold (ms)", "slowResponseNotificationThresholdDescription": "Send a notification if calculated response time is greater than {0} ms.", + "slowResponseNotificationThresholdMethod": "Threshold Calculation", + "slowResponseNotificationThresholdMethodStatic": "Static Threshold", + "slowResponseNotificationThresholdMethodStaticDescription": "Define a static threshold.", + "slowResponseNotificationThresholdMethodRelative24Hour": "Relative to Avg. Response", + "slowResponseNotificationThresholdMethodRelative24HourDescription": "Calculate the threshold ({0}x the 24-hour average response time).", + "slowResponseNotificationThresholdMultiplier": "Threshold Multiplier", + "slowResponseNotificationThresholdMultiplierDescription": "Send notification if response time is greater than {0}x the 24-hour average.", "slowResponseNotificationRange": "Window Duration (seconds)", "slowResponseNotificationRangeDescription": "Window duration for calculating the {0}.", - "slowResponseNotificationMethod": "Calculation Method", + "slowResponseNotificationMethod": "Response Time Calculation", "slowResponseNotificationMethodAverage": "Average", "slowResponseNotificationMethodAverageDescription": "Get the average response time over the last {0} seconds.", "slowResponseNotificationMethodMax": "Max", diff --git a/src/pages/EditMonitor.vue b/src/pages/EditMonitor.vue index 0ab30742..eb420a60 100644 --- a/src/pages/EditMonitor.vue +++ b/src/pages/EditMonitor.vue @@ -426,8 +426,8 @@
- - @@ -449,7 +449,36 @@
+ +
+ + +
+ {{ $t("slowResponseNotificationRangeDescription", [monitor.slowResponseNotificationMethod]) }} +
+
+ +
+ + +
+ {{ $t("slowResponseNotificationThresholdMethodStaticDescription") }} +
+
+ {{ $t("slowResponseNotificationThresholdMethodRelative24HourDescription", [monitor.slowResponseNotificationThresholdMultiplier]) }} +
+
+ + +
@@ -457,14 +486,16 @@
-
- - + +
+ +
- {{ $t("slowResponseNotificationRangeDescription", [monitor.slowResponseNotificationMethod]) }} + {{ $t("slowResponseNotificationThresholdMultiplierDescription", [monitor.slowResponseNotificationThresholdMultiplier]) }}
+
+ +
+ + +
+ {{ $t("slowResponseNotificationRangeDescription", [monitor.slowResponseNotificationMethod]) }} +
+
+ +
+ + +
+ {{ $t("slowResponseNotificationThresholdMethodStaticDescription") }} +
+
+ {{ $t("slowResponseNotificationThresholdMethodRelative24HourDescription", [monitor.slowResponseNotificationThresholdMultiplier]) }} +
+
+ + +
@@ -457,14 +486,16 @@
-
- - + +
+ +
- {{ $t("slowResponseNotificationRangeDescription", [monitor.slowResponseNotificationMethod]) }} + {{ $t("slowResponseNotificationThresholdMultiplierDescription", [monitor.slowResponseNotificationThresholdMultiplier]) }}
+