Extend Prometheus Labels to include tags (requires restart for NEW labels on the monitor to be visible) (#4704)

Co-authored-by: Frank Elsinga <frank@elsinga.de>

server/model/monitor.js

@@ -336,7 +336,7 @@ class Monitor extends BeanModel {
         let previousBeat = null;
         let retries = 0;
 
-        this.prometheus = new Prometheus(this);
+        this.prometheus = await Prometheus.createAndInitMetrics(this);
 
         const beat = async () => {
 
@@ -980,7 +980,7 @@ class Monitor extends BeanModel {
             await R.store(bean);
 
             log.debug("monitor", `[${this.name}] prometheus.update`);
-            this.prometheus?.update(bean, tlsInfo);
+            await this.prometheus?.update(bean, tlsInfo);
 
             previousBeat = bean;
 

server/prometheus.js

@@ -1,3 +1,4 @@
+const { R } = require("redbean-node");
 const PrometheusClient = require("prom-client");
 const { log } = require("../src/util");
 
@@ -9,36 +10,102 @@ const commonLabels = [
     "monitor_port",
 ];
 
-const monitorCertDaysRemaining = new PrometheusClient.Gauge({
-    name: "monitor_cert_days_remaining",
-    help: "The number of days remaining until the certificate expires",
-    labelNames: commonLabels
-});
-
-const monitorCertIsValid = new PrometheusClient.Gauge({
-    name: "monitor_cert_is_valid",
-    help: "Is the certificate still valid? (1 = Yes, 0= No)",
-    labelNames: commonLabels
-});
-const monitorResponseTime = new PrometheusClient.Gauge({
-    name: "monitor_response_time",
-    help: "Monitor Response Time (ms)",
-    labelNames: commonLabels
-});
-
-const monitorStatus = new PrometheusClient.Gauge({
-    name: "monitor_status",
-    help: "Monitor Status (1 = UP, 0= DOWN, 2= PENDING, 3= MAINTENANCE)",
-    labelNames: commonLabels
-});
-
 class Prometheus {
-    monitorLabelValues = {};
 
     /**
-     * @param {object} monitor Monitor object to monitor
+     * Metric: monitor_cert_days_remaining
+     * @type {PrometheusClient.Gauge<string> | null}
+     */
+    static monitorCertDaysRemaining = null;
+
+    /**
+     * Metric: monitor_cert_is_valid
+     * @type {PrometheusClient.Gauge<string> | null}
+     */
+    static monitorCertIsValid = null;
+
+    /**
+     * Metric: monitor_response_time
+     * @type {PrometheusClient.Gauge<string> | null}
+     */
+    static monitorResponseTime = null;
+
+    /**
+     * Metric: monitor_status
+     * @type {PrometheusClient.Gauge<string> | null}
+     */
+    static monitorStatus = null;
+
+    /**
+     * All registered metric labels.
+     * @type {string[] | null}
+     */
+    static monitorLabelNames = null;
+
+    /**
+     * Monitor labels/values combination.
+     * @type {{}}
+     */
+    monitorLabelValues;
+
+    /**
+     * Initialize metrics and get all label names the first time called.
+     * @returns {void}
      */
-    constructor(monitor) {
+    static async initMetrics() {
+        if (!this.monitorLabelNames) {
+            let labelNames = await R.getCol("SELECT name FROM tag");
+            this.monitorLabelNames = [ ...commonLabels, ...labelNames ];
+        }
+        if (!this.monitorCertDaysRemaining) {
+            this.monitorCertDaysRemaining = new PrometheusClient.Gauge({
+                name: "monitor_cert_days_remaining",
+                help: "The number of days remaining until the certificate expires",
+                labelNames: this.monitorLabelNames
+            });
+        }
+        if (!this.monitorCertIsValid) {
+            this.monitorCertIsValid = new PrometheusClient.Gauge({
+                name: "monitor_cert_is_valid",
+                help: "Is the certificate still valid? (1 = Yes, 0 = No)",
+                labelNames: this.monitorLabelNames
+            });
+        }
+        if (!this.monitorResponseTime) {
+            this.monitorResponseTime = new PrometheusClient.Gauge({
+                name: "monitor_response_time",
+                help: "Monitor Response Time (ms)",
+                labelNames: this.monitorLabelNames
+            });
+        }
+        if (!this.monitorStatus) {
+            this.monitorStatus = new PrometheusClient.Gauge({
+                name: "monitor_status",
+                help: "Monitor Status (1 = UP, 0 = DOWN, 2 = PENDING, 3 = MAINTENANCE)",
+                labelNames: this.monitorLabelNames
+            });
+        }
+    }
+
+    /**
+     * Wrapper to create a `Prometheus` instance and ensure metrics are initialized.
+     * @param {Monitor} monitor Monitor object to monitor
+     * @returns {Promise<Prometheus>} `Prometheus` instance
+     */
+    static async createAndInitMetrics(monitor) {
+        await Prometheus.initMetrics();
+        let tags = await monitor.getTags();
+        return new Prometheus(monitor, tags);
+    }
+
+    /**
+     * Creates a prometheus metric instance.
+     *
+     * Note: Make sure to call `Prometheus.initMetrics()` once prior creating Prometheus instances.
+     * @param {Monitor} monitor Monitor object to monitor
+     * @param {Promise<LooseObject<any>[]>} tags Tags of the monitor
+     */
+    constructor(monitor, tags) {
         this.monitorLabelValues = {
             monitor_name: monitor.name,
             monitor_type: monitor.type,
@@ -46,6 +113,12 @@ class Prometheus {
             monitor_hostname: monitor.hostname,
             monitor_port: monitor.port
         };
+        Object.values(tags)
+            // only label names that were known at first metric creation.
+            .filter(tag => Prometheus.monitorLabelNames.includes(tag.name))
+            .forEach(tag => {
+                this.monitorLabelValues[tag.name] = tag.value;
+            });
     }
 
     /**
@@ -55,7 +128,6 @@ class Prometheus {
      * @returns {void}
      */
     update(heartbeat, tlsInfo) {
-
         if (typeof tlsInfo !== "undefined") {
             try {
                 let isValid;
@@ -64,7 +136,7 @@ class Prometheus {
                 } else {
                     isValid = 0;
                 }
-                monitorCertIsValid.set(this.monitorLabelValues, isValid);
+                Prometheus.monitorCertIsValid.set(this.monitorLabelValues, isValid);
             } catch (e) {
                 log.error("prometheus", "Caught error");
                 log.error("prometheus", e);
@@ -72,7 +144,7 @@ class Prometheus {
 
             try {
                 if (tlsInfo.certInfo != null) {
-                    monitorCertDaysRemaining.set(this.monitorLabelValues, tlsInfo.certInfo.daysRemaining);
+                    Prometheus.monitorCertDaysRemaining.set(this.monitorLabelValues, tlsInfo.certInfo.daysRemaining);
                 }
             } catch (e) {
                 log.error("prometheus", "Caught error");
@@ -82,7 +154,7 @@ class Prometheus {
 
         if (heartbeat) {
             try {
-                monitorStatus.set(this.monitorLabelValues, heartbeat.status);
+                Prometheus.monitorStatus.set(this.monitorLabelValues, heartbeat.status);
             } catch (e) {
                 log.error("prometheus", "Caught error");
                 log.error("prometheus", e);
@@ -90,10 +162,10 @@ class Prometheus {
 
             try {
                 if (typeof heartbeat.ping === "number") {
-                    monitorResponseTime.set(this.monitorLabelValues, heartbeat.ping);
+                    Prometheus.monitorResponseTime.set(this.monitorLabelValues, heartbeat.ping);
                 } else {
                     // Is it good?
-                    monitorResponseTime.set(this.monitorLabelValues, -1);
+                    Prometheus.monitorResponseTime.set(this.monitorLabelValues, -1);
                 }
             } catch (e) {
                 log.error("prometheus", "Caught error");
@@ -108,10 +180,10 @@ class Prometheus {
      */
     remove() {
         try {
-            monitorCertDaysRemaining.remove(this.monitorLabelValues);
+            Prometheus.monitorCertDaysRemaining?.remove(this.monitorLabelValues);
-            monitorCertIsValid.remove(this.monitorLabelValues);
+            Prometheus.monitorCertIsValid?.remove(this.monitorLabelValues);
-            monitorResponseTime.remove(this.monitorLabelValues);
+            Prometheus.monitorResponseTime?.remove(this.monitorLabelValues);
-            monitorStatus.remove(this.monitorLabelValues);
+            Prometheus.monitorStatus?.remove(this.monitorLabelValues);
         } catch (e) {
             console.error(e);
         }