WIP: building database in knex.js

db/knex_migrations/README.md

@@ -0,0 +1,46 @@
+## Info
+
+https://knexjs.org/guide/migrations.html#knexfile-in-other-languages
+
+
+## Template
+
+Filename: YYYYMMDDHHMMSS_name.js
+
+```js
+exports.up = function(knex) {
+
+};
+
+exports.down = function(knex) {
+
+};
+
+// exports.config = { transaction: false };
+```
+
+## Example
+
+20230211120000_create_users_products.js
+
+```js
+exports.up = function(knex) {
+  return knex.schema
+    .createTable('users', function (table) {
+        table.increments('id');
+        table.string('first_name', 255).notNullable();
+        table.string('last_name', 255).notNullable();
+    })
+    .createTable('products', function (table) {
+        table.increments('id');
+        table.decimal('price').notNullable();
+        table.string('name', 1000).notNullable();
+    });
+};
+
+exports.down = function(knex) {
+  return knex.schema
+      .dropTable("products")
+      .dropTable("users");
+};
+```

db/kuma.js

@@ -0,0 +1,213 @@
+const { R } = require("redbean-node");
+const { log, sleep } = require("../src/util");
+
+/**
+ * DO NOT ADD ANYTHING HERE!
+ * IF YOU NEED TO ADD SOMETHING, ADD IT TO ./db/knex_migrations
+ * @returns {Promise<void>}
+ */
+async function createTables() {
+    log.info("mariadb", "Creating basic tables for MariaDB");
+    const knex = R.knex;
+
+    // Up to `patch-add-google-analytics-status-page-tag.sql`
+
+    // docker_host
+    await knex.schema.createTable("docker_host", (table) => {
+        table.increments("id");
+        table.integer("user_id").unsigned().notNullable();
+        table.string("docker_daemon", 255);
+        table.string("docker_type", 255);
+        table.string("name", 255);
+    });
+
+    // group
+    await knex.schema.createTable("group", (table) => {
+        table.increments("id");
+        table.string("name", 255).notNullable();
+        table.datetime("created_date").notNullable().defaultTo(knex.fn.now());
+        table.boolean("public").notNullable().defaultTo(false);
+        table.boolean("active").notNullable().defaultTo(true);
+        table.integer("weight").notNullable().defaultTo(1000);
+    });
+
+    // proxy
+    await knex.schema.createTable("proxy", (table) => {
+        table.increments("id");
+        table.integer("user_id").unsigned().notNullable();
+        table.string("protocol", 10).notNullable();
+        table.string("host", 255).notNullable();
+        table.smallint("port").notNullable();           // Maybe a issue with MariaDB, need migration to int
+        table.boolean("auth").notNullable();
+        table.string("username", 255).nullable();
+        table.string("password", 255).nullable();
+        table.boolean("active").notNullable().defaultTo(true);
+        table.boolean("default").notNullable().defaultTo(false);
+        table.datetime("created_date").notNullable().defaultTo(knex.fn.now());
+
+        table.index("user_id", "proxy_user_id");
+    });
+
+    // user
+    await knex.schema.createTable("user", (table) => {
+        table.increments("id");
+        table.string("username", 255).notNullable().unique().collate("utf8_general_ci");
+        table.string("password", 255);
+        table.boolean("active").notNullable().defaultTo(true);
+        table.string("timezone", 150);
+        table.string("twofa_secret", 64);
+        table.boolean("twofa_status").notNullable().defaultTo(false);
+        table.string("twofa_last_token", 6);
+    });
+
+    // monitor
+    await knex.schema.createTable("monitor", (table) => {
+        table.increments("id");
+        table.string("name", 150);
+        table.boolean("active").notNullable().defaultTo(true);
+        table.integer("user_id").unsigned()
+            .references("id").inTable("user")
+            .onDelete("SET NULL")
+            .onUpdate("CASCADE");
+        table.integer("interval").notNullable().defaultTo(20);
+        table.text("url");
+        table.string("type", 20);
+        table.integer("weight").defaultTo(2000);
+        table.string("hostname", 255);
+        table.integer("port");
+        table.datetime("created_date").notNullable().defaultTo(knex.fn.now());
+        table.string("keyword", 255);
+        table.integer("maxretries").notNullable().defaultTo(0);
+        table.boolean("ignore_tls").notNullable().defaultTo(false);
+        table.boolean("upside_down").notNullable().defaultTo(false);
+        table.integer("maxredirects").notNullable().defaultTo(10);
+        table.text("accepted_statuscodes_json").notNullable().defaultTo("[\"200-299\"]");
+        table.string("dns_resolve_type", 5);
+        table.string("dns_resolve_server", 255);
+        table.string("dns_last_result", 255);
+        table.integer("retry_interval").notNullable().defaultTo(0);
+        table.string("push_token", 20).defaultTo(null);
+        table.text("method").notNullable().defaultTo("GET");
+        table.text("body").defaultTo(null);
+        table.text("headers").defaultTo(null);
+        table.text("basic_auth_user").defaultTo(null);
+        table.text("basic_auth_pass").defaultTo(null);
+        table.integer("docker_host").unsigned()
+            .references("id").inTable("docker_host");
+        table.string("docker_container", 255);
+        table.integer("proxy_id").unsigned()
+            .references("id").inTable("proxy");
+        table.boolean("expiry_notification").defaultTo(true);
+        table.text("mqtt_topic");
+        table.string("mqtt_success_message", 255);
+        table.string("mqtt_username", 255);
+        table.string("mqtt_password", 255);
+        table.string("database_connection_string", 2000);
+        table.text("database_query");
+        table.string("auth_method", 250);
+        table.text("auth_domain");
+        table.text("auth_workstation");
+        table.string("grpc_url", 255).defaultTo(null);
+        table.text("grpc_protobuf").defaultTo(null);
+        table.text("grpc_body").defaultTo(null);
+        table.text("grpc_metadata").defaultTo(null);
+        table.text("grpc_method").defaultTo(null);
+        table.text("grpc_service_name").defaultTo(null);
+        table.boolean("grpc_enable_tls").notNullable().defaultTo(false);
+        table.string("radius_username", 255);
+        table.string("radius_password", 255);
+        table.string("radius_calling_station_id", 50);
+        table.string("radius_called_station_id", 50);
+        table.string("radius_secret", 255);
+        table.integer("resend_interval").notNullable().defaultTo(0);
+        table.integer("packet_size").notNullable().defaultTo(56);
+        table.string("game", 255);
+    });
+
+    // heartbeat
+    await knex.schema.createTable("heartbeat", (table) => {
+        table.increments("id");
+        table.boolean("important").notNullable().defaultTo(false);
+        table.integer("monitor_id").unsigned().notNullable()
+            .references("id").inTable("monitor")
+            .onDelete("CASCADE")
+            .onUpdate("CASCADE");
+        table.smallint("status").notNullable();
+
+        table.text("msg");
+        table.datetime("time").notNullable();
+        table.integer("ping");
+        table.integer("duration").notNullable().defaultTo(0);
+        table.integer("down_count").notNullable().defaultTo(0);
+
+        table.index("important");
+        table.index([ "monitor_id", "time" ], "monitor_time_index");
+        table.index("monitor_id");
+        table.index([ "monitor_id", "important", "time" ], "monitor_important_time_index");
+    });
+
+    // incident
+    await knex.schema.createTable("incident", (table) => {
+        table.increments("id");
+        table.string("title", 255).notNullable();
+        table.text("content", 255).notNullable();
+        table.string("style", 30).notNullable().defaultTo("warning");
+        table.datetime("created_date").notNullable().defaultTo(knex.fn.now());
+        table.datetime("last_updated_date");
+        table.boolean("pin").notNullable().defaultTo(true);
+        table.boolean("active").notNullable().defaultTo(true);
+        table.integer("status_page_id").unsigned();
+    });
+
+    // maintenance
+    await knex.schema.createTable("maintenance", (table) => {
+        table.increments("id");
+        table.string("title", 150).notNullable();
+        table.text("description").notNullable();
+        table.integer("user_id").unsigned()
+            .references("id").inTable("user")
+            .onDelete("SET NULL")
+            .onUpdate("CASCADE");
+        table.boolean("active").notNullable().defaultTo(true);
+        table.string("strategy", 50).notNullable().defaultTo("single");
+        table.datetime("start_date");
+        table.datetime("end_date");
+        table.time("start_time");
+        table.time("end_time");
+        table.string("weekdays", 250).defaultTo("[]");
+        table.text("days_of_month").defaultTo("[]");
+        table.integer("interval_day");
+
+        table.index("active");
+        table.index([ "strategy", "active" ], "manual_active");
+        table.index("user_id", "maintenance_user_id");
+    });
+
+    // maintenance_status_page
+    // maintenance_timeslot
+    // monitor_group
+    // monitor_maintenance
+    // monitor_notification
+    // monitor_tag
+    // monitor_tls_info
+    // notification
+    // notification_sent_history
+    // setting
+    await knex.schema.createTable("setting", (table) => {
+        table.increments("id");
+        table.string("key", 200).notNullable().unique().collate("utf8_general_ci");
+        table.text("value");
+        table.string("type", 20);
+    });
+
+    // status_page
+    // status_page_cname
+    // tag
+    // user
+
+    log.info("mariadb", "Created basic tables for MariaDB");
+}
+
+module.exports = {
+    createTables,
+};

package-lock.json

@@ -40,6 +40,7 @@
                 "jsesc": "~3.0.2",
                 "jsonwebtoken": "~9.0.0",
                 "jwt-decode": "~3.1.2",
+                "knex": "^2.4.2",
                 "limiter": "~2.1.0",
                 "mongodb": "~4.13.0",
                 "mqtt": "~4.3.7",

package.json

@@ -95,6 +95,7 @@
         "jsesc": "~3.0.2",
         "jsonwebtoken": "~9.0.0",
         "jwt-decode": "~3.1.2",
+        "knex": "^2.4.2",
         "limiter": "~2.1.0",
         "mongodb": "~4.13.0",
         "mqtt": "~4.3.7",
@@ -148,8 +149,8 @@
         "eslint": "~8.14.0",
         "eslint-plugin-vue": "~8.7.1",
         "favico.js": "~0.3.10",
-        "marked": "~4.2.5",
         "jest": "~27.2.5",
+        "marked": "~4.2.5",
         "postcss-html": "~1.5.0",
         "postcss-rtlcss": "~3.7.2",
         "postcss-scss": "~4.0.4",

server/database.js

@@ -38,11 +38,13 @@ class Database {
     static backupPath = null;
 
     /**
+     * SQLite only
      * Add patch filename in key
      * Values:
      *      true: Add it regardless of order
      *      false: Do nothing
      *      { parents: []}: Need parents before add it
+     *  @deprecated
      */
     static patchList = {
         "patch-setting-value-type.sql": true,
@@ -82,6 +84,10 @@ class Database {
 
     static noReject = true;
 
+    static dbConfig = {};
+
+    static knexMigrationsPath = "./db/knex_migrations";
+
     /**
      * Initialize the data directory
      * @param {Object} args Arguments to initialize DB with
@@ -144,17 +150,23 @@ class Database {
         let dbConfig;
         try {
             dbConfig = this.readDBConfig();
+            Database.dbConfig = dbConfig;
         } catch (err) {
             log.warn("db", err.message);
             dbConfig = {
                 type: "sqlite",
-                //type: "embedded-mariadb",
             };
         }
 
         let config = {};
 
         if (dbConfig.type === "sqlite") {
+
+            if (! fs.existsSync(Database.sqlitePath)) {
+                log.info("server", "Copying Database");
+                fs.copyFileSync(Database.templatePath, Database.sqlitePath);
+            }
+
             const Dialect = require("knex/lib/dialects/sqlite3/index.js");
             Dialect.prototype._driver = () => require("@louislam/sqlite3");
 
@@ -173,6 +185,17 @@ class Database {
                     acquireTimeoutMillis: acquireConnectionTimeout,
                 }
             };
+        } else if (dbConfig.type === "mariadb") {
+            config = {
+                client: "mysql2",
+                connection: {
+                    host: dbConfig.hostname,
+                    port: dbConfig.port,
+                    user: dbConfig.username,
+                    password: dbConfig.password,
+                    database: dbConfig.dbName,
+                }
+            };
         } else if (dbConfig.type === "embedded-mariadb") {
             let embeddedMariaDB = EmbeddedMariaDB.getInstance();
             await embeddedMariaDB.start();
@@ -182,13 +205,22 @@ class Database {
                 connection: {
                     socketPath: embeddedMariaDB.socketPath,
                     user: "node",
-                    database: "kuma"
+                    database: "kuma",
                 }
             };
         } else {
             throw new Error("Unknown Database type: " + dbConfig.type);
         }
 
+        // Set to utf8mb4 for MariaDB
+        if (dbConfig.type.endsWith("mariadb")) {
+            config.pool = {
+                afterCreate(conn, done) {
+                    conn.query("SET CHARACTER SET utf8mb4;", (err) => done(err, conn));
+                },
+            };
+        }
+
         const knexInstance = knex(config);
 
         R.setup(knexInstance);
@@ -204,6 +236,14 @@ class Database {
             await R.autoloadModels("./server/model");
         }
 
+        if (dbConfig.type === "sqlite") {
+            await this.initSQLite(testMode, noLog);
+        } else if (dbConfig.type.endsWith("mariadb")) {
+            await this.initMariaDB();
+        }
+    }
+
+    static async initSQLite(testMode, noLog) {
         await R.exec("PRAGMA foreign_keys = ON");
         if (testMode) {
             // Change to MEMORY
@@ -228,8 +268,36 @@ class Database {
         }
     }
 
-    /** Patch the database */
+    static async initMariaDB() {
+        log.debug("db", "Checking if MariaDB database exists...");
+
+        let hasTable = await R.hasTable("docker_host");
+        if (!hasTable) {
+            const { createTables } = require("../db/kuma");
+            await createTables();
+        } else {
+            log.debug("db", "MariaDB database already exists");
+        }
+    }
+
     static async patch() {
+        if (Database.dbConfig.type === "sqlite") {
+            await this.patchSqlite();
+        }
+
+        // TODO: Using knex migrations
+        // https://knexjs.org/guide/migrations.html
+        // https://gist.github.com/NigelEarle/70db130cc040cc2868555b29a0278261
+        await R.knex.migrate.latest({
+            directory: Database.knexMigrationsPath,
+        });
+    }
+
+    /**
+     * Patch the database for SQLite
+     * @deprecated
+     */
+    static async patchSqlite() {
         let version = parseInt(await setting("database_version"));
 
         if (! version) {
@@ -275,17 +343,18 @@ class Database {
             }
         }
 
-        await this.patch2();
+        await this.patchSqlite2();
         await this.migrateNewStatusPage();
     }
 
     /**
      * Patch DB using new process
      * Call it from patch() only
+     * @deprecated
      * @private
      * @returns {Promise<void>}
      */
-    static async patch2() {
+    static async patchSqlite2() {
         log.info("db", "Database Patch 2.0 Process");
         let databasePatchedFiles = await setting("databasePatchedFiles");
 
@@ -321,6 +390,7 @@ class Database {
     }
 
     /**
+     * SQlite only
      * Migrate status page value in setting to "status_page" table
      * @returns {Promise<void>}
      */

server/model/maintenance.js

@@ -212,8 +212,8 @@ class Maintenance extends BeanModel {
     static getActiveMaintenanceSQLCondition() {
         return `
             (
-                (maintenance_timeslot.start_date <= DATETIME('now')
+                (maintenance_timeslot.start_date <= CURRENT_TIMESTAMP
-                AND maintenance_timeslot.end_date >= DATETIME('now')
+                AND maintenance_timeslot.end_date >= CURRENT_TIMESTAMP
                 AND maintenance.active = 1)
                 OR
                 (maintenance.strategy = 'manual' AND active = 1)
@@ -228,7 +228,7 @@ class Maintenance extends BeanModel {
     static getActiveAndFutureMaintenanceSQLCondition() {
         return `
             (
-                ((maintenance_timeslot.end_date >= DATETIME('now')
+                ((maintenance_timeslot.end_date >= CURRENT_TIMESTAMP
                 AND maintenance.active = 1)
                 OR
                 (maintenance.strategy = 'manual' AND active = 1))

server/server.js

@@ -180,7 +180,12 @@ let needSetup = false;
     }
 
     // Connect to database
+    try {
         await initDatabase(testMode);
+    } catch (e) {
+        log.error("server", "Failed to prepare your database: " + e.message);
+        process.exit(1);
+    }
 
     // Database should be ready now
     await server.initAfterDatabaseReady();
@@ -1658,11 +1663,6 @@ async function afterLogin(socket, user) {
  * @returns {Promise<void>}
  */
 async function initDatabase(testMode = false) {
-    if (! fs.existsSync(Database.sqlitePath)) {
-        log.info("server", "Copying Database");
-        fs.copyFileSync(Database.templatePath, Database.sqlitePath);
-    }
-
     log.info("server", "Connecting to the Database");
     await Database.connect(testMode);
     log.info("server", "Connected");

server/setup-database.js

@@ -34,6 +34,9 @@ class SetupDatabase {
 
         try {
             dbConfig = Database.readDBConfig();
+            log.info("setup-database", "db-config.json is found and is valid");
+            this.needSetup = false;
+
         } catch (e) {
             log.info("setup-database", "db-config.json is not found or invalid: " + e.message);
 

server/uptime-kuma-server.js

@@ -272,7 +272,7 @@ class UptimeKumaServer {
     /** Load the timeslots for maintenance */
     async generateMaintenanceTimeslots() {
 
-        let list = await R.find("maintenance_timeslot", " generated_next = 0 AND start_date <= DATETIME('now') ");
+        let list = await R.find("maintenance_timeslot", " generated_next = 0 AND start_date <= CURRENT_TIMESTAMP ");
 
         for (let maintenanceTimeslot of list) {
             let maintenance = await maintenanceTimeslot.maintenance;