From 40569519155e563357b2d9c0a0d4ce34fd5c9bb2 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sat, 11 Feb 2023 22:21:06 +0800 Subject: [PATCH] WIP: building database in knex.js --- db/knex_migrations/README.md | 46 ++++++++ db/kuma.js | 213 +++++++++++++++++++++++++++++++++++ package-lock.json | 1 + package.json | 3 +- server/database.js | 80 ++++++++++++- server/model/maintenance.js | 6 +- server/server.js | 12 +- server/setup-database.js | 3 + server/uptime-kuma-server.js | 2 +- 9 files changed, 350 insertions(+), 16 deletions(-) create mode 100644 db/knex_migrations/README.md create mode 100644 db/kuma.js diff --git a/db/knex_migrations/README.md b/db/knex_migrations/README.md new file mode 100644 index 00000000..bcad0468 --- /dev/null +++ b/db/knex_migrations/README.md @@ -0,0 +1,46 @@ +## Info + +https://knexjs.org/guide/migrations.html#knexfile-in-other-languages + + +## Template + +Filename: YYYYMMDDHHMMSS_name.js + +```js +exports.up = function(knex) { + +}; + +exports.down = function(knex) { + +}; + +// exports.config = { transaction: false }; +``` + +## Example + +20230211120000_create_users_products.js + +```js +exports.up = function(knex) { + return knex.schema + .createTable('users', function (table) { + table.increments('id'); + table.string('first_name', 255).notNullable(); + table.string('last_name', 255).notNullable(); + }) + .createTable('products', function (table) { + table.increments('id'); + table.decimal('price').notNullable(); + table.string('name', 1000).notNullable(); + }); +}; + +exports.down = function(knex) { + return knex.schema + .dropTable("products") + .dropTable("users"); +}; +``` diff --git a/db/kuma.js b/db/kuma.js new file mode 100644 index 00000000..8d1f3309 --- /dev/null +++ b/db/kuma.js @@ -0,0 +1,213 @@ +const { R } = require("redbean-node"); +const { log, sleep } = require("../src/util"); + +/** + * DO NOT ADD ANYTHING HERE! + * IF YOU NEED TO ADD SOMETHING, ADD IT TO ./db/knex_migrations + * @returns {Promise} + */ +async function createTables() { + log.info("mariadb", "Creating basic tables for MariaDB"); + const knex = R.knex; + + // Up to `patch-add-google-analytics-status-page-tag.sql` + + // docker_host + await knex.schema.createTable("docker_host", (table) => { + table.increments("id"); + table.integer("user_id").unsigned().notNullable(); + table.string("docker_daemon", 255); + table.string("docker_type", 255); + table.string("name", 255); + }); + + // group + await knex.schema.createTable("group", (table) => { + table.increments("id"); + table.string("name", 255).notNullable(); + table.datetime("created_date").notNullable().defaultTo(knex.fn.now()); + table.boolean("public").notNullable().defaultTo(false); + table.boolean("active").notNullable().defaultTo(true); + table.integer("weight").notNullable().defaultTo(1000); + }); + + // proxy + await knex.schema.createTable("proxy", (table) => { + table.increments("id"); + table.integer("user_id").unsigned().notNullable(); + table.string("protocol", 10).notNullable(); + table.string("host", 255).notNullable(); + table.smallint("port").notNullable(); // Maybe a issue with MariaDB, need migration to int + table.boolean("auth").notNullable(); + table.string("username", 255).nullable(); + table.string("password", 255).nullable(); + table.boolean("active").notNullable().defaultTo(true); + table.boolean("default").notNullable().defaultTo(false); + table.datetime("created_date").notNullable().defaultTo(knex.fn.now()); + + table.index("user_id", "proxy_user_id"); + }); + + // user + await knex.schema.createTable("user", (table) => { + table.increments("id"); + table.string("username", 255).notNullable().unique().collate("utf8_general_ci"); + table.string("password", 255); + table.boolean("active").notNullable().defaultTo(true); + table.string("timezone", 150); + table.string("twofa_secret", 64); + table.boolean("twofa_status").notNullable().defaultTo(false); + table.string("twofa_last_token", 6); + }); + + // monitor + await knex.schema.createTable("monitor", (table) => { + table.increments("id"); + table.string("name", 150); + table.boolean("active").notNullable().defaultTo(true); + table.integer("user_id").unsigned() + .references("id").inTable("user") + .onDelete("SET NULL") + .onUpdate("CASCADE"); + table.integer("interval").notNullable().defaultTo(20); + table.text("url"); + table.string("type", 20); + table.integer("weight").defaultTo(2000); + table.string("hostname", 255); + table.integer("port"); + table.datetime("created_date").notNullable().defaultTo(knex.fn.now()); + table.string("keyword", 255); + table.integer("maxretries").notNullable().defaultTo(0); + table.boolean("ignore_tls").notNullable().defaultTo(false); + table.boolean("upside_down").notNullable().defaultTo(false); + table.integer("maxredirects").notNullable().defaultTo(10); + table.text("accepted_statuscodes_json").notNullable().defaultTo("[\"200-299\"]"); + table.string("dns_resolve_type", 5); + table.string("dns_resolve_server", 255); + table.string("dns_last_result", 255); + table.integer("retry_interval").notNullable().defaultTo(0); + table.string("push_token", 20).defaultTo(null); + table.text("method").notNullable().defaultTo("GET"); + table.text("body").defaultTo(null); + table.text("headers").defaultTo(null); + table.text("basic_auth_user").defaultTo(null); + table.text("basic_auth_pass").defaultTo(null); + table.integer("docker_host").unsigned() + .references("id").inTable("docker_host"); + table.string("docker_container", 255); + table.integer("proxy_id").unsigned() + .references("id").inTable("proxy"); + table.boolean("expiry_notification").defaultTo(true); + table.text("mqtt_topic"); + table.string("mqtt_success_message", 255); + table.string("mqtt_username", 255); + table.string("mqtt_password", 255); + table.string("database_connection_string", 2000); + table.text("database_query"); + table.string("auth_method", 250); + table.text("auth_domain"); + table.text("auth_workstation"); + table.string("grpc_url", 255).defaultTo(null); + table.text("grpc_protobuf").defaultTo(null); + table.text("grpc_body").defaultTo(null); + table.text("grpc_metadata").defaultTo(null); + table.text("grpc_method").defaultTo(null); + table.text("grpc_service_name").defaultTo(null); + table.boolean("grpc_enable_tls").notNullable().defaultTo(false); + table.string("radius_username", 255); + table.string("radius_password", 255); + table.string("radius_calling_station_id", 50); + table.string("radius_called_station_id", 50); + table.string("radius_secret", 255); + table.integer("resend_interval").notNullable().defaultTo(0); + table.integer("packet_size").notNullable().defaultTo(56); + table.string("game", 255); + }); + + // heartbeat + await knex.schema.createTable("heartbeat", (table) => { + table.increments("id"); + table.boolean("important").notNullable().defaultTo(false); + table.integer("monitor_id").unsigned().notNullable() + .references("id").inTable("monitor") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + table.smallint("status").notNullable(); + + table.text("msg"); + table.datetime("time").notNullable(); + table.integer("ping"); + table.integer("duration").notNullable().defaultTo(0); + table.integer("down_count").notNullable().defaultTo(0); + + table.index("important"); + table.index([ "monitor_id", "time" ], "monitor_time_index"); + table.index("monitor_id"); + table.index([ "monitor_id", "important", "time" ], "monitor_important_time_index"); + }); + + // incident + await knex.schema.createTable("incident", (table) => { + table.increments("id"); + table.string("title", 255).notNullable(); + table.text("content", 255).notNullable(); + table.string("style", 30).notNullable().defaultTo("warning"); + table.datetime("created_date").notNullable().defaultTo(knex.fn.now()); + table.datetime("last_updated_date"); + table.boolean("pin").notNullable().defaultTo(true); + table.boolean("active").notNullable().defaultTo(true); + table.integer("status_page_id").unsigned(); + }); + + // maintenance + await knex.schema.createTable("maintenance", (table) => { + table.increments("id"); + table.string("title", 150).notNullable(); + table.text("description").notNullable(); + table.integer("user_id").unsigned() + .references("id").inTable("user") + .onDelete("SET NULL") + .onUpdate("CASCADE"); + table.boolean("active").notNullable().defaultTo(true); + table.string("strategy", 50).notNullable().defaultTo("single"); + table.datetime("start_date"); + table.datetime("end_date"); + table.time("start_time"); + table.time("end_time"); + table.string("weekdays", 250).defaultTo("[]"); + table.text("days_of_month").defaultTo("[]"); + table.integer("interval_day"); + + table.index("active"); + table.index([ "strategy", "active" ], "manual_active"); + table.index("user_id", "maintenance_user_id"); + }); + + // maintenance_status_page + // maintenance_timeslot + // monitor_group + // monitor_maintenance + // monitor_notification + // monitor_tag + // monitor_tls_info + // notification + // notification_sent_history + // setting + await knex.schema.createTable("setting", (table) => { + table.increments("id"); + table.string("key", 200).notNullable().unique().collate("utf8_general_ci"); + table.text("value"); + table.string("type", 20); + }); + + // status_page + // status_page_cname + // tag + // user + + log.info("mariadb", "Created basic tables for MariaDB"); +} + +module.exports = { + createTables, +}; diff --git a/package-lock.json b/package-lock.json index 5d172288..ed3a8527 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,6 +40,7 @@ "jsesc": "~3.0.2", "jsonwebtoken": "~9.0.0", "jwt-decode": "~3.1.2", + "knex": "^2.4.2", "limiter": "~2.1.0", "mongodb": "~4.13.0", "mqtt": "~4.3.7", diff --git a/package.json b/package.json index 31276c43..c19577e7 100644 --- a/package.json +++ b/package.json @@ -95,6 +95,7 @@ "jsesc": "~3.0.2", "jsonwebtoken": "~9.0.0", "jwt-decode": "~3.1.2", + "knex": "^2.4.2", "limiter": "~2.1.0", "mongodb": "~4.13.0", "mqtt": "~4.3.7", @@ -148,8 +149,8 @@ "eslint": "~8.14.0", "eslint-plugin-vue": "~8.7.1", "favico.js": "~0.3.10", - "marked": "~4.2.5", "jest": "~27.2.5", + "marked": "~4.2.5", "postcss-html": "~1.5.0", "postcss-rtlcss": "~3.7.2", "postcss-scss": "~4.0.4", diff --git a/server/database.js b/server/database.js index 954882a6..b28bdfd0 100644 --- a/server/database.js +++ b/server/database.js @@ -38,11 +38,13 @@ class Database { static backupPath = null; /** + * SQLite only * Add patch filename in key * Values: * true: Add it regardless of order * false: Do nothing * { parents: []}: Need parents before add it + * @deprecated */ static patchList = { "patch-setting-value-type.sql": true, @@ -82,6 +84,10 @@ class Database { static noReject = true; + static dbConfig = {}; + + static knexMigrationsPath = "./db/knex_migrations"; + /** * Initialize the data directory * @param {Object} args Arguments to initialize DB with @@ -144,17 +150,23 @@ class Database { let dbConfig; try { dbConfig = this.readDBConfig(); + Database.dbConfig = dbConfig; } catch (err) { log.warn("db", err.message); dbConfig = { type: "sqlite", - //type: "embedded-mariadb", }; } let config = {}; if (dbConfig.type === "sqlite") { + + if (! fs.existsSync(Database.sqlitePath)) { + log.info("server", "Copying Database"); + fs.copyFileSync(Database.templatePath, Database.sqlitePath); + } + const Dialect = require("knex/lib/dialects/sqlite3/index.js"); Dialect.prototype._driver = () => require("@louislam/sqlite3"); @@ -173,6 +185,17 @@ class Database { acquireTimeoutMillis: acquireConnectionTimeout, } }; + } else if (dbConfig.type === "mariadb") { + config = { + client: "mysql2", + connection: { + host: dbConfig.hostname, + port: dbConfig.port, + user: dbConfig.username, + password: dbConfig.password, + database: dbConfig.dbName, + } + }; } else if (dbConfig.type === "embedded-mariadb") { let embeddedMariaDB = EmbeddedMariaDB.getInstance(); await embeddedMariaDB.start(); @@ -182,13 +205,22 @@ class Database { connection: { socketPath: embeddedMariaDB.socketPath, user: "node", - database: "kuma" + database: "kuma", } }; } else { throw new Error("Unknown Database type: " + dbConfig.type); } + // Set to utf8mb4 for MariaDB + if (dbConfig.type.endsWith("mariadb")) { + config.pool = { + afterCreate(conn, done) { + conn.query("SET CHARACTER SET utf8mb4;", (err) => done(err, conn)); + }, + }; + } + const knexInstance = knex(config); R.setup(knexInstance); @@ -204,6 +236,14 @@ class Database { await R.autoloadModels("./server/model"); } + if (dbConfig.type === "sqlite") { + await this.initSQLite(testMode, noLog); + } else if (dbConfig.type.endsWith("mariadb")) { + await this.initMariaDB(); + } + } + + static async initSQLite(testMode, noLog) { await R.exec("PRAGMA foreign_keys = ON"); if (testMode) { // Change to MEMORY @@ -228,8 +268,36 @@ class Database { } } - /** Patch the database */ + static async initMariaDB() { + log.debug("db", "Checking if MariaDB database exists..."); + + let hasTable = await R.hasTable("docker_host"); + if (!hasTable) { + const { createTables } = require("../db/kuma"); + await createTables(); + } else { + log.debug("db", "MariaDB database already exists"); + } + } + static async patch() { + if (Database.dbConfig.type === "sqlite") { + await this.patchSqlite(); + } + + // TODO: Using knex migrations + // https://knexjs.org/guide/migrations.html + // https://gist.github.com/NigelEarle/70db130cc040cc2868555b29a0278261 + await R.knex.migrate.latest({ + directory: Database.knexMigrationsPath, + }); + } + + /** + * Patch the database for SQLite + * @deprecated + */ + static async patchSqlite() { let version = parseInt(await setting("database_version")); if (! version) { @@ -275,17 +343,18 @@ class Database { } } - await this.patch2(); + await this.patchSqlite2(); await this.migrateNewStatusPage(); } /** * Patch DB using new process * Call it from patch() only + * @deprecated * @private * @returns {Promise} */ - static async patch2() { + static async patchSqlite2() { log.info("db", "Database Patch 2.0 Process"); let databasePatchedFiles = await setting("databasePatchedFiles"); @@ -321,6 +390,7 @@ class Database { } /** + * SQlite only * Migrate status page value in setting to "status_page" table * @returns {Promise} */ diff --git a/server/model/maintenance.js b/server/model/maintenance.js index 45db63d1..6c371fd2 100644 --- a/server/model/maintenance.js +++ b/server/model/maintenance.js @@ -212,8 +212,8 @@ class Maintenance extends BeanModel { static getActiveMaintenanceSQLCondition() { return ` ( - (maintenance_timeslot.start_date <= DATETIME('now') - AND maintenance_timeslot.end_date >= DATETIME('now') + (maintenance_timeslot.start_date <= CURRENT_TIMESTAMP + AND maintenance_timeslot.end_date >= CURRENT_TIMESTAMP AND maintenance.active = 1) OR (maintenance.strategy = 'manual' AND active = 1) @@ -228,7 +228,7 @@ class Maintenance extends BeanModel { static getActiveAndFutureMaintenanceSQLCondition() { return ` ( - ((maintenance_timeslot.end_date >= DATETIME('now') + ((maintenance_timeslot.end_date >= CURRENT_TIMESTAMP AND maintenance.active = 1) OR (maintenance.strategy = 'manual' AND active = 1)) diff --git a/server/server.js b/server/server.js index 627c586b..8d9ea76d 100644 --- a/server/server.js +++ b/server/server.js @@ -180,7 +180,12 @@ let needSetup = false; } // Connect to database - await initDatabase(testMode); + try { + await initDatabase(testMode); + } catch (e) { + log.error("server", "Failed to prepare your database: " + e.message); + process.exit(1); + } // Database should be ready now await server.initAfterDatabaseReady(); @@ -1658,11 +1663,6 @@ async function afterLogin(socket, user) { * @returns {Promise} */ async function initDatabase(testMode = false) { - if (! fs.existsSync(Database.sqlitePath)) { - log.info("server", "Copying Database"); - fs.copyFileSync(Database.templatePath, Database.sqlitePath); - } - log.info("server", "Connecting to the Database"); await Database.connect(testMode); log.info("server", "Connected"); diff --git a/server/setup-database.js b/server/setup-database.js index 53d68fd0..e40649fa 100644 --- a/server/setup-database.js +++ b/server/setup-database.js @@ -34,6 +34,9 @@ class SetupDatabase { try { dbConfig = Database.readDBConfig(); + log.info("setup-database", "db-config.json is found and is valid"); + this.needSetup = false; + } catch (e) { log.info("setup-database", "db-config.json is not found or invalid: " + e.message); diff --git a/server/uptime-kuma-server.js b/server/uptime-kuma-server.js index 0573f0d8..6ba11f59 100644 --- a/server/uptime-kuma-server.js +++ b/server/uptime-kuma-server.js @@ -272,7 +272,7 @@ class UptimeKumaServer { /** Load the timeslots for maintenance */ async generateMaintenanceTimeslots() { - let list = await R.find("maintenance_timeslot", " generated_next = 0 AND start_date <= DATETIME('now') "); + let list = await R.find("maintenance_timeslot", " generated_next = 0 AND start_date <= CURRENT_TIMESTAMP "); for (let maintenanceTimeslot of list) { let maintenance = await maintenanceTimeslot.maintenance;