add a better approach for patching db, change setting.value from varchar to TEXT, restore Database.close() to 1.2.0

db/patch-setting-value-type.sql

@@ -0,0 +1,22 @@
+-- You should not modify if this have pushed to Github, unless it does serious wrong with the db.
+BEGIN TRANSACTION;
+
+-- Generated by Intellij IDEA
+create table setting_dg_tmp
+(
+    id INTEGER
+        primary key autoincrement,
+    key VARCHAR(200) not null
+        unique,
+    value TEXT,
+    type VARCHAR(20)
+);
+
+insert into setting_dg_tmp(id, key, value, type) select id, key, value, type from setting;
+
+drop table setting;
+
+alter table setting_dg_tmp rename to setting;
+
+
+COMMIT;

server/database.js

@@ -1,15 +1,44 @@
 const fs = require("fs");
 const { R } = require("redbean-node");
 const { setSetting, setting } = require("./util-server");
+const { debug, sleep } = require("../src/util");
+const dayjs = require("dayjs");
 
 class Database {
 
-    static templatePath = "./db/kuma.db"
+    static templatePath = "./db/kuma.db";
     static dataDir;
     static path;
+
+    /**
+     * @type {boolean}
+     */
+    static patched = false;
+
+    /**
+     * For Backup only
+     */
+    static backupPath = null;
+
+    /**
+     * Add patch filename in key
+     * Values:
+     *      true: Add it regardless of order
+     *      false: Do nothing
+     *      { parents: []}: Need parents before add it
+     */
+    static patchList = {
+        "patch-setting-value-type.sql": true,
+        "patch-improve-performance.sql": true,
+    }
+
+    /**
+     * The finally version should be 10 after merged tag feature
+     * @deprecated Use patchList for any new feature
+     */
     static latestVersion = 9;
+
     static noReject = true;
-    static sqliteInstance = null;
 
     static async connect() {
         const acquireConnectionTimeout = 120 * 1000;
@@ -60,19 +89,7 @@ class Database {
         } else {
             console.info("Database patch is needed")
 
-            console.info("Backup the db")
+            this.backup(version);
-            const backupPath = this.dataDir + "kuma.db.bak" + version;
-            fs.copyFileSync(Database.path, backupPath);
-
-            const shmPath = Database.path + "-shm";
-            if (fs.existsSync(shmPath)) {
-                fs.copyFileSync(shmPath, shmPath + ".bak" + version);
-            }
-
-            const walPath = Database.path + "-wal";
-            if (fs.existsSync(walPath)) {
-                fs.copyFileSync(walPath, walPath + ".bak" + version);
-            }
 
             // Try catch anything here, if gone wrong, restore the backup
             try {
@@ -83,18 +100,92 @@ class Database {
                     console.info(`Patched ${sqlFile}`);
                     await setSetting("database_version", i);
                 }
-                console.log("Database Patched Successfully");
             } catch (ex) {
                 await Database.close();
-                console.error("Patch db failed!!! Restoring the backup")
+                this.restore();
-                fs.copyFileSync(backupPath, Database.path);
-                console.error(ex)
 
+                console.error(ex)
                 console.error("Start Uptime-Kuma failed due to patch db failed")
                 console.error("Please submit the bug report if you still encounter the problem after restart: https://github.com/louislam/uptime-kuma/issues")
                 process.exit(1);
             }
         }
+
+        await this.patch2();
+    }
+
+    /**
+     * Call it from patch() only
+     * @returns {Promise<void>}
+     */
+    static async patch2() {
+        console.log("Database Patch 2.0 Process");
+        let databasePatchedFiles = await setting("databasePatchedFiles");
+
+        if (! databasePatchedFiles) {
+            databasePatchedFiles = {};
+        }
+
+        debug("Patched files:");
+        debug(databasePatchedFiles);
+
+        try {
+            for (let sqlFilename in this.patchList) {
+                await this.patch2Recursion(sqlFilename, databasePatchedFiles)
+            }
+
+            if (this.patched) {
+                console.log("Database Patched Successfully");
+            }
+
+        } catch (ex) {
+            await Database.close();
+            this.restore();
+
+            console.error(ex)
+            console.error("Start Uptime-Kuma failed due to patch db failed");
+            console.error("Please submit the bug report if you still encounter the problem after restart: https://github.com/louislam/uptime-kuma/issues");
+            process.exit(1);
+        }
+
+        await setSetting("databasePatchedFiles", databasePatchedFiles);
+    }
+
+    /**
+     * Used it patch2() only
+     * @param sqlFilename
+     * @param databasePatchedFiles
+     */
+    static async patch2Recursion(sqlFilename, databasePatchedFiles) {
+        let value = this.patchList[sqlFilename];
+
+        if (! value) {
+            console.log(sqlFilename + " skip");
+            return;
+        }
+
+        // Check if patched
+        if (! databasePatchedFiles[sqlFilename]) {
+            console.log(sqlFilename + " is not patched");
+
+            if (value.parents) {
+                console.log(sqlFilename + " need parents");
+                for (let parentSQLFilename of value.parents) {
+                    await this.patch2Recursion(parentSQLFilename, databasePatchedFiles);
+                }
+            }
+
+            this.backup(dayjs().format("YYYYMMDDHHmmss"));
+
+            console.log(sqlFilename + " is patching");
+            this.patched = true;
+            await this.importSQLFile("./db/" + sqlFilename);
+            databasePatchedFiles[sqlFilename] = true;
+            console.log(sqlFilename + " is patched successfully");
+
+        } else {
+            console.log(sqlFilename + " is already patched, skip");
+        }
     }
 
     /**
@@ -140,10 +231,96 @@ class Database {
      * @returns {Promise<void>}
      */
     static async close() {
-        if (this.sqliteInstance) {
+        const listener = (reason, p) => {
-            this.sqliteInstance.close();
+            Database.noReject = false;
+        };
+        process.addListener("unhandledRejection", listener);
+
+        console.log("Closing DB");
+
+        while (true) {
+            Database.noReject = true;
+            await R.close();
+            await sleep(2000);
+
+            if (Database.noReject) {
+                break;
+            } else {
+                console.log("Waiting to close the db");
+            }
+        }
+        console.log("SQLite closed");
+
+        process.removeListener("unhandledRejection", listener);
+    }
+
+    /**
+     * One backup one time in this process.
+     * Reset this.backupPath if you want to backup again
+     * @param version
+     */
+    static backup(version) {
+        if (! this.backupPath) {
+            console.info("Backup the db")
+            this.backupPath = this.dataDir + "kuma.db.bak" + version;
+            fs.copyFileSync(Database.path, this.backupPath);
+
+            const shmPath = Database.path + "-shm";
+            if (fs.existsSync(shmPath)) {
+                this.backupShmPath = shmPath + ".bak" + version;
+                fs.copyFileSync(shmPath, this.backupShmPath);
+            }
+
+            const walPath = Database.path + "-wal";
+            if (fs.existsSync(walPath)) {
+                this.backupWalPath = walPath + ".bak" + version;
+                fs.copyFileSync(walPath, this.backupWalPath);
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    static restore() {
+        if (this.backupPath) {
+            console.error("Patch db failed!!! Restoring the backup");
+
+            const shmPath = Database.path + "-shm";
+            const walPath = Database.path + "-wal";
+
+            // Delete patch failed db
+            try {
+                if (fs.existsSync(Database.path)) {
+                    fs.unlinkSync(Database.path);
+                }
+
+                if (fs.existsSync(shmPath)) {
+                    fs.unlinkSync(shmPath);
+                }
+
+                if (fs.existsSync(walPath)) {
+                    fs.unlinkSync(walPath);
+                }
+            } catch (e) {
+                console.log("Restore failed, you may need to restore the backup manually");
+                process.exit(1);
+            }
+
+            // Restore backup
+            fs.copyFileSync(this.backupPath, Database.path);
+
+            if (this.backupShmPath) {
+                fs.copyFileSync(this.backupShmPath, shmPath);
+            }
+
+            if (this.backupWalPath) {
+                fs.copyFileSync(this.backupWalPath, walPath);
+            }
+
+        } else {
+            console.log("Nothing to restore");
         }
-        console.log("Stopped database");
     }
 }