From dc4d2a77bb71550c51bb1021118c93999cf64a63 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 5 Feb 2023 17:45:36 +0800 Subject: [PATCH] WIP --- docker/debian-base.dockerfile | 12 +++- docker/docker-compose-dev.yml | 13 ++++ docker/docker-compose.yml | 15 ++-- docker/dockerfile | 28 ++------ docker/my.cnf | 0 package.json | 14 ++-- server/database.js | 30 +++++--- server/embedded-mariadb.js | 118 ++++++++++++++++++++++++++++---- test/ubuntu-nodejs16.dockerfile | 10 --- 9 files changed, 171 insertions(+), 69 deletions(-) create mode 100644 docker/docker-compose-dev.yml delete mode 100644 docker/my.cnf delete mode 100644 test/ubuntu-nodejs16.dockerfile diff --git a/docker/debian-base.dockerfile b/docker/debian-base.dockerfile index 543f52bc..c2b8bfb4 100644 --- a/docker/debian-base.dockerfile +++ b/docker/debian-base.dockerfile @@ -1,6 +1,6 @@ # DON'T UPDATE TO node:14-bullseye-slim, see #372. # If the image changed, the second stage image should be changed too -FROM node:18-buster-slim +FROM node:18-buster-slim AS base2-slim ARG TARGETPLATFORM # Install Curl @@ -24,3 +24,13 @@ RUN node ./extra/download-cloudflared.js $TARGETPLATFORM && \ rm -f cloudflared.deb && \ apt --yes autoremove +FROM base2-slim AS base2 +RUN apt update && \ + apt --yes --no-install-recommends install curl && \ + curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | bash -s -- --mariadb-server-version="mariadb-10.11" && \ + apt --yes --no-install-recommends install mariadb-server && \ + apt --yes remove curl && \ + rm -rf /var/lib/apt/lists/* && \ + apt --yes autoremove +RUN chown -R node:node /var/lib/mysql + diff --git a/docker/docker-compose-dev.yml b/docker/docker-compose-dev.yml new file mode 100644 index 00000000..5510b0d8 --- /dev/null +++ b/docker/docker-compose-dev.yml @@ -0,0 +1,13 @@ +version: '3.8' + +services: + uptime-kuma: + container_name: uptime-kuma-dev + image: louislam/uptime-kuma:nightly2 + volumes: + - ./data:/app/data + - ../server:/app/server + ports: + - "3001:3001" # : + - "3307:3306" + diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index f5c8f366..20e37329 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,14 +1,15 @@ -# Simple docker-compose.yml -# You can change your port or volume location - -version: '3.3' +version: '3.8' services: uptime-kuma: - image: louislam/uptime-kuma:1 + image: louislam/uptime-kuma:2 container_name: uptime-kuma volumes: - - ./uptime-kuma-data:/app/data + - uptime-kuma:/app/data ports: - - 3001:3001 # : + - "3001:3001" # : restart: always + +volumes: + uptime-kuma: + diff --git a/docker/dockerfile b/docker/dockerfile index a5f4ed8a..a62f2340 100644 --- a/docker/dockerfile +++ b/docker/dockerfile @@ -1,6 +1,8 @@ +ARG BASE_IMAGE=louislam/uptime-kuma:base2 + ############################################ # Build in Golang -# Run npm run build-healthcheck-armv7 in the host first, another it will be super slow where it is building the armv7 healthcheck +# Run npm run build-healthcheck-armv7 in the host first, otherwise it will be super slow where it is building the armv7 healthcheck # Check file: builder-go.dockerfile ############################################ FROM louislam/uptime-kuma:builder-go AS build_healthcheck @@ -9,6 +11,7 @@ FROM louislam/uptime-kuma:builder-go AS build_healthcheck # Build in Node.js ############################################ FROM louislam/uptime-kuma:base2 AS build +USER node WORKDIR /app ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 @@ -20,34 +23,20 @@ COPY . . COPY --chown=node:node --from=build_healthcheck /app/extra/healthcheck /app/extra/healthcheck ############################################ -# ⭐ Main Image (Slim) +# ⭐ Main Image ############################################ -FROM louislam/uptime-kuma:base2 AS release-slim +FROM $BASE_IMAGE AS release USER node WORKDIR /app # Copy app files from build layer COPY --chown=node:node --from=build /app /app - EXPOSE 3001 -VOLUME ["/app/data"] HEALTHCHECK --interval=60s --timeout=30s --start-period=180s --retries=5 CMD extra/healthcheck ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["node", "server/server.js"] -############################################ -# ⭐ Main Image (With MariaDB) -############################################ -FROM release-slim AS release -RUN apt update && \ - apt --yes --no-install-recommends install curl && \ - curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | bash -s -- --mariadb-server-version="mariadb-10.11" && \ - apt --yes --no-install-recommends install mariadb-server && \ - apt remove curl && \ - rm -rf /var/lib/apt/lists/* && \ - apt --yes autoremove - ############################################ # Mark as Nightly ############################################ @@ -58,10 +47,8 @@ RUN npm run mark-as-nightly ############################################ # Build an image for testing pr ############################################ -FROM louislam/uptime-kuma:base2 AS pr-test - +FROM louislam/uptime-kuma:base2 AS pr-test2 WORKDIR /app - ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 ## Install Git @@ -83,7 +70,6 @@ RUN git clone https://github.com/louislam/uptime-kuma.git . RUN npm ci EXPOSE 3000 3001 -VOLUME ["/app/data"] HEALTHCHECK --interval=60s --timeout=30s --start-period=180s --retries=5 CMD node extra/healthcheck.js CMD ["npm", "run", "start-pr-test"] diff --git a/docker/my.cnf b/docker/my.cnf deleted file mode 100644 index e69de29b..00000000 diff --git a/package.json b/package.json index b578145a..48e61003 100644 --- a/package.json +++ b/package.json @@ -29,11 +29,13 @@ "tsc": "tsc", "vite-preview-dist": "vite preview --host --config ./config/vite.config.js", "build-docker": "npm run build && npm run build-docker-full && npm run build-docker-slim", - "build-docker-base": "docker buildx build -f docker/debian-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base2 . --push", + "build-docker-base": "docker buildx build -f docker/debian-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base2 --target base2 . --push", + "build-docker-base-slim": "docker buildx build -f docker/debian-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base2-slim --target base2-slim . --push", "build-docker-builder-go": "docker buildx build -f docker/builder-go.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:builder-go . --push", - "build-docker-slim": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2-slim -t louislam/uptime-kuma:$VERSION-slim --target release-slim . --push", + "build-docker-slim": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2-slim -t louislam/uptime-kuma:$VERSION-slim --target release --build-arg BASE_IMAGE=louislam/uptime-kuma:base2-slim . --push", "build-docker-full": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2 -t louislam/uptime-kuma:$VERSION --target release . --push", - "build-docker-nightly": "npm run build && docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:nightly2 --target nightly . --push", + "build-docker-nightly": "npm run build && docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:nightly2 --target nightly --build-arg . --push", + "build-docker-nightly-local": "docker build -f docker/dockerfile -t louislam/uptime-kuma:nightly2 --target nightly .", "build-docker-pr-test": "docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64 -t louislam/uptime-kuma:pr-test --target pr-test . --push", "upload-artifacts": "docker buildx build -f docker/dockerfile --platform linux/amd64 -t louislam/uptime-kuma:upload-artifact --build-arg VERSION --build-arg GITHUB_TOKEN --target upload-artifact . --progress plain", "setup": "git checkout 1.19.6 && npm ci --production && npm run download-dist", @@ -45,11 +47,9 @@ "test-install-script-centos7": "npm run compile-install-script && docker build --progress plain -f test/test_install_script/centos7.dockerfile .", "test-install-script-ubuntu": "npm run compile-install-script && docker build --progress plain -f test/test_install_script/ubuntu.dockerfile .", "test-install-script-ubuntu1604": "npm run compile-install-script && docker build --progress plain -f test/test_install_script/ubuntu1604.dockerfile .", - "test-nodejs16": "docker build --progress plain -f test/ubuntu-nodejs16.dockerfile .", "simple-dns-server": "node extra/simple-dns-server.js", "simple-mqtt-server": "node extra/simple-mqtt-server.js", "update-language-files": "cd extra/update-language-files && node index.js && cross-env-shell eslint ../../src/languages/$npm_config_language.js --fix", - "ncu-patch": "npm-check-updates -u -t patch", "release-final": "node extra/update-version.js && npm run build-docker && node ./extra/press-any-key.js && npm run upload-artifacts && node ./extra/update-wiki-version.js", "release-beta": "node extra/beta/update-version.js && npm run build && node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:$VERSION -t louislam/uptime-kuma:beta . --target release --push && node ./extra/press-any-key.js && npm run upload-artifacts", "git-remove-tag": "git tag -d", @@ -59,7 +59,9 @@ "cy:run": "npx cypress run --browser chrome --headless --config-file ./config/cypress.config.js", "cy:run:unit": "npx cypress run --browser chrome --headless --config-file ./config/cypress.frontend.config.js", "cypress-open": "concurrently -k -r \"node test/prepare-test-server.js && node server/server.js --port=3002 --data-dir=./data/test/\" \"cypress open --config-file ./config/cypress.config.js\"", - "build-healthcheck-armv7": "cross-env GOOS=linux GOARCH=arm GOARM=7 go build -x -o ./extra/healthcheck-armv7 ./extra/healthcheck.go" + "build-healthcheck-armv7": "cross-env GOOS=linux GOARCH=arm GOARM=7 go build -x -o ./extra/healthcheck-armv7 ./extra/healthcheck.go", + "quick-run-nightly": "docker run --rm -p 3001:3001 louislam/uptime-kuma:nightly2", + "start-dev-container": "cd docker && docker-compose -f docker-compose-dev.yml up" }, "dependencies": { "@grpc/grpc-js": "~1.7.3", diff --git a/server/database.js b/server/database.js index 974e7cc4..bb1151cf 100644 --- a/server/database.js +++ b/server/database.js @@ -6,6 +6,7 @@ const dayjs = require("dayjs"); const knex = require("knex"); const { PluginsManager } = require("./plugins-manager"); const path = require("path"); +const { EmbeddedMariaDB } = require("./embedded-mariadb"); /** * Database & App Data Folder @@ -123,10 +124,20 @@ class Database { let dbConfig; try { - dbConfig = fs.readFileSync(path.join(Database.dataDir, "db-config.json")); + let dbConfigString = fs.readFileSync(path.join(Database.dataDir, "db-config.json")).toString("utf-8"); + dbConfig = JSON.parse(dbConfigString); + + if (typeof dbConfig !== "object") { + throw new Error("Invalid db-config.json, it must be an object"); + } + + if (typeof dbConfig.type !== "string") { + throw new Error("Invalid db-config.json, type must be a string"); + } } catch (_) { dbConfig = { - type: "sqlite", + //type: "sqlite", + type: "embedded-mariadb", }; } @@ -151,19 +162,20 @@ class Database { acquireTimeoutMillis: acquireConnectionTimeout, } }; - } else if (dbConfig === "embedded-mariadb") { + } else if (dbConfig.type === "embedded-mariadb") { + let embeddedMariaDB = EmbeddedMariaDB.getInstance(); + await embeddedMariaDB.start(); + log.info("mariadb", "Embedded MariaDB started"); config = { - client: "mysql", + client: "mysql2", connection: { - host: "127.0.0.1", - port: 3306, - user: "your_database_user", - password: "your_database_password", + socketPath: embeddedMariaDB.socketPath, + user: "node", database: "kuma" } }; } else { - throw new Error("Unknown Database type"); + throw new Error("Unknown Database type: " + dbConfig.type); } const knexInstance = knex(config); diff --git a/server/embedded-mariadb.js b/server/embedded-mariadb.js index eef78920..50fbfb94 100644 --- a/server/embedded-mariadb.js +++ b/server/embedded-mariadb.js @@ -1,47 +1,131 @@ const { log } = require("../src/util"); const childProcess = require("child_process"); +const fs = require("fs"); +/** + * It is only used inside the docker container + */ class EmbeddedMariaDB { - static childProcess = null; - static running = false; + static instance = null; - static init() { + exec = "mariadbd"; + mariadbDataDir = "/app/data/mariadb"; + + runDir = "/app/data/run/mariadb"; + + socketPath = this.runDir + "/mysqld.sock"; + + childProcess = null; + running = false; + + started = false; + + /** + * + * @returns {EmbeddedMariaDB} + */ + static getInstance() { + if (!EmbeddedMariaDB.instance) { + EmbeddedMariaDB.instance = new EmbeddedMariaDB(); + } + return EmbeddedMariaDB.instance; } - static start() { + static hasInstance() { + return !!EmbeddedMariaDB.instance; + } + + /** + * + */ + start() { if (this.childProcess) { - log.log("mariadb", "Already started"); + log.info("mariadb", "Already started"); return; } + if (!fs.existsSync(this.mariadbDataDir)) { + log.info("mariadb", `Embedded MariaDB: ${this.mariadbDataDir} is not found, create one now.`); + fs.mkdirSync(this.mariadbDataDir, { + recursive: true, + }); + + let result = childProcess.spawnSync("mysql_install_db", [ + "--user=node", + "--ldata=" + this.mariadbDataDir, + ]); + + if (result.status !== 0) { + let error = result.stderr.toString("utf-8"); + log.error("mariadb", error); + return; + } else { + log.info("mariadb", "Embedded MariaDB: mysql_install_db done:" + result.stdout.toString("utf-8")); + } + } + + if (!fs.existsSync(this.runDir)) { + log.info("mariadb", `Embedded MariaDB: ${this.runDir} is not found, create one now.`); + fs.mkdirSync(this.runDir, { + recursive: true, + }); + } + this.running = true; - this.emitChange("Starting cloudflared"); - this.childProcess = childProcess.spawn(this.cloudflaredPath, args); - this.childProcess.stdout.pipe(process.stdout); - this.childProcess.stderr.pipe(process.stderr); + log.info("mariadb", "Starting Embedded MariaDB"); + this.childProcess = childProcess.spawn(this.exec, [ + "--user=node", + "--datadir=" + this.mariadbDataDir, + `--socket=${this.socketPath}`, + `--pid-file=${this.runDir}/mysqld.pid`, + ]); this.childProcess.on("close", (code) => { this.running = false; this.childProcess = null; - this.emitChange("Stopped cloudflared", code); + this.started = false; + log.info("mariadb", "Stopped Embedded MariaDB: " + code); + + if (code !== 0) { + log.info("mariadb", "Try to restart Embedded MariaDB as it is not stopped by user"); + this.start(); + } }); this.childProcess.on("error", (err) => { if (err.code === "ENOENT") { - this.emitError(`Cloudflared error: ${this.cloudflaredPath} is not found`); + log.error("mariadb", `Embedded MariaDB: ${this.exec} is not found`); } else { - this.emitError(err); + log.error("mariadb", err); } }); - this.childProcess.stderr.on("data", (data) => { - this.emitError(data.toString()); + let handler = (data) => { + log.debug("mariadb", data.toString("utf-8")); + if (data.toString("utf-8").includes("ready for connections")) { + log.info("mariadb", "Embedded MariaDB is ready for connections"); + this.started = true; + } + }; + + this.childProcess.stdout.on("data", handler); + this.childProcess.stderr.on("data", handler); + + return new Promise((resolve) => { + let interval = setInterval(() => { + if (this.started) { + clearInterval(interval); + resolve(); + } else { + log.info("mariadb", "Waiting for Embedded MariaDB to start..."); + } + }, 1000); }); } - static stop() { + stop() { if (this.childProcess) { this.childProcess.kill("SIGINT"); this.childProcess = null; @@ -49,3 +133,7 @@ class EmbeddedMariaDB { } } + +module.exports = { + EmbeddedMariaDB, +}; diff --git a/test/ubuntu-nodejs16.dockerfile b/test/ubuntu-nodejs16.dockerfile deleted file mode 100644 index a2dd2ec8..00000000 --- a/test/ubuntu-nodejs16.dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM ubuntu -WORKDIR /app -RUN apt update && apt --yes install git curl -RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - -RUN apt --yes install nodejs -RUN git clone https://github.com/louislam/uptime-kuma.git . -RUN npm run setup - -# Option 1. Try it -RUN node server/server.js