diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..417d081 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +node_modules +/autorestic diff --git a/bun.lockb b/bun.lockb new file mode 100755 index 0000000..e4da522 Binary files /dev/null and b/bun.lockb differ diff --git a/migration.md b/migration.md new file mode 100644 index 0000000..706dad9 --- /dev/null +++ b/migration.md @@ -0,0 +1,14 @@ +# Rename backend to repository + +# Env variables + +AUTORESTIC_BB_B2_ACCOUNT_ID=123 -> AUTORESTIC_BACKENDS_BB_ENV_B2**ACCOUNT**ID=123 + +- All fields can be configured by env now +- To escape `_` replace it with double underscore `__` + +# Rest property on backend config + +No rest property anymore, can be used in string extrapolation + +# Every string is now replaceable with env variables diff --git a/package.json b/package.json new file mode 100644 index 0000000..1bae615 --- /dev/null +++ b/package.json @@ -0,0 +1,23 @@ +{ + "private": true, + "name": "autorestic", + "module": "src/index.ts", + "type": "module", + "scripts": { + "schema:gen": "bun run ./scripts/generateSchema.ts", + "bin": "bun build ./src/index.ts --compile --outfile autorestic" + }, + "devDependencies": { + "bun-types": "^0.6.0", + "typescript": "^5.0.0", + "zod-to-json-schema": "^3.21.2" + }, + "dependencies": { + "@commander-js/extra-typings": "^11.0.0", + "commander": "^11.0.0", + "pino": "^8.14.1", + "pino-pretty": "^10.0.0", + "yaml": "^2.3.1", + "zod": "^3.21.4" + } +} diff --git a/schema/config.json b/schema/config.json new file mode 100644 index 0000000..16bae68 --- /dev/null +++ b/schema/config.json @@ -0,0 +1,252 @@ +{ + "$ref": "#/definitions/mySchema", + "definitions": { + "mySchema": { + "type": "object", + "properties": { + "version": { + "type": "number", + "description": "version number" + }, + "repos": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "local", + "sftp", + "rest", + "swift", + "s3", + "b2", + "azure", + "gs", + "rclone" + ], + "description": "type of repository" + }, + "path": { + "type": "string", + "minLength": 1, + "description": "restic path" + }, + "key": { + "$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path", + "description": "encryption key for the repository" + }, + "env": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path", + "description": "value of the environment variable" + }, + "description": "environment variables" + }, + "options": { + "type": "object", + "properties": { + "all": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "boolean", + "const": true, + "description": "boolean flag" + }, + { + "anyOf": [ + { + "$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path", + "description": "non-empty string that can extrapolate env variables inside it" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/options/properties/all/additionalProperties/anyOf/1/anyOf/0" + }, + "minItems": 1 + } + ] + } + ], + "description": "value of option" + } + }, + "backup": { + "$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/options/properties/all" + }, + "forget": { + "$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/options/properties/all" + } + }, + "additionalProperties": false, + "description": "options" + } + }, + "required": [ + "type", + "path", + "key" + ], + "additionalProperties": false + }, + "description": "available repositories" + }, + "locations": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "from": { + "anyOf": [ + { + "$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path", + "description": "local path to backup" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/from/anyOf/0" + }, + "minItems": 1 + } + ] + }, + "to": { + "anyOf": [ + { + "$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path", + "description": "repository to backup to" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/to/anyOf/0" + }, + "minItems": 1 + } + ] + }, + "copy": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path", + "description": "destination repository" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/copy/additionalProperties/anyOf/0" + }, + "minItems": 1 + } + ] + } + }, + "cron": { + "$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path", + "description": "execute backups for the given cron job" + }, + "hooks": { + "type": "object", + "properties": { + "before": { + "anyOf": [ + { + "$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path", + "description": "command to be executed" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/hooks/properties/before/anyOf/0" + }, + "minItems": 1 + } + ], + "description": "list of commands" + }, + "after": { + "$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/hooks/properties/before", + "description": "list of commands" + }, + "failure": { + "$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/hooks/properties/before", + "description": "list of commands" + }, + "success": { + "$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/hooks/properties/before", + "description": "list of commands" + } + }, + "additionalProperties": false, + "description": "hooks to be executed" + }, + "options": { + "type": "object", + "properties": { + "all": { + "$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/options/properties/all" + }, + "backup": { + "$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/options/properties/backup" + }, + "forget": { + "$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/options/properties/forget" + } + }, + "additionalProperties": false, + "description": "native restic options" + }, + "forget": { + "anyOf": [ + { + "type": "boolean", + "description": "automatically run \"forget\" when backing up" + }, + { + "type": "string", + "const": "prune", + "description": "also prune when forgetting" + } + ] + } + }, + "required": [ + "from", + "to" + ], + "additionalProperties": false, + "description": "Location" + }, + "description": "available locations" + }, + "global": { + "type": "object", + "properties": { + "options": { + "$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/options", + "description": "native restic options" + } + }, + "additionalProperties": false, + "description": "global configuration" + }, + "extras": {} + }, + "required": [ + "version", + "repos", + "locations" + ], + "additionalProperties": false + } + }, + "$schema": "http://json-schema.org/draft-07/schema#" +} \ No newline at end of file diff --git a/scripts/generateSchema.ts b/scripts/generateSchema.ts new file mode 100644 index 0000000..50a8e3c --- /dev/null +++ b/scripts/generateSchema.ts @@ -0,0 +1,17 @@ +import { mkdir, rm, writeFile } from 'node:fs/promises' +import { zodToJsonSchema } from 'zod-to-json-schema' +import { ConfigSchema } from '../src/config/schema/config' + +const OUTPUT = './schema' + +await rm(OUTPUT, { recursive: true, force: true }) +await mkdir(OUTPUT, { recursive: true }) + +const Schemas = { + config: ConfigSchema, +} + +for (const [name, schema] of Object.entries(Schemas)) { + const jsonSchema = zodToJsonSchema(schema, 'mySchema') + await writeFile(`${OUTPUT}/${name}.json`, JSON.stringify(jsonSchema, null, 2), { encoding: 'utf-8' }) +} diff --git a/src/cmd/backup.ts b/src/cmd/backup.ts new file mode 100644 index 0000000..f7d0e9e --- /dev/null +++ b/src/cmd/backup.ts @@ -0,0 +1,14 @@ +import { Log } from '../logger' +import { Context } from '../models/context' + +export async function backup(ctx: Context) { + const log = Log.child({ cmd: 'check' }) + log.trace('starting') + + // Locations + for (const location of ctx.locations) { + await location.backup() + } + + log.trace('done') +} diff --git a/src/cmd/check.ts b/src/cmd/check.ts new file mode 100644 index 0000000..edb2e65 --- /dev/null +++ b/src/cmd/check.ts @@ -0,0 +1,25 @@ +import { unlockRepo, waitForRepo } from '../lock' +import { Log } from '../logger' +import { Context } from '../models/context' +import { isResticAvailable } from '../restic' + +export async function check(ctx: Context) { + const l = Log.child({ cmd: 'check' }) + l.trace('starting') + + // Restic + isResticAvailable() + + // Repos + for (const repo of ctx.repos) { + await waitForRepo(ctx, repo.name) + try { + await repo.init() + await repo.check() + } finally { + unlockRepo(ctx, repo.name) + } + } + + l.trace('done') +} diff --git a/src/config/env/file.test.ts b/src/config/env/file.test.ts new file mode 100644 index 0000000..44ea81f --- /dev/null +++ b/src/config/env/file.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, test } from 'bun:test' +import { InvalidEnvFileLine } from '../../errors' +import { parseFile } from './file' + +describe('env file', () => { + test('simple', () => { + expect(parseFile(`test_foo=ok`)).toEqual({ test_foo: 'ok' }) + }) + + test('multiple values', () => { + expect(parseFile(`test_foo=ok\n \n spacing = foo \n`)).toEqual({ test_foo: 'ok', spacing: 'foo' }) + }) + + test('invalid: key', () => { + expect(() => parseFile(`a=123\na f=ok`)).toThrow(new InvalidEnvFileLine('a f=ok')) + }) + + test('invalid: missing =', () => { + expect(() => parseFile(`a=123\na ok`)).toThrow(new InvalidEnvFileLine('a ok')) + }) +}) diff --git a/src/config/env/file.ts b/src/config/env/file.ts new file mode 100644 index 0000000..f323904 --- /dev/null +++ b/src/config/env/file.ts @@ -0,0 +1,55 @@ +import { exists, readFile } from 'node:fs/promises' +import { InvalidEnvFileLine } from '../../errors' +import { setByPath } from '../../utils/path' +import { relativePath } from '../resolution' + +export function parseFile(contents: string) { + const variables: Record = {} + const lines = contents + .trim() + .split('\n') + .map((l) => l.trim()) + const matcher = /^\s*(?\w+)\s*=(?.*)$/ + for (const line of lines) { + if (!line) continue + const match = matcher.exec(line) + if (!match) throw new InvalidEnvFileLine(line) + variables[match.groups!.variable] = match.groups!.value.trim() + } + return variables +} + +const PREFIX = 'AUTORESTIC_' + +function envVariableToObjectPath(env: string): string { + if (env.startsWith(PREFIX)) env = env.replace(PREFIX, '') + return ( + env + // Convert to object path + .replaceAll('_', '.') + // Escape the double unterscore. __ -> .. -> _ + .replaceAll('..', '_') + .toLowerCase() + ) +} + +/** + * Fill the config file with the env file variables. + * These take precedence before the config file itself. + */ +export async function enrichConfig(rawConfig: any, path: string) { + const envFilePath = relativePath(path, '.autorestic.env') + let variables: Record = {} + + if (await exists(envFilePath)) { + const envFile = parseFile(await readFile(envFilePath, 'utf-8')) + Object.assign(variables, envFile) + } + + Object.assign(variables, process.env) + + for (const [key, value] of Object.entries(variables)) { + if (!key.startsWith(PREFIX)) continue + setByPath(rawConfig, envVariableToObjectPath(key), value) + } +} diff --git a/src/config/index.ts b/src/config/index.ts new file mode 100644 index 0000000..5b45968 --- /dev/null +++ b/src/config/index.ts @@ -0,0 +1,29 @@ +import { exists, readFile } from 'node:fs/promises' +import yaml from 'yaml' +import { ConfigFileNotFound, CustomError, InvalidConfigFile } from '../errors' +import { enrichConfig } from './env/file' +import { autoLocateConfig } from './resolution' +import { Config, ConfigWithMetaSchema } from './schema/config' +import { basename } from 'node:path' + +export async function loadConfig(customPath?: string): Promise { + let path: string + if (customPath) { + path = customPath + if (!(await exists(path))) throw new ConfigFileNotFound([path]) + } else { + path = await autoLocateConfig() + } + + const rawConfig = await readFile(path, 'utf-8') + const config = yaml.parse(rawConfig) + await enrichConfig(config, path) + config.meta = { path: basename(path) } + const parsed = ConfigWithMetaSchema.safeParse(config) + if (!parsed.success) + throw new InvalidConfigFile(parsed.error.errors.map((e) => `${e.path.join(' > ')}: ${e.message}`)) + + // Check for semantics + + return parsed.data +} diff --git a/src/config/resolution.ts b/src/config/resolution.ts new file mode 100644 index 0000000..0dc742f --- /dev/null +++ b/src/config/resolution.ts @@ -0,0 +1,23 @@ +import { exists } from 'node:fs/promises' +import { dirname, isAbsolute, join, resolve } from 'node:path' +import { ConfigFileNotFound } from '../errors' + +const DEFAULT_DIRS = ['./', '~/', '~/.config/autorestic'] +const FILENAMES = ['.autorestic.yaml', '.autorestic.yml', '.autorestic.json'] + +export async function autoLocateConfig(): Promise { + const paths = DEFAULT_DIRS + const xdgHome = process.env['XDG_CONFIG_HOME'] + if (xdgHome) paths.push(xdgHome) + for (const path in paths) { + for (const filename in FILENAMES) { + const file = join(path, filename) + if (await exists(file)) return file + } + } + throw new ConfigFileNotFound(paths) +} + +export function relativePath(base: string, path: string): string { + return isAbsolute(path) ? path : resolve(base, path) +} diff --git a/src/config/schema/common.ts b/src/config/schema/common.ts new file mode 100644 index 0000000..d0a6f01 --- /dev/null +++ b/src/config/schema/common.ts @@ -0,0 +1,20 @@ +import { ZodTypeAny, z } from 'zod' +import { Log } from '../../logger' + +export const NonEmptyString = z + .string() + .min(1) + // Extrapolate env variables from a string + .transform((s) => { + return s.replaceAll(/\$(\w+)|\${(\w+)}/g, (_, g0, g1) => { + const variable = g0 || g1 + const value = process.env[variable] ?? '' + if (!value) Log.error(`cannot find environment variable "${variable}" to replace in ${s}`) + return value + }) + }) + .describe('non-empty string that can extrapolate env variables inside it') + +export function OptionallyArray(type: T) { + return z.union([type, z.array(type).min(1)]) +} diff --git a/src/config/schema/config.ts b/src/config/schema/config.ts new file mode 100644 index 0000000..3c86e9a --- /dev/null +++ b/src/config/schema/config.ts @@ -0,0 +1,69 @@ +import { z } from 'zod' +import { asArray } from '../../utils/array' +import { RepositorySchema } from './repository' +import { NonEmptyString } from './common' +import { LocationSchema } from './location' +import { OptionsSchema } from './options' + +export const ConfigSchema = z.strictObject({ + version: z.number().describe('version number'), + repos: z.record(NonEmptyString.describe('repository name'), RepositorySchema).describe('available repositories'), + locations: z.record(NonEmptyString.describe('location name'), LocationSchema).describe('available locations'), + global: z + .strictObject({ + options: OptionsSchema.optional(), + }) + .describe('global configuration') + .optional(), + extras: z.any().optional(), +}) + +const ConfigMeta = z + .strictObject({ + path: NonEmptyString.describe('The path of the loaded config'), + }) + .describe('Meta information about the config') + +export const ConfigWithMetaSchema = ConfigSchema.extend({ + meta: ConfigMeta, +}).superRefine((config, ctx) => { + const availableRepos = Object.keys(config.repos) + for (const [name, location] of Object.entries(config.locations)) { + const locationPath = [...ctx.path, 'locations', name] + const toRepos = asArray(location.to) + // Check if all target repos are valid + for (const to of toRepos) { + if (!availableRepos.includes(to)) { + const message = `location "${name}" has an invalid repository "${to}"` + ctx.addIssue({ message, code: 'custom', path: [...locationPath, 'to'] }) + } + } + // Check copy field + if (!location.copy) continue + for (const [source, destinations] of Object.entries(location.copy)) { + const path = [...locationPath, 'copy', source] + if (!toRepos.includes(source)) + ctx.addIssue({ + code: 'custom', + path, + message: `copy source "${source}" must be also a backup target`, + }) + for (const destination of asArray(destinations)) { + if (destination === source) + ctx.addIssue({ + code: 'custom', + path: [...path, destination], + message: `destination repository "${destination}" cannot be also the source in copy field`, + }) + if (!availableRepos.includes(destination)) + ctx.addIssue({ + code: 'custom', + path: [...path, destination], + message: `destination repository "${destination}" does not exist`, + }) + } + } + } +}) + +export type Config = z.infer diff --git a/src/config/schema/hooks.ts b/src/config/schema/hooks.ts new file mode 100644 index 0000000..9d5d98a --- /dev/null +++ b/src/config/schema/hooks.ts @@ -0,0 +1,14 @@ +import { z } from 'zod' +import { NonEmptyString, OptionallyArray } from './common' + +const Command = NonEmptyString.describe('command to be executed') +const Commands = OptionallyArray(Command).describe('list of commands') + +export const HooksSchema = z + .strictObject({ + before: Commands.optional(), + after: Commands.optional(), + failure: Commands.optional(), + success: Commands.optional(), + }) + .describe('hooks to be executed') diff --git a/src/config/schema/location.ts b/src/config/schema/location.ts new file mode 100644 index 0000000..dc99893 --- /dev/null +++ b/src/config/schema/location.ts @@ -0,0 +1,28 @@ +import { z } from 'zod' +import { NonEmptyString, OptionallyArray } from './common' +import { HooksSchema } from './hooks' +import { OptionsSchema } from './options' + +export const LocationSchema = z + .strictObject({ + from: OptionallyArray(NonEmptyString.describe('local path to backup')), + to: OptionallyArray(NonEmptyString.describe('repository to backup to')), + copy: z + .record( + NonEmptyString.describe('source repository from which to copy from'), + OptionallyArray(NonEmptyString.describe('destination repository')) + ) + .optional(), + + // adapter: + cron: NonEmptyString.describe('execute backups for the given cron job').optional(), + hooks: HooksSchema.optional(), + options: OptionsSchema.optional(), + forget: z + .union([ + z.boolean().describe('automatically run "forget" when backing up'), + z.literal('prune').describe('also prune when forgetting'), + ]) + .optional(), + }) + .describe('Location') diff --git a/src/config/schema/options.ts b/src/config/schema/options.ts new file mode 100644 index 0000000..17710f4 --- /dev/null +++ b/src/config/schema/options.ts @@ -0,0 +1,15 @@ +import { z } from 'zod' +import { NonEmptyString, OptionallyArray } from './common' + +const OptionSchema = z.record( + NonEmptyString.describe('native restic option'), + z.union([z.literal(true).describe('boolean flag'), OptionallyArray(NonEmptyString)]).describe('value of option') +) + +export const OptionsSchema = z + .strictObject({ + all: OptionSchema.optional(), + backup: OptionSchema.optional(), + forget: OptionSchema.optional(), + }) + .describe('native restic options') diff --git a/src/config/schema/repository.ts b/src/config/schema/repository.ts new file mode 100644 index 0000000..5f3a684 --- /dev/null +++ b/src/config/schema/repository.ts @@ -0,0 +1,18 @@ +import { z } from 'zod' +import { NonEmptyString } from './common' +import { OptionsSchema } from './options' + +export const RepositorySchema = z.strictObject({ + type: z.enum(['local', 'sftp', 'rest', 'swift', 's3', 'b2', 'azure', 'gs', 'rclone']).describe('type of repository'), + path: NonEmptyString.describe('restic path'), + key: NonEmptyString.describe('encryption key for the repository'), + env: z + .record( + NonEmptyString.describe('environment variable'), + NonEmptyString.describe('value of the environment variable') + ) + .transform((env) => Object.fromEntries(Object.entries(env).map(([key, value]) => [key.toUpperCase(), value]))) + .describe('environment variables') + .optional(), + options: OptionsSchema.describe('options').optional(), +}) diff --git a/src/errors/index.ts b/src/errors/index.ts new file mode 100644 index 0000000..3f683fd --- /dev/null +++ b/src/errors/index.ts @@ -0,0 +1,47 @@ +function formatLines(lines: string[]) { + return lines.map((p) => ` ▶ ${p}`).join('\n') +} + +export class CustomError extends Error {} + +export class InvalidEnvFileLine extends CustomError { + constructor(line: string) { + super(`invalid env file line: "${line}"`) + } +} + +export class NotImplemented extends CustomError { + constructor(functionality: string) { + super(`not implemented: ${functionality}`) + } +} + +export class ConfigFileNotFound extends CustomError { + constructor(paths: string[]) { + super(`could not locate config file.\nthe following paths were tried:\n${formatLines(paths)}`) + } +} + +export class InvalidConfigFile extends CustomError { + constructor(errors: string[]) { + super(`could not parse the config file.\n${formatLines(errors)}`) + } +} + +export class BinaryNotAvailable extends CustomError { + constructor(binary: string) { + super(`binary "${binary}" is not available in $PATH`) + } +} + +export class ResticError extends CustomError { + constructor(errors: string[]) { + super(`internal restic error.\n${formatLines(errors)}`) + } +} + +export class LockfileAlreadyLocked extends CustomError { + constructor(repo: string) { + super(`cannot acquire lock for repository "${repo}", already in use`) + } +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..b9cf7cd --- /dev/null +++ b/src/index.ts @@ -0,0 +1,138 @@ +import { Command, Help, Option, program } from '@commander-js/extra-typings' +import { loadConfig } from './config' +import { CustomError, NotImplemented } from './errors' +import { Log, LogLevel, setLevelFromFlag } from './logger' +import { check } from './cmd/check' +import { Context } from './models/context' +import { backup } from './cmd/backup' + +export const helpConfig: Partial = { + showGlobalOptions: true, + sortOptions: true, + sortSubcommands: true, + helpWidth: 1, +} + +program + .name('autorestic') + .description('configuration manager and runner for restic') + .version('2.0.0-alpha.0') + .configureHelp(helpConfig) + .allowExcessArguments(false) + .allowUnknownOption(false) + +// Global options +program.option('-c, --config ', 'specify custom configuration file') +program.option('-v', 'verbosity', (_, previous) => previous + 1, 1) +program.addOption(new Option('--ci', 'CI mode').env('CI').default(false)) + +// Common Options +const specificLocation = new Option('-l, --location ', 'location name, multiple possible') +specificLocation.variadic = true +const allLocations = new Option('-a, --all', 'all locations') +const specificRepo = new Option('-r, --repository ', 'repository name, multiple possible') +specificLocation.variadic = true +const allRepos = new Option('-a, --all', 'all repositories') + +function mergeOptions(local: T, p: Command) { + const globals = p.optsWithGlobals() as { config?: string; verbosity: number; ci: boolean } + return { + ...globals, + ...local, + } +} + +program.hook('preAction', (command) => { + // @ts-ignore + const v: number = command.opts().v + setLevelFromFlag(v) +}) + +program + .command('check') + .description('check if the config is valid and sets up the repositories') + .configureHelp(helpConfig) + .action(async (options, p) => { + const merged = mergeOptions(options, p) + const config = await loadConfig(merged.config) + const ctx = new Context(config) + await check(ctx) + }) + +program + .command('backup') + .description('create backups') + .configureHelp(helpConfig) + .addOption(specificLocation) + .addOption(allLocations) + .action(async (options, p) => { + // throw new NotImplemented('backup') + const merged = mergeOptions(options, p) + const config = await loadConfig(merged.config) + const ctx = new Context(config) + await backup(ctx) + }) + +program + .command('exec') + .description('execute arbitrary native restic commands for given repositories') + .configureHelp(helpConfig) + .addOption(specificRepo) + .addOption(allRepos) + .allowExcessArguments(true) + .action((options, p) => { + throw new NotImplemented('exec') + }) + +program + .command('forget') + .description('forget snapshots according to the specified policies') + .configureHelp(helpConfig) + .addOption(specificLocation) + .addOption(allLocations) + // Pass natively + // .option('--dry-run', 'do not write changes, show what would be affected') + // .option('--prune', 'also prune repository') + .action((options) => { + throw new NotImplemented('backup') + }) + +program + .command('restore') + .description('restore a snapshot to a given location') + .option('--force', 'overwrite target folder') + .option('--from ', 'repository from which to restore') + .option('--to ', 'path where to restore the data') + .option('-l, --location ', 'location to be restored') + .argument('[snapshot-id]', 'snapshot to be restored. if empty latest will be taken') + .action(() => { + throw new NotImplemented('restore') + }) + +const self = new Command('self').description('utility commands for managing autorestic').configureHelp(helpConfig) +self.command('install').action(() => { + throw new NotImplemented('install') +}) +self.command('uninstall').action(() => { + throw new NotImplemented('uninstall') +}) +self.command('upgrade').action(() => { + throw new NotImplemented('upgrade') +}) +self.command('completion').action(() => { + throw new NotImplemented('completion') +}) +program.addCommand(self) + +try { + await program.parseAsync() +} catch (e) { + if (e instanceof CustomError) { + Log.fatal(e.message) + } else if (e instanceof Error) { + Log.fatal(`unknown error: ${e.message}`) + } + process.exit(1) +} finally { + // TODO: Unlock +} diff --git a/src/lock/index.test.ts b/src/lock/index.test.ts new file mode 100644 index 0000000..0884e54 --- /dev/null +++ b/src/lock/index.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, mock, test, beforeEach } from 'bun:test' +import { lockRepo } from '.' +import { Context } from '../models/context' +import { mkdir, rm } from 'node:fs/promises' + +const mockPath = './test/' +const mockContext: Context = { config: { meta: { path: mockPath } } } as any + +describe('lock', () => { + beforeEach(async () => { + // Cleanup lock file + await rm(mockPath, { recursive: true, force: true }) + await mkdir(mockPath, { recursive: true }) + }) + + test('lock', () => { + lockRepo(mockContext, 'foo') + // lockRepo(mockContext, 'foo') + }) +}) diff --git a/src/lock/index.ts b/src/lock/index.ts new file mode 100644 index 0000000..42da55d --- /dev/null +++ b/src/lock/index.ts @@ -0,0 +1,78 @@ +import { readFileSync, writeFileSync } from 'node:fs' +import yaml from 'yaml' +import { relativePath } from '../config/resolution' +import { LockfileAlreadyLocked } from '../errors' +import { Log } from '../logger' +import { Context } from '../models/context' +import { Lockfile, LockfileSchema } from './schema' +import { wait } from '../utils/time' + +const LOCKFILE = '.autorestic.lock' +const VERSION = 2 +const l = Log.child({ command: 'lock' }) + +function load(ctx: Context): Lockfile { + const defaultLockfile = { version: VERSION, cron: {}, running: {} } + try { + const path = relativePath(ctx.config.meta.path, LOCKFILE) + l.trace('looking for lock file', { path }) + // throw new Error(path) + const rawConfig = readFileSync(path, 'utf-8') + const config = yaml.parse(rawConfig) + const parsed = LockfileSchema.safeParse(config) + if (!parsed.success) return defaultLockfile + if (parsed.data.version < VERSION) { + l.debug('lockfile is old and will be overwritten') + return defaultLockfile + } + return parsed.data + } catch { + return defaultLockfile + } +} + +function write(ctx: Context, lockfile: Lockfile) { + const path = relativePath(ctx.config.meta.path, LOCKFILE) + writeFileSync(path, yaml.stringify(lockfile), 'utf-8') +} + +export function lockRepo(ctx: Context, repo: string) { + const lock = load(ctx) + l.trace('trying to lock repository', { repo }) + if (lock.running[repo]) throw new LockfileAlreadyLocked(repo) + lock.running[repo] = true + write(ctx, lock) +} + +/** + * Waits for a repo to become unlocked, and errors if it does not succeed in the given timeout. + * + * @param [timeout=10] max seconds to wait for repo to become unlocked + */ +export async function waitForRepo(ctx: Context, repo: string, timeout = 10) { + const now = Date.now() + while (Date.now() - now < timeout * 1_000) { + try { + lockRepo(ctx, repo) + l.trace('repo is free again', { repo }) + break + } catch { + l.trace('waiting for repo to be unlocked', { repo }) + await wait(0.1) // Wait for 100ms + } + } + throw new LockfileAlreadyLocked(repo) +} + +export function updateLastRun(ctx: Context, location: string) { + const lock = load(ctx) + lock.cron[location] = Date.now() + write(ctx, lock) +} + +export function unlockRepo(ctx: Context, repo: string) { + l.trace('unlocking repository', { repo }) + const lock = load(ctx) + lock.running[repo] = false + write(ctx, lock) +} diff --git a/src/lock/schema.ts b/src/lock/schema.ts new file mode 100644 index 0000000..8bcc6ce --- /dev/null +++ b/src/lock/schema.ts @@ -0,0 +1,13 @@ +import { z } from 'zod' + +export const LockfileSchema = z.strictObject({ + version: z.number().min(0).describe('lockfile version'), + running: z + .record(z.string().describe('repository'), z.boolean().describe('whether repository is running')) + .describe('running information for each repository'), + cron: z + .record(z.string().describe('location'), z.number().describe('timestamp of last backup')) + .describe('information about last run for a given location. in milliseconds'), +}) + +export type Lockfile = z.infer diff --git a/src/logger.ts b/src/logger.ts new file mode 100644 index 0000000..53306ad --- /dev/null +++ b/src/logger.ts @@ -0,0 +1,37 @@ +import Pino, { type LoggerOptions } from 'pino' +import Pretty from 'pino-pretty' + +// https://getpino.io/#/docs/api?id=loggerlevel-string-gettersetter +export enum LogLevel { + Trace = 'trace', + Debug = 'debug', + Info = 'info', + Warn = 'warn', + Error = 'error', + Fatal = 'fatal', + Silent = 'silent', +} + +const pretty = !process.env.CI +const options: LoggerOptions = { + base: undefined, + level: LogLevel.Info, +} + +export const Log = pretty ? Pino(options, Pretty({ colorize: true })) : Pino(options) + +export function setLevelFromFlag(flag: number) { + switch (flag) { + case 1: + Log.level = LogLevel.Info + break + case 2: + Log.level = LogLevel.Debug + break + case 3: + Log.level = LogLevel.Trace + break + default: + Log.error('invalid logging level') + } +} diff --git a/src/models/context.ts b/src/models/context.ts new file mode 100644 index 0000000..5c31235 --- /dev/null +++ b/src/models/context.ts @@ -0,0 +1,25 @@ +import { Config } from '../config/schema/config' +import { CustomError } from '../errors' +import { Location } from './location' +import { Repository } from './repository' + +export class Context { + repos: Repository[] + locations: Location[] + + constructor(public config: Config) { + this.repos = Object.entries(config.repos).map(([name, r]) => new Repository(this, name, r)) + this.locations = Object.entries(config.locations).map(([name, l]) => new Location(this, name, l)) + } + + getRepo(name: string) { + const repo = this.repos.find((r) => r.name === name) + if (!repo) throw new CustomError(`could not find backend "${name}"`) + return repo + } + getLocation(name: string) { + const location = this.locations.find((l) => l.name === name) + if (!location) throw new CustomError(`could not find location "${name}"`) + return location + } +} diff --git a/src/models/location.ts b/src/models/location.ts new file mode 100644 index 0000000..acbafcb --- /dev/null +++ b/src/models/location.ts @@ -0,0 +1,28 @@ +import { Logger } from 'pino' +import { z } from 'zod' +import { LocationSchema } from '../config/schema/location' +import { Log } from '../logger' +import { asArray } from '../utils/array' +import { Context } from './context' +import { execute } from '../restic' + +export class Location { + l: Logger + + constructor(public ctx: Context, public name: string, public data: z.infer) { + this.l = Log.child({ location: name }) + } + + async backup() { + this.l.trace('backing up location') + for (const name of asArray(this.data.to)) { + const repo = this.ctx.getRepo(name) + this.l.debug(repo.name) + await execute({ + command: 'restic', + args: ['backup', '--dry-run'], + env: repo.env, + }) + } + } +} diff --git a/src/models/repository.ts b/src/models/repository.ts new file mode 100644 index 0000000..ea20f49 --- /dev/null +++ b/src/models/repository.ts @@ -0,0 +1,66 @@ +import { Logger } from 'pino' +import { z } from 'zod' +import { relativePath } from '../config/resolution' +import { Config } from '../config/schema/config' +import { RepositorySchema } from '../config/schema/repository' +import { ResticError } from '../errors' +import { Log } from '../logger' +import { execute } from '../restic' +import { Context } from './context' + +export class Repository { + l: Logger + + constructor(public ctx: Context, public name: string, public data: z.infer) { + this.l = Log.child({ repository: this.name }) + } + + get repository(): string { + switch (this.data.type) { + case 'local': + return relativePath(this.ctx.config.meta.path, this.data.path) + case 'b2': + case 'azure': + case 'gs': + case 's3': + case 'sftp': + case 'rclone': + case 'swift': + case 'rest': + return `${this.data.type}:${this.data.path}` + break + } + } + + get env() { + return { + ...this.data.env, + RESTIC_PASSWORD: this.data.key, + RESTIC_REPOSITORY: this.repository, + } + } + + /** + * true if initialized + * false if already initialized + */ + async init(): Promise { + this.l.trace('initializing') + const output = await execute({ command: 'restic', args: ['init'], env: this.env }) + if (!output.ok) { + if (output.stderr.includes('config file already exists')) { + this.l.debug('already initialized') + return false + } + throw new ResticError([output.stderr]) + } + this.l.debug('initialized repository') + return true + } + + async check() { + this.l.trace('checking') + const output = await execute({ command: 'restic', args: ['check'], env: this.env }) + if (!output.ok) throw new ResticError(['could not check repository', output.stderr]) + } +} diff --git a/src/restic/index.ts b/src/restic/index.ts new file mode 100644 index 0000000..bf8dd53 --- /dev/null +++ b/src/restic/index.ts @@ -0,0 +1,44 @@ +import { execFile } from 'node:child_process' +import { Log } from '../logger' +import { BinaryNotAvailable } from '../errors' + +export type ExecutionContext = { + command: string + args?: string[] + env?: Record +} +export async function execute({ + env, + args, + command, +}: ExecutionContext): Promise<{ code: number; stderr: string; stdout: string; ok: boolean }> { + return new Promise((resolve) => { + execFile(command, args ?? [], { env }, (err, stdout, stderr) => { + const code = err?.code ?? 0 + resolve({ + code, + ok: code === 0, + stderr, + stdout, + }) + }) + }) +} + +export async function isBinaryAvailable(command: string): Promise { + const l = Log.child({ command }) + try { + l.trace('checking if command is installed') + const result = await execute({ command }) + return result.ok + } catch { + l.trace('not installed') + return false + } +} + +export async function isResticAvailable() { + const bin = 'restic' + const installed = await isBinaryAvailable(bin) + if (!installed) throw new BinaryNotAvailable(bin) +} diff --git a/src/utils/array.test.ts b/src/utils/array.test.ts new file mode 100644 index 0000000..0105bf3 --- /dev/null +++ b/src/utils/array.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, test } from 'bun:test' +import { isSubset } from './array' + +describe('set theory', () => { + test('subset', () => { + expect(isSubset([1], [1, 2])).toBe(true) + expect(isSubset([1], [2])).toBe(false) + expect(isSubset([], [])).toBe(true) + expect(isSubset([1, 2, 3], [1, 2])).toBe(false) + expect(isSubset([1, 2], [1, 2])).toBe(true) + }) +}) diff --git a/src/utils/array.ts b/src/utils/array.ts new file mode 100644 index 0000000..5fef293 --- /dev/null +++ b/src/utils/array.ts @@ -0,0 +1,7 @@ +export function asArray(singleOrArray: T | T[]): T[] { + return Array.isArray(singleOrArray) ? singleOrArray : [singleOrArray] +} + +export function isSubset(subset: T[], set: T[]): boolean { + return subset.every((v) => set.includes(v)) +} diff --git a/src/utils/path.test.ts b/src/utils/path.test.ts new file mode 100644 index 0000000..1762de7 --- /dev/null +++ b/src/utils/path.test.ts @@ -0,0 +1,33 @@ +import { expect, test, describe } from 'bun:test' +import { setByPath } from './path' + +describe('set by path', () => { + test('simple', () => { + expect(setByPath({}, 'a', true)).toEqual({ a: true }) + expect(setByPath({}, 'f', { ok: true })).toEqual({ f: { ok: true } }) + expect(setByPath([], '0', true)).toEqual([true]) + expect(setByPath([], '2', false)).toEqual([undefined, undefined, false]) + }) + + test('object', () => { + expect(setByPath({}, 'a.b', true)).toEqual({ a: { b: true } }) + expect(setByPath({}, 'a.b.c', true)).toEqual({ a: { b: { c: true } } }) + expect(setByPath({ a: true }, 'b', false)).toEqual({ a: true, b: false }) + expect(setByPath({ a: { b: true } }, 'a.c', false)).toEqual({ a: { b: true, c: false } }) + + expect(() => setByPath({ a: 'foo' }, 'a.b', true)).toThrow() + expect(setByPath({ a: 'foo' }, 'a', true)).toEqual({ a: true }) + }) + + test('array', () => { + expect(() => setByPath([], 'a', true)).toThrow() + expect(setByPath([], '0', true)).toEqual([true]) + expect(setByPath([], '0.0.0', true)).toEqual([[[true]]]) + expect(setByPath([], '0.1.2', true)).toEqual([[undefined, [undefined, undefined, true]]]) + }) + + test('mixed', () => { + expect(setByPath({ items: [] }, 'items.0.name', 'John')).toEqual({ items: [{ name: 'John' }] }) + expect(setByPath([], '0.name', 'John')).toEqual([{ name: 'John' }]) + }) +}) diff --git a/src/utils/path.ts b/src/utils/path.ts new file mode 100644 index 0000000..64528e3 --- /dev/null +++ b/src/utils/path.ts @@ -0,0 +1,24 @@ +function parseKey(key: any) { + const asNumber = parseInt(key) + const isString = isNaN(asNumber) + return [isString ? key : asNumber, isString] +} + +export function setByPath(source: object, path: string, value: unknown) { + const segments = path.split('.') + const last = segments.length - 1 + let node: any = source + for (const [i, segment] of segments.entries()) { + const [key, isString] = parseKey(segment) + if (Array.isArray(node) && isString) throw new Error(`array require a numeric index`) + if (typeof node !== 'object') throw new Error(`could not set path "${segment}" on ${node}.`) + if (i === last) { + node[key] = value + } else { + const [_, isNextString] = parseKey(segments[i + 1]) + if (node[key] === undefined) node[key] = isNextString ? {} : [] + node = node[key] + } + } + return source +} diff --git a/src/utils/time.ts b/src/utils/time.ts new file mode 100644 index 0000000..e6cadb3 --- /dev/null +++ b/src/utils/time.ts @@ -0,0 +1,3 @@ +export function wait(seconds: number): Promise { + return new Promise((resolve) => setTimeout(resolve, seconds * 1000)) +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..8b61f2b --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "./node_modules/bun-types/tsconfig.json", + "compilerOptions": { + "strict": true, + "noFallthroughCasesInSwitch": true, + "noEmit": true + } +}