From 8c8d48ccde2de9824eacf7a05054e943344b7663 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Thu, 20 Jun 2019 23:09:47 +0200 Subject: [PATCH] source --- src/autorestic.ts | 41 ++++++++++ src/backend.ts | 57 ++++++++++++++ src/backup.ts | 37 +++++++++ src/config.ts | 58 ++++++++++++++ src/handlers.ts | 196 ++++++++++++++++++++++++++++++++++++++++++++++ src/types.ts | 70 +++++++++++++++++ src/utils.ts | 45 +++++++++++ 7 files changed, 504 insertions(+) create mode 100644 src/autorestic.ts create mode 100644 src/backend.ts create mode 100644 src/backup.ts create mode 100644 src/config.ts create mode 100644 src/handlers.ts create mode 100644 src/types.ts create mode 100644 src/utils.ts diff --git a/src/autorestic.ts b/src/autorestic.ts new file mode 100644 index 0000000..a073416 --- /dev/null +++ b/src/autorestic.ts @@ -0,0 +1,41 @@ +import 'colors' +import minimist from 'minimist' +import { resolve } from 'path' + +import { init } from './config' +import handlers, { error, help } from './handlers' +import { Config } from './types' + + +process.on('uncaughtException', err => { + console.log(err.message) + process.exit(1) +}) + +export const { _: commands, ...flags } = minimist(process.argv.slice(2), { + alias: { + 'c': 'config', + 'v': 'verbose', + 'h': 'help', + 'a': 'all', + 'l': 'location', + 'b': 'backend', + }, + boolean: ['a'], + string: ['l', 'b'], +}) + +export const DEFAULT_CONFIG = 'config.yml' +export const INSTALL_DIR = '/usr/local/bin' +export const CONFIG_FILE: string = resolve(flags.config || DEFAULT_CONFIG) +export const VERBOSE = flags.verbose + +export const config: Config = init() + +if (commands.length < 1) + help() +else { + const command: string = commands[0] + const args: string[] = commands.slice(1) + ;(handlers[command] || error)(args, flags) +} diff --git a/src/backend.ts b/src/backend.ts new file mode 100644 index 0000000..7a4da2d --- /dev/null +++ b/src/backend.ts @@ -0,0 +1,57 @@ +import { Writer } from 'clitastic' + +import { config, VERBOSE } from './autorestic' +import { Backend, Backends } from './types' +import { exec } from './utils' + +const ALREADY_EXISTS = /(?=.*exists)(?=.*already)(?=.*config).*/ + + +export const getPathFromBackend = (backend: Backend): string => { + switch (backend.type) { + case 'local': + return backend.path + case 'b2': + case 'azure': + case 'gs': + case 's3': + return `${backend.type}:${backend.path}` + case 'sftp': + case 'rest': + throw new Error(`Unsupported backend type: "${backend.type}"`) + default: + throw new Error(`Unknown backend type.`) + } +} + + +export const getEnvFromBackend = (backend: Backend) => { + const { type, path, key, ...rest } = backend + return { + RESTIC_PASSWORD: key, + RESTIC_REPOSITORY: getPathFromBackend(backend), + ...rest, + } +} + + +export const checkAndConfigureBackend = (name: string, backend: Backend) => { + const writer = new Writer(name.blue + ' : ' + 'Configuring... ⏳') + const env = getEnvFromBackend(backend) + + const { out, err } = exec('restic', ['init'], { env }) + + if (err.length > 0 && !ALREADY_EXISTS.test(err)) + throw new Error(`Could not load the backend "${name}": ${err}`) + + if (VERBOSE && out.length > 0) console.log(out) + + writer.done(name.blue + ' : ' + 'Done ✓'.green) +} + + +export const checkAndConfigureBackends = (backends: Backends = config.backends) => { + console.log('\nConfiguring Backends'.grey.underline) + for (const [name, backend] of Object.entries(backends)) + checkAndConfigureBackend(name, backend) +} \ No newline at end of file diff --git a/src/backup.ts b/src/backup.ts new file mode 100644 index 0000000..352aa89 --- /dev/null +++ b/src/backup.ts @@ -0,0 +1,37 @@ +import { Writer } from 'clitastic' + +import { config, VERBOSE } from './autorestic' +import { getEnvFromBackend } from './backend' +import { Locations, Location } from './types' +import { exec } from './utils' + + +export const backupSingle = (name: string, from: string, to: string) => { + const writer = new Writer(name + to.blue + ' : ' + 'Backing up... ⏳') + const backend = config.backends[to] + const cmd = exec('restic', ['backup', from], { env: getEnvFromBackend(backend) }) + + if (VERBOSE) console.log(cmd.out, cmd.err) + writer.done(name + to.blue + ' : ' + 'Done ✓'.green) +} + + +export const backupLocation = (name: string, backup: Location) => { + const display = name.yellow + ' ▶ ' + if (Array.isArray(backup.to)) { + let first = true + for (const t of backup.to) { + const nameOrBlankSpaces: string = first ? display : new Array(name.length + 3).fill(' ').join('') + backupSingle(nameOrBlankSpaces, backup.from, t) + if (first) first = false + } + } else + backupSingle(display, backup.from, backup.to) +} + + +export const backupAll = (backups: Locations = config.locations) => { + console.log('\nBacking Up'.underline.grey) + for (const [name, backup] of Object.entries(backups)) + backupLocation(name, backup) +} \ No newline at end of file diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..4901a4a --- /dev/null +++ b/src/config.ts @@ -0,0 +1,58 @@ +import { readFileSync, writeFileSync } from 'fs' +import yaml from 'js-yaml' +import { CONFIG_FILE } from './autorestic' +import { Backend, Config } from './types' +import { makeObjectKeysLowercase, rand } from './utils' + + +export const normalizeAndCheckBackends = (config: Config) => { + config.backends = makeObjectKeysLowercase(config.backends) + + for (const [name, { type, path, key, ...rest }] of Object.entries(config.backends)) { + + if (!type || !path) throw new Error(`The backend "${name}" is missing some required attributes`) + + const tmp: any = { + type, + path, + key: key || rand(128), + } + for (const [key, value] of Object.entries(rest)) + tmp[key.toUpperCase()] = value + + config.backends[name] = tmp as Backend + } +} + + +export const normalizeAndCheckBackups = (config: Config) => { + config.locations = makeObjectKeysLowercase(config.locations) + const backends = Object.keys(config.backends) + + const checkDestination = (backend: string, backup: string) => { + if (!backends.includes(backend)) + throw new Error(`Cannot find the backend "${backend}" for "${backup}"`) + } + + for (const [name, { from, to, ...rest }] of Object.entries(config.locations)) { + if (!from || !to) throw new Error(`The backup "${name}" is missing some required attributes`) + + if (Array.isArray(to)) + for (const t of to) + checkDestination(t, name) + else + checkDestination(to, name) + } +} + + +export const init = (): Config => { + const raw: Config = makeObjectKeysLowercase(yaml.safeLoad(readFileSync(CONFIG_FILE).toString())) + + normalizeAndCheckBackends(raw) + normalizeAndCheckBackups(raw) + + writeFileSync(CONFIG_FILE, yaml.safeDump(raw)) + + return raw +} diff --git a/src/handlers.ts b/src/handlers.ts new file mode 100644 index 0000000..d2c8b18 --- /dev/null +++ b/src/handlers.ts @@ -0,0 +1,196 @@ +import axios from 'axios' +import { Writer } from 'clitastic' +import { createWriteStream, unlinkSync } from 'fs' +import { arch, platform, tmpdir } from 'os' +import { join, resolve } from 'path' + +import { config, INSTALL_DIR } from './autorestic' +import { checkAndConfigureBackends, getEnvFromBackend } from './backend' +import { backupAll } from './backup' +import { Backends, Flags, Locations } from './types' +import { checkIfCommandIsAvailable, checkIfResticIsAvailable, exec, filterObjectByKey, singleToArray } from './utils' + +export type Handlers = { [command: string]: (args: string[], flags: Flags) => void } + +const parseBackend = (flags: Flags): Backends => { + if (!flags.all && !flags.backend) + throw new Error('No backends specified.'.red + + '\n--all [-a]\t\t\t\tCheck all.' + + '\n--backend [-b] myBackend\t\tSpecify one or more backend', + ) + if (flags.all) + return config.backends + else { + const backends = singleToArray(flags.backend) + for (const backend of backends) + if (!config.backends[backend]) + throw new Error('Invalid backend: '.red + backend) + return filterObjectByKey(config.backends, backends) + } +} + +const parseLocations = (flags: Flags): Locations => { + if (!flags.all && !flags.location) + throw new Error('No locations specified.'.red + + '\n--all [-a]\t\t\t\tBackup all.' + + '\n--location [-l] site1\t\t\tSpecify one or more locations', + ) + + if (flags.all) { + return config.locations + } else { + const locations = singleToArray(flags.location) + for (const location of locations) + if (!config.locations[location]) + throw new Error('Invalid location: '.red + location) + return filterObjectByKey(config.locations, locations) + } +} + +const handlers: Handlers = { + check(args, flags) { + checkIfResticIsAvailable() + const backends = parseBackend(flags) + checkAndConfigureBackends(backends) + }, + backup(args, flags) { + checkIfResticIsAvailable() + const locations: Locations = parseLocations(flags) + + const backends = new Set() + for (const to of Object.values(locations).map(location => location.to)) + Array.isArray(to) ? to.forEach(t => backends.add(t)) : backends.add(to) + + checkAndConfigureBackends(filterObjectByKey(config.backends, Array.from(backends))) + backupAll(locations) + + console.log('\nFinished!'.underline + ' 🎉') + }, + restore(args, flags) { + checkIfResticIsAvailable() + const locations = parseLocations(flags) + for (const [name, location] of Object.entries(locations)) { + const w = new Writer(name.green + `\t\tRestoring... ⏳`) + const env = getEnvFromBackend(config.backends[Array.isArray(location.to) ? location.to[0] : location.to]) + + exec( + 'restic', + ['restore', 'latest', '--path', resolve(location.from), ...args], + { env }, + ) + w.done(name.green + '\t\tDone 🎉') + } + }, + exec(args, flags) { + checkIfResticIsAvailable() + const backends = parseBackend(flags) + for (const [name, backend] of Object.entries(backends)) { + console.log(`\n${name}:\n`.grey.underline) + const env = getEnvFromBackend(backend) + + const { out, err } = exec('restic', args, { env }) + console.log(out, err) + } + }, + async install() { + try { + checkIfResticIsAvailable() + console.log('Restic is already installed') + return + } catch (e) { + } + + const w = new Writer('Checking latest version... ⏳') + checkIfCommandIsAvailable('bzip2') + const { data: json } = await axios({ + method: 'get', + url: 'https://api.github.com/repos/restic/restic/releases/latest', + responseType: 'json', + }) + + const archMap: { [a: string]: string } = { + 'x32': '386', + 'x64': 'amd64', + } + + w.replaceLn('Downloading binary... 🌎') + const name = `${json.name.replace(' ', '_')}_${platform()}_${archMap[arch()]}.bz2` + const dl = json.assets.find((asset: any) => asset.name === name) + if (!dl) return console.log( + 'Cannot get the right binary.'.red, + 'Please see https://bit.ly/2Y1Rzai', + ) + + const { data: file } = await axios({ + method: 'get', + url: dl.browser_download_url, + responseType: 'stream', + }) + + const from = join(tmpdir(), name) + const to = from.slice(0, -4) + + w.replaceLn('Decompressing binary... 📦') + const stream = createWriteStream(from) + await new Promise(res => { + const writer = file.pipe(stream) + writer.on('close', res) + }) + stream.close() + + w.replaceLn(`Moving to ${INSTALL_DIR} 🚙`) + // TODO: Native bz2 + // Decompress + exec('bzip2', ['-dk', from]) + // Remove .bz2 + exec('chmod', ['+x', to]) + exec('mv', [to, INSTALL_DIR + '/restic']) + + unlinkSync(from) + + w.done(`\nFinished! restic is installed under: ${INSTALL_DIR}`.underline + ' 🎉') + }, + uninstall() { + try { + unlinkSync(INSTALL_DIR + '/restic') + console.log(`Finished! restic was uninstalled`) + } catch (e) { + console.log('restic is already uninstalled'.red) + } + }, + update() { + checkIfResticIsAvailable() + const w = new Writer('Checking for new restic version... ⏳') + exec('restic', ['self-update']) + w.done('All up to date! 🚀') + }, +} + +export const help = () => { + console.log('\nAutorestic'.blue + ' - Easy Restic CLI Utility' + + '\n' + + '\nOptions:'.yellow + + '\n -c, --config [default=config.yml] Specify config file' + + '\n' + + '\nCommands:'.yellow + + '\n check [-b, --backend] [-a, --all] Check backends' + + '\n backup [-l, --location] [-a, --all] Backup all or specified locations' + + '\n restore [-l, --location] [-- --target ] Check backends' + + '\n' + + '\n exec [-b, --backend] [-a, --all] -- [native options] Execute native restic command' + + '\n' + + '\n install install restic' + + '\n uninstall uninstall restic' + + '\n update update restic' + + '\n help Show help' + + '\n' + + '\nExamples: '.yellow + 'https://git.io/fjVbg' + + '\n', + ) +} +export const error = () => { + help() + console.log(`Invalid Command:`.red.underline, `${process.argv.slice(2).join(' ')}`) +} + +export default handlers \ No newline at end of file diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..607b995 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,70 @@ +type BackendLocal = { + type: 'local', + key: string, + path: string +} + +type BackendSFTP = { + type: 'sftp', + key: string, + path: string, + password?: string, +} + +type BackendREST = { + type: 'rest', + key: string, + path: string, + user?: string, + password?: string +} + +type BackendS3 = { + type: 's3', + key: string, + path: string, + aws_access_key_id: string, + aws_secret_access_key: string, +} + +type BackendB2 = { + type: 'b2', + key: string, + path: string, + b2_account_id: string, + b2_account_key: string +} + +type BackendAzure = { + type: 'azure', + key: string, + path: string, + azure_account_name: string, + azure_account_key: string +} + +type BackendGS = { + type: 'gs', + key: string, + path: string, + google_project_id: string, + google_application_credentials: string +} + +export type Backend = BackendAzure | BackendB2 | BackendGS | BackendLocal | BackendREST | BackendS3 | BackendSFTP + +export type Backends = { [name: string]: Backend } + +export type Location = { + from: string, + to: string | string[] +} + +export type Locations = { [name: string]: Location } + +export type Config = { + locations: Locations + backends: Backends +} + +export type Flags = { [arg: string]: any } \ No newline at end of file diff --git a/src/utils.ts b/src/utils.ts new file mode 100644 index 0000000..13e1b53 --- /dev/null +++ b/src/utils.ts @@ -0,0 +1,45 @@ +import { spawnSync, SpawnSyncOptions } from 'child_process' +import { randomBytes } from 'crypto' + +export const exec = (command: string, args: string[], { env, ...rest }: SpawnSyncOptions = {}) => { + + const cmd = spawnSync(command, args, { + ...rest, + env: { + ...process.env, + ...env, + }, + }) + + const out = cmd.stdout && cmd.stdout.toString().trim() + const err = cmd.stderr && cmd.stderr.toString().trim() + + return { out, err } +} + +export const checkIfResticIsAvailable = () => checkIfCommandIsAvailable( + 'restic', + 'Restic is not installed'.red + ' https://restic.readthedocs.io/en/latest/020_installation.html#stable-releases', +) + +export const checkIfCommandIsAvailable = (cmd: string, errorMsg?: string) => { + if (require('child_process').spawnSync(cmd).error) + throw new Error(errorMsg ? errorMsg : `"${errorMsg}" is not installed`.red) +} + +export const makeObjectKeysLowercase = (object: Object): any => + Object.fromEntries( + Object.entries(object) + .map(([key, value]) => [key.toLowerCase(), value]), + ) + + +export function rand(length = 32): string { + return randomBytes(length / 2).toString('hex') +} + +export const singleToArray = (singleOrArray: T | T[]): T[] => Array.isArray(singleOrArray) ? singleOrArray : [singleOrArray] + +export const filterObject = (obj: { [key: string]: T }, filter: (item: [string, T]) => boolean): { [key: string]: T } => Object.fromEntries(Object.entries(obj).filter(filter)) + +export const filterObjectByKey = (obj: { [key: string]: T }, keys: string[]) => filterObject(obj, ([key]) => keys.includes(key)) \ No newline at end of file