From d6bfd63deb07205c56a129c9f0fafeba6f0912bb Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 14 Jan 2022 15:13:28 -0500 Subject: [PATCH] feat(database): Implement database migration flow for App * refactor bot init to include removing any old running ones (out of server and into app) * run database prep before init bots --- src/App.ts | 97 ++++++++++++++++++++++++++++++++++++-- src/Common/interfaces.ts | 14 +++++- src/ConfigBuilder.ts | 12 +++-- src/Utils/databaseUtils.ts | 10 ++-- src/Web/Server/server.ts | 31 ++++++------ 5 files changed, 136 insertions(+), 28 deletions(-) diff --git a/src/App.ts b/src/App.ts index ebc4ce0..516c04d 100644 --- a/src/App.ts +++ b/src/App.ts @@ -1,21 +1,31 @@ import winston, {Logger} from "winston"; import dayjs, {Dayjs} from "dayjs"; import {getLogger} from "./Utils/loggerFactory"; -import {Invokee, OperatorConfig} from "./Common/interfaces"; +import {DatabaseConfig, DatabaseMigrationOptions, Invokee, OperatorConfig} from "./Common/interfaces"; import Bot from "./Bot"; import LoggedError from "./Utils/LoggedError"; -import {sleep} from "./util"; +import {mergeArr, sleep} from "./util"; +import {copyFile} from "fs/promises"; +import {constants} from "fs"; +import {Connection} from "typeorm"; export class App { - bots: Bot[] + bots: Bot[] = []; logger: Logger; + dbLogger: Logger; + database: Connection startedAt: Dayjs = dayjs(); + ranMigrations: boolean = false; + migrationBlocker?: string; + + config: OperatorConfig; error: any; constructor(config: OperatorConfig) { const { + database, operator: { name, }, @@ -23,12 +33,13 @@ export class App { bots = [], } = config; + this.config = config; this.logger = getLogger(config.logging); + this.dbLogger = this.logger.child({leaf: 'Database'}, mergeArr); + this.database = database; this.logger.info(`Operators: ${name.length === 0 ? 'None Specified' : name.join(', ')}`) - this.bots = bots.map(x => new Bot(x, this.logger)); - process.on('uncaughtException', (e) => { this.error = e; }); @@ -61,7 +72,83 @@ export class App { } } + async doMigration() { + if (this.database.options.type === 'sqljs' && this.database.options.location !== undefined) { + const ts = Date.now(); + const backupLocation = `${this.database.options.location}.${ts}.bak` + this.dbLogger.info(`Detected sqljs (sqlite) database. Will try to make a backup at ${backupLocation} before migrating.`); + try { + await copyFile(this.database.options.location, backupLocation, constants.COPYFILE_EXCL); + this.dbLogger.info('Successfully created backup!'); + } catch (e: any) { + this.dbLogger.error(`Could not create a backup but will continue with migration: ${e.message}`); + } + } + this.dbLogger.info('Beginning migrations...'); + await this.database.runMigrations(); + this.migrationBlocker = undefined; + this.ranMigrations = true; + } + + async initDatabase(confirm: boolean = false) { + const { + databaseConfig: { + migrations: { + force = false, + } = {} + } = {}, + } = this.config; + + this.dbLogger.info('Checking if migrations are required...'); + + const runner = this.database.createQueryRunner(); + const tables = await runner.getTables(); + if (tables.length === 0 || (tables.length === 1 && tables.map(x => x.name).includes('migrations'))) { + this.dbLogger.info('Detected a new database! Starting migrations...'); + await this.database.showMigrations(); + await this.doMigration(); + return true; + } else if (!tables.map(x => x.name).includes('migrations') && !force && !confirm) { + this.dbLogger.warn(`DANGER! Your database has existing tables but none of them include a 'migrations' table. + Are you sure this is the correct database? Continuing with migrations will most likely drop any existing data and recreate all tables.`); + this.migrationBlocker = 'unknownTables'; + return false; + } else if (await this.database.showMigrations()) { + this.dbLogger.info('Detected pending migrations.'); + if (!force && !confirm) { + this.dbLogger.error(`You must confirm migrations. Either set 'force: true' in database config or confirm migrations from web interface. +YOU SHOULD BACKUP YOUR EXISTING DATABASE BEFORE CONTINUING WITH MIGRATIONS.`); + this.migrationBlocker = 'pending'; + return false; + } + if (force && !confirm) { + this.dbLogger.info('Migration was forced'); + } + await this.doMigration(); + return true; + } else { + this.dbLogger.info('No migrations required!'); + this.ranMigrations = true; + return true; + } + } + async initBots(causedBy: Invokee = 'system') { + if(!this.ranMigrations) { + this.logger.error('Must run migrations before starting bots'); + return; + } + + if(this.bots.length > 0) { + this.logger.info('Bots already exist, will stop and destroy these before building new ones.'); + await this.destroy(causedBy); + } + const { + bots = [], + } = this.config; + + this.bots = bots.map(x => new Bot(x, this.logger)); + for (const b of this.bots) { if (b.error === undefined) { try { diff --git a/src/Common/interfaces.ts b/src/Common/interfaces.ts index 3ad2606..10176c6 100644 --- a/src/Common/interfaces.ts +++ b/src/Common/interfaces.ts @@ -1630,7 +1630,10 @@ export interface OperatorJsonConfig { * * Defaults to 'sqljs' which stores data in a file * */ - databaseConfig?: DatabaseDriver | DatabaseConfig + databaseConfig?: { + connection?: DatabaseDriver | DatabaseConfig, + migrations?: DatabaseMigrationOptions + } /** * Set global snoowrap options as well as default snoowrap config for all bots that don't specify their own @@ -1829,7 +1832,10 @@ export interface OperatorConfig extends OperatorJsonConfig { path?: string, }, caching: StrongCache, - databaseConfig: DatabaseConfig + databaseConfig: { + connection: DatabaseConfig, + migrations: DatabaseMigrationOptions + } database: Connection web: { port: number, @@ -2046,3 +2052,7 @@ export interface StringComparisonOptions { lengthWeight?: number, transforms?: ((str: string) => string)[] } + +export interface DatabaseMigrationOptions { + force?: boolean +} diff --git a/src/ConfigBuilder.ts b/src/ConfigBuilder.ts index 4691743..129edb9 100644 --- a/src/ConfigBuilder.ts +++ b/src/ConfigBuilder.ts @@ -586,7 +586,10 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig): path, } = {}, caching: opCache, - databaseConfig = 'sqljs', + databaseConfig: { + connection: dbConnection = 'sqljs', + migrations = {}, + } = {}, web: { port = 8085, maxLogs = 200, @@ -665,7 +668,7 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig): const logger = getLogger(loggingOptions); - const dbConfig = createDatabaseConfig(databaseConfig); + const dbConfig = createDatabaseConfig(dbConnection); const database = await createDatabaseConnection(dbConfig); @@ -837,7 +840,10 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig): }, logging: loggingOptions, caching: cache, - databaseConfig: dbConfig, + databaseConfig: { + connection: dbConfig, + migrations, + }, database, web: { port, diff --git a/src/Utils/databaseUtils.ts b/src/Utils/databaseUtils.ts index 522b385..3f8afc3 100644 --- a/src/Utils/databaseUtils.ts +++ b/src/Utils/databaseUtils.ts @@ -8,7 +8,7 @@ import "reflect-metadata"; import {Connection, createConnection} from "typeorm"; import fs, {promises, constants} from "fs"; import {parse} from 'path'; -import {getLogger} from "./loggerFactory"; +import {getDatabaseLogger, getLogger} from "./loggerFactory"; import SimpleError from "./SimpleError"; export const isDatabaseDriver = (val: any): val is DatabaseDriver => { @@ -100,7 +100,11 @@ export const createDatabaseConnection = async (rawConfig: DatabaseConfig): Promi return await createConnection({ ...config, - synchronize: true, - entities: [`${resolve(__dirname, '../Common/Entities')}/*.js`] + synchronize: false, + entities: [`${resolve(__dirname, '../Common/Entities')}/*.js`], + migrations: [`${resolve(__dirname, '../Common/Migrations')}/*.js`], + migrationsRun: false, + logging: ['error','warn','migration'], + logger: getDatabaseLogger({}, 'app', ['error','warn','migration', 'schema']) }); } diff --git a/src/Web/Server/server.ts b/src/Web/Server/server.ts index d9cb7cc..978dda8 100644 --- a/src/Web/Server/server.ts +++ b/src/Web/Server/server.ts @@ -10,9 +10,7 @@ import passport from 'passport'; import tcpUsed from 'tcp-port-used'; import { - intersect, - LogEntry, parseBotLogName, - parseSubredditLogName + LogEntry } from "../../util"; import {getLogger} from "../../Utils/loggerFactory"; import LoggedError from "../../Utils/LoggedError"; @@ -210,31 +208,34 @@ const rcbServer = async function (options: OperatorConfig) { server.deleteAsync('/bot/invite', ...deleteInviteRoute); + app = new App(options); + const initBot = async (causedBy: Invokee = 'system') => { - if (app !== undefined) { - logger.info('A bot instance already exists. Attempting to stop event/queue processing first before building new bot.'); - await app.destroy(causedBy); - } - const newApp = new App(options); - newApp.initBots(causedBy).catch((err: any) => { - if (newApp.error === undefined) { - newApp.error = err.message; + app.initBots(causedBy).catch((err: any) => { + if (app.error === undefined) { + app.error = err.message; } logger.error('Server is still ONLINE but bot cannot recover from this error and must be re-built'); if (!err.logged || !(err instanceof LoggedError)) { logger.error(err); } }); - return newApp; } server.postAsync('/init', authUserCheck(), async (req, res) => { logger.info(`${(req.user as Express.User).name} requested the app to be re-built. Starting rebuild now...`, {subreddit: (req.user as Express.User).name}); - app = await initBot('user'); + await initBot('user'); }); - logger.info('Beginning bot init on startup...'); - app = await initBot(); + logger.info('Beginning bot init...'); + try { + const dbReady = await app.initDatabase(); + if(dbReady) { + await initBot(); + } + } catch (e: any) { + logger.error('Error occurred during database connection or migration. Cannot continue with starting bots.'); + } }; export default rcbServer;