github/context-mod
1
1
Fork 0
mirror of https://github.com/FoxxMD/context-mod.git synced 2026-04-19 03:00:07 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(database): Implement database migration flow for App

Browse Source 
* refactor bot init to include removing any old running ones (out of server and into app)
* run database prep before init bots
This commit is contained in:
FoxxMD
2022-01-14 15:13:28 -05:00
parent d62ed3daf5
commit d6bfd63deb
5 changed files with 136 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
src/App.ts
View File

@@ -1,21 +1,31 @@
import winston, {Logger} from "winston";
import dayjs, {Dayjs} from "dayjs";
import {getLogger} from "./Utils/loggerFactory";
import {Invokee, OperatorConfig} from "./Common/interfaces";
import {DatabaseConfig, DatabaseMigrationOptions, Invokee, OperatorConfig} from "./Common/interfaces";
import Bot from "./Bot";
import LoggedError from "./Utils/LoggedError";
import {sleep} from "./util";
import {mergeArr, sleep} from "./util";
import {copyFile} from "fs/promises";
import {constants} from "fs";
import {Connection} from "typeorm";
export class App {
bots: Bot[]
bots: Bot[] = [];
logger: Logger;
dbLogger: Logger;
database: Connection
startedAt: Dayjs = dayjs();
ranMigrations: boolean = false;
migrationBlocker?: string;
config: OperatorConfig;
error: any;
constructor(config: OperatorConfig) {
const {
database,
operator: {
name,
},
@@ -23,12 +33,13 @@ export class App {
bots = [],
} = config;
this.config = config;
this.logger = getLogger(config.logging);
this.dbLogger = this.logger.child({leaf: 'Database'}, mergeArr);
this.database = database;
this.logger.info(`Operators: ${name.length === 0 ? 'None Specified' : name.join(', ')}`)
this.bots = bots.map(x => new Bot(x, this.logger));
process.on('uncaughtException', (e) => {
this.error = e;
});
@@ -61,7 +72,83 @@ export class App {
}
}
async doMigration() {
if (this.database.options.type === 'sqljs' && this.database.options.location !== undefined) {
const ts = Date.now();
const backupLocation = `${this.database.options.location}.${ts}.bak`
this.dbLogger.info(`Detected sqljs (sqlite) database. Will try to make a backup at ${backupLocation} before migrating.`);
try {
await copyFile(this.database.options.location, backupLocation, constants.COPYFILE_EXCL);
this.dbLogger.info('Successfully created backup!');
} catch (e: any) {
this.dbLogger.error(`Could not create a backup but will continue with migration: ${e.message}`);
}
}
this.dbLogger.info('Beginning migrations...');
await this.database.runMigrations();
this.migrationBlocker = undefined;
this.ranMigrations = true;
}
async initDatabase(confirm: boolean = false) {
const {
databaseConfig: {
migrations: {
force = false,
} = {}
} = {},
} = this.config;
this.dbLogger.info('Checking if migrations are required...');
const runner = this.database.createQueryRunner();
const tables = await runner.getTables();
if (tables.length === 0 || (tables.length === 1 && tables.map(x => x.name).includes('migrations'))) {
this.dbLogger.info('Detected a new database! Starting migrations...');
await this.database.showMigrations();
await this.doMigration();
return true;
} else if (!tables.map(x => x.name).includes('migrations') && !force && !confirm) {
this.dbLogger.warn(`DANGER! Your database has existing tables but none of them include a 'migrations' table.
Are you sure this is the correct database? Continuing with migrations will most likely drop any existing data and recreate all tables.`);
this.migrationBlocker = 'unknownTables';
return false;
} else if (await this.database.showMigrations()) {
this.dbLogger.info('Detected pending migrations.');
if (!force && !confirm) {
this.dbLogger.error(`You must confirm migrations. Either set 'force: true' in database config or confirm migrations from web interface.
YOU SHOULD BACKUP YOUR EXISTING DATABASE BEFORE CONTINUING WITH MIGRATIONS.`);
this.migrationBlocker = 'pending';
return false;
}
if (force && !confirm) {
this.dbLogger.info('Migration was forced');
}
await this.doMigration();
return true;
} else {
this.dbLogger.info('No migrations required!');
this.ranMigrations = true;
return true;
}
}
async initBots(causedBy: Invokee = 'system') {
if(!this.ranMigrations) {
this.logger.error('Must run migrations before starting bots');
return;
}
if(this.bots.length > 0) {
this.logger.info('Bots already exist, will stop and destroy these before building new ones.');
await this.destroy(causedBy);
}
const {
bots = [],
} = this.config;
this.bots = bots.map(x => new Bot(x, this.logger));
for (const b of this.bots) {
if (b.error === undefined) {
try {

14
src/Common/interfaces.ts
View File

@@ -1630,7 +1630,10 @@ export interface OperatorJsonConfig {
*
* Defaults to 'sqljs' which stores data in a file
* */
databaseConfig?: DatabaseDriver | DatabaseConfig
databaseConfig?: {
connection?: DatabaseDriver | DatabaseConfig,
migrations?: DatabaseMigrationOptions
}
/**
* Set global snoowrap options as well as default snoowrap config for all bots that don't specify their own
@@ -1829,7 +1832,10 @@ export interface OperatorConfig extends OperatorJsonConfig {
path?: string,
},
caching: StrongCache,
databaseConfig: DatabaseConfig
databaseConfig: {
connection: DatabaseConfig,
migrations: DatabaseMigrationOptions
}
database: Connection
web: {
port: number,
@@ -2046,3 +2052,7 @@ export interface StringComparisonOptions {
lengthWeight?: number,
transforms?: ((str: string) => string)[]
}
export interface DatabaseMigrationOptions {
force?: boolean
}

12
src/ConfigBuilder.ts
View File

@@ -586,7 +586,10 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig):
path,
} = {},
caching: opCache,
databaseConfig = 'sqljs',
databaseConfig: {
connection: dbConnection = 'sqljs',
migrations = {},
} = {},
web: {
port = 8085,
maxLogs = 200,
@@ -665,7 +668,7 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig):
const logger = getLogger(loggingOptions);
const dbConfig = createDatabaseConfig(databaseConfig);
const dbConfig = createDatabaseConfig(dbConnection);
const database = await createDatabaseConnection(dbConfig);
@@ -837,7 +840,10 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig):
},
logging: loggingOptions,
caching: cache,
databaseConfig: dbConfig,
databaseConfig: {
connection: dbConfig,
migrations,
},
database,
web: {
port,

10
src/Utils/databaseUtils.ts
View File

@@ -8,7 +8,7 @@ import "reflect-metadata";
import {Connection, createConnection} from "typeorm";
import fs, {promises, constants} from "fs";
import {parse} from 'path';
import {getLogger} from "./loggerFactory";
import {getDatabaseLogger, getLogger} from "./loggerFactory";
import SimpleError from "./SimpleError";
export const isDatabaseDriver = (val: any): val is DatabaseDriver => {
@@ -100,7 +100,11 @@ export const createDatabaseConnection = async (rawConfig: DatabaseConfig): Promi
return await createConnection({
...config,
synchronize: true,
entities: [`${resolve(__dirname, '../Common/Entities')}/*.js`]
synchronize: false,
entities: [`${resolve(__dirname, '../Common/Entities')}/*.js`],
migrations: [`${resolve(__dirname, '../Common/Migrations')}/*.js`],
migrationsRun: false,
logging: ['error','warn','migration'],
logger: getDatabaseLogger({}, 'app', ['error','warn','migration', 'schema'])
});
}

31
src/Web/Server/server.ts
View File

@@ -10,9 +10,7 @@ import passport from 'passport';
import tcpUsed from 'tcp-port-used';
import {
intersect,
LogEntry, parseBotLogName,
parseSubredditLogName
LogEntry
} from "../../util";
import {getLogger} from "../../Utils/loggerFactory";
import LoggedError from "../../Utils/LoggedError";
@@ -210,31 +208,34 @@ const rcbServer = async function (options: OperatorConfig) {
server.deleteAsync('/bot/invite', ...deleteInviteRoute);
app = new App(options);
const initBot = async (causedBy: Invokee = 'system') => {
if (app !== undefined) {
logger.info('A bot instance already exists. Attempting to stop event/queue processing first before building new bot.');
await app.destroy(causedBy);
}
const newApp = new App(options);
newApp.initBots(causedBy).catch((err: any) => {
if (newApp.error === undefined) {
newApp.error = err.message;
app.initBots(causedBy).catch((err: any) => {
if (app.error === undefined) {
app.error = err.message;
}
logger.error('Server is still ONLINE but bot cannot recover from this error and must be re-built');
if (!err.logged || !(err instanceof LoggedError)) {
logger.error(err);
}
});
return newApp;
}
server.postAsync('/init', authUserCheck(), async (req, res) => {
logger.info(`${(req.user as Express.User).name} requested the app to be re-built. Starting rebuild now...`, {subreddit: (req.user as Express.User).name});
app = await initBot('user');
await initBot('user');
});
logger.info('Beginning bot init on startup...');
app = await initBot();
logger.info('Beginning bot init...');
try {
const dbReady = await app.initDatabase();
if(dbReady) {
await initBot();
}
} catch (e: any) {
logger.error('Error occurred during database connection or migration. Cannot continue with starting bots.');
}
};
export default rcbServer;