From 5cbc669dfdb2beb8a12464e1575644539622b386 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 6 Apr 2022 11:00:17 -0400 Subject: [PATCH] refactor(database): Revert f4e5f1a2 and allow specifying cache/database for web independently Since web DB needs to be built separately from server to allow migrations anyway might as well provide *all* the flexibility * Reverts breaking change removing caching from web * Add database config override in web config --- src/Common/MigrationService.ts | 2 +- src/Common/interfaces.ts | 49 ++++++++++--- src/ConfigBuilder.ts | 42 +++++++++-- src/Schema/OperatorConfig.json | 111 +++++++++++++++++++++++++++++- src/Utils/databaseUtils.ts | 99 ++++++++++++++++++-------- src/Web/Client/StorageProvider.ts | 2 +- src/Web/Client/index.ts | 16 ++--- 7 files changed, 264 insertions(+), 57 deletions(-) diff --git a/src/Common/MigrationService.ts b/src/Common/MigrationService.ts index 8e776a2..7f8226f 100644 --- a/src/Common/MigrationService.ts +++ b/src/Common/MigrationService.ts @@ -17,7 +17,7 @@ export class MigrationService { database: DataSource, options: DatabaseMigrationOptions }) { - this.dbLogger = data.logger.child({labels: [`Database`, (data.type === 'app' ? 'App' : 'Web')]}, mergeArr); + this.dbLogger = data.logger.child({labels: [(data.type === 'app' ? 'App' : 'Web'), `Database`]}, mergeArr); this.database = data.database; this.options = data.options; } diff --git a/src/Common/interfaces.ts b/src/Common/interfaces.ts index 5937dfd..5b4e899 100644 --- a/src/Common/interfaces.ts +++ b/src/Common/interfaces.ts @@ -710,8 +710,19 @@ export interface OperatorCacheConfig extends CacheConfig { actionedEventsDefault?: number } -export type DatabaseDriver = 'sqljs' | 'better-sqlite3' | 'mysql' | 'mariadb' | 'postgres'; +export type DatabaseDriverType = 'sqljs' | 'better-sqlite3' | 'mysql' | 'mariadb' | 'postgres'; export type DatabaseConfig = SqljsConnectionOptions | MysqlConnectionOptions | PostgresConnectionOptions | BetterSqlite3ConnectionOptions; +export type DatabaseDriverConfig = { + type: DatabaseDriverType, + [key: string]: any + /** + * Set the type of logging typeorm should output + * + * Defaults to errors, warnings, and schema (migration progress) + * */ + logging?: LoggerOptions +} +export type DatabaseDriver = DatabaseDriverType | DatabaseDriverConfig; export interface Footer { /** @@ -1802,21 +1813,15 @@ export interface OperatorJsonConfig { caching?: OperatorCacheConfig /** - * Database backend to use for persistent data + * Database backend to use for persistent APPLICATION data * * Defaults to 'sqljs' which stores data in a file * */ databaseConfig?: { // can't use DatabaseConfig here because generating the schema complains about unsupported symbol and a circular reference // ...also including all those options makes the schema huge - connection?: DatabaseDriver | {type: DatabaseDriver, [key: string]: any}, + connection?: DatabaseDriverType | DatabaseDriverConfig, migrations?: DatabaseMigrationOptions - /** - * Set the type of logging typeorm should output - * - * Defaults to errors, warnings, and schema (migration progress) - * */ - logging?: LoggerOptions } /** @@ -1853,9 +1858,26 @@ export interface OperatorJsonConfig { port?: number, /** - * Storage provider to use for sessions and other web client specific data + * Database backend to use for persistent WEB data * - * If none is provided the top-level database is used + * If none is provided the top-level database provider is used + * */ + databaseConfig?: { + connection?: DatabaseDriver, + migrations?: DatabaseMigrationOptions + } + + /** + * Caching provider to use for session and invite data + * + * If none is provided the top-level caching provider is used + * */ + caching?: 'memory' | 'redis' | CacheOptions + + /** + * Storage provider type to use for sessions and other web client specific data + * + * Defaults to `database` if none is provided * * * Specify `database` to use top-level database * * Specify `cache` to use top-level cache @@ -2043,6 +2065,11 @@ export interface OperatorConfig extends OperatorJsonConfig { database: DataSource web: { database: DataSource, + databaseConfig: { + connection: DatabaseConfig, + migrations: DatabaseMigrationOptions + } + caching: CacheOptions, port: number, storage?: 'database' | 'cache' session: { diff --git a/src/ConfigBuilder.ts b/src/ConfigBuilder.ts index e21ba8d..8979664 100644 --- a/src/ConfigBuilder.ts +++ b/src/ConfigBuilder.ts @@ -35,7 +35,13 @@ import { RedditCredentials, BotCredentialsJsonConfig, BotCredentialsConfig, - FilterCriteriaDefaults, TypedActivityStates, OperatorFileConfig, PostBehavior, StrongLoggingOptions + FilterCriteriaDefaults, + TypedActivityStates, + OperatorFileConfig, + PostBehavior, + StrongLoggingOptions, + DatabaseDriverConfig, + DatabaseDriver, DatabaseDriverType } from "./Common/interfaces"; import {isRuleSetJSON, RuleSetJson, RuleSetObjectJson} from "./Rule/RuleSet"; import deepEqual from "fast-deep-equal"; @@ -55,7 +61,12 @@ import { } from "./Common/defaults"; import objectHash from "object-hash"; import {AuthorCriteria, AuthorOptions} from "./Author/Author"; -import {createDatabaseConfig, createDatabaseConnection} from "./Utils/databaseUtils"; +import { + createAppDatabaseConnection, + createDatabaseConfig, + createDatabaseConnection, + createWebDatabaseConnection +} from "./Utils/databaseUtils"; import path from 'path'; import { JsonOperatorConfigDocument, @@ -746,13 +757,17 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig): caching: opCache, userAgent, databaseConfig: { - connection: dbConnection = process.env.DB_DRIVER ?? 'sqljs', + connection: dbConnection = (process.env.DB_DRIVER ?? 'sqljs') as DatabaseDriverType, migrations = {}, - logging: dbLogging, } = {}, web: { port = 8085, maxLogs = 200, + databaseConfig: { + connection: dbConnectionWeb = dbConnection, + migrations: migrationsWeb = migrations, + } = {}, + caching: webCaching = {}, storage: webStorage = undefined, session: { secret: sessionSecretFromConfig = undefined, @@ -854,6 +869,13 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig): const appLogger = getLogger(loggingOptions, 'app'); const dbConfig = createDatabaseConfig(dbConnection); + let realdbConnectionWeb: DatabaseDriver = dbConnectionWeb; + if(typeof dbConnectionWeb === 'string') { + realdbConnectionWeb = dbConnectionWeb as DatabaseDriverType; + } else if(!(typeof dbConnection === 'string')) { + realdbConnectionWeb = {...dbConnection, ...dbConnectionWeb}; + } + const webDbConfig = createDatabaseConfig(realdbConnectionWeb); const config: OperatorConfig = { mode, @@ -864,14 +886,22 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig): logging: loggingOptions, caching: cache, snoowrap: snoowrapOp, - database: await createDatabaseConnection('app', dbConfig, appLogger, dbLogging), + database: await createAppDatabaseConnection(dbConfig, appLogger), databaseConfig: { connection: dbConfig, migrations, }, userAgent, web: { - database: await createDatabaseConnection('web', dbConfig, appLogger, dbLogging), + database: await createWebDatabaseConnection(webDbConfig, appLogger), + databaseConfig: { + connection: webDbConfig, + migrations: migrationsWeb, + }, + caching: { + ...defaultProvider, + ...webCaching + }, port, storage: webStorage, invites: { diff --git a/src/Schema/OperatorConfig.json b/src/Schema/OperatorConfig.json index 625377e..d10bd68 100644 --- a/src/Schema/OperatorConfig.json +++ b/src/Schema/OperatorConfig.json @@ -590,7 +590,7 @@ }, "type": "object" }, - "DatabaseDriver": { + "DatabaseDriverType": { "enum": [ "better-sqlite3", "mariadb", @@ -1521,15 +1521,42 @@ "$ref": "#/definitions/ThirdPartyCredentialsJsonConfig" }, "databaseConfig": { - "description": "Database backend to use for persistent data\n\nDefaults to 'sqljs' which stores data in a file", + "description": "Database backend to use for persistent APPLICATION data\n\nDefaults to 'sqljs' which stores data in a file", "properties": { "connection": { "anyOf": [ { "additionalProperties": {}, "properties": { + "logging": { + "anyOf": [ + { + "items": { + "enum": [ + "error", + "info", + "log", + "migration", + "query", + "schema", + "warn" + ], + "type": "string" + }, + "type": "array" + }, + { + "enum": [ + "all", + false, + true + ] + } + ], + "description": "Set the type of logging typeorm should output\n\nDefaults to errors, warnings, and schema (migration progress)" + }, "type": { - "$ref": "#/definitions/DatabaseDriver" + "$ref": "#/definitions/DatabaseDriverType" } }, "required": [ @@ -1658,6 +1685,68 @@ } ] }, + "databaseConfig": { + "description": "Database backend to use for persistent WEB data\n\nIf none is provided the top-level database provider is used", + "properties": { + "connection": { + "anyOf": [ + { + "additionalProperties": {}, + "properties": { + "logging": { + "anyOf": [ + { + "items": { + "enum": [ + "error", + "info", + "log", + "migration", + "query", + "schema", + "warn" + ], + "type": "string" + }, + "type": "array" + }, + { + "enum": [ + "all", + false, + true + ] + } + ], + "description": "Set the type of logging typeorm should output\n\nDefaults to errors, warnings, and schema (migration progress)" + }, + "type": { + "$ref": "#/definitions/DatabaseDriverType" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + { + "enum": [ + "better-sqlite3", + "mariadb", + "mysql", + "postgres", + "sqljs" + ], + "type": "string" + } + ] + }, + "migrations": { + "$ref": "#/definitions/DatabaseMigrationOptions" + } + }, + "type": "object" + }, "invites": { "description": "Settings related to oauth flow invites", "properties": { @@ -1729,9 +1818,25 @@ "definitelyARandomString" ], "type": "string" + }, + "storage": { + "description": "Specify backend storage to use for persisting client sessions. If specified this will overwrite parent-level `storage` settings.\n\nMay be useful if using `database` for general web client storage but have heavy traffic and want sessions to be more performant (using `cache`)", + "enum": [ + "cache", + "database" + ], + "type": "string" } }, "type": "object" + }, + "storage": { + "description": "Storage provider type to use for sessions and other web client specific data\n\nDefaults to `database` if none is provided\n\n* Specify `database` to use top-level database\n* Specify `cache` to use top-level cache\n\nNOTE: `database` should almost always be used. Cache would only be necessary if this instance experiences heavy traffic", + "enum": [ + "cache", + "database" + ], + "type": "string" } }, "type": "object" diff --git a/src/Utils/databaseUtils.ts b/src/Utils/databaseUtils.ts index c407baa..28b7923 100644 --- a/src/Utils/databaseUtils.ts +++ b/src/Utils/databaseUtils.ts @@ -1,4 +1,4 @@ -import {DatabaseConfig, DatabaseDriver} from "../Common/interfaces"; +import {DatabaseConfig, DatabaseDriverType} from "../Common/interfaces"; import {SqljsConnectionOptions} from "typeorm/driver/sqljs/SqljsConnectionOptions"; import {MysqlConnectionOptions} from "typeorm/driver/mysql/MysqlConnectionOptions"; import {MongoConnectionOptions} from "typeorm/driver/mongodb/MongoConnectionOptions"; @@ -15,18 +15,19 @@ import {WinstonAdaptor} from "typeorm-logger-adaptor/logger/winston"; import process from "process"; import {defaultDataDir} from "../Common/defaults"; import {LoggerOptions} from "typeorm/logger/LoggerOptions"; +import {DataSourceOptions} from "typeorm/data-source/DataSourceOptions"; const validDrivers = ['sqljs', 'better-sqlite3', 'mysql', 'mariadb', 'postgres']; -export const isDatabaseDriver = (val: any): val is DatabaseDriver => { +export const isDatabaseDriver = (val: any): val is DatabaseDriverType => { if (typeof val !== 'string') { return false; } return validDrivers.some(x => x === val.toLocaleLowerCase()); } -export const asDatabaseDriver = (val: string): DatabaseDriver => { +export const asDatabaseDriver = (val: string): DatabaseDriverType => { const cleanVal = val.trim().toLocaleLowerCase(); if(isDatabaseDriver(cleanVal)) { return cleanVal; @@ -34,9 +35,9 @@ export const asDatabaseDriver = (val: string): DatabaseDriver => { throw new Error(`Value '${cleanVal}' is not a valid driver. Must be one of: ${validDrivers.join(', ')}`); } -export const createDatabaseConfig = (val: DatabaseDriver | any): DatabaseConfig => { +export const createDatabaseConfig = (val: DatabaseDriverType | any): DatabaseConfig => { try { - let dbType: DatabaseDriver; + let dbType: DatabaseDriverType; let userDbConfig: any = {}; if (typeof val === 'string') { dbType = asDatabaseDriver(val); @@ -105,11 +106,13 @@ export const createDatabaseConfig = (val: DatabaseDriver | any): DatabaseConfig } } -export const createDatabaseConnection = async (type: 'app' | 'web', rawConfig: DatabaseConfig, logger: Logger, dbLogLevels?: LoggerOptions): Promise => { +type DomainSpecificDataSourceOptions = Required>; + +export const createDatabaseConnection = async (rawConfig: DatabaseConfig, domainOptions: DomainSpecificDataSourceOptions, logger: Logger): Promise => { let config = {...rawConfig}; - const dbLogger = logger.child({labels: ['Database', (type === 'app' ? 'App' : 'Web')]}, mergeArr); + const dbLogger = logger.child({labels: ['Database']}, mergeArr); dbLogger.info(`Using '${rawConfig.type}' database type`); @@ -121,21 +124,15 @@ export const createDatabaseConnection = async (type: 'app' | 'web', rawConfig: D const rawPath = rawConfig.type === 'sqljs' ? rawConfig.location : rawConfig.database; if (typeof rawPath !== 'string' || (typeof rawPath === 'string' && rawPath.trim().toLocaleLowerCase() === ':memory:')) { - dbLogger.info('Will use IN-MEMORY database'); - } else if (typeof rawPath === 'string' && rawPath.trim().toLocaleLowerCase() !== ':memory:') { + dbLogger.warn('Will use IN-MEMORY database. All data will be lost on application restart.'); + } else { try { - let sqlLitePath = rawPath; - if(rawConfig.type === 'sqljs') { - const pathInfo = parsePath(rawPath); - dbLogger.info(`Converting to domain-specific database file (${pathInfo.name}-${type}.sqlite) due to how sqljs works.`) - sqlLitePath = resolve(pathInfo.dir, `${pathInfo.name}-${type}${pathInfo.ext}`); - } dbLogger.debug('Testing that database path is writeable...'); - fileOrDirectoryIsWriteable(sqlLitePath); - dbPath = sqlLitePath; - dbLogger.info(`Using database at path: ${sqlLitePath}`); + fileOrDirectoryIsWriteable(rawPath); + dbPath = rawPath; + dbLogger.verbose(`Using database at path: ${rawPath}`); } catch (e: any) { - dbLogger.error(new ErrorWithCause(`Falling back to IN-MEMORY database due to error while trying to access database`, {cause: e})); + dbLogger.error(new ErrorWithCause(`Falling back to IN-MEMORY database due to error while trying to access database. All data will be lost on application restart`, {cause: e})); if(castToBool(process.env.IS_DOCKER) === true) { dbLogger.info(`Make sure you have specified user in docker run command! See https://github.com/FoxxMD/context-mod/blob/master/docs/gettingStartedOperator.md#docker-recommended`); } @@ -156,25 +153,73 @@ export const createDatabaseConnection = async (type: 'app' | 'web', rawConfig: D config = {...config, ...dbOptions} as SqljsConnectionOptions | BetterSqlite3ConnectionOptions; } - const entitiesDir = type === 'app' ? '../Common/Entities' : '../Common/WebEntities' - const migrationsDir = type === 'app' ? '../Common/Migrations/Database/Server' : '../Common/Migrations/Database/Web'; - const migrationTable = type === 'app' ? 'migrationsApp' : 'migrationsWeb'; + const { + logging = ['error', 'warn', 'schema'], + ...rest + } = config; + + let logTypes: any = logging; + if(logTypes === true) { + logTypes = ['ALL (logging=true)']; + } else if (logTypes === false) { + logTypes = ['NONE (logging=false)']; + } + dbLogger.debug(`Will log the follow types from typeorm: ${logTypes.join(',')}`); const source = new DataSource({ - ...config, + ...rest, synchronize: false, - entities: [`${resolve(__dirname, entitiesDir)}/**/*.js`], - migrations: [`${resolve(__dirname, migrationsDir)}/*.js`], - migrationsTableName: migrationTable, + ...domainOptions, migrationsRun: false, logging: ['error', 'warn', 'migration', 'schema', 'log'], - logger: new WinstonAdaptor(dbLogger, dbLogLevels ?? ['error', 'warn', 'schema'], false, ormLoggingAdaptorLevelMappings(dbLogger)), + logger: new WinstonAdaptor(dbLogger, logging, false, ormLoggingAdaptorLevelMappings(dbLogger)), namingStrategy: new CMNamingStrategy(), }); await source.initialize(); return source; } +export const convertSqlJsLocation = (suffix: string, rawConfig: DatabaseConfig, logger: Logger) => { + if (rawConfig.type === 'sqljs' && typeof rawConfig.location === 'string' && rawConfig.location.trim().toLocaleLowerCase() !== ':memory:') { + const pathInfo = parsePath(rawConfig.location); + const suffixedFilename = `${pathInfo.name}-${suffix}${pathInfo.ext}`; + logger.debug(`Converting to domain-specific database file (${suffixedFilename}) due to how sqljs works.`, {leaf: 'Database'}); + return {...rawConfig, location: resolve(pathInfo.dir, suffixedFilename)} + } + return rawConfig; +} + +export const domainDatabaseOptions = { + app: { + entities: '../Common/Entities', + migrations: '../Common/Migrations/Database/Server', + migrationsTableName: 'migrationsApp' + }, + web: { + entities: '../Common/WebEntities', + migrations: '../Common/Migrations/Database/Web', + migrationsTableName: 'migrationsWeb' + } +} + +export const createAppDatabaseConnection = async (rawConfig: DatabaseConfig, logger: Logger) => { + const domainLogger = logger.child({labels: ['App']}, mergeArr); + return createDatabaseConnection(convertSqlJsLocation('app', rawConfig, domainLogger), { + entities: [`${resolve(__dirname, domainDatabaseOptions.app.entities)}/**/*.js`], + migrations: [`${resolve(__dirname, domainDatabaseOptions.app.migrations)}/*.js`], + migrationsTableName: domainDatabaseOptions.app.migrationsTableName + }, domainLogger); +} + +export const createWebDatabaseConnection = async (rawConfig: DatabaseConfig, logger: Logger) => { + const domainLogger = logger.child({labels: ['Web']}, mergeArr); + return createDatabaseConnection(convertSqlJsLocation('web', rawConfig, domainLogger), { + entities: [`${resolve(__dirname, domainDatabaseOptions.web.entities)}/**/*.js`], + migrations: [`${resolve(__dirname, domainDatabaseOptions.web.migrations)}/*.js`], + migrationsTableName: domainDatabaseOptions.web.migrationsTableName + }, domainLogger); +} + const ormLoggingAdaptorLevelMappings = (logger: Logger) => ({ log: (first: any, ...rest: any) => logger.debug(first, ...rest), info: (first: any, ...rest: any) => logger.info(first, ...rest), diff --git a/src/Web/Client/StorageProvider.ts b/src/Web/Client/StorageProvider.ts index 4daaf87..667cad4 100644 --- a/src/Web/Client/StorageProvider.ts +++ b/src/Web/Client/StorageProvider.ts @@ -52,7 +52,7 @@ abstract class StorageProvider { loggerLabels = [], } = data; this.invitesMaxAge = invitesMaxAge; - this.logger = logger.child({labels: ['Web Storage', ...loggerLabels]}, mergeArr); + this.logger = logger.child({labels: ['Web', 'Storage', ...loggerLabels]}, mergeArr); } protected abstract getInvite(id: string): Promise; diff --git a/src/Web/Client/index.ts b/src/Web/Client/index.ts index 5c2f162..c422aa1 100644 --- a/src/Web/Client/index.ts +++ b/src/Web/Client/index.ts @@ -155,17 +155,17 @@ const webClient = async (options: OperatorConfig) => { display, }, userAgent: uaFragment, - caching: { - provider: caching - }, + // caching: { + // provider: caching + // }, web: { database, + databaseConfig: { + migrations + }, port, storage: webStorage = 'database', - // caching, - // caching: { - // prefix - // }, + caching, invites: { maxAge: invitesMaxAge, }, @@ -209,7 +209,7 @@ const webClient = async (options: OperatorConfig) => { type: 'web', logger, database, - options: options.databaseConfig.migrations + options: migrations }); if (await tcpUsed.check(port)) {