github/context-mod
1
1
Fork 0
mirror of https://github.com/FoxxMD/context-mod.git synced 2026-04-19 03:00:07 -04:00
Code Issues Packages Projects Releases Wiki Activity

refactor(database): Revert f4e5f1a2 and allow specifying cache/database for web independently

Browse Source 
Since web DB needs to be built separately from server to allow migrations anyway might as well provide *all* the flexibility

* Reverts breaking change removing caching from web
* Add database config override in web config
This commit is contained in:
FoxxMD
2022-04-06 11:00:17 -04:00
parent 2b39644202
commit 5cbc669dfd
7 changed files with 264 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/Common/MigrationService.ts
View File

@@ -17,7 +17,7 @@ export class MigrationService {
database: DataSource,
options: DatabaseMigrationOptions
}) {
this.dbLogger = data.logger.child({labels: [`Database`, (data.type === 'app' ? 'App' : 'Web')]}, mergeArr);
this.dbLogger = data.logger.child({labels: [(data.type === 'app' ? 'App' : 'Web'), `Database`]}, mergeArr);
this.database = data.database;
this.options = data.options;
}

49
src/Common/interfaces.ts
View File

@@ -710,8 +710,19 @@ export interface OperatorCacheConfig extends CacheConfig {
actionedEventsDefault?: number
}
export type DatabaseDriver = 'sqljs' | 'better-sqlite3' | 'mysql' | 'mariadb' | 'postgres';
export type DatabaseDriverType = 'sqljs' | 'better-sqlite3' | 'mysql' | 'mariadb' | 'postgres';
export type DatabaseConfig = SqljsConnectionOptions | MysqlConnectionOptions | PostgresConnectionOptions | BetterSqlite3ConnectionOptions;
export type DatabaseDriverConfig = {
type: DatabaseDriverType,
[key: string]: any
/**
* Set the type of logging typeorm should output
*
* Defaults to errors, warnings, and schema (migration progress)
* */
logging?: LoggerOptions
}
export type DatabaseDriver = DatabaseDriverType | DatabaseDriverConfig;
export interface Footer {
/**
@@ -1802,21 +1813,15 @@ export interface OperatorJsonConfig {
caching?: OperatorCacheConfig
/**
* Database backend to use for persistent data
* Database backend to use for persistent APPLICATION data
*
* Defaults to 'sqljs' which stores data in a file
* */
databaseConfig?: {
// can't use DatabaseConfig here because generating the schema complains about unsupported symbol and a circular reference
// ...also including all those options makes the schema huge
connection?: DatabaseDriver | {type: DatabaseDriver, [key: string]: any},
connection?: DatabaseDriverType | DatabaseDriverConfig,
migrations?: DatabaseMigrationOptions
/**
* Set the type of logging typeorm should output
*
* Defaults to errors, warnings, and schema (migration progress)
* */
logging?: LoggerOptions
}
/**
@@ -1853,9 +1858,26 @@ export interface OperatorJsonConfig {
port?: number,
/**
* Storage provider to use for sessions and other web client specific data
* Database backend to use for persistent WEB data
*
* If none is provided the top-level database is used
* If none is provided the top-level database provider is used
* */
databaseConfig?: {
connection?: DatabaseDriver,
migrations?: DatabaseMigrationOptions
}
/**
* Caching provider to use for session and invite data
*
* If none is provided the top-level caching provider is used
* */
caching?: 'memory' | 'redis' | CacheOptions
/**
* Storage provider type to use for sessions and other web client specific data
*
* Defaults to `database` if none is provided
*
* * Specify `database` to use top-level database
* * Specify `cache` to use top-level cache
@@ -2043,6 +2065,11 @@ export interface OperatorConfig extends OperatorJsonConfig {
database: DataSource
web: {
database: DataSource,
databaseConfig: {
connection: DatabaseConfig,
migrations: DatabaseMigrationOptions
}
caching: CacheOptions,
port: number,
storage?: 'database' | 'cache'
session: {

42
src/ConfigBuilder.ts
View File

@@ -35,7 +35,13 @@ import {
RedditCredentials,
BotCredentialsJsonConfig,
BotCredentialsConfig,
FilterCriteriaDefaults, TypedActivityStates, OperatorFileConfig, PostBehavior, StrongLoggingOptions
FilterCriteriaDefaults,
TypedActivityStates,
OperatorFileConfig,
PostBehavior,
StrongLoggingOptions,
DatabaseDriverConfig,
DatabaseDriver, DatabaseDriverType
} from "./Common/interfaces";
import {isRuleSetJSON, RuleSetJson, RuleSetObjectJson} from "./Rule/RuleSet";
import deepEqual from "fast-deep-equal";
@@ -55,7 +61,12 @@ import {
} from "./Common/defaults";
import objectHash from "object-hash";
import {AuthorCriteria, AuthorOptions} from "./Author/Author";
import {createDatabaseConfig, createDatabaseConnection} from "./Utils/databaseUtils";
import {
createAppDatabaseConnection,
createDatabaseConfig,
createDatabaseConnection,
createWebDatabaseConnection
} from "./Utils/databaseUtils";
import path from 'path';
import {
JsonOperatorConfigDocument,
@@ -746,13 +757,17 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig):
caching: opCache,
userAgent,
databaseConfig: {
connection: dbConnection = process.env.DB_DRIVER ?? 'sqljs',
connection: dbConnection = (process.env.DB_DRIVER ?? 'sqljs') as DatabaseDriverType,
migrations = {},
logging: dbLogging,
} = {},
web: {
port = 8085,
maxLogs = 200,
databaseConfig: {
connection: dbConnectionWeb = dbConnection,
migrations: migrationsWeb = migrations,
} = {},
caching: webCaching = {},
storage: webStorage = undefined,
session: {
secret: sessionSecretFromConfig = undefined,
@@ -854,6 +869,13 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig):
const appLogger = getLogger(loggingOptions, 'app');
const dbConfig = createDatabaseConfig(dbConnection);
let realdbConnectionWeb: DatabaseDriver = dbConnectionWeb;
if(typeof dbConnectionWeb === 'string') {
realdbConnectionWeb = dbConnectionWeb as DatabaseDriverType;
} else if(!(typeof dbConnection === 'string')) {
realdbConnectionWeb = {...dbConnection, ...dbConnectionWeb};
}
const webDbConfig = createDatabaseConfig(realdbConnectionWeb);
const config: OperatorConfig = {
mode,
@@ -864,14 +886,22 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig):
logging: loggingOptions,
caching: cache,
snoowrap: snoowrapOp,
database: await createDatabaseConnection('app', dbConfig, appLogger, dbLogging),
database: await createAppDatabaseConnection(dbConfig, appLogger),
databaseConfig: {
connection: dbConfig,
migrations,
},
userAgent,
web: {
database: await createDatabaseConnection('web', dbConfig, appLogger, dbLogging),
database: await createWebDatabaseConnection(webDbConfig, appLogger),
databaseConfig: {
connection: webDbConfig,
migrations: migrationsWeb,
},
caching: {
...defaultProvider,
...webCaching
},
port,
storage: webStorage,
invites: {

111
src/Schema/OperatorConfig.json
View File

@@ -590,7 +590,7 @@
},
"type": "object"
},
"DatabaseDriver": {
"DatabaseDriverType": {
"enum": [
"better-sqlite3",
"mariadb",
@@ -1521,15 +1521,42 @@
"$ref": "#/definitions/ThirdPartyCredentialsJsonConfig"
},
"databaseConfig": {
"description": "Database backend to use for persistent data\n\nDefaults to 'sqljs' which stores data in a file",
"description": "Database backend to use for persistent APPLICATION data\n\nDefaults to 'sqljs' which stores data in a file",
"properties": {
"connection": {
"anyOf": [
{
"additionalProperties": {},
"properties": {
"logging": {
"anyOf": [
{
"items": {
"enum": [
"error",
"info",
"log",
"migration",
"query",
"schema",
"warn"
],
"type": "string"
},
"type": "array"
},
{
"enum": [
"all",
false,
true
]
}
],
"description": "Set the type of logging typeorm should output\n\nDefaults to errors, warnings, and schema (migration progress)"
},
"type": {
"$ref": "#/definitions/DatabaseDriver"
"$ref": "#/definitions/DatabaseDriverType"
}
},
"required": [
@@ -1658,6 +1685,68 @@
}
]
},
"databaseConfig": {
"description": "Database backend to use for persistent WEB data\n\nIf none is provided the top-level database provider is used",
"properties": {
"connection": {
"anyOf": [
{
"additionalProperties": {},
"properties": {
"logging": {
"anyOf": [
{
"items": {
"enum": [
"error",
"info",
"log",
"migration",
"query",
"schema",
"warn"
],
"type": "string"
},
"type": "array"
},
{
"enum": [
"all",
false,
true
]
}
],
"description": "Set the type of logging typeorm should output\n\nDefaults to errors, warnings, and schema (migration progress)"
},
"type": {
"$ref": "#/definitions/DatabaseDriverType"
}
},
"required": [
"type"
],
"type": "object"
},
{
"enum": [
"better-sqlite3",
"mariadb",
"mysql",
"postgres",
"sqljs"
],
"type": "string"
}
]
},
"migrations": {
"$ref": "#/definitions/DatabaseMigrationOptions"
}
},
"type": "object"
},
"invites": {
"description": "Settings related to oauth flow invites",
"properties": {
@@ -1729,9 +1818,25 @@
"definitelyARandomString"
],
"type": "string"
},
"storage": {
"description": "Specify backend storage to use for persisting client sessions. If specified this will overwrite parent-level `storage` settings.\n\nMay be useful if using `database` for general web client storage but have heavy traffic and want sessions to be more performant (using `cache`)",
"enum": [
"cache",
"database"
],
"type": "string"
}
},
"type": "object"
},
"storage": {
"description": "Storage provider type to use for sessions and other web client specific data\n\nDefaults to `database` if none is provided\n\n* Specify `database` to use top-level database\n* Specify `cache` to use top-level cache\n\nNOTE: `database` should almost always be used. Cache would only be necessary if this instance experiences heavy traffic",
"enum": [
"cache",
"database"
],
"type": "string"
}
},
"type": "object"

99
src/Utils/databaseUtils.ts
View File

@@ -1,4 +1,4 @@
import {DatabaseConfig, DatabaseDriver} from "../Common/interfaces";
import {DatabaseConfig, DatabaseDriverType} from "../Common/interfaces";
import {SqljsConnectionOptions} from "typeorm/driver/sqljs/SqljsConnectionOptions";
import {MysqlConnectionOptions} from "typeorm/driver/mysql/MysqlConnectionOptions";
import {MongoConnectionOptions} from "typeorm/driver/mongodb/MongoConnectionOptions";
@@ -15,18 +15,19 @@ import {WinstonAdaptor} from "typeorm-logger-adaptor/logger/winston";
import process from "process";
import {defaultDataDir} from "../Common/defaults";
import {LoggerOptions} from "typeorm/logger/LoggerOptions";
import {DataSourceOptions} from "typeorm/data-source/DataSourceOptions";
const validDrivers = ['sqljs', 'better-sqlite3', 'mysql', 'mariadb', 'postgres'];
export const isDatabaseDriver = (val: any): val is DatabaseDriver => {
export const isDatabaseDriver = (val: any): val is DatabaseDriverType => {
if (typeof val !== 'string') {
return false;
}
return validDrivers.some(x => x === val.toLocaleLowerCase());
}
export const asDatabaseDriver = (val: string): DatabaseDriver => {
export const asDatabaseDriver = (val: string): DatabaseDriverType => {
const cleanVal = val.trim().toLocaleLowerCase();
if(isDatabaseDriver(cleanVal)) {
return cleanVal;
@@ -34,9 +35,9 @@ export const asDatabaseDriver = (val: string): DatabaseDriver => {
throw new Error(`Value '${cleanVal}' is not a valid driver. Must be one of: ${validDrivers.join(', ')}`);
}
export const createDatabaseConfig = (val: DatabaseDriver | any): DatabaseConfig => {
export const createDatabaseConfig = (val: DatabaseDriverType | any): DatabaseConfig => {
try {
let dbType: DatabaseDriver;
let dbType: DatabaseDriverType;
let userDbConfig: any = {};
if (typeof val === 'string') {
dbType = asDatabaseDriver(val);
@@ -105,11 +106,13 @@ export const createDatabaseConfig = (val: DatabaseDriver | any): DatabaseConfig
}
}
export const createDatabaseConnection = async (type: 'app' | 'web', rawConfig: DatabaseConfig, logger: Logger, dbLogLevels?: LoggerOptions): Promise<DataSource> => {
type DomainSpecificDataSourceOptions = Required<Pick<DataSourceOptions, 'migrations' | 'migrationsTableName' | 'entities'>>;
export const createDatabaseConnection = async (rawConfig: DatabaseConfig, domainOptions: DomainSpecificDataSourceOptions, logger: Logger): Promise<DataSource> => {
let config = {...rawConfig};
const dbLogger = logger.child({labels: ['Database', (type === 'app' ? 'App' : 'Web')]}, mergeArr);
const dbLogger = logger.child({labels: ['Database']}, mergeArr);
dbLogger.info(`Using '${rawConfig.type}' database type`);
@@ -121,21 +124,15 @@ export const createDatabaseConnection = async (type: 'app' | 'web', rawConfig: D
const rawPath = rawConfig.type === 'sqljs' ? rawConfig.location : rawConfig.database;
if (typeof rawPath !== 'string' || (typeof rawPath === 'string' && rawPath.trim().toLocaleLowerCase() === ':memory:')) {
dbLogger.info('Will use IN-MEMORY database');
} else if (typeof rawPath === 'string' && rawPath.trim().toLocaleLowerCase() !== ':memory:') {
dbLogger.warn('Will use IN-MEMORY database. All data will be lost on application restart.');
} else {
try {
let sqlLitePath = rawPath;
if(rawConfig.type === 'sqljs') {
const pathInfo = parsePath(rawPath);
dbLogger.info(`Converting to domain-specific database file (${pathInfo.name}-${type}.sqlite) due to how sqljs works.`)
sqlLitePath = resolve(pathInfo.dir, `${pathInfo.name}-${type}${pathInfo.ext}`);
}
dbLogger.debug('Testing that database path is writeable...');
fileOrDirectoryIsWriteable(sqlLitePath);
dbPath = sqlLitePath;
dbLogger.info(`Using database at path: ${sqlLitePath}`);
fileOrDirectoryIsWriteable(rawPath);
dbPath = rawPath;
dbLogger.verbose(`Using database at path: ${rawPath}`);
} catch (e: any) {
dbLogger.error(new ErrorWithCause(`Falling back to IN-MEMORY database due to error while trying to access database`, {cause: e}));
dbLogger.error(new ErrorWithCause(`Falling back to IN-MEMORY database due to error while trying to access database. All data will be lost on application restart`, {cause: e}));
if(castToBool(process.env.IS_DOCKER) === true) {
dbLogger.info(`Make sure you have specified user in docker run command! See https://github.com/FoxxMD/context-mod/blob/master/docs/gettingStartedOperator.md#docker-recommended`);
}
@@ -156,25 +153,73 @@ export const createDatabaseConnection = async (type: 'app' | 'web', rawConfig: D
config = {...config, ...dbOptions} as SqljsConnectionOptions | BetterSqlite3ConnectionOptions;
}
const entitiesDir = type === 'app' ? '../Common/Entities' : '../Common/WebEntities'
const migrationsDir = type === 'app' ? '../Common/Migrations/Database/Server' : '../Common/Migrations/Database/Web';
const migrationTable = type === 'app' ? 'migrationsApp' : 'migrationsWeb';
const {
logging = ['error', 'warn', 'schema'],
...rest
} = config;
let logTypes: any = logging;
if(logTypes === true) {
logTypes = ['ALL (logging=true)'];
} else if (logTypes === false) {
logTypes = ['NONE (logging=false)'];
}
dbLogger.debug(`Will log the follow types from typeorm: ${logTypes.join(',')}`);
const source = new DataSource({
...config,
...rest,
synchronize: false,
entities: [`${resolve(__dirname, entitiesDir)}/**/*.js`],
migrations: [`${resolve(__dirname, migrationsDir)}/*.js`],
migrationsTableName: migrationTable,
...domainOptions,
migrationsRun: false,
logging: ['error', 'warn', 'migration', 'schema', 'log'],
logger: new WinstonAdaptor(dbLogger, dbLogLevels ?? ['error', 'warn', 'schema'], false, ormLoggingAdaptorLevelMappings(dbLogger)),
logger: new WinstonAdaptor(dbLogger, logging, false, ormLoggingAdaptorLevelMappings(dbLogger)),
namingStrategy: new CMNamingStrategy(),
});
await source.initialize();
return source;
}
export const convertSqlJsLocation = (suffix: string, rawConfig: DatabaseConfig, logger: Logger) => {
if (rawConfig.type === 'sqljs' && typeof rawConfig.location === 'string' && rawConfig.location.trim().toLocaleLowerCase() !== ':memory:') {
const pathInfo = parsePath(rawConfig.location);
const suffixedFilename = `${pathInfo.name}-${suffix}${pathInfo.ext}`;
logger.debug(`Converting to domain-specific database file (${suffixedFilename}) due to how sqljs works.`, {leaf: 'Database'});
return {...rawConfig, location: resolve(pathInfo.dir, suffixedFilename)}
}
return rawConfig;
}
export const domainDatabaseOptions = {
app: {
entities: '../Common/Entities',
migrations: '../Common/Migrations/Database/Server',
migrationsTableName: 'migrationsApp'
},
web: {
entities: '../Common/WebEntities',
migrations: '../Common/Migrations/Database/Web',
migrationsTableName: 'migrationsWeb'
}
}
export const createAppDatabaseConnection = async (rawConfig: DatabaseConfig, logger: Logger) => {
const domainLogger = logger.child({labels: ['App']}, mergeArr);
return createDatabaseConnection(convertSqlJsLocation('app', rawConfig, domainLogger), {
entities: [`${resolve(__dirname, domainDatabaseOptions.app.entities)}/**/*.js`],
migrations: [`${resolve(__dirname, domainDatabaseOptions.app.migrations)}/*.js`],
migrationsTableName: domainDatabaseOptions.app.migrationsTableName
}, domainLogger);
}
export const createWebDatabaseConnection = async (rawConfig: DatabaseConfig, logger: Logger) => {
const domainLogger = logger.child({labels: ['Web']}, mergeArr);
return createDatabaseConnection(convertSqlJsLocation('web', rawConfig, domainLogger), {
entities: [`${resolve(__dirname, domainDatabaseOptions.web.entities)}/**/*.js`],
migrations: [`${resolve(__dirname, domainDatabaseOptions.web.migrations)}/*.js`],
migrationsTableName: domainDatabaseOptions.web.migrationsTableName
}, domainLogger);
}
const ormLoggingAdaptorLevelMappings = (logger: Logger) => ({
log: (first: any, ...rest: any) => logger.debug(first, ...rest),
info: (first: any, ...rest: any) => logger.info(first, ...rest),

2
src/Web/Client/StorageProvider.ts
View File

@@ -52,7 +52,7 @@ abstract class StorageProvider {
loggerLabels = [],
} = data;
this.invitesMaxAge = invitesMaxAge;
this.logger = logger.child({labels: ['Web Storage', ...loggerLabels]}, mergeArr);
this.logger = logger.child({labels: ['Web', 'Storage', ...loggerLabels]}, mergeArr);
}
protected abstract getInvite(id: string): Promise<InviteData | undefined | null>;

16
src/Web/Client/index.ts
View File

@@ -155,17 +155,17 @@ const webClient = async (options: OperatorConfig) => {
display,
},
userAgent: uaFragment,
caching: {
provider: caching
},
// caching: {
// provider: caching
// },
web: {
database,
databaseConfig: {
migrations
},
port,
storage: webStorage = 'database',
// caching,
// caching: {
// prefix
// },
caching,
invites: {
maxAge: invitesMaxAge,
},
@@ -209,7 +209,7 @@ const webClient = async (options: OperatorConfig) => {
type: 'web',
logger,
database,
options: options.databaseConfig.migrations
options: migrations
});
if (await tcpUsed.check(port)) {