diff --git a/src/Common/Entities/Invite.ts b/src/Common/Entities/Invite.ts new file mode 100644 index 0000000..0d23e90 --- /dev/null +++ b/src/Common/Entities/Invite.ts @@ -0,0 +1,72 @@ +import {Entity, Column, PrimaryColumn} from "typeorm"; +import {TimeAwareBaseEntity} from "./Base/TimeAwareBaseEntity"; +import {InviteData} from "../../Web/Common/interfaces"; +import dayjs, {Dayjs} from "dayjs"; + +@Entity() +export class Invite extends TimeAwareBaseEntity implements InviteData { + + @PrimaryColumn('varchar', {length: 255}) + id!: string + + @Column("varchar", {length: 50}) + clientId!: string; + + @Column("varchar", {length: 50}) + clientSecret!: string; + + @Column("text") + redirectUri!: string; + + @Column("varchar", {length: 255}) + creator!: string; + + @Column("simple-json") + permissions!: string[]; + + @Column("varchar", {length: 200, nullable: true}) + instance?: string; + + @Column() + overwrite?: boolean; + + @Column("simple-json", {nullable: true}) + subreddits?: string[]; + + @Column({name: 'expiresAt', nullable: true}) + _expiresAt?: Date; + + public get expiresAt(): Dayjs | undefined { + if (this._expiresAt === undefined) { + return undefined; + } + return dayjs(this._expiresAt); + } + + public set expiresAt(d: Dayjs | undefined) { + if (d === undefined) { + this._expiresAt = d; + } else { + this._expiresAt = d.utc().toDate(); + } + } + + constructor(data?: InviteData & { id: string, expiresIn?: number }) { + super(); + if (data !== undefined) { + this.id = data.id; + this.permissions = data.permissions; + this.subreddits = data.subreddits; + this.instance = data.instance; + this.clientId = data.clientId; + this.clientSecret = data.clientSecret; + this.redirectUri = data.redirectUri; + this.creator = data.creator; + this.overwrite = data.overwrite; + + if (data.expiresIn !== undefined && data.expiresIn !== 0) { + this.expiresAt = dayjs().add(data.expiresIn, 'seconds'); + } + } + } +} diff --git a/src/Common/Migrations/Database/1642180264563-initApi.ts b/src/Common/Migrations/Database/1642180264563-initApi.ts index 88df7b7..6dbfa1a 100644 --- a/src/Common/Migrations/Database/1642180264563-initApi.ts +++ b/src/Common/Migrations/Database/1642180264563-initApi.ts @@ -121,6 +121,64 @@ export class initApi1642180264563 implements MigrationInterface { true ); + await queryRunner.createTable( + new Table({ + name: 'Invite', + columns: [ + { + name: 'id', + type: 'varchar', + length: '255', + isPrimary: true, + }, + { + name: 'clientId', + type: 'varchar', + length: '255', + }, + { + name: 'clientSecret', + type: 'varchar', + length: '255', + }, + { + name: 'redirectUri', + type: 'text', + }, + { + name: 'creator', + type: 'varchar', + length: '255', + }, + { + name: 'permissions', + type: 'text' + }, + { + name: 'instance', + type: 'varchar', + length: '255', + isNullable: true + }, + { + name: 'overwrite', + type: 'boolean', + isNullable: true, + }, + { + name: 'subreddits', + type: 'text', + isNullable: true + }, + createdAtColumn(dbType), + timeAtColumn('expiresAt', dbType, true) + ], + }), + true, + true, + true + ); + await queryRunner.createTable( new Table({ diff --git a/src/Common/interfaces.ts b/src/Common/interfaces.ts index 6848ed9..4226d5a 100644 --- a/src/Common/interfaces.ts +++ b/src/Common/interfaces.ts @@ -8,7 +8,7 @@ import {SqljsConnectionOptions} from "typeorm/driver/sqljs/SqljsConnectionOption import {MysqlConnectionOptions} from "typeorm/driver/mysql/MysqlConnectionOptions"; import {MongoConnectionOptions} from "typeorm/driver/mongodb/MongoConnectionOptions"; import {PostgresConnectionOptions} from "typeorm/driver/postgres/PostgresConnectionOptions"; -import {Connection} from "typeorm"; +import {Connection, DataSource} from "typeorm"; import {AuthorCriteria, AuthorOptions} from "../Author/Author"; import {JsonOperatorConfigDocument, YamlOperatorConfigDocument} from "./Config/Operator"; import {ConsoleTransportOptions} from "winston/lib/winston/transports"; @@ -1853,11 +1853,17 @@ export interface OperatorJsonConfig { port?: number, /** - * Caching provider to use for session and invite data + * Storage provider to use for sessions and other web client specific data + * + * If none is provided the top-level database is used + * + * * Specify `database` to use top-level database + * * Specify `cache` to use top-level cache + * + * NOTE: `database` should almost always be used. Cache would only be necessary if this instance experiences heavy traffic * - * If none is provided the top-level caching provider is used * */ - caching?: 'memory' | 'redis' | CacheOptions + storage?: 'database' | 'cache' /** * Settings to configure the behavior of user sessions -- the session is what the web interface uses to identify logged in users. * */ @@ -1881,6 +1887,13 @@ export interface OperatorJsonConfig { * @examples ["definitelyARandomString"] * */ secret?: string, + + /** + * Specify backend storage to use for persisting client sessions. If specified this will overwrite parent-level `storage` settings. + * + * May be useful if using `database` for general web client storage but have heavy traffic and want sessions to be more performant (using `cache`) + * */ + storage?: 'database' | 'cache' } /** @@ -2027,13 +2040,14 @@ export interface OperatorConfig extends OperatorJsonConfig { connection: DatabaseConfig, migrations: DatabaseMigrationOptions } - database: Connection + database: DataSource web: { port: number, - caching: CacheOptions, + storage?: 'database' | 'cache' session: { maxAge: number, secret: string, + storage?: 'database' | 'cache' }, invites: { maxAge: number diff --git a/src/ConfigBuilder.ts b/src/ConfigBuilder.ts index 4c1af0f..2cca92e 100644 --- a/src/ConfigBuilder.ts +++ b/src/ConfigBuilder.ts @@ -753,10 +753,11 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig): web: { port = 8085, maxLogs = 200, - caching: webCaching = {}, + storage: webStorage = undefined, session: { secret = randomId(), maxAge: sessionMaxAge = 86400, + storage: sessionStorage = undefined, } = {}, invites: { maxAge: inviteMaxAge = 0, @@ -874,16 +875,14 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig): userAgent, web: { port, - caching: { - ...defaultProvider, - ...webCaching - }, + storage: webStorage, invites: { maxAge: inviteMaxAge, }, session: { secret, maxAge: sessionMaxAge, + storage: sessionStorage }, maxLogs, clients: clients === undefined ? [{host: 'localhost:8095', secret: apiSecret}] : clients, diff --git a/src/Web/Client/StorageProvider.ts b/src/Web/Client/StorageProvider.ts new file mode 100644 index 0000000..889a895 --- /dev/null +++ b/src/Web/Client/StorageProvider.ts @@ -0,0 +1,137 @@ +import {SessionOptions, Store} from "express-session"; +import {TypeormStore} from "connect-typeorm"; +import {InviteData} from "../Common/interfaces"; +import {buildCachePrefix, createCacheManager, mergeArr} from "../../util"; +import {Cache} from "cache-manager"; +// @ts-ignore +import CacheManagerStore from 'express-session-cache-manager' +import {CacheOptions} from "../../Common/interfaces"; +import {Brackets, DataSource, IsNull, LessThanOrEqual} from "typeorm"; +import {DateUtils} from 'typeorm/util/DateUtils'; +import {ClientSession} from "../../Common/Entities/ClientSession"; +import {Invite} from "../../Common/Entities/Invite"; +import dayjs from "dayjs"; +import {Logger} from "winston"; + +export interface CacheManagerStoreOptions { + prefix?: string +} + +export type TypeormStoreOptions = Partial void; + ttl: number | ((store: TypeormStore, sess: any, sid?: string) => number); +}>; + +interface IWebStorageProvider { + createSessionStore(options?: CacheManagerStoreOptions | TypeormStoreOptions): Store + + inviteGet(id: string): Promise + + inviteDelete(id: string): Promise + + inviteCreate(id: string, data: InviteData): Promise +} + +interface StorageProviderOptions { + logger: Logger + invitesMaxAge?: number + loggerLabels?: string[] +} + +abstract class StorageProvider { + + invitesMaxAge?: number + logger: Logger; + + protected constructor(data: StorageProviderOptions) { + const { + logger, + invitesMaxAge, + loggerLabels = [], + } = data; + this.invitesMaxAge = invitesMaxAge; + this.logger = logger.child({labels: ['Web Storage', ...loggerLabels]}, mergeArr); + } + + protected abstract getInvite(id: string): Promise; + + async inviteGet(id: string) { + const data = await this.getInvite(id); + if (data === undefined || data === null) { + return undefined; + } + return data; + } +} + +export class CacheStorageProvider extends StorageProvider implements IWebStorageProvider { + + protected cache: Cache; + + constructor(caching: CacheOptions & StorageProviderOptions) { + super(caching); + this.cache = createCacheManager({...caching, prefix: buildCachePrefix(['web'])}) as Cache; + this.logger.debug('Using CACHE'); + if (caching.store === 'none') { + this.logger.warn(`Using 'none' as cache provider means no one will be able to access the interface since sessions will never be persisted!`); + } + } + + createSessionStore(options?: CacheManagerStoreOptions): Store { + return new CacheManagerStore(this.cache, {prefix: 'sess:'}); + } + + protected async getInvite(id: string) { + return await this.cache.get(`invite:${id}`) as InviteData | undefined | null; + } + + async inviteCreate(id: string, data: InviteData): Promise { + await this.cache.set(`invite:${id}`, data, {ttl: (this.invitesMaxAge ?? 0) * 1000}); + return data; + } + + async inviteDelete(id: string): Promise { + return await this.cache.del(`invite:${id}`); + } + +} + +export class DatabaseStorageProvider extends StorageProvider implements IWebStorageProvider { + + database: DataSource; + inviteRepo; + + constructor(data: { database: DataSource } & StorageProviderOptions) { + super(data); + this.database = data.database; + this.inviteRepo = this.database.getRepository(Invite); + this.logger.debug('Using DATABASE'); + } + + createSessionStore(options?: TypeormStoreOptions): Store { + return new TypeormStore(options).connect(this.database.getRepository(ClientSession)) + } + + protected async getInvite(id: string): Promise { + const qb = this.inviteRepo.createQueryBuilder('invite'); + return await qb + .andWhere({id}) + .andWhere(new Brackets((qb) => { + qb.where({_expiresAt: LessThanOrEqual(DateUtils.mixedDateToDatetimeString(dayjs().toDate()))}) + .orWhere({_expiresAt: IsNull()}) + }) + ).getOne(); + } + + async inviteCreate(id: string, data: InviteData): Promise { + await this.inviteRepo.save(new Invite({...data, id})); + return data; + } + + async inviteDelete(id: string): Promise { + await this.inviteRepo.delete(id); + } + +} diff --git a/src/Web/Client/index.ts b/src/Web/Client/index.ts index 992b0de..6081c57 100644 --- a/src/Web/Client/index.ts +++ b/src/Web/Client/index.ts @@ -54,7 +54,7 @@ import Autolinker from "autolinker"; import path from "path"; import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients"; import ClientUser from "../Common/User/ClientUser"; -import {BotStatusResponse} from "../Common/interfaces"; +import {BotStatusResponse, InviteData} from "../Common/interfaces"; import {TransformableInfo} from "logform"; import {SimpleError} from "../../Utils/Errors"; import {ErrorWithCause} from "pony-cause"; @@ -65,6 +65,8 @@ import { RulePremise } from "../../Common/Entities/RulePremise"; import { ActionPremise } from "../../Common/Entities/ActionPremise"; import {TypeormStore} from "connect-typeorm"; import {ClientSession} from "../../Common/Entities/ClientSession"; +import {CacheStorageProvider, DatabaseStorageProvider} from "./StorageProvider"; +import {nanoid} from "nanoid"; const emitter = new EventEmitter(); @@ -151,18 +153,23 @@ const webClient = async (options: OperatorConfig) => { display, }, userAgent: uaFragment, + caching: { + provider: caching + }, web: { port, - caching, - caching: { - prefix - }, + storage: webStorage = 'database', + // caching, + // caching: { + // prefix + // }, invites: { maxAge: invitesMaxAge, }, session: { secret, maxAge: sessionMaxAge, + storage: sessionStorage = 'database', }, maxLogs, clients, @@ -195,14 +202,10 @@ const webClient = async (options: OperatorConfig) => { throw new SimpleError(`Specified port for web interface (${port}) is in use or not available. Cannot start web server.`); } - if (caching.store === 'none') { - logger.warn(`Cannot use 'none' for web caching or else no one can use the interface...falling back to 'memory'`); - caching.store = 'memory'; - } - //const webCachePrefix = buildCachePrefix([prefix, 'web']); - const webCache = createCacheManager({...caching, prefix: buildCachePrefix([prefix, 'web'])}) as Cache; + //const webCache = createCacheManager({...caching, prefix: buildCachePrefix([prefix, 'web'])}) as Cache; + + const storage = webStorage === 'database' ? new DatabaseStorageProvider({database, invitesMaxAge, logger}) : new CacheStorageProvider({...caching, invitesMaxAge, logger}); - //const previousSessions = await webCache.get const connectedUsers: ConnectUserObj = {}; // @@ -212,19 +215,12 @@ const webClient = async (options: OperatorConfig) => { passport.serializeUser(async function (data: any, done) { const {user, subreddits, scope, token} = data; - //await webCache.set(`userSession-${user}`, { subreddits: subreddits.map((x: Subreddit) => x.display_name), isOperator: webOps.includes(user.toLowerCase()) }, {ttl: provider.ttl as number}); done(null, { subreddits: subreddits.map((x: Subreddit) => x.display_name), isOperator: webOps.includes(user.toLowerCase()), name: user, scope, token, tokenExpiresAt: dayjs().unix() + (60 * 60) }); }); passport.deserializeUser(async function (obj: any, done) { const user = new ClientUser(obj.name, obj.subreddits, {token: obj.token, scope: obj.scope, webOperator: obj.isOperator, tokenExpiresAt: obj.tokenExpiresAt}); done(null, user); - // const data = await webCache.get(`userSession-${obj}`) as object; - // if (data === undefined) { - // done('Not Found'); - // } - // - // done(null, {...data, name: obj as string} as Express.User); }); passport.use('snoowrap', new CustomStrategy( @@ -260,15 +256,19 @@ const webClient = async (options: OperatorConfig) => { } )); + + let sessionStoreProvider = storage; + if(sessionStorage !== webStorage) { + sessionStoreProvider = sessionStorage === 'database' ? new DatabaseStorageProvider({database, invitesMaxAge, logger, loggerLabels: ['Session']}) : new CacheStorageProvider({...caching, invitesMaxAge, logger, loggerLabels: ['Session']}); + } const sessionObj = session({ cookie: { maxAge: sessionMaxAge * 1000, }, - //store: new CacheManagerStore(webCache, {prefix: 'sess:'}), - store: new TypeormStore({ + store: sessionStoreProvider.createSessionStore(sessionStorage === 'database' ? { cleanupLimit: 2, ttl: sessionMaxAge - }).connect(database.getRepository(ClientSession)), + } : {}), resave: false, saveUninitialized: false, secret, @@ -338,7 +338,11 @@ const webClient = async (options: OperatorConfig) => { return res.render('error', {error: errContent}); } // @ts-ignore - const invite = await webCache.get(`invite:${req.session.inviteId}`) as InviteData; + const invite = await storage.inviteGet(req.session.inviteId); + if(invite === undefined) { + // @ts-ignore + return res.render('error', {error: `Could not find invite with id ${req.session.inviteId}?? This should happen!`}); + } const client = await Snoowrap.fromAuthCode({ userAgent, clientId: invite.clientId, @@ -350,7 +354,7 @@ const webClient = async (options: OperatorConfig) => { const user = await client.getMe(); const userName = `u/${user.name}`; // @ts-ignore - await webCache.del(`invite:${req.session.inviteId}`); + await storage.inviteDelete(req.session.inviteId); let data: any = { accessToken: client.accessToken, refreshToken: client.refreshToken, @@ -422,16 +426,6 @@ const webClient = async (options: OperatorConfig) => { }); let token = randomId(); - interface InviteData { - permissions: string[], - subreddits?: string, - instance?: string, - clientId: string - clientSecret: string - redirectUri: string - creator: string - overwrite?: boolean - } const helperAuthed = async (req: express.Request, res: express.Response, next: Function) => { @@ -469,7 +463,7 @@ const webClient = async (options: OperatorConfig) => { if(inviteId === undefined) { return res.render('error', {error: '`invite` param is missing from URL'}); } - const invite = await webCache.get(`invite:${inviteId}`) as InviteData | undefined | null; + const invite = await storage.inviteGet(inviteId as string); if(invite === undefined || invite === null) { return res.render('error', {error: 'Invite with the given id does not exist'}); } @@ -505,8 +499,8 @@ const webClient = async (options: OperatorConfig) => { return res.status(400).send('redirectUrl is required'); } - const inviteId = code || randomId(); - await webCache.set(`invite:${inviteId}`, { + const inviteId = code || nanoid(20); + await storage.inviteCreate(inviteId, { permissions, clientId: (ci || clientId).trim(), clientSecret: (ce || clientSecret).trim(), @@ -514,7 +508,7 @@ const webClient = async (options: OperatorConfig) => { instance, subreddits: subreddits.trim() === '' ? [] : subreddits.split(',').map((x: string) => parseRedditEntity(x).name), creator: (req.user as Express.User).name, - }, {ttl: invitesMaxAge * 1000}); + }); return res.send(inviteId); }); @@ -523,7 +517,7 @@ const webClient = async (options: OperatorConfig) => { if(inviteId === undefined) { return res.render('error', {error: '`invite` param is missing from URL'}); } - const invite = await webCache.get(`invite:${inviteId}`) as InviteData | undefined | null; + const invite = await storage.inviteGet(inviteId as string); if(invite === undefined || invite === null) { return res.render('error', {error: 'Invite with the given id does not exist'}); } diff --git a/src/Web/Common/interfaces.ts b/src/Web/Common/interfaces.ts index b5aa315..7a6e398 100644 --- a/src/Web/Common/interfaces.ts +++ b/src/Web/Common/interfaces.ts @@ -85,3 +85,15 @@ export interface HeartbeatResponse { friendly?: string bots: BotInstance[] } + + +export interface InviteData { + permissions: string[], + subreddits?: string[], + instance?: string, + clientId: string + clientSecret: string + redirectUri: string + creator: string + overwrite?: boolean +}