github/context-mod
1
1
Fork 0
mirror of https://github.com/FoxxMD/context-mod.git synced 2026-01-11 22:57:57 -05:00
Code Issues Packages Projects Releases Wiki Activity

refactor(web)!: Migrate all web storage to use either database or session

Browse Source 
* BREAKING: Replace web.caching with web.storage and restrict options to specifying either top-level database or cache
  * Simplifies init but provides options for different use cases
  * Default to database
* Implement generic storage provider for web persistence. Uses either cache or database
* Refactor session/invites to use storage provider
* Implement invite entity for database
* Allow specifying different storage provider for sessions
This commit is contained in:
FoxxMD
2022-04-05 13:18:18 -04:00
parent c81703720b
commit f4e5f1a22f
7 changed files with 336 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
src/Common/Entities/Invite.ts Normal file
View File

@@ -0,0 +1,72 @@
import {Entity, Column, PrimaryColumn} from "typeorm";
import {TimeAwareBaseEntity} from "./Base/TimeAwareBaseEntity";
import {InviteData} from "../../Web/Common/interfaces";
import dayjs, {Dayjs} from "dayjs";
@Entity()
export class Invite extends TimeAwareBaseEntity implements InviteData {
@PrimaryColumn('varchar', {length: 255})
id!: string
@Column("varchar", {length: 50})
clientId!: string;
@Column("varchar", {length: 50})
clientSecret!: string;
@Column("text")
redirectUri!: string;
@Column("varchar", {length: 255})
creator!: string;
@Column("simple-json")
permissions!: string[];
@Column("varchar", {length: 200, nullable: true})
instance?: string;
@Column()
overwrite?: boolean;
@Column("simple-json", {nullable: true})
subreddits?: string[];
@Column({name: 'expiresAt', nullable: true})
_expiresAt?: Date;
public get expiresAt(): Dayjs | undefined {
if (this._expiresAt === undefined) {
return undefined;
}
return dayjs(this._expiresAt);
}
public set expiresAt(d: Dayjs | undefined) {
if (d === undefined) {
this._expiresAt = d;
} else {
this._expiresAt = d.utc().toDate();
}
}
constructor(data?: InviteData & { id: string, expiresIn?: number }) {
super();
if (data !== undefined) {
this.id = data.id;
this.permissions = data.permissions;
this.subreddits = data.subreddits;
this.instance = data.instance;
this.clientId = data.clientId;
this.clientSecret = data.clientSecret;
this.redirectUri = data.redirectUri;
this.creator = data.creator;
this.overwrite = data.overwrite;
if (data.expiresIn !== undefined && data.expiresIn !== 0) {
this.expiresAt = dayjs().add(data.expiresIn, 'seconds');
}
}
}
}

58
src/Common/Migrations/Database/1642180264563-initApi.ts
View File

@@ -121,6 +121,64 @@ export class initApi1642180264563 implements MigrationInterface {
true
);
await queryRunner.createTable(
new Table({
name: 'Invite',
columns: [
{
name: 'id',
type: 'varchar',
length: '255',
isPrimary: true,
},
{
name: 'clientId',
type: 'varchar',
length: '255',
},
{
name: 'clientSecret',
type: 'varchar',
length: '255',
},
{
name: 'redirectUri',
type: 'text',
},
{
name: 'creator',
type: 'varchar',
length: '255',
},
{
name: 'permissions',
type: 'text'
},
{
name: 'instance',
type: 'varchar',
length: '255',
isNullable: true
},
{
name: 'overwrite',
type: 'boolean',
isNullable: true,
},
{
name: 'subreddits',
type: 'text',
isNullable: true
},
createdAtColumn(dbType),
timeAtColumn('expiresAt', dbType, true)
],
}),
true,
true,
true
);
await queryRunner.createTable(
new Table({

26
src/Common/interfaces.ts
View File

@@ -8,7 +8,7 @@ import {SqljsConnectionOptions} from "typeorm/driver/sqljs/SqljsConnectionOption
import {MysqlConnectionOptions} from "typeorm/driver/mysql/MysqlConnectionOptions";
import {MongoConnectionOptions} from "typeorm/driver/mongodb/MongoConnectionOptions";
import {PostgresConnectionOptions} from "typeorm/driver/postgres/PostgresConnectionOptions";
import {Connection} from "typeorm";
import {Connection, DataSource} from "typeorm";
import {AuthorCriteria, AuthorOptions} from "../Author/Author";
import {JsonOperatorConfigDocument, YamlOperatorConfigDocument} from "./Config/Operator";
import {ConsoleTransportOptions} from "winston/lib/winston/transports";
@@ -1853,11 +1853,17 @@ export interface OperatorJsonConfig {
port?: number,
/**
* Caching provider to use for session and invite data
* Storage provider to use for sessions and other web client specific data
*
* If none is provided the top-level database is used
*
* * Specify `database` to use top-level database
* * Specify `cache` to use top-level cache
*
* NOTE: `database` should almost always be used. Cache would only be necessary if this instance experiences heavy traffic
*
* If none is provided the top-level caching provider is used
* */
caching?: 'memory' | 'redis' | CacheOptions
storage?: 'database' | 'cache'
/**
* Settings to configure the behavior of user sessions -- the session is what the web interface uses to identify logged in users.
* */
@@ -1881,6 +1887,13 @@ export interface OperatorJsonConfig {
* @examples ["definitelyARandomString"]
* */
secret?: string,
/**
* Specify backend storage to use for persisting client sessions. If specified this will overwrite parent-level `storage` settings.
*
* May be useful if using `database` for general web client storage but have heavy traffic and want sessions to be more performant (using `cache`)
* */
storage?: 'database' | 'cache'
}
/**
@@ -2027,13 +2040,14 @@ export interface OperatorConfig extends OperatorJsonConfig {
connection: DatabaseConfig,
migrations: DatabaseMigrationOptions
}
database: Connection
database: DataSource
web: {
port: number,
caching: CacheOptions,
storage?: 'database' | 'cache'
session: {
maxAge: number,
secret: string,
storage?: 'database' | 'cache'
},
invites: {
maxAge: number

9
src/ConfigBuilder.ts
View File

@@ -753,10 +753,11 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig):
web: {
port = 8085,
maxLogs = 200,
caching: webCaching = {},
storage: webStorage = undefined,
session: {
secret = randomId(),
maxAge: sessionMaxAge = 86400,
storage: sessionStorage = undefined,
} = {},
invites: {
maxAge: inviteMaxAge = 0,
@@ -874,16 +875,14 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig):
userAgent,
web: {
port,
caching: {
...defaultProvider,
...webCaching
},
storage: webStorage,
invites: {
maxAge: inviteMaxAge,
},
session: {
secret,
maxAge: sessionMaxAge,
storage: sessionStorage
},
maxLogs,
clients: clients === undefined ? [{host: 'localhost:8095', secret: apiSecret}] : clients,

137
src/Web/Client/StorageProvider.ts Normal file
View File

@@ -0,0 +1,137 @@
import {SessionOptions, Store} from "express-session";
import {TypeormStore} from "connect-typeorm";
import {InviteData} from "../Common/interfaces";
import {buildCachePrefix, createCacheManager, mergeArr} from "../../util";
import {Cache} from "cache-manager";
// @ts-ignore
import CacheManagerStore from 'express-session-cache-manager'
import {CacheOptions} from "../../Common/interfaces";
import {Brackets, DataSource, IsNull, LessThanOrEqual} from "typeorm";
import {DateUtils} from 'typeorm/util/DateUtils';
import {ClientSession} from "../../Common/Entities/ClientSession";
import {Invite} from "../../Common/Entities/Invite";
import dayjs from "dayjs";
import {Logger} from "winston";
export interface CacheManagerStoreOptions {
prefix?: string
}
export type TypeormStoreOptions = Partial<SessionOptions & {
cleanupLimit: number;
limitSubquery: boolean;
onError: (s: TypeormStore, e: Error) => void;
ttl: number | ((store: TypeormStore, sess: any, sid?: string) => number);
}>;
interface IWebStorageProvider {
createSessionStore(options?: CacheManagerStoreOptions | TypeormStoreOptions): Store
inviteGet(id: string): Promise<InviteData | undefined>
inviteDelete(id: string): Promise<void>
inviteCreate(id: string, data: InviteData): Promise<InviteData>
}
interface StorageProviderOptions {
logger: Logger
invitesMaxAge?: number
loggerLabels?: string[]
}
abstract class StorageProvider {
invitesMaxAge?: number
logger: Logger;
protected constructor(data: StorageProviderOptions) {
const {
logger,
invitesMaxAge,
loggerLabels = [],
} = data;
this.invitesMaxAge = invitesMaxAge;
this.logger = logger.child({labels: ['Web Storage', ...loggerLabels]}, mergeArr);
}
protected abstract getInvite(id: string): Promise<InviteData | undefined | null>;
async inviteGet(id: string) {
const data = await this.getInvite(id);
if (data === undefined || data === null) {
return undefined;
}
return data;
}
}
export class CacheStorageProvider extends StorageProvider implements IWebStorageProvider {
protected cache: Cache;
constructor(caching: CacheOptions & StorageProviderOptions) {
super(caching);
this.cache = createCacheManager({...caching, prefix: buildCachePrefix(['web'])}) as Cache;
this.logger.debug('Using CACHE');
if (caching.store === 'none') {
this.logger.warn(`Using 'none' as cache provider means no one will be able to access the interface since sessions will never be persisted!`);
}
}
createSessionStore(options?: CacheManagerStoreOptions): Store {
return new CacheManagerStore(this.cache, {prefix: 'sess:'});
}
protected async getInvite(id: string) {
return await this.cache.get(`invite:${id}`) as InviteData | undefined | null;
}
async inviteCreate(id: string, data: InviteData): Promise<InviteData> {
await this.cache.set(`invite:${id}`, data, {ttl: (this.invitesMaxAge ?? 0) * 1000});
return data;
}
async inviteDelete(id: string): Promise<void> {
return await this.cache.del(`invite:${id}`);
}
}
export class DatabaseStorageProvider extends StorageProvider implements IWebStorageProvider {
database: DataSource;
inviteRepo;
constructor(data: { database: DataSource } & StorageProviderOptions) {
super(data);
this.database = data.database;
this.inviteRepo = this.database.getRepository(Invite);
this.logger.debug('Using DATABASE');
}
createSessionStore(options?: TypeormStoreOptions): Store {
return new TypeormStore(options).connect(this.database.getRepository(ClientSession))
}
protected async getInvite(id: string): Promise<InviteData | undefined | null> {
const qb = this.inviteRepo.createQueryBuilder('invite');
return await qb
.andWhere({id})
.andWhere(new Brackets((qb) => {
qb.where({_expiresAt: LessThanOrEqual(DateUtils.mixedDateToDatetimeString(dayjs().toDate()))})
.orWhere({_expiresAt: IsNull()})
})
).getOne();
}
async inviteCreate(id: string, data: InviteData): Promise<InviteData> {
await this.inviteRepo.save(new Invite({...data, id}));
return data;
}
async inviteDelete(id: string): Promise<void> {
await this.inviteRepo.delete(id);
}
}

72
src/Web/Client/index.ts
View File

@@ -54,7 +54,7 @@ import Autolinker from "autolinker";
import path from "path";
import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients";
import ClientUser from "../Common/User/ClientUser";
import {BotStatusResponse} from "../Common/interfaces";
import {BotStatusResponse, InviteData} from "../Common/interfaces";
import {TransformableInfo} from "logform";
import {SimpleError} from "../../Utils/Errors";
import {ErrorWithCause} from "pony-cause";
@@ -65,6 +65,8 @@ import { RulePremise } from "../../Common/Entities/RulePremise";
import { ActionPremise } from "../../Common/Entities/ActionPremise";
import {TypeormStore} from "connect-typeorm";
import {ClientSession} from "../../Common/Entities/ClientSession";
import {CacheStorageProvider, DatabaseStorageProvider} from "./StorageProvider";
import {nanoid} from "nanoid";
const emitter = new EventEmitter();
@@ -151,18 +153,23 @@ const webClient = async (options: OperatorConfig) => {
display,
},
userAgent: uaFragment,
caching: {
provider: caching
},
web: {
port,
caching,
caching: {
prefix
},
storage: webStorage = 'database',
// caching,
// caching: {
// prefix
// },
invites: {
maxAge: invitesMaxAge,
},
session: {
secret,
maxAge: sessionMaxAge,
storage: sessionStorage = 'database',
},
maxLogs,
clients,
@@ -195,14 +202,10 @@ const webClient = async (options: OperatorConfig) => {
throw new SimpleError(`Specified port for web interface (${port}) is in use or not available. Cannot start web server.`);
}
if (caching.store === 'none') {
logger.warn(`Cannot use 'none' for web caching or else no one can use the interface...falling back to 'memory'`);
caching.store = 'memory';
}
//const webCachePrefix = buildCachePrefix([prefix, 'web']);
const webCache = createCacheManager({...caching, prefix: buildCachePrefix([prefix, 'web'])}) as Cache;
//const webCache = createCacheManager({...caching, prefix: buildCachePrefix([prefix, 'web'])}) as Cache;
const storage = webStorage === 'database' ? new DatabaseStorageProvider({database, invitesMaxAge, logger}) : new CacheStorageProvider({...caching, invitesMaxAge, logger});
//const previousSessions = await webCache.get
const connectedUsers: ConnectUserObj = {};
//<editor-fold desc=Session and Auth>
@@ -212,19 +215,12 @@ const webClient = async (options: OperatorConfig) => {
passport.serializeUser(async function (data: any, done) {
const {user, subreddits, scope, token} = data;
//await webCache.set(`userSession-${user}`, { subreddits: subreddits.map((x: Subreddit) => x.display_name), isOperator: webOps.includes(user.toLowerCase()) }, {ttl: provider.ttl as number});
done(null, { subreddits: subreddits.map((x: Subreddit) => x.display_name), isOperator: webOps.includes(user.toLowerCase()), name: user, scope, token, tokenExpiresAt: dayjs().unix() + (60 * 60) });
});
passport.deserializeUser(async function (obj: any, done) {
const user = new ClientUser(obj.name, obj.subreddits, {token: obj.token, scope: obj.scope, webOperator: obj.isOperator, tokenExpiresAt: obj.tokenExpiresAt});
done(null, user);
// const data = await webCache.get(`userSession-${obj}`) as object;
// if (data === undefined) {
// done('Not Found');
// }
//
// done(null, {...data, name: obj as string} as Express.User);
});
passport.use('snoowrap', new CustomStrategy(
@@ -260,15 +256,19 @@ const webClient = async (options: OperatorConfig) => {
}
));
let sessionStoreProvider = storage;
if(sessionStorage !== webStorage) {
sessionStoreProvider = sessionStorage === 'database' ? new DatabaseStorageProvider({database, invitesMaxAge, logger, loggerLabels: ['Session']}) : new CacheStorageProvider({...caching, invitesMaxAge, logger, loggerLabels: ['Session']});
}
const sessionObj = session({
cookie: {
maxAge: sessionMaxAge * 1000,
},
//store: new CacheManagerStore(webCache, {prefix: 'sess:'}),
store: new TypeormStore({
store: sessionStoreProvider.createSessionStore(sessionStorage === 'database' ? {
cleanupLimit: 2,
ttl: sessionMaxAge
}).connect(database.getRepository(ClientSession)),
} : {}),
resave: false,
saveUninitialized: false,
secret,
@@ -338,7 +338,11 @@ const webClient = async (options: OperatorConfig) => {
return res.render('error', {error: errContent});
}
// @ts-ignore
const invite = await webCache.get(`invite:${req.session.inviteId}`) as InviteData;
const invite = await storage.inviteGet(req.session.inviteId);
if(invite === undefined) {
// @ts-ignore
return res.render('error', {error: `Could not find invite with id ${req.session.inviteId}?? This should happen!`});
}
const client = await Snoowrap.fromAuthCode({
userAgent,
clientId: invite.clientId,
@@ -350,7 +354,7 @@ const webClient = async (options: OperatorConfig) => {
const user = await client.getMe();
const userName = `u/${user.name}`;
// @ts-ignore
await webCache.del(`invite:${req.session.inviteId}`);
await storage.inviteDelete(req.session.inviteId);
let data: any = {
accessToken: client.accessToken,
refreshToken: client.refreshToken,
@@ -422,16 +426,6 @@ const webClient = async (options: OperatorConfig) => {
});
let token = randomId();
interface InviteData {
permissions: string[],
subreddits?: string,
instance?: string,
clientId: string
clientSecret: string
redirectUri: string
creator: string
overwrite?: boolean
}
const helperAuthed = async (req: express.Request, res: express.Response, next: Function) => {
@@ -469,7 +463,7 @@ const webClient = async (options: OperatorConfig) => {
if(inviteId === undefined) {
return res.render('error', {error: '`invite` param is missing from URL'});
}
const invite = await webCache.get(`invite:${inviteId}`) as InviteData | undefined | null;
const invite = await storage.inviteGet(inviteId as string);
if(invite === undefined || invite === null) {
return res.render('error', {error: 'Invite with the given id does not exist'});
}
@@ -505,8 +499,8 @@ const webClient = async (options: OperatorConfig) => {
return res.status(400).send('redirectUrl is required');
}
const inviteId = code || randomId();
await webCache.set(`invite:${inviteId}`, {
const inviteId = code || nanoid(20);
await storage.inviteCreate(inviteId, {
permissions,
clientId: (ci || clientId).trim(),
clientSecret: (ce || clientSecret).trim(),
@@ -514,7 +508,7 @@ const webClient = async (options: OperatorConfig) => {
instance,
subreddits: subreddits.trim() === '' ? [] : subreddits.split(',').map((x: string) => parseRedditEntity(x).name),
creator: (req.user as Express.User).name,
}, {ttl: invitesMaxAge * 1000});
});
return res.send(inviteId);
});
@@ -523,7 +517,7 @@ const webClient = async (options: OperatorConfig) => {
if(inviteId === undefined) {
return res.render('error', {error: '`invite` param is missing from URL'});
}
const invite = await webCache.get(`invite:${inviteId}`) as InviteData | undefined | null;
const invite = await storage.inviteGet(inviteId as string);
if(invite === undefined || invite === null) {
return res.render('error', {error: 'Invite with the given id does not exist'});
}

12
src/Web/Common/interfaces.ts
View File

@@ -85,3 +85,15 @@ export interface HeartbeatResponse {
friendly?: string
bots: BotInstance[]
}
export interface InviteData {
permissions: string[],
subreddits?: string[],
instance?: string,
clientId: string
clientSecret: string
redirectUri: string
creator: string
overwrite?: boolean
}