github/context-mod
1
1
Fork 0
mirror of https://github.com/FoxxMD/context-mod.git synced 2026-04-19 03:00:07 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'edge'

Browse Source
This commit is contained in:
FoxxMD
2021-07-29 13:25:16 -04:00
parent acc7c49e0e fd4c2a38e7
commit 0c7218571c
13 changed files with 237 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
docs/operatorConfiguration.md
View File

@@ -57,6 +57,9 @@ const config = {
// ENV => OPERATOR_DISPLAY
// ARG => --operator <name>
display: undefined,
// Name to display for the bot in web interface, logs, notifications...
// If not defined will use authenticated Reddit account IE u/yourBot
botName: undefined
},
// Values required to interact with Reddit's API
credentials: {

33
src/App.ts
View File

@@ -48,7 +48,8 @@ export class App {
hardLimit: number | string = 50;
nannyMode?: 'soft' | 'hard';
nextExpiration!: Dayjs;
botName?: string;
botName!: string;
maxWorkers: number;
startedAt: Dayjs = dayjs();
sharedModqueue: boolean = false;
@@ -60,6 +61,9 @@ export class App {
constructor(config: OperatorConfig) {
const {
operator: {
botName,
},
subreddits: {
names = [],
wikiConfig,
@@ -79,6 +83,9 @@ export class App {
polling: {
sharedMod,
},
queue: {
maxWorkers,
},
caching: {
authorTTL,
provider: {
@@ -100,9 +107,19 @@ export class App {
this.hardLimit = hardLimit;
this.wikiLocation = wikiConfig;
this.sharedModqueue = sharedMod;
if(botName !== undefined) {
this.botName = botName;
}
this.logger = getLogger(config.logging);
let mw = maxWorkers;
if(maxWorkers < 1) {
this.logger.warn(`Max queue workers must be greater than or equal to 1 (Specified: ${maxWorkers})`);
mw = 1;
}
this.maxWorkers = mw;
if (this.dryRun) {
this.logger.info('Running in DRYRUN mode');
}
@@ -200,13 +217,19 @@ export class App {
let availSubs = [];
const name = await this.client.getMe().name;
this.logger.info(`Reddit API Limit Remaining: ${this.client.ratelimitRemaining}`);
this.logger.info(`Authenticated Account: /u/${name}`);
this.botName = name;
this.logger.info(`Authenticated Account: u/${name}`);
const botNameFromConfig = this.botName !== undefined;
if(this.botName === undefined) {
this.botName = `u/${name}`;
}
this.logger.info(`Bot Name${botNameFromConfig ? ' (from config)' : ''}: ${this.botName}`);
for (const sub of await this.client.getModeratedSubreddits()) {
// TODO don't know a way to check permissions yet
availSubs.push(sub);
}
this.logger.info(`/u/${name} is a moderator of these subreddits: ${availSubs.map(x => x.display_name_prefixed).join(', ')}`);
this.logger.info(`${this.botName} is a moderator of these subreddits: ${availSubs.map(x => x.display_name_prefixed).join(', ')}`);
let subsToRun: Subreddit[] = [];
const subsToUse = subreddits.length > 0 ? subreddits.map(parseSubredditName) : this.subreddits;
@@ -231,7 +254,7 @@ export class App {
let subSchedule: Manager[] = [];
// get configs for subs we want to run on and build/validate them
for (const sub of subsToRun) {
const manager = new Manager(sub, this.client, this.logger, {dryRun: this.dryRun, sharedModqueue: this.sharedModqueue, wikiLocation: this.wikiLocation});
const manager = new Manager(sub, this.client, this.logger, {dryRun: this.dryRun, sharedModqueue: this.sharedModqueue, wikiLocation: this.wikiLocation, botName: this.botName, maxWorkers: this.maxWorkers});
try {
await manager.parseConfiguration('system', true, {suppressNotification: true});
} catch (err) {

40
src/Common/interfaces.ts
View File

@@ -460,6 +460,23 @@ export interface ManagerOptions {
* */
polling?: (string | PollingOptions)[]
queue?: {
/**
* The maximum number of events that can be processed simultaneously.
*
* **Do not modify this setting unless you know what you are doing.** The default of `1` is suitable for the majority of use-cases.
*
* Raising the max above `1` could be useful if you require very fast response time to short bursts of high-volume events. However logs may become unreadable as many events are processed at the same time. Additionally, any events that depend on past actions from your bot may not be processed correctly given the concurrent nature of this use case.
*
* **Note:** Max workers are also enforced at the operator level so a subreddit cannot raise their max above what is specified by the operator.
*
* @default 1
* @minimum 1
* @examples [1]
* */
maxWorkers?: number
}
/**
* Per-subreddit config for caching TTL values. If set to `false` caching is disabled.
* */
@@ -701,8 +718,24 @@ export interface ManagerStateChangeOption {
export interface OperatorJsonConfig {
operator?: {
/**
* The name of the Reddit account, without prefix, that the operator of this bot uses.
*
* This is used for showing more information in the web interface IE show all logs/subreddits if even not a moderator.
*
* EX -- User is /u/FoxxMD then `"name": "FoxxMD"`
* */
name?: string,
/**
* A **public** name to display to users of the web interface. Use this to help moderators using your bot identify who is the operator in case they need to contact you.
*
* Leave undefined for no public name to be displayed.
* */
display?: string,
/**
* The name to use when identifying the bot. Defaults to name of the authenticated Reddit account IE `u/yourBotAccount`
* */
botName?: string,
},
credentials?: {
clientId?: string,
@@ -731,6 +764,9 @@ export interface OperatorJsonConfig {
limit?: number,
interval?: number,
},
queue?: {
maxWorkers?: number,
},
web?: {
enabled?: boolean,
port?: number,
@@ -776,6 +812,7 @@ export interface OperatorConfig extends OperatorJsonConfig {
operator: {
name?: string
display?: string,
botName?: string,
},
credentials: {
clientId: string,
@@ -804,6 +841,9 @@ export interface OperatorConfig extends OperatorJsonConfig {
limit: number,
interval: number,
},
queue: {
maxWorkers: number,
},
web: {
enabled: boolean,
port: number,

12
src/ConfigBuilder.ts
View File

@@ -463,7 +463,8 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
const {
operator: {
name,
display = 'Anonymous'
display = 'Anonymous',
botName,
} = {},
credentials: {
clientId: ci,
@@ -495,6 +496,9 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
limit = 100,
interval = 30,
} = {},
queue: {
maxWorkers = 1,
} = {},
caching = 'memory',
api: {
softLimit = 250,
@@ -546,7 +550,8 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
const config: OperatorConfig = {
operator: {
name,
display
display,
botName,
},
credentials: {
clientId: (ci as string),
@@ -588,6 +593,9 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
limit,
interval,
},
queue: {
maxWorkers,
},
api: {
softLimit,
hardLimit

6
src/Notification/DiscordNotifier.ts
View File

@@ -3,12 +3,14 @@ import {NotificationContent} from "../Common/interfaces";
class DiscordNotifier {
name: string
botName: string
type: string = 'Discord';
url: string;
constructor(name: string, url: string) {
constructor(name: string, botName: string, url: string) {
this.name = name;
this.url = url;
this.botName = botName;
}
async handle(val: NotificationContent) {
@@ -18,7 +20,7 @@ class DiscordNotifier {
const {logLevel, title, footer, body = ''} = val;
hook.setName('RCB')
hook.setName(this.botName === 'ContextMod' ? 'ContextMod' : `(ContextMod) ${this.botName}`)
.setTitle(title)
.setDescription(body)

4
src/Notification/NotificationManager.ts
View File

@@ -17,7 +17,7 @@ class NotificationManager {
subreddit: Subreddit;
name: string;
constructor(logger: Logger, subreddit: Subreddit, displayName: string, config?: NotificationConfig) {
constructor(logger: Logger, subreddit: Subreddit, displayName: string, botName: string, config?: NotificationConfig) {
this.logger = logger.child({leaf: 'Notifications'}, mergeArr);
this.subreddit = subreddit;
this.name = displayName;
@@ -27,7 +27,7 @@ class NotificationManager {
for (const p of providers) {
switch (p.type) {
case 'discord':
this.notifiers.push(new DiscordNotifier(p.name, p.url));
this.notifiers.push(new DiscordNotifier(p.name, botName, p.url));
break;
default:
this.logger.warn(`Notification provider type of ${p.type} not recognized.`);

14
src/Schema/App.json
View File

@@ -2609,6 +2609,20 @@
]
},
"type": "array"
},
"queue": {
"properties": {
"maxWorkers": {
"default": 1,
"description": "The maximum number of events that can be processed simultaneously.\n\n**Do not modify this setting unless you know what you are doing.** The default of `1` is suitable for the majority of use-cases.\n\nRaising the max above `1` could be useful if you require very fast response time to short bursts of high-volume events. However logs may become unreadable as many events are processed at the same time. Additionally, any events that depend on past actions from your bot may not be processed correctly given the concurrent nature of this use case.\n\n**Note:** Max workers are also enforced at the operator level so a subreddit cannot raise their max above what is specified by the operator.",
"examples": [
1
],
"minimum": 1,
"type": "number"
}
},
"type": "object"
}
},
"required": [

14
src/Schema/OperatorConfig.json
View File

@@ -261,10 +261,16 @@
},
"operator": {
"properties": {
"botName": {
"description": "The name to use when identifying the bot. Defaults to name of the authenticated Reddit account IE `u/yourBotAccount`",
"type": "string"
},
"display": {
"description": "A **public** name to display to users of the web interface. Use this to help moderators using your bot identify who is the operator in case they need to contact you.\n\nLeave undefined for no public name to be displayed.",
"type": "string"
},
"name": {
"description": "The name of the Reddit account, without prefix, that the operator of this bot uses.\n\nThis is used for showing more information in the web interface IE show all logs/subreddits if even not a moderator.\n\nEX -- User is /u/FoxxMD then `\"name\": \"FoxxMD\"`",
"type": "string"
}
},
@@ -291,6 +297,14 @@
}
]
},
"queue": {
"properties": {
"maxWorkers": {
"type": "number"
}
},
"type": "object"
},
"snoowrap": {
"properties": {
"debug": {

27
src/Server/server.ts
View File

@@ -327,6 +327,8 @@ const rcbServer = function (options: OperatorConfig): ([() => Promise<void>, App
queuedActivities: m.queue.length(),
runningActivities: m.queue.running(),
maxWorkers: m.queue.concurrency,
subMaxWorkers: m.subMaxWorkers || bot.maxWorkers,
globalMaxWorkers: bot.maxWorkers,
validConfig: boolToString(m.validConfigLoaded),
dryRun: boolToString(m.dryRun === true),
pollingInfo: m.pollOptions.length === 0 ? ['nothing :('] : m.pollOptions.map(pollingInfo),
@@ -392,6 +394,11 @@ const rcbServer = function (options: OperatorConfig): ([() => Promise<void>, App
rulesCachedTotal: acc.rulesCachedTotal + curr.stats.rulesCachedTotal,
rulesTriggeredTotal: acc.rulesTriggeredTotal + curr.stats.rulesTriggeredTotal,
actionsRunTotal: acc.actionsRunTotal + curr.stats.actionsRunTotal,
maxWorkers: acc.maxWorkers + curr.maxWorkers,
subMaxWorkers: acc.subMaxWorkers + curr.subMaxWorkers,
globalMaxWorkers: acc.globalMaxWorkers + curr.globalMaxWorkers,
runningActivities: acc.runningActivities + curr.runningActivities,
queuedActivities: acc.queuedActivities + curr.queuedActivities,
};
}, {
checks: {
@@ -405,8 +412,21 @@ const rcbServer = function (options: OperatorConfig): ([() => Promise<void>, App
rulesCachedTotal: 0,
rulesTriggeredTotal: 0,
actionsRunTotal: 0,
maxWorkers: 0,
subMaxWorkers: 0,
globalMaxWorkers: 0,
runningActivities: 0,
queuedActivities: 0,
});
const {checks, ...rest} = totalStats;
const {
checks,
maxWorkers,
globalMaxWorkers,
subMaxWorkers,
runningActivities,
queuedActivities,
...rest
} = totalStats;
let cumRaw = subManagerData.reduce((acc, curr) => {
Object.keys(curr.stats.cache.types as ResourceStats).forEach((k) => {
@@ -425,6 +445,11 @@ const rcbServer = function (options: OperatorConfig): ([() => Promise<void>, App
name: 'All',
linkName: 'All',
indicator: 'green',
maxWorkers,
globalMaxWorkers,
subMaxWorkers,
runningActivities,
queuedActivities,
botState: {
state: RUNNING,
causedBy: SYSTEM

2
src/Server/views/partials/authTitle.ejs
View File

@@ -4,7 +4,7 @@
<div class="flex items-center flex-grow pr-4">
<div class="px-4 width-full relative">
<span>
<a href="https://github.com/FoxxMD/context-mod">ContextMod</a> for <a href="https://reddit.com/user/<%= botName %>">/u/<%= botName %></a>
<a href="https://github.com/FoxxMD/context-mod">ContextMod</a> for <a href="https://reddit.com/user/<%= botName %>"><%= botName %></a>
</span>
<span class="inline-block -mb-3 ml-2">
<label style="font-size:2.5px;">

2
src/Server/views/partials/head.ejs
View File

@@ -77,7 +77,7 @@
display: none;
}
</style>
<title><%= title !== undefined ? title : `CM for /u/${botName}`%></title>
<title><%= title !== undefined ? title : `CM for ${botName}`%></title>
<!--<title><%# `CM for /u/${botName}`%></title>-->
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">

38
src/Server/views/status.ejs
View File

@@ -177,15 +177,6 @@
</div>
</div>
</div>
<label>Activities</label>
<span class="has-tooltip">
<span style="margin-top:-55px"
class='tooltip rounded shadow-lg p-1 bg-gray-100 text-black'>
<div>Max Concurrent Processing</div>
<div>Config: <%= data.maxWorkers %></div>
</span>
<span><%= `${data.runningActivities} Processing / ${data.queuedActivities} Queued` %></span>
</span>
<label>Slow Mode</label>
<span><%= data.delayBy %></span>
<% } %>
@@ -214,6 +205,35 @@
<span id="nextHeartbeatHuman"><%= data.nextHeartbeatHuman %></span>
</span>
<% } %>
<label>
<span class="has-tooltip">
<span style="margin-top:35px"
class='tooltip rounded shadow-lg p-1 bg-gray-100 text-black -mt-2 space-y-3 p-2 text-left'>
<div>The total number of <b>Activities</b> (Comment/Submission) currently being processed by the bot or queued to be processed.</div>
<div>
Max Concurrent Processing
<ul class="list-inside list-disc">
<li>Real Max: <%= data.maxWorkers %></li>
<% if (data.name !== 'All') { %>
<li>Config Max: <%= data.subMaxWorkers %></li>
<% } %>
<li>Global Max: <%= data.globalMaxWorkers %></li>
</ul>
</div>
</span>
<span>Activities <svg xmlns="http://www.w3.org/2000/svg"
class="h-4 w-4 inline-block cursor-help"
fill="none"
viewBox="0 0 24 24" stroke="currentColor">
<path stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
d="M8.228 9c.549-1.165 2.03-2 3.772-2 2.21 0 4 1.343 4 3 0 1.4-1.278 2.575-3.006 2.907-.542.104-.994.54-.994 1.093m0 3h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z"/>
</svg>
</span>
</span>
</label>
<span><%= `${data.runningActivities} Processing / ${data.queuedActivities} Queued` %></span>
</div>
<% if (data.name !== 'All') { %>
<ul class="list-disc list-inside mt-4">

85
src/Subreddit/Manager.ts
View File

@@ -56,12 +56,15 @@ export interface CheckTask {
export interface RuntimeManagerOptions extends ManagerOptions {
sharedModqueue?: boolean;
wikiLocation?: string;
botName: string;
maxWorkers: number;
}
export class Manager {
subreddit: Subreddit;
client: Snoowrap;
logger: Logger;
botName: string;
pollOptions: PollingOptionsStrong[] = [];
submissionChecks!: SubmissionCheck[];
commentChecks!: CommentCheck[];
@@ -79,6 +82,8 @@ export class Manager {
globalDryRun?: boolean;
emitter: EventEmitter = new EventEmitter();
queue: QueueObject<CheckTask>;
globalMaxWorkers: number;
subMaxWorkers?: number;
displayLabel: string;
currentLabels: string[] = [];
@@ -177,8 +182,8 @@ export class Manager {
return this.displayLabel;
}
constructor(sub: Subreddit, client: Snoowrap, logger: Logger, opts: RuntimeManagerOptions = {}) {
const {dryRun, sharedModqueue = false, wikiLocation = 'botconfig/contextbot'} = opts;
constructor(sub: Subreddit, client: Snoowrap, logger: Logger, opts: RuntimeManagerOptions = {botName: 'ContextMod', maxWorkers: 1}) {
const {dryRun, sharedModqueue = false, wikiLocation = 'botconfig/contextbot', botName, maxWorkers} = opts;
this.displayLabel = opts.nickname || `${sub.display_name_prefixed}`;
const getLabels = this.getCurrentLabels;
const getDisplay = this.getDisplay;
@@ -197,24 +202,11 @@ export class Manager {
this.sharedModqueue = sharedModqueue;
this.subreddit = sub;
this.client = client;
this.notificationManager = new NotificationManager(this.logger, this.subreddit, this.displayLabel);
this.botName = botName;
this.globalMaxWorkers = maxWorkers;
this.notificationManager = new NotificationManager(this.logger, this.subreddit, this.displayLabel, botName);
this.queue = queue(async (task: CheckTask, cb) => {
if(this.delayBy !== undefined) {
this.logger.debug(`SOFT API LIMIT MODE: Delaying Event run by ${this.delayBy} seconds`);
await sleep(this.delayBy * 1000);
}
await this.runChecks(task.checkType, task.activity, task.options);
}
// TODO allow concurrency??
, 1);
this.queue.error((err, task) => {
this.logger.error('Encountered unhandled error while processing Activity, processing stopped early');
this.logger.error(err);
});
this.queue.drain(() => {
this.logger.debug('All queued activities have been processed.');
});
this.queue = this.generateQueue(this.getMaxWorkers(this.globalMaxWorkers));
this.queue.pause();
this.eventsSampleInterval = setInterval((function(self) {
@@ -258,6 +250,49 @@ export class Manager {
})(this), 10000);
}
protected getMaxWorkers(subMaxWorkers?: number) {
let maxWorkers = this.globalMaxWorkers;
if (subMaxWorkers !== undefined) {
if (subMaxWorkers > maxWorkers) {
this.logger.warn(`Config specified ${subMaxWorkers} max queue workers but global max is set to ${this.globalMaxWorkers} -- will use global max`);
} else {
maxWorkers = subMaxWorkers;
}
}
if (maxWorkers < 1) {
this.logger.warn(`Max queue workers must be greater than or equal to 1, specified: ${maxWorkers}. Will use 1.`);
maxWorkers = 1;
}
return maxWorkers;
}
protected generateQueue(maxWorkers: number) {
if (maxWorkers > 1) {
this.logger.warn(`Setting max queue workers above 1 (specified: ${maxWorkers}) may have detrimental effects to log readability and api usage. Consult the documentation before using this advanced/experimental feature.`);
}
const q = queue(async (task: CheckTask, cb) => {
if (this.delayBy !== undefined) {
this.logger.debug(`SOFT API LIMIT MODE: Delaying Event run by ${this.delayBy} seconds`);
await sleep(this.delayBy * 1000);
}
await this.runChecks(task.checkType, task.activity, task.options);
}
, maxWorkers);
q.error((err, task) => {
this.logger.error('Encountered unhandled error while processing Activity, processing stopped early');
this.logger.error(err);
});
q.drain(() => {
this.logger.debug('All queued activities have been processed.');
});
this.logger.info(`Generated new Queue with ${maxWorkers} max workers`);
return q;
}
protected parseConfigurationFromObject(configObj: object) {
try {
const configBuilder = new ConfigBuilder({logger: this.logger});
@@ -270,6 +305,9 @@ export class Manager {
footer,
nickname,
notifications,
queue: {
maxWorkers = undefined,
} = {},
} = configManagerOpts || {};
this.pollOptions = buildPollingOptions(polling);
this.dryRun = this.globalDryRun || dryRun;
@@ -280,12 +318,19 @@ export class Manager {
this.resources.footer = footer;
}
this.subMaxWorkers = maxWorkers;
const realMax = this.getMaxWorkers(this.subMaxWorkers);
if(realMax !== this.queue.concurrency) {
this.queue = this.generateQueue(realMax);
this.queue.pause();
}
this.logger.info(`Dry Run: ${this.dryRun === true}`);
for (const p of this.pollOptions) {
this.logger.info(`Polling Info => ${pollingInfo(p)}`)
}
this.notificationManager = new NotificationManager(this.logger, this.subreddit, this.displayLabel, notifications);
this.notificationManager = new NotificationManager(this.logger, this.subreddit, this.displayLabel, this.botName, notifications);
const {events, notifiers} = this.notificationManager.getStats();
const notifierContent = notifiers.length === 0 ? 'None' : notifiers.join(', ');
const eventContent = events.length === 0 ? 'None' : events.join(', ');