Merge branch 'edge'

* Fixes an issue where the cached notes for a user only contain the last added note instead of all notes + new
* Also reduced api calls by caching moderator adding new note instead of calling each time

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: [FoxxMD]
+custom: ["bitcoincash:qqmpsh365r8n9jhp4p8ks7f7qdr7203cws4kmkmr8q"]

docs/webInterface.md

@@ -0,0 +1,30 @@
+## Editing/Updating Your Config
+
+* Open the editor for your subreddit
+    * In the web dashboard \-> r/YourSubreddit \-> Config -> **View** [(here)](/docs/screenshots/config/config.jpg)
+* Follow the directions on the [link at the top of the window](/docs/screenshots/config/save.png) to enable config editing using your moderator account
+    * After enabling editing just click "save" at any time to save your config
+    * After you have added/edited your config the bot will detect changes within 5 minutes or you can manually trigger it by clicking **Update**
+
+## General Config (Editor) Tips
+
+* The editor will automatically validate your [syntax (formatting)](/docs/screenshots/config/syntax.png) and [config correctness](/docs/screenshots/config/correctness.png) (property names, required properties, etc.)
+    * These show up as squiggly lines like in Microsoft Word and as a [list at the bottom of the editor](/docs/screenshots/config/errors.png)
+* In your config all **Checks** and **Actions** have two properties that control how they behave:
+    * [**Enable**](/docs/screenshots/config/enable.png) (defaults to `enable: true`) -- Determines if the check or action is run, at all
+    * **Dryrun** (defaults to `dryRun: false`) -- When `true` the check or action will run but any **Actions** that may be triggered will "pretend" to execute but not actually talk to the Reddit API.
+      * Use `dryRun` to test your config without the bot making any changes on reddit
+    * When starting out with a new config it is recommended running the bot with remove/ban actions **disabled**
+        * Use `report` actions to get reports in your modqueue from the bot that describe what it detected and what it would do about it
+        * Once the bot is behaving as desired (no false positives or weird behavior) destructive actions can be enabled or turned off of dryrun
+
+## Web Dashboard Tips
+
+* Use the [**Overview** section](/docs/screenshots/botOperations.png) to control the bot at a high-level
+* You can **manually run** the bot on any activity (comment/submission) by pasting its permalink into the [input field below the Overview section](/docs/screenshots/runInput.png) and hitting one of the **run buttons**
+    * **Dry run** will make the bot run on the activity but it will only **pretend** to run actions, if triggered. This is super useful for testing your config without consequences
+    * **Run** will do everything
+* All of the bot's activity is shown in real-time in the [log section](/docs/screenshots/logs.png)
+    * This will output the results of all run checks/rules and any actions that run
+* You can view summaries of all activities that triggered a check (had actions run) by clicking on [Actioned Events](/docs/screenshots/actionsEvents.png)
+    * This includes activities run with dry run

package.json

@@ -49,6 +49,7 @@
     "express-socket.io-session": "^1.3.5",
     "fast-deep-equal": "^3.1.3",
     "fuse.js": "^6.4.6",
+    "globrex": "^0.1.2",
     "got": "^11.8.2",
     "he": "^1.2.0",
     "http-proxy": "^1.18.1",
@@ -68,6 +69,7 @@
     "passport-custom": "^1.1.1",
     "passport-jwt": "^4.0.0",
     "pixelmatch": "^5.2.1",
+    "pony-cause": "^1.1.1",
     "pretty-print-json": "^1.0.3",
     "safe-stable-stringify": "^1.1.1",
     "snoostorm": "^1.5.2",
@@ -94,6 +96,7 @@
     "@types/express": "^4.17.13",
     "@types/express-session": "^1.17.4",
     "@types/express-socket.io-session": "^1.3.6",
+    "@types/globrex": "^0.1.1",
     "@types/he": "^1.1.1",
     "@types/http-proxy": "^1.17.7",
     "@types/js-yaml": "^4.0.1",
@@ -114,7 +117,7 @@
     "@types/triple-beam": "^1.3.2",
     "ts-essentials": "^9.1.2",
     "ts-json-schema-generator": "^0.93.0",
-    "typescript-json-schema": "^0.50.1"
+    "typescript-json-schema": "~0.53"
   },
   "optionalDependencies": {
     "sharp": "^0.29.1"

src/Action/ApproveAction.ts

@@ -36,7 +36,7 @@ export class ApproveAction extends Action {
             }
 
             // @ts-ignore
-            if (item.approved) {
+            if (targetItem.approved) {
                 const msg = `${target === 'self' ? 'Item' : 'Comment\'s parent Submission'} is already approved`;
                 this.logger.warn(msg);
                 return {
@@ -54,6 +54,16 @@ export class ApproveAction extends Action {
                 }
                 // @ts-ignore
                 touchedEntities.push(await targetItem.approve());
+
+                if(target === 'self') {
+                    // @ts-ignore
+                    item.approved = true;
+                    await this.resources.resetCacheForItem(item);
+                } else if(await this.resources.hasActivity(targetItem)) {
+                    // @ts-ignore
+                    targetItem.approved = true;
+                    await this.resources.resetCacheForItem(targetItem);
+                }
             }
         }
 

src/Action/LockAction.ts

@@ -26,6 +26,9 @@ export class LockAction extends Action {
             //snoowrap typing issue, thinks comments can't be locked
             // @ts-ignore
             await item.lock();
+            // @ts-ignore
+            item.locked = true;
+            await this.resources.resetCacheForItem(item);
             touchedEntities.push(item);
         }
         return {

src/Action/MessageAction.ts

@@ -12,7 +12,8 @@ import {
     REDDIT_ENTITY_REGEX_URL,
     truncateStringToLength
 } from "../util";
-import SimpleError from "../Utils/SimpleError";
+import {SimpleError} from "../Utils/Errors";
+import {ErrorWithCause} from "pony-cause";
 
 export class MessageAction extends Action {
     content: string;
@@ -65,10 +66,7 @@ export class MessageAction extends Action {
                     recipient = `/r/${entityData.name}`;
                 }
             } catch (err: any) {
-                this.logger.error(`'to' field for message was not in a valid format. See ${REDDIT_ENTITY_REGEX_URL} for valid examples`);
-                this.logger.error(err);
-                err.logged = true;
-                throw err;
+                throw new ErrorWithCause(`'to' field for message was not in a valid format. See ${REDDIT_ENTITY_REGEX_URL} for valid examples`, {cause: err});
             }
             if(recipient.includes('/r/') && this.asSubreddit) {
                 throw new SimpleError(`Cannot send a message as a subreddit to another subreddit. Requested recipient: ${recipient}`);

src/Action/RemoveAction.ts

@@ -1,30 +1,48 @@
-import {ActionJson, ActionConfig} from "./index";
+import {ActionJson, ActionConfig, ActionOptions} from "./index";
 import Action from "./index";
 import Snoowrap, {Comment, Submission} from "snoowrap";
 import {RuleResult} from "../Rule";
 import {activityIsRemoved} from "../Utils/SnoowrapUtils";
 import {ActionProcessResult} from "../Common/interfaces";
+import dayjs from "dayjs";
+import {isSubmission} from "../util";
 
 export class RemoveAction extends Action {
+    spam: boolean;
+
     getKind() {
         return 'Remove';
     }
 
+    constructor(options: RemoveOptions) {
+        super(options);
+        const {
+            spam = false,
+        } = options;
+        this.spam = spam;
+    }
+
     async process(item: Comment | Submission, ruleResults: RuleResult[], runtimeDryrun?: boolean): Promise<ActionProcessResult> {
         const dryRun = runtimeDryrun || this.dryRun;
         const touchedEntities = [];
         // issue with snoowrap typings, doesn't think prop exists on Submission
         // @ts-ignore
         if (activityIsRemoved(item)) {
-            return {
-                dryRun,
-                success: false,
-                result: 'Item is already removed',
-            }
+            this.logger.warn('It looks like this Item is already removed!');
+        }
+        if (this.spam) {
+            this.logger.verbose('Marking as spam on removal');
         }
         if (!dryRun) {
             // @ts-ignore
-            await item.remove();
+            await item.remove({spam: this.spam});
+            item.banned_at_utc = dayjs().unix();
+            item.spam = this.spam;
+            if(!isSubmission(item)) {
+                // @ts-ignore
+                item.removed = true;
+            }
+            await this.resources.resetCacheForItem(item);
             touchedEntities.push(item);
         }
 
@@ -36,13 +54,16 @@ export class RemoveAction extends Action {
     }
 }
 
-export interface RemoveActionConfig extends ActionConfig {
+export interface RemoveOptions extends RemoveActionConfig, ActionOptions {
+}
 
+export interface RemoveActionConfig extends ActionConfig {
+    spam?: boolean
 }
 
 /**
  * Remove the Activity
  * */
 export interface RemoveActionJson extends RemoveActionConfig, ActionJson {
-kind: 'remove'
+    kind: 'remove'
 }

src/Action/ReportAction.ts

@@ -35,6 +35,7 @@ export class ReportAction extends Action {
             await item.report({reason: truncatedContent});
             // due to reddit not updating this in response (maybe)?? just increment stale activity
             item.num_reports++;
+            await this.resources.resetCacheForItem(item);
             touchedEntities.push(item);
         }
 

src/Action/SubmissionAction/FlairAction.ts

@@ -33,16 +33,26 @@ export class FlairAction extends Action {
         if(this.css !== '') {
             flairParts.push(`CSS: ${this.css}`);
         }
+        if(this.flair_template_id !== '') {
+            flairParts.push(`Template: ${this.flair_template_id}`);
+        }
         const flairSummary = flairParts.length === 0 ? 'No flair (unflaired)' : flairParts.join(' | ');
         this.logger.verbose(flairSummary);
         if (item instanceof Submission) {
             if(!this.dryRun) {
                 if (this.flair_template_id) {
-                    await item.selectFlair({flair_template_id: this.flair_template_id}).then(() => {});
+                    // typings are wrong for this function, flair_template_id should be accepted
+                    // assignFlair uses /api/flair (mod endpoint)
+                    // selectFlair uses /api/selectflair (self endpoint for user to choose their own flair for submission)
+                    // @ts-ignore
+                    await item.assignFlair({flair_template_id: this.flair_template_id}).then(() => {});
+                    item.link_flair_template_id = this.flair_template_id;
                 } else {
                     await item.assignFlair({text: this.text, cssClass: this.css}).then(() => {});
+                    item.link_flair_css_class = this.css;
+                    item.link_flair_text = this.text;
                 }
-
+                await this.resources.resetCacheForItem(item);
             }
         } else {
             this.logger.warn('Cannot flair Comment');

src/Action/UserFlairAction.ts

@@ -50,6 +50,7 @@ export class UserFlairAction extends Action {
             flairTemplateId: this.flair_template_id,
             username: item.author.name,
           });
+          item.author_flair_template_id = this.flair_template_id
         } catch (err: any) {
           this.logger.error('Either the flair template ID is incorrect or you do not have permission to access it.');
           throw err;
@@ -57,6 +58,9 @@ export class UserFlairAction extends Action {
       } else if (this.text === undefined && this.css === undefined) {
         // @ts-ignore
         await item.subreddit.deleteUserFlair(item.author.name);
+        item.author_flair_css_class = null;
+        item.author_flair_text = null;
+        item.author_flair_template_id = null;
       } else {
         // @ts-ignore
         await item.author.assignFlair({
@@ -64,7 +68,11 @@ export class UserFlairAction extends Action {
           cssClass: this.css,
           text: this.text,
         });
+        item.author_flair_text = this.text ?? null;
+        item.author_flair_css_class = this.css ?? null;
       }
+      await this.resources.resetCacheForItem(item);
+      await this.resources.resetCacheForItem(item.author);
     }
 
     return {

src/Action/index.ts

@@ -7,6 +7,7 @@ import Author, {AuthorOptions} from "../Author/Author";
 import {mergeArr} from "../util";
 import LoggedError from "../Utils/LoggedError";
 import {ExtendedSnoowrap} from '../Utils/SnoowrapClients';
+import {ErrorWithCause} from "pony-cause";
 
 export abstract class Action {
     name?: string;
@@ -86,7 +87,8 @@ export abstract class Action {
             return {...actRes, ...results};
         } catch (err: any) {
             if(!(err instanceof LoggedError)) {
-                this.logger.error(`Encountered error while running`, err);
+                const actionError = new ErrorWithCause('Action did not run successfully due to unexpected error', {cause: err});
+                this.logger.error(actionError);
             }
             actRes.success = false;
             actRes.result = err.message;

src/Bot/index.ts

@@ -16,10 +16,10 @@ import {
 } from "../Common/interfaces";
 import {
     createRetryHandler,
-    formatNumber,
+    formatNumber, getExceptionMessage,
     mergeArr,
     parseBool,
-    parseDuration,
+    parseDuration, parseMatchMessage,
     parseSubredditName, RetryOptions,
     sleep,
     snooLogWrapper
@@ -30,8 +30,8 @@ import {CommentStream, ModQueueStream, SPoll, SubmissionStream, UnmoderatedStrea
 import {BotResourcesManager} from "../Subreddit/SubredditResources";
 import LoggedError from "../Utils/LoggedError";
 import pEvent from "p-event";
-import SimpleError from "../Utils/SimpleError";
-import {isRateLimitError, isStatusError} from "../Utils/Errors";
+import {SimpleError, isRateLimitError, isRequestError, isScopeError, isStatusError, CMError} from "../Utils/Errors";
+import {ErrorWithCause} from "pony-cause";
 
 
 class Bot {
@@ -234,10 +234,9 @@ class Bot {
     }
 
     createSharedStreamErrorListener = (name: string) => async (err: any) => {
-        this.logger.error(`Polling error occurred on stream ${name.toUpperCase()}`, err);
         const shouldRetry = await this.sharedStreamRetryHandler(err);
         if(shouldRetry) {
-            (this.cacheManager.modStreams.get(name) as SPoll<any>).startInterval(false);
+            (this.cacheManager.modStreams.get(name) as SPoll<any>).startInterval(false, 'Within retry limits');
         } else {
             for(const m of this.subManagers) {
                 if(m.sharedStreamCallbacks.size > 0) {
@@ -255,7 +254,7 @@ class Bot {
             return;
         }
         for(const i of listing) {
-            const foundManager = this.subManagers.find(x => x.subreddit.display_name === i.subreddit.display_name && x.sharedStreamCallbacks.get(name) !== undefined);
+            const foundManager = this.subManagers.find(x => x.subreddit.display_name === i.subreddit.display_name && x.sharedStreamCallbacks.get(name) !== undefined && x.eventsState.state === RUNNING);
             if(foundManager !== undefined) {
                 foundManager.sharedStreamCallbacks.get(name)(i);
                 if(this.stagger !== undefined) {
@@ -280,22 +279,16 @@ class Bot {
             if (initial) {
                 this.logger.error('An error occurred while trying to initialize the Reddit API Client which would prevent the entire application from running.');
             }
-            if (err.name === 'StatusCodeError') {
-                const authHeader = err.response.headers['www-authenticate'];
-                if (authHeader !== undefined && authHeader.includes('insufficient_scope')) {
-                    this.logger.error('Reddit responded with a 403 insufficient_scope. Please ensure you have chosen the correct scopes when authorizing your account.');
-                } else if (err.statusCode === 401) {
-                    this.logger.error('It is likely a credential is missing or incorrect. Check clientId, clientSecret, refreshToken, and accessToken');
-                } else if(err.statusCode === 400) {
-                    this.logger.error('Credentials may have been invalidated due to prior behavior. The error message may contain more information.');
-                }
-                this.logger.error(`Error Message: ${err.message}`);
-            } else {
-                this.logger.error(err);
-            }
-            this.error = `Error occurred while testing Reddit API client: ${err.message}`;
-            err.logged = true;
-            throw err;
+            const hint = getExceptionMessage(err, {
+                401: 'Likely a credential is missing or incorrect. Check clientId, clientSecret, refreshToken, and accessToken',
+                400: 'Credentials may have been invalidated manually or by reddit due to behavior',
+            });
+            let msg = `Error occurred while testing Reddit API client${hint !== undefined ? `: ${hint}` : ''}`;
+            this.error = msg;
+            const clientError = new CMError(msg, {cause: err});
+            clientError.logged = true;
+            this.logger.error(clientError);
+            throw clientError;
         }
     }
 
@@ -375,7 +368,7 @@ class Bot {
                 let processed;
                 if (stream !== undefined) {
                     this.logger.info('Restarting SHARED COMMENT STREAM due to a subreddit config change');
-                    stream.end();
+                    stream.end('Replacing with a new stream with updated subreddits');
                     processed = stream.processed;
                 }
                 if (sharedCommentsSubreddits.length > 100) {
@@ -397,7 +390,7 @@ class Bot {
         } else {
             const stream = this.cacheManager.modStreams.get('newComm');
             if (stream !== undefined) {
-                stream.end();
+                stream.end('Determined no managers are listening on shared stream parsing');
             }
         }
 
@@ -408,7 +401,7 @@ class Bot {
                 let processed;
                 if (stream !== undefined) {
                     this.logger.info('Restarting SHARED SUBMISSION STREAM due to a subreddit config change');
-                    stream.end();
+                    stream.end('Replacing with a new stream with updated subreddits');
                     processed = stream.processed;
                 }
                 if (sharedSubmissionsSubreddits.length > 100) {
@@ -430,7 +423,7 @@ class Bot {
         } else {
             const stream = this.cacheManager.modStreams.get('newSub');
             if (stream !== undefined) {
-                stream.end();
+                stream.end('Determined no managers are listening on shared stream parsing');
             }
         }
 
@@ -448,7 +441,7 @@ class Bot {
             defaultUnmoderatedStream.on('listing', this.createSharedStreamListingListener('unmoderated'));
             this.cacheManager.modStreams.set('unmoderated', defaultUnmoderatedStream);
         } else if (!isUnmoderatedShared && unmoderatedstream !== undefined) {
-            unmoderatedstream.end();
+            unmoderatedstream.end('Determined no managers are listening on shared stream parsing');
         }
 
         const isModqueueShared = !this.sharedStreams.includes('modqueue') ? false : this.subManagers.some(x => x.isPollingShared('modqueue'));
@@ -465,7 +458,7 @@ class Bot {
             defaultModqueueStream.on('listing', this.createSharedStreamListingListener('modqueue'));
             this.cacheManager.modStreams.set('modqueue', defaultModqueueStream);
         } else if (isModqueueShared && modqueuestream !== undefined) {
-            modqueuestream.end();
+            modqueuestream.end('Determined no managers are listening on shared stream parsing');
         }
     }
 
@@ -473,10 +466,12 @@ class Bot {
         try {
             await manager.parseConfiguration('system', true, {suppressNotification: true, suppressChangeEvent: true});
         } catch (err: any) {
-            if (!(err instanceof LoggedError)) {
-                this.logger.error(`Config was not valid:`, {subreddit: manager.subreddit.display_name_prefixed});
-                this.logger.error(err, {subreddit: manager.subreddit.display_name_prefixed});
-                err.logged = true;
+            if(err.logged !== true) {
+                const normalizedError = new ErrorWithCause(`Bot could not start manager because config was not valid`, {cause: err});
+                // @ts-ignore
+                this.logger.error(normalizedError, {subreddit: manager.subreddit.display_name_prefixed});
+            } else {
+                this.logger.error('Bot could not start manager because config was not valid', {subreddit: manager.subreddit.display_name_prefixed});
             }
         }
     }
@@ -666,9 +661,11 @@ class Bot {
                     }
                 }
             } catch (err: any) {
-                this.logger.info('Stopping event polling to prevent activity processing queue from backing up. Will be restarted when config update succeeds.')
-                await s.stopEvents('system', {reason: 'Invalid config will cause events to pile up in queue. Will be restarted when config update succeeds (next heartbeat).'});
-                if(!(err instanceof LoggedError)) {
+                if(s.eventsState.state === RUNNING) {
+                    this.logger.info('Stopping event polling to prevent activity processing queue from backing up. Will be restarted when config update succeeds.')
+                    await s.stopEvents('system', {reason: 'Invalid config will cause events to pile up in queue. Will be restarted when config update succeeds (next heartbeat).'});
+                }
+                if(err.logged !== true) {
                     this.logger.error(err, {subreddit: s.displayLabel});
                 }
                 if(this.nextHeartbeat !== undefined) {
@@ -744,6 +741,10 @@ class Bot {
                     m.notificationManager.handle('runStateChanged', 'Hard Limit Triggered', `Hard Limit of ${this.hardLimit} hit (API Remaining: ${this.client.ratelimitRemaining}). Subreddit event polling has been paused.`, 'system', 'warn');
                 }
 
+                for(const [k,v] of this.cacheManager.modStreams) {
+                    v.end('Hard limit cutoff');
+                }
+
                 this.nannyMode = 'hard';
                 return;
             }
@@ -810,6 +811,7 @@ class Bot {
                         await m.startEvents('system', {reason: 'API Nanny has been turned off due to better API conditions'});
                     }
                 }
+                await this.runSharedStreams(true);
                 this.nannyMode = undefined;
             }
 

src/Check/index.ts

@@ -31,6 +31,8 @@ import {ActionObjectJson, RuleJson, RuleObjectJson, ActionJson as ActionTypeJson
 import {checkAuthorFilter, SubredditResources} from "../Subreddit/SubredditResources";
 import {Author, AuthorCriteria, AuthorOptions} from '..';
 import {ExtendedSnoowrap} from '../Utils/SnoowrapClients';
+import {isRateLimitError} from "../Utils/Errors";
+import {ErrorWithCause} from "pony-cause";
 
 const checkLogName = truncateStringToLength(25);
 
@@ -248,9 +250,7 @@ export abstract class Check implements ICheck {
             this.logger.info(`${PASS} => Rules: ${resultsSummary(allResults, this.condition)}`);
             return [true, allRuleResults];
         } catch (e: any) {
-            e.logged = true;
-            this.logger.warn(`Running rules failed due to uncaught exception`, e);
-            throw e;
+            throw new ErrorWithCause('Running rules failed due to error', {cause: e});
         }
     }
 

src/Common/ImageData.ts

@@ -3,9 +3,9 @@ import {Submission} from "snoowrap/dist/objects";
 import {URL} from "url";
 import {absPercentDifference, getSharpAsync, isValidImageURL} from "../util";
 import sizeOf from "image-size";
-import SimpleError from "../Utils/SimpleError";
 import {Sharp} from "sharp";
 import {blockhash} from "./blockhash/blockhash";
+import {SimpleError} from "../Utils/Errors";
 
 export interface ImageDataOptions {
     width?: number,

src/Common/interfaces.ts

@@ -13,6 +13,9 @@ import {ConfigFormat} from "./types";
 import AbstractConfigDocument, {ConfigDocumentInterface} from "./Config/AbstractConfigDocument";
 import {Document as YamlDocument} from 'yaml';
 import {JsonOperatorConfigDocument, YamlOperatorConfigDocument} from "./Config/Operator";
+import {ConsoleTransportOptions} from "winston/lib/winston/transports";
+import {DailyRotateFileTransportOptions} from "winston-daily-rotate-file";
+import {DuplexTransportOptions} from "winston-duplex/dist/DuplexTransport";
 
 /**
  * An ISO 8601 Duration
@@ -946,6 +949,10 @@ export interface SubmissionState extends ActivityState {
     link_flair_text?: string | string[]
     link_flair_css_class?: string | string[]
     flairTemplate?: string | string[]
+    /**
+     * Is the submission a reddit-hosted image or video?
+     * */
+    isRedditMediaDomain?: boolean
 }
 
 // properties calculated/derived by CM -- not provided as plain values by reddit
@@ -1071,6 +1078,92 @@ export interface RegExResult {
 }
 
 type LogLevel = "error" | "warn" | "info" | "verbose" | "debug";
+
+export type LogConsoleOptions = Pick<ConsoleTransportOptions, 'silent' | 'eol' | 'stderrLevels' | 'consoleWarnLevels'> & {
+    level?: LogLevel
+}
+
+export type LogFileOptions = Omit<DailyRotateFileTransportOptions, 'stream' | 'handleRejections' | 'options' | 'handleExceptions' | 'format' | 'log' | 'logv' | 'close' | 'dirname'> & {
+    level?: LogLevel
+    /**
+     * The absolute path to a directory where rotating log files should be stored.
+     *
+     * * If not present or `null` or `false` no log files will be created
+     * * If `true` logs will be stored at `[working directory]/logs`
+     *
+     * * ENV => `LOG_DIR`
+     * * ARG => `--logDir [dir]`
+     *
+     * @examples ["/var/log/contextmod"]
+     * */
+    dirname?: string | boolean | null
+}
+
+// export type StrongFileOptions = LogFileOptions & {
+//     dirname?: string
+// }
+
+export type LogStreamOptions = Omit<DuplexTransportOptions, 'name' | 'stream' | 'handleRejections' | 'handleExceptions' | 'format' | 'log' | 'logv' | 'close'> & {
+    level?: LogLevel
+}
+
+export interface LoggingOptions  {
+    /**
+     * The minimum log level to output. The log level set will output logs at its level **and all levels above it:**
+     *
+     *  * `error`
+     *  * `warn`
+     *  * `info`
+     *  * `verbose`
+     *  * `debug`
+     *
+     *  Note: `verbose` will display *a lot* of information on the status/result of run rules/checks/actions etc. which is very useful for testing configurations. Once your bot is stable changing the level to `info` will reduce log noise.
+     *
+     *  * ENV => `LOG_LEVEL`
+     *  * ARG => `--logLevel <level>`
+     *
+     *  @default "verbose"
+     *  @examples ["verbose"]
+     * */
+    level?: LogLevel,
+    /**
+     * **DEPRECATED** - Use `file.dirname` instead
+     * The absolute path to a directory where rotating log files should be stored.
+     *
+     * * If not present or `null` or `false` no log files will be created
+     * * If `true` logs will be stored at `[working directory]/logs`
+     *
+     * * ENV => `LOG_DIR`
+     * * ARG => `--logDir [dir]`
+     *
+     * @examples ["/var/log/contextmod"]
+     * @deprecated
+     * @see logging.file.dirname
+     * */
+    path?: string | boolean | null
+
+    /**
+     * Options for Rotating File logging
+     * */
+    file?: LogFileOptions
+    /**
+     * Options for logging to api/web
+     * */
+    stream?: LogStreamOptions
+    /**
+     * Options for logging to console
+     * */
+    console?: LogConsoleOptions
+}
+
+export type StrongLoggingOptions = Required<Pick<LoggingOptions, 'stream' | 'console' | 'file'>> & {
+    level?: LogLevel
+};
+
+export type LoggerFactoryOptions = StrongLoggingOptions & {
+    additionalTransports?: any[]
+    defaultLabel?: string
+}
 /**
  * Available cache providers
  * */
@@ -1581,38 +1674,7 @@ export interface OperatorJsonConfig {
     /**
      * Settings to configure global logging defaults
      * */
-    logging?: {
-        /**
-         * The minimum log level to output. The log level set will output logs at its level **and all levels above it:**
-         *
-         *  * `error`
-         *  * `warn`
-         *  * `info`
-         *  * `verbose`
-         *  * `debug`
-         *
-         *  Note: `verbose` will display *a lot* of information on the status/result of run rules/checks/actions etc. which is very useful for testing configurations. Once your bot is stable changing the level to `info` will reduce log noise.
-         *
-         *  * ENV => `LOG_LEVEL`
-         *  * ARG => `--logLevel <level>`
-         *
-         *  @default "verbose"
-         *  @examples ["verbose"]
-         * */
-        level?: LogLevel,
-        /**
-         * The absolute path to a directory where rotating log files should be stored.
-         *
-         * * If not present or `null` no log files will be created
-         * * If `true` logs will be stored at `[working directory]/logs`
-         *
-         * * ENV => `LOG_DIR`
-         * * ARG => `--logDir [dir]`
-         *
-         * @examples ["/var/log/contextmod"]
-         * */
-        path?: string,
-    },
+    logging?: LoggingOptions,
 
     /**
      * Settings to configure the default caching behavior globally
@@ -1812,10 +1874,7 @@ export interface OperatorConfig extends OperatorJsonConfig {
         display?: string,
     },
     notifications?: NotificationConfig
-    logging: {
-        level: LogLevel,
-        path?: string,
-    },
+    logging: StrongLoggingOptions,
     caching: StrongCache,
     web: {
         port: number,
@@ -1918,22 +1977,6 @@ export interface RedditEntity {
     type: RedditEntityType
 }
 
-export interface StatusCodeError extends Error {
-    name: 'StatusCodeError',
-    statusCode: number,
-    message: string,
-    response: IncomingMessage,
-    error: Error
-}
-
-export interface RequestError extends Error {
-    name: 'RequestError',
-    statusCode: number,
-    message: string,
-    response: IncomingMessage,
-    error: Error
-}
-
 export interface HistoricalStatsDisplay extends HistoricalStats {
     checksRunTotal: number
     checksFromCacheTotal: number
@@ -2061,3 +2104,56 @@ export interface FilterResult<T> {
     join: JoinOperands
     passed: boolean
 }
+
+export interface TextTransformOptions {
+    /**
+     * A set of search-and-replace operations to perform on text values before performing a match. Transformations are performed in the order they are defined.
+     *
+     * * If `transformationsActivity` IS NOT defined then these transformations will be performed on BOTH the activity text (submission title or comment) AND the repost candidate text
+     * * If `transformationsActivity` IS defined then these transformations are only performed on repost candidate text
+     * */
+    transformations?: SearchAndReplaceRegExp[]
+
+    /**
+     * Specify a separate set of transformations for the activity text (submission title or comment)
+     *
+     * To perform no transformations when `transformations` is defined set this to an empty array (`[]`)
+     * */
+    transformationsActivity?: SearchAndReplaceRegExp[]
+}
+
+export interface TextMatchOptions {
+    /**
+     * The percentage, as a whole number, of a repost title/comment that must match the title/comment being checked in order to consider both a match
+     *
+     * Note: Setting to 0 will make every candidate considered a match -- useful if you want to match if the URL has been reposted anywhere
+     *
+     * Defaults to `85` (85%)
+     *
+     * @default 85
+     * @example [85]
+     * */
+    matchScore?: number
+
+    /**
+     * The minimum number of words in the activity being checked for which this rule will run on
+     *
+     * If the word count is below the minimum the rule fails
+     *
+     * Defaults to 2
+     *
+     * @default 2
+     * @example [2]
+     * */
+    minWordCount?: number
+
+    /**
+     * Should text matching be case sensitive?
+     *
+     * Defaults to false
+     *
+     * @default false
+     * @example [false]
+     **/
+    caseSensitive?: boolean
+}

src/Common/typings/support.d.ts

@@ -0,0 +1,26 @@
+declare module 'snoowrap/dist/errors' {
+
+    export interface InvalidUserError extends Error {
+
+    }
+    export interface NoCredentialsError extends Error {
+
+    }
+    export interface InvalidMethodCallError extends Error {
+
+    }
+
+    export interface RequestError extends Error {
+        statusCode: number,
+        response: http.IncomingMessage
+        error: Error
+    }
+
+    export interface StatusCodeError extends RequestError {
+        name: 'StatusCodeError',
+    }
+
+    export interface RateLimitError extends RequestError {
+        name: 'RateLimitError',
+    }
+}

src/ConfigBuilder.ts

@@ -55,9 +55,10 @@ import {
     OperatorConfigDocumentInterface,
     YamlOperatorConfigDocument
 } from "./Common/Config/Operator";
-import SimpleError from "./Utils/SimpleError";
 import {ConfigDocumentInterface} from "./Common/Config/AbstractConfigDocument";
 import {Document as YamlDocument} from "yaml";
+import {SimpleError} from "./Utils/Errors";
+import {ErrorWithCause} from "pony-cause";
 
 export interface ConfigBuilderOptions {
     logger: Logger,
@@ -373,7 +374,16 @@ export const parseOpConfigFromArgs = (args: any): OperatorJsonConfig => {
         },
         logging: {
             level: logLevel,
-            path: logDir === true ? `${process.cwd()}/logs` : undefined,
+            file: {
+                level: logLevel,
+                dirName: logDir,
+            },
+            stream: {
+                level: logLevel,
+            },
+            console: {
+                level: logLevel,
+            }
         },
         caching: {
             provider: caching,
@@ -457,9 +467,17 @@ export const parseOpConfigFromEnv = (): OperatorJsonConfig => {
             display: process.env.OPERATOR_DISPLAY
         },
         logging: {
-            // @ts-ignore
             level: process.env.LOG_LEVEL,
-            path: process.env.LOG_DIR === 'true' ? `${process.cwd()}/logs` : undefined,
+            file: {
+                level: process.env.LOG_LEVEL,
+                dirname: process.env.LOG_DIR,
+            },
+            stream: {
+                level: process.env.LOG_LEVEL,
+            },
+            console: {
+                level: process.env.LOG_LEVEL,
+            }
         },
         caching: {
             provider: {
@@ -501,11 +519,25 @@ export const parseOpConfigFromEnv = (): OperatorJsonConfig => {
 // json config
 // args from cli
 export const parseOperatorConfigFromSources = async (args: any): Promise<[OperatorJsonConfig, OperatorFileConfig]> => {
-    const {logLevel = process.env.LOG_LEVEL, logDir = process.env.LOG_DIR || false} = args || {};
+    const {logLevel = process.env.LOG_LEVEL ?? 'debug', logDir = process.env.LOG_DIR} = args || {};
     const envPath = process.env.OPERATOR_ENV;
+    const initLoggerOptions = {
+        level: logLevel,
+        console: {
+            level: logLevel
+        },
+        file: {
+            level: logLevel,
+            dirname: logDir,
+        },
+        stream: {
+            level: logLevel
+        }
+    }
 
     // create a pre config logger to help with debugging
-    const initLogger = getLogger({logLevel, logDir: logDir === true ? `${process.cwd()}/logs` : logDir}, 'init');
+    // default to debug if nothing is provided
+    const initLogger = getLogger(initLoggerOptions, 'init');
 
     try {
         const vars = await GetEnvVars({
@@ -556,9 +588,7 @@ export const parseOperatorConfigFromSources = async (args: any): Promise<[Operat
                 fileConfigFormat = err.extension
             }
         } else {
-            initLogger.error('Cannot continue app startup because operator config file exists but was not parseable.');
-            err.logged = true;
-            throw err;
+            throw new ErrorWithCause('Cannot continue app startup because operator config file exists but was not parseable.', {cause: err});
         }
     }
     const [format, doc, jsonErr, yamlErr] = parseFromJsonOrYamlToObject(rawConfig, {
@@ -591,7 +621,15 @@ export const parseOperatorConfigFromSources = async (args: any): Promise<[Operat
 
         try {
             configFromFile = validateJson(configDoc.toJS(), operatorSchema, initLogger) as OperatorJsonConfig;
-            const {bots = []} = configFromFile || {};
+            const {
+                bots = [],
+                logging: {
+                    path = undefined
+                } = {}
+            } = configFromFile || {};
+            if(path !== undefined) {
+                initLogger.warn(`'path' property in top-level 'logging' object is DEPRECATED and will be removed in next minor version. Use 'logging.file.dirname' instead`);
+            }
             for (const b of bots) {
                 const {
                     polling: {
@@ -628,9 +666,13 @@ export const parseOperatorConfigFromSources = async (args: any): Promise<[Operat
         defaultBotInstance.caching = configFromFile.caching;
     }
 
-    let botInstances = [];
+    let botInstances: BotInstanceJsonConfig[] = [];
     if (botInstancesFromFile.length === 0) {
-        botInstances = [defaultBotInstance];
+        // only add default bot if user supplied any credentials
+        // otherwise its most likely just default, empty settings
+        if(defaultBotInstance.credentials !== undefined) {
+            botInstances = [defaultBotInstance];
+        }
     } else {
         botInstances = botInstancesFromFile.map(x => merge.all([defaultBotInstance, x], {arrayMerge: overwriteMerge}));
     }
@@ -651,6 +693,9 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
         logging: {
             level = 'verbose',
             path,
+            file = {},
+            console = {},
+            stream = {},
         } = {},
         caching: opCache,
         web: {
@@ -726,6 +771,16 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
 
     const defaultOperators = typeof name === 'string' ? [name] : name;
 
+    const {
+        dirname = path,
+        ...fileRest
+    } = file;
+
+     const defaultWebCredentials = {
+         redirectUri: 'http://localhost:8085/callback'
+     };
+
+
     const config: OperatorConfig = {
         mode,
         operator: {
@@ -734,7 +789,19 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
         },
         logging: {
             level,
-            path
+            file: {
+                level: level,
+                dirname,
+                ...fileRest,
+            },
+            stream: {
+                level: level,
+                ...stream,
+            },
+            console: {
+                level: level,
+                ...console,
+            }
         },
         caching: cache,
         web: {
@@ -752,7 +819,7 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
             },
             maxLogs,
             clients: clients === undefined ? [{host: 'localhost:8095', secret: apiSecret}] : clients,
-            credentials: webCredentials as RequiredWebRedditCredentials,
+            credentials: {...defaultWebCredentials, ...webCredentials} as RequiredWebRedditCredentials,
             operators: operators || defaultOperators,
         },
         api: {

src/Rule/AttributionRule.ts

@@ -14,8 +14,8 @@ import {
     PASS
 } from "../util";
 import { Comment } from "snoowrap/dist/objects";
-import SimpleError from "../Utils/SimpleError";
 import as from "async";
+import {SimpleError} from "../Utils/Errors";
 
 
 export interface AttributionCriteria {

src/Rule/RegexRule.ts

@@ -11,7 +11,7 @@ import {
     ActivityWindowType, JoinOperands,
 } from "../Common/interfaces";
 import dayjs from 'dayjs';
-import SimpleError from "../Utils/SimpleError";
+import {SimpleError} from "../Utils/Errors";
 
 export interface RegexCriteria {
     /**
@@ -95,6 +95,15 @@ export interface RegexCriteria {
      * */
     totalMatchThreshold?: string,
 
+    /**
+     * When `true` the Activity being checked MUST pass the `matchThreshold` before the Rule considers any history
+     *
+     * For use with `activityMatchThreshold`/`totalMatchThreshold` -- useful to conserve API calls
+     *
+     * @default false
+     * */
+    mustMatchCurrent?: boolean
+
     window?: ActivityWindowType
 }
 
@@ -140,6 +149,7 @@ export class RegexRule extends Rule {
                 matchThreshold = '> 0',
                 activityMatchThreshold = '> 0',
                 totalMatchThreshold = null,
+                mustMatchCurrent = false,
                 window,
             } = criteria;
 
@@ -184,6 +194,8 @@ export class RegexRule extends Rule {
             if (singleMatched) {
                 activitiesMatchedCount++;
             }
+            const singleCriteriaPass = !mustMatchCurrent || (mustMatchCurrent && singleMatched);
+
             if (activityMatchComparison !== undefined) {
                 activityThresholdMet = !activityMatchComparison.isPercent && comparisonTextOp(activitiesMatchedCount, activityMatchComparison.operator, activityMatchComparison.value);
             }
@@ -192,7 +204,7 @@ export class RegexRule extends Rule {
             }
 
             let history: (Submission | Comment)[] = [];
-            if ((activityThresholdMet === false || totalThresholdMet === false) && window !== undefined) {
+            if ((activityThresholdMet === false || totalThresholdMet === false) && window !== undefined && singleCriteriaPass) {
                 // our checking activity didn't meet threshold requirements and criteria does define window
                 // leh go
 
@@ -263,7 +275,8 @@ export class RegexRule extends Rule {
                     matchThreshold,
                     activityMatchThreshold,
                     totalMatchThreshold,
-                    window: humanWindow
+                    window: humanWindow,
+                    mustMatchCurrent,
                 },
                 matches,
                 matchCount,

src/Rule/RepeatActivityRule.ts

@@ -1,17 +1,28 @@
 import {Rule, RuleJSONConfig, RuleOptions, RuleResult} from "./index";
 import {Comment} from "snoowrap";
 import {
-    activityWindowText, asSubmission,
-    comparisonTextOp, FAIL, getActivitySubredditName, isExternalUrlSubmission, isRedditMedia,
-    parseGenericValueComparison, parseSubredditName,
-    parseUsableLinkIdentifier as linkParser, PASS, subredditStateIsNameOnly, toStrongSubredditState
+    activityWindowText,
+    asSubmission,
+    comparisonTextOp,
+    FAIL,
+    getActivitySubredditName,
+    isExternalUrlSubmission,
+    isRedditMedia,
+    parseGenericValueComparison,
+    parseSubredditName,
+    parseUsableLinkIdentifier as linkParser,
+    PASS,
+    searchAndReplace,
+    stringSameness,
+    subredditStateIsNameOnly,
+    toStrongSubredditState
 } from "../util";
 import {
     ActivityWindow,
     ActivityWindowType,
-    ReferenceSubmission,
+    ReferenceSubmission, SearchAndReplaceRegExp,
     StrongSubredditState,
-    SubredditState
+    SubredditState, TextMatchOptions, TextTransformOptions
 } from "../Common/interfaces";
 import Submission from "snoowrap/dist/objects/Submission";
 import dayjs from "dayjs";
@@ -29,27 +40,6 @@ interface RepeatActivityReducer {
     allSets: RepeatActivityData[]
 }
 
-const getActivityIdentifier = (activity: (Submission | Comment), length = 200) => {
-    let identifier: string;
-    if (asSubmission(activity)) {
-        if (activity.is_self) {
-            identifier = `${activity.title}${activity.selftext.slice(0, length)}`;
-        } else if(isRedditMedia(activity)) {
-            identifier = activity.title;
-        } else {
-            identifier = parseUsableLinkIdentifier(activity.url) as string;
-        }
-    } else {
-        identifier = activity.body.slice(0, length);
-    }
-    return identifier;
-}
-
-const fuzzyOptions = {
-    includeScore: true,
-    distance: 15
-};
-
 export class RepeatActivityRule extends Rule {
     threshold: string;
     window: ActivityWindowType;
@@ -62,6 +52,9 @@ export class RepeatActivityRule extends Rule {
     activityFilterFunc: (x: Submission|Comment) => Promise<boolean> = async (x) => true;
     keepRemoved: boolean;
     minWordCount: number;
+    transformations: SearchAndReplaceRegExp[]
+    caseSensitive: boolean
+    matchScore: number
 
     constructor(options: RepeatActivityOptions) {
         super(options);
@@ -75,7 +68,13 @@ export class RepeatActivityRule extends Rule {
             include = [],
             exclude = [],
             keepRemoved = false,
+            transformations = [],
+            caseSensitive = true,
+            matchScore = 85,
         } = options;
+        this.matchScore = matchScore;
+        this.transformations = transformations;
+        this.caseSensitive = caseSensitive;
         this.minWordCount = minWordCount;
         this.keepRemoved = keepRemoved;
         this.threshold = threshold;
@@ -136,6 +135,37 @@ export class RepeatActivityRule extends Rule {
         }
     }
 
+    getActivityIdentifier(activity: (Submission | Comment), length = 200, transform = true) {
+        let identifier: string;
+        if (asSubmission(activity)) {
+            if (activity.is_self) {
+                identifier = `${activity.title}${activity.selftext.slice(0, length)}`;
+            } else if(isRedditMedia(activity)) {
+                identifier = activity.title;
+            } else {
+                identifier = parseUsableLinkIdentifier(activity.url) as string;
+            }
+        } else {
+            identifier = activity.body.slice(0, length);
+        }
+
+        if(!transform) {
+            return identifier;
+        }
+
+        // apply any transforms
+        if (this.transformations.length > 0) {
+            identifier = searchAndReplace(identifier, this.transformations);
+        }
+
+        // perform after transformations so as not to mess up regex's depending on case
+        if(!this.caseSensitive) {
+            identifier = identifier.toLowerCase();
+        }
+
+        return identifier;
+    }
+
     async process(item: Submission|Comment): Promise<[boolean, RuleResult]> {
         let referenceUrl;
         if(asSubmission(item) && this.useSubmissionAsReference) {
@@ -162,9 +192,10 @@ export class RepeatActivityRule extends Rule {
             const acc = await accProm;
             const {openSets = [], allSets = []} = acc;
 
-            let identifier = getActivityIdentifier(activity);
+            let identifier = this.getActivityIdentifier(activity);
+
             const isUrl = isExternalUrlSubmission(activity);
-            let fu = new Fuse([identifier], !isUrl ? fuzzyOptions : {...fuzzyOptions, distance: 5});
+            //let fu = new Fuse([identifier], !isUrl ? fuzzyOptions : {...fuzzyOptions, distance: 5});
             const validSub = await this.activityFilterFunc(activity);
             let minMet = identifier.length >= this.minWordCount;
 
@@ -174,12 +205,15 @@ export class RepeatActivityRule extends Rule {
             let currIdentifierInOpen = false;
             const bufferedActivities = this.gapAllowance === undefined || this.gapAllowance === 0 ? [] : activities.slice(Math.max(0, index - this.gapAllowance), Math.max(0, index));
             for (const o of openSets) {
-                const res = fu.search(o.identifier);
-                const match = res.length > 0;
-                if (match && validSub && minMet) {
+                const strMatchResults = stringSameness(o.identifier, identifier);
+                if (strMatchResults.highScoreWeighted >= this.matchScore && minMet) {
                     updatedOpenSets.push({...o, sets: [...o.sets, activity]});
                     currIdentifierInOpen = true;
-                } else if (bufferedActivities.some(x => fu.search(getActivityIdentifier(x)).length > 0) && validSub && minMet) {
+                } else if (bufferedActivities.some(x => {
+                    let buffIdentifier = this.getActivityIdentifier(x);
+                    const buffMatch = stringSameness(identifier, buffIdentifier);
+                    return buffMatch.highScoreWeighted >= this.matchScore;
+                }) && validSub && minMet) {
                     updatedOpenSets.push(o);
                 } else if(!currIdentifierInOpen && !isUrl) {
                     updatedAllSets.push(o);
@@ -193,15 +227,18 @@ export class RepeatActivityRule extends Rule {
                     // could be that a spammer is using different URLs for each submission but similar submission titles so search by title as well
                     const sub = activity as Submission;
                     identifier = sub.title;
-                    fu = new Fuse([identifier], !isUrl ? fuzzyOptions : {...fuzzyOptions, distance: 5});
+                    //fu = new Fuse([identifier], !isUrl ? fuzzyOptions : {...fuzzyOptions, distance: 5});
                     minMet = identifier.length >= this.minWordCount;
                     for (const o of openSets) {
-                        const res = fu.search(o.identifier);
-                        const match = res.length > 0;
-                        if (match && validSub && minMet) {
+                        const strMatchResults = stringSameness(o.identifier, identifier);
+                        if (strMatchResults.highScoreWeighted >= this.matchScore && minMet) {
                             updatedOpenSets.push({...o, sets: [...o.sets, activity]});
                             currIdentifierInOpen = true;
-                        } else if (bufferedActivities.some(x => fu.search(getActivityIdentifier(x)).length > 0) && validSub && minMet && !updatedOpenSets.includes(o)) {
+                        } else if (bufferedActivities.some(x => {
+                            let buffIdentifier = this.getActivityIdentifier(x);
+                            const buffMatch = stringSameness(identifier, buffIdentifier);
+                            return buffMatch.highScoreWeighted >= this.matchScore;
+                        }) && validSub && minMet && !updatedOpenSets.includes(o)) {
                             updatedOpenSets.push(o);
                         } else if(!updatedAllSets.includes(o)) {
                             updatedAllSets.push(o);
@@ -232,7 +269,7 @@ export class RepeatActivityRule extends Rule {
         let applicableGroupedActivities = identifierGroupedActivities;
         if (this.useSubmissionAsReference) {
             applicableGroupedActivities = new Map();
-            let identifier = getActivityIdentifier(item);
+            let identifier = this.getActivityIdentifier(item);
             let referenceSubmissions = identifierGroupedActivities.get(identifier);
             if(referenceSubmissions === undefined && isExternalUrlSubmission(item)) {
                 // if external url sub then try by title
@@ -240,7 +277,7 @@ export class RepeatActivityRule extends Rule {
                 referenceSubmissions = identifierGroupedActivities.get(identifier);
                 if(referenceSubmissions === undefined) {
                     // didn't get by title so go back to url since that's the default
-                    identifier = getActivityIdentifier(item);
+                    identifier = this.getActivityIdentifier(item);
                 }
             }
 
@@ -265,7 +302,7 @@ export class RepeatActivityRule extends Rule {
             };
             for (let set of value) {
                 const test = comparisonTextOp(set.length, operator, thresholdValue);
-                const md = set.map((x: (Comment | Submission)) => `[${asSubmission(x) ? x.title : getActivityIdentifier(x, 50)}](https://reddit.com${x.permalink}) in ${x.subreddit_name_prefixed} on ${dayjs(x.created_utc * 1000).utc().format()}`);
+                const md = set.map((x: (Comment | Submission)) => `[${asSubmission(x) ? x.title : this.getActivityIdentifier(x, 50)}](https://reddit.com${x.permalink}) in ${x.subreddit_name_prefixed} on ${dayjs(x.created_utc * 1000).utc().format()}`);
 
                 summaryData.sets.push(set);
                 summaryData.largestTrigger = Math.max(summaryData.largestTrigger, set.length);
@@ -325,7 +362,7 @@ interface SummaryData {
     triggeringSetsMarkdown: string[]
 }
 
-interface RepeatActivityConfig extends ActivityWindow, ReferenceSubmission {
+interface RepeatActivityConfig extends ActivityWindow, ReferenceSubmission, TextMatchOptions {
     /**
      * The number of repeat submissions that will trigger the rule
      * @default ">= 5"
@@ -383,18 +420,9 @@ interface RepeatActivityConfig extends ActivityWindow, ReferenceSubmission {
     keepRemoved?: boolean
 
     /**
-     * For activities that are text-based this is the minimum number of words required for the activity to be considered for a repeat
-     *
-     * EX if `minimumWordCount=5` and a comment is `what about you` then it is ignored because `3 is less than 5`
-     *
-     * **For self-text submissions** -- title + body text
-     *
-     * **For comments* -- body text
-     *
-     * @default 1
-     * @example [1]
+     * A set of search-and-replace operations to perform on text values before performing a match. Transformations are performed in the order they are defined.
      * */
-    minWordCount?: number,
+    transformations?: SearchAndReplaceRegExp[]
 }
 
 export interface RepeatActivityOptions extends RepeatActivityConfig, RuleOptions {

src/Rule/RepostRule.ts

@@ -18,7 +18,7 @@ import {
     RepostItem,
     RepostItemResult,
     SearchAndReplaceRegExp,
-    SearchFacetType,
+    SearchFacetType, TextMatchOptions, TextTransformOptions,
 } from "../Common/interfaces";
 import objectHash from "object-hash";
 import {getActivities, getAttributionIdentifier} from "../Utils/SnoowrapUtils";
@@ -30,59 +30,6 @@ import {rest} from "lodash";
 
 const parseYtIdentifier = parseUsableLinkIdentifier();
 
-export interface TextMatchOptions {
-    /**
-     * The percentage, as a whole number, of a repost title/comment that must match the title/comment being checked in order to consider both a match
-     *
-     * Note: Setting to 0 will make every candidate considered a match -- useful if you want to match if the URL has been reposted anywhere
-     *
-     * Defaults to `85` (85%)
-     *
-     * @default 85
-     * @example [85]
-     * */
-    matchScore?: number
-
-    /**
-     * The minimum number of words in the activity being checked for which this rule will run on
-     *
-     * If the word count is below the minimum the rule fails
-     *
-     * Defaults to 2
-     *
-     * @default 2
-     * @example [2]
-     * */
-    minWordCount?: number
-
-    /**
-     * Should text matching be case sensitive?
-     *
-     * Defaults to false
-     *
-     * @default false
-     * @example [false]
-     **/
-    caseSensitive?: boolean
-}
-
-export interface TextTransformOptions {
-    /**
-     * A set of search-and-replace operations to perform on text values before performing a match. Transformations are performed in the order they are defined.
-     *
-     * * If `transformationsActivity` IS NOT defined then these transformations will be performed on BOTH the activity text (submission title or comment) AND the repost candidate text
-     * * If `transformationsActivity` IS defined then these transformations are only performed on repost candidate text
-     * */
-    transformations?: SearchAndReplaceRegExp[]
-
-    /**
-     * Specify a separate set of transformations for the activity text (submission title or comment)
-     *
-     * To perform no transformations when `transformations` is defined set this to an empty array (`[]`)
-     * */
-    transformationsActivity?: SearchAndReplaceRegExp[]
-}
-
 export interface SearchFacetJSONConfig extends TextMatchOptions, TextTransformOptions, ActivityWindow {
     kind: SearchFacetType | SearchFacetType[]
 }

src/Schema/Action.json

@@ -290,6 +290,10 @@
                         }
                     ]
                 },
+                "isRedditMediaDomain": {
+                    "description": "Is the submission a reddit-hosted image or video?",
+                    "type": "boolean"
+                },
                 "is_self": {
                     "type": "boolean"
                 },

src/Schema/App.json

@@ -246,8 +246,7 @@
                     "default": "undefined",
                     "description": "This list determines which categories of domains should be aggregated on. All aggregated domains will be tested against `threshold`\n\n* If `media` is included then aggregate author's submission history which reddit recognizes as media (youtube, vimeo, etc.)\n* If `redditMedia` is included then aggregate on author's submissions history which are media hosted on reddit: galleries, videos, and images (i.redd.it / v.redd.it)\n* If `self` is included then aggregate on author's submission history which are self-post (`self.[subreddit]`) or domain is `reddit.com`\n* If `link` is included then aggregate author's submission history which is external links and not recognized as `media` by reddit\n\nIf nothing is specified or list is empty (default) rule will only aggregate on `link` and `media` (ignores reddit-hosted content and self-posts)",
                     "examples": [
-                        [
-                        ]
+                        []
                     ],
                     "items": {
                         "enum": [
@@ -280,8 +279,7 @@
                 },
                 "domains": {
                     "default": [
-                        [
-                        ]
+                        []
                     ],
                     "description": "A list of domains whose Activities will be tested against `threshold`.\n\nThe values are tested as partial strings so you do not need to include full URLs, just the part that matters.\n\nEX `[\"youtube\"]` will match submissions with the domain `https://youtube.com/c/aChannel`\nEX `[\"youtube.com/c/bChannel\"]` will NOT match submissions with the domain `https://youtube.com/c/aChannel`\n\nIf you wish to aggregate on self-posts for a subreddit use the syntax `self.[subreddit]` EX `self.AskReddit`\n\n**If this Rule is part of a Check for a Submission and you wish to aggregate on the domain of the Submission use the special string `AGG:SELF`**\n\nIf nothing is specified or list is empty (default) aggregate using `aggregateOn`",
                     "items": {
@@ -963,8 +961,7 @@
             "type": "object"
         },
         "CacheOptions": {
-            "additionalProperties": {
-            },
+            "additionalProperties": {},
             "description": "Configure granular settings for a cache provider with this object",
             "properties": {
                 "auth_pass": {
@@ -2416,6 +2413,11 @@
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)(\\s+.*)*$",
                     "type": "string"
                 },
+                "mustMatchCurrent": {
+                    "default": false,
+                    "description": "When `true` the Activity being checked MUST pass the `matchThreshold` before the Rule considers any history\n\nFor use with `activityMatchThreshold`/`totalMatchThreshold` -- useful to conserve API calls",
+                    "type": "boolean"
+                },
                 "name": {
                     "description": "A descriptive name that will be used in logging and be available for templating",
                     "examples": [
@@ -2638,6 +2640,9 @@
                     ],
                     "pattern": "^[a-zA-Z]([\\w -]*[\\w])?$",
                     "type": "string"
+                },
+                "spam": {
+                    "type": "boolean"
                 }
             },
             "required": [
@@ -2667,6 +2672,11 @@
                         }
                     ]
                 },
+                "caseSensitive": {
+                    "default": false,
+                    "description": "Should text matching be case sensitive?\n\nDefaults to false",
+                    "type": "boolean"
+                },
                 "exclude": {
                     "description": "If present, activities will be counted only if they are **NOT** found in this list of Subreddits\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`",
                     "examples": [
@@ -2757,9 +2767,14 @@
                     ],
                     "type": "string"
                 },
+                "matchScore": {
+                    "default": 85,
+                    "description": "The percentage, as a whole number, of a repost title/comment that must match the title/comment being checked in order to consider both a match\n\nNote: Setting to 0 will make every candidate considered a match -- useful if you want to match if the URL has been reposted anywhere\n\nDefaults to `85` (85%)",
+                    "type": "number"
+                },
                 "minWordCount": {
-                    "default": 1,
-                    "description": "For activities that are text-based this is the minimum number of words required for the activity to be considered for a repeat\n\nEX if `minimumWordCount=5` and a comment is `what about you` then it is ignored because `3 is less than 5`\n\n**For self-text submissions** -- title + body text\n\n**For comments* -- body text",
+                    "default": 2,
+                    "description": "The minimum number of words in the activity being checked for which this rule will run on\n\nIf the word count is below the minimum the rule fails\n\nDefaults to 2",
                     "type": "number"
                 },
                 "name": {
@@ -2775,6 +2790,13 @@
                     "description": "The number of repeat submissions that will trigger the rule",
                     "type": "string"
                 },
+                "transformations": {
+                    "description": "A set of search-and-replace operations to perform on text values before performing a match. Transformations are performed in the order they are defined.",
+                    "items": {
+                        "$ref": "#/definitions/SearchAndReplaceRegExp"
+                    },
+                    "type": "array"
+                },
                 "useSubmissionAsReference": {
                     "default": true,
                     "description": "If activity is a Submission and is a link (not self-post) then only look at Submissions that contain this link, otherwise consider all activities.",
@@ -3491,6 +3513,10 @@
                         }
                     ]
                 },
+                "isRedditMediaDomain": {
+                    "description": "Is the submission a reddit-hosted image or video?",
+                    "type": "boolean"
+                },
                 "is_self": {
                     "type": "boolean"
                 },
@@ -3601,8 +3627,7 @@
             "type": "object"
         },
         "ThirdPartyCredentialsJsonConfig": {
-            "additionalProperties": {
-            },
+            "additionalProperties": {},
             "properties": {
                 "youtube": {
                     "properties": {

src/Schema/OperatorConfig.json

@@ -397,8 +397,7 @@
             "type": "object"
         },
         "CacheOptions": {
-            "additionalProperties": {
-            },
+            "additionalProperties": {},
             "description": "Configure granular settings for a cache provider with this object",
             "properties": {
                 "auth_pass": {
@@ -607,6 +606,117 @@
             },
             "type": "object"
         },
+        "LoggingOptions": {
+            "properties": {
+                "console": {
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/Pick<Transports.ConsoleTransportOptions,\"silent\"|\"eol\"|\"stderrLevels\"|\"consoleWarnLevels\">"
+                        },
+                        {
+                            "properties": {
+                                "level": {
+                                    "enum": [
+                                        "debug",
+                                        "error",
+                                        "info",
+                                        "verbose",
+                                        "warn"
+                                    ],
+                                    "type": "string"
+                                }
+                            },
+                            "type": "object"
+                        }
+                    ],
+                    "description": "Options for logging to console"
+                },
+                "file": {
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/Omit<DailyRotateFileTransportOptions,\"stream\"|\"dirname\"|\"options\"|\"handleRejections\"|\"format\"|\"handleExceptions\"|\"log\"|\"logv\"|\"close\">"
+                        },
+                        {
+                            "properties": {
+                                "dirname": {
+                                    "description": "The absolute path to a directory where rotating log files should be stored.\n\n* If not present or `null` or `false` no log files will be created\n* If `true` logs will be stored at `[working directory]/logs`\n\n* ENV => `LOG_DIR`\n* ARG => `--logDir [dir]`",
+                                    "examples": [
+                                        "/var/log/contextmod"
+                                    ],
+                                    "type": [
+                                        "null",
+                                        "string",
+                                        "boolean"
+                                    ]
+                                },
+                                "level": {
+                                    "enum": [
+                                        "debug",
+                                        "error",
+                                        "info",
+                                        "verbose",
+                                        "warn"
+                                    ],
+                                    "type": "string"
+                                }
+                            },
+                            "type": "object"
+                        }
+                    ],
+                    "description": "Options for Rotating File logging"
+                },
+                "level": {
+                    "default": "verbose",
+                    "description": "The minimum log level to output. The log level set will output logs at its level **and all levels above it:**\n\n * `error`\n * `warn`\n * `info`\n * `verbose`\n * `debug`\n\n Note: `verbose` will display *a lot* of information on the status/result of run rules/checks/actions etc. which is very useful for testing configurations. Once your bot is stable changing the level to `info` will reduce log noise.\n\n * ENV => `LOG_LEVEL`\n * ARG => `--logLevel <level>`",
+                    "enum": [
+                        "debug",
+                        "error",
+                        "info",
+                        "verbose",
+                        "warn"
+                    ],
+                    "examples": [
+                        "verbose"
+                    ],
+                    "type": "string"
+                },
+                "path": {
+                    "description": "**DEPRECATED** - Use `file.dirname` instead\nThe absolute path to a directory where rotating log files should be stored.\n\n* If not present or `null` or `false` no log files will be created\n* If `true` logs will be stored at `[working directory]/logs`\n\n* ENV => `LOG_DIR`\n* ARG => `--logDir [dir]`",
+                    "examples": [
+                        "/var/log/contextmod"
+                    ],
+                    "type": [
+                        "null",
+                        "string",
+                        "boolean"
+                    ]
+                },
+                "stream": {
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/Omit<DuplexTransportOptions,\"name\"|\"stream\"|\"handleRejections\"|\"format\"|\"handleExceptions\"|\"log\"|\"logv\"|\"close\">"
+                        },
+                        {
+                            "properties": {
+                                "level": {
+                                    "enum": [
+                                        "debug",
+                                        "error",
+                                        "info",
+                                        "verbose",
+                                        "warn"
+                                    ],
+                                    "type": "string"
+                                }
+                            },
+                            "type": "object"
+                        }
+                    ],
+                    "description": "Options for logging to api/web"
+                }
+            },
+            "type": "object"
+        },
         "NotificationConfig": {
             "properties": {
                 "events": {
@@ -672,6 +782,90 @@
             ],
             "type": "object"
         },
+        "Omit<DailyRotateFileTransportOptions,\"stream\"|\"dirname\"|\"options\"|\"handleRejections\"|\"format\"|\"handleExceptions\"|\"log\"|\"logv\"|\"close\">": {
+            "properties": {
+                "auditFile": {
+                    "description": "A string representing the name of the name of the audit file. (default: './hash-audit.json')",
+                    "type": "string"
+                },
+                "createSymlink": {
+                    "description": "Create a tailable symlink to the current active log file. (default: false)",
+                    "type": "boolean"
+                },
+                "datePattern": {
+                    "description": "A string representing the moment.js date format to be used for rotating. The meta characters used in this string will dictate the frequency of the file rotation. For example, if your datePattern is simply 'HH' you will end up with 24 log files that are picked up and appended to every day. (default 'YYYY-MM-DD')",
+                    "type": "string"
+                },
+                "eol": {
+                    "type": "string"
+                },
+                "extension": {
+                    "description": "A string representing an extension to be added to the filename, if not included in the filename property. (default: '')",
+                    "type": "string"
+                },
+                "filename": {
+                    "description": "Filename to be used to log to. This filename can include the %DATE% placeholder which will include the formatted datePattern at that point in the filename. (default: 'winston.log.%DATE%)",
+                    "type": "string"
+                },
+                "frequency": {
+                    "description": "A string representing the frequency of rotation. (default: 'custom')",
+                    "type": "string"
+                },
+                "json": {
+                    "type": "boolean"
+                },
+                "level": {
+                    "type": "string"
+                },
+                "maxFiles": {
+                    "description": "Maximum number of logs to keep. If not set, no logs will be removed. This can be a number of files or number of days. If using days, add 'd' as the suffix. (default: null)",
+                    "type": [
+                        "string",
+                        "number"
+                    ]
+                },
+                "maxSize": {
+                    "description": "Maximum size of the file after which it will rotate. This can be a number of bytes, or units of kb, mb, and gb. If using the units, add 'k', 'm', or 'g' as the suffix. The units need to directly follow the number. (default: null)",
+                    "type": [
+                        "string",
+                        "number"
+                    ]
+                },
+                "silent": {
+                    "type": "boolean"
+                },
+                "symlinkName": {
+                    "description": "The name of the tailable symlink. (default: 'current.log')",
+                    "type": "string"
+                },
+                "utc": {
+                    "description": "A boolean whether or not to generate file name from \"datePattern\" in UTC format. (default: false)",
+                    "type": "boolean"
+                },
+                "zippedArchive": {
+                    "description": "A boolean to define whether or not to gzip archived log files. (default 'false')",
+                    "type": "boolean"
+                }
+            },
+            "type": "object"
+        },
+        "Omit<DuplexTransportOptions,\"name\"|\"stream\"|\"handleRejections\"|\"format\"|\"handleExceptions\"|\"log\"|\"logv\"|\"close\">": {
+            "properties": {
+                "dump": {
+                    "type": "boolean"
+                },
+                "eol": {
+                    "type": "string"
+                },
+                "level": {
+                    "type": "string"
+                },
+                "silent": {
+                    "type": "boolean"
+                }
+            },
+            "type": "object"
+        },
         "OperatorCacheConfig": {
             "properties": {
                 "actionedEventsDefault": {
@@ -791,6 +985,29 @@
             },
             "type": "object"
         },
+        "Pick<Transports.ConsoleTransportOptions,\"silent\"|\"eol\"|\"stderrLevels\"|\"consoleWarnLevels\">": {
+            "properties": {
+                "consoleWarnLevels": {
+                    "items": {
+                        "type": "string"
+                    },
+                    "type": "array"
+                },
+                "eol": {
+                    "type": "string"
+                },
+                "silent": {
+                    "type": "boolean"
+                },
+                "stderrLevels": {
+                    "items": {
+                        "type": "string"
+                    },
+                    "type": "array"
+                }
+            },
+            "type": "object"
+        },
         "PollingDefaults": {
             "properties": {
                 "delayUntil": {
@@ -914,6 +1131,10 @@
                         }
                     ]
                 },
+                "isRedditMediaDomain": {
+                    "description": "Is the submission a reddit-hosted image or video?",
+                    "type": "boolean"
+                },
                 "is_self": {
                     "type": "boolean"
                 },
@@ -983,8 +1204,7 @@
             "type": "object"
         },
         "ThirdPartyCredentialsJsonConfig": {
-            "additionalProperties": {
-            },
+            "additionalProperties": {},
             "properties": {
                 "youtube": {
                     "properties": {
@@ -1110,32 +1330,8 @@
             "$ref": "#/definitions/ThirdPartyCredentialsJsonConfig"
         },
         "logging": {
-            "description": "Settings to configure global logging defaults",
-            "properties": {
-                "level": {
-                    "default": "verbose",
-                    "description": "The minimum log level to output. The log level set will output logs at its level **and all levels above it:**\n\n * `error`\n * `warn`\n * `info`\n * `verbose`\n * `debug`\n\n Note: `verbose` will display *a lot* of information on the status/result of run rules/checks/actions etc. which is very useful for testing configurations. Once your bot is stable changing the level to `info` will reduce log noise.\n\n * ENV => `LOG_LEVEL`\n * ARG => `--logLevel <level>`",
-                    "enum": [
-                        "debug",
-                        "error",
-                        "info",
-                        "verbose",
-                        "warn"
-                    ],
-                    "examples": [
-                        "verbose"
-                    ],
-                    "type": "string"
-                },
-                "path": {
-                    "description": "The absolute path to a directory where rotating log files should be stored.\n\n* If not present or `null` no log files will be created\n* If `true` logs will be stored at `[working directory]/logs`\n\n* ENV => `LOG_DIR`\n* ARG => `--logDir [dir]`",
-                    "examples": [
-                        "/var/log/contextmod"
-                    ],
-                    "type": "string"
-                }
-            },
-            "type": "object"
+            "$ref": "#/definitions/LoggingOptions",
+            "description": "Settings to configure global logging defaults"
         },
         "mode": {
             "default": "all",

src/Schema/Rule.json

@@ -184,8 +184,7 @@
                     "default": "undefined",
                     "description": "This list determines which categories of domains should be aggregated on. All aggregated domains will be tested against `threshold`\n\n* If `media` is included then aggregate author's submission history which reddit recognizes as media (youtube, vimeo, etc.)\n* If `redditMedia` is included then aggregate on author's submissions history which are media hosted on reddit: galleries, videos, and images (i.redd.it / v.redd.it)\n* If `self` is included then aggregate on author's submission history which are self-post (`self.[subreddit]`) or domain is `reddit.com`\n* If `link` is included then aggregate author's submission history which is external links and not recognized as `media` by reddit\n\nIf nothing is specified or list is empty (default) rule will only aggregate on `link` and `media` (ignores reddit-hosted content and self-posts)",
                     "examples": [
-                        [
-                        ]
+                        []
                     ],
                     "items": {
                         "enum": [
@@ -218,8 +217,7 @@
                 },
                 "domains": {
                     "default": [
-                        [
-                        ]
+                        []
                     ],
                     "description": "A list of domains whose Activities will be tested against `threshold`.\n\nThe values are tested as partial strings so you do not need to include full URLs, just the part that matters.\n\nEX `[\"youtube\"]` will match submissions with the domain `https://youtube.com/c/aChannel`\nEX `[\"youtube.com/c/bChannel\"]` will NOT match submissions with the domain `https://youtube.com/c/aChannel`\n\nIf you wish to aggregate on self-posts for a subreddit use the syntax `self.[subreddit]` EX `self.AskReddit`\n\n**If this Rule is part of a Check for a Submission and you wish to aggregate on the domain of the Submission use the special string `AGG:SELF`**\n\nIf nothing is specified or list is empty (default) aggregate using `aggregateOn`",
                     "items": {
@@ -1288,6 +1286,11 @@
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)(\\s+.*)*$",
                     "type": "string"
                 },
+                "mustMatchCurrent": {
+                    "default": false,
+                    "description": "When `true` the Activity being checked MUST pass the `matchThreshold` before the Rule considers any history\n\nFor use with `activityMatchThreshold`/`totalMatchThreshold` -- useful to conserve API calls",
+                    "type": "boolean"
+                },
                 "name": {
                     "description": "A descriptive name that will be used in logging and be available for templating",
                     "examples": [
@@ -1462,6 +1465,11 @@
                         }
                     ]
                 },
+                "caseSensitive": {
+                    "default": false,
+                    "description": "Should text matching be case sensitive?\n\nDefaults to false",
+                    "type": "boolean"
+                },
                 "exclude": {
                     "description": "If present, activities will be counted only if they are **NOT** found in this list of Subreddits\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`",
                     "examples": [
@@ -1552,9 +1560,14 @@
                     ],
                     "type": "string"
                 },
+                "matchScore": {
+                    "default": 85,
+                    "description": "The percentage, as a whole number, of a repost title/comment that must match the title/comment being checked in order to consider both a match\n\nNote: Setting to 0 will make every candidate considered a match -- useful if you want to match if the URL has been reposted anywhere\n\nDefaults to `85` (85%)",
+                    "type": "number"
+                },
                 "minWordCount": {
-                    "default": 1,
-                    "description": "For activities that are text-based this is the minimum number of words required for the activity to be considered for a repeat\n\nEX if `minimumWordCount=5` and a comment is `what about you` then it is ignored because `3 is less than 5`\n\n**For self-text submissions** -- title + body text\n\n**For comments* -- body text",
+                    "default": 2,
+                    "description": "The minimum number of words in the activity being checked for which this rule will run on\n\nIf the word count is below the minimum the rule fails\n\nDefaults to 2",
                     "type": "number"
                 },
                 "name": {
@@ -1570,6 +1583,13 @@
                     "description": "The number of repeat submissions that will trigger the rule",
                     "type": "string"
                 },
+                "transformations": {
+                    "description": "A set of search-and-replace operations to perform on text values before performing a match. Transformations are performed in the order they are defined.",
+                    "items": {
+                        "$ref": "#/definitions/SearchAndReplaceRegExp"
+                    },
+                    "type": "array"
+                },
                 "useSubmissionAsReference": {
                     "default": true,
                     "description": "If activity is a Submission and is a link (not self-post) then only look at Submissions that contain this link, otherwise consider all activities.",
@@ -1963,6 +1983,10 @@
                         }
                     ]
                 },
+                "isRedditMediaDomain": {
+                    "description": "Is the submission a reddit-hosted image or video?",
+                    "type": "boolean"
+                },
                 "is_self": {
                     "type": "boolean"
                 },

src/Schema/RuleSet.json

@@ -158,8 +158,7 @@
                     "default": "undefined",
                     "description": "This list determines which categories of domains should be aggregated on. All aggregated domains will be tested against `threshold`\n\n* If `media` is included then aggregate author's submission history which reddit recognizes as media (youtube, vimeo, etc.)\n* If `redditMedia` is included then aggregate on author's submissions history which are media hosted on reddit: galleries, videos, and images (i.redd.it / v.redd.it)\n* If `self` is included then aggregate on author's submission history which are self-post (`self.[subreddit]`) or domain is `reddit.com`\n* If `link` is included then aggregate author's submission history which is external links and not recognized as `media` by reddit\n\nIf nothing is specified or list is empty (default) rule will only aggregate on `link` and `media` (ignores reddit-hosted content and self-posts)",
                     "examples": [
-                        [
-                        ]
+                        []
                     ],
                     "items": {
                         "enum": [
@@ -192,8 +191,7 @@
                 },
                 "domains": {
                     "default": [
-                        [
-                        ]
+                        []
                     ],
                     "description": "A list of domains whose Activities will be tested against `threshold`.\n\nThe values are tested as partial strings so you do not need to include full URLs, just the part that matters.\n\nEX `[\"youtube\"]` will match submissions with the domain `https://youtube.com/c/aChannel`\nEX `[\"youtube.com/c/bChannel\"]` will NOT match submissions with the domain `https://youtube.com/c/aChannel`\n\nIf you wish to aggregate on self-posts for a subreddit use the syntax `self.[subreddit]` EX `self.AskReddit`\n\n**If this Rule is part of a Check for a Submission and you wish to aggregate on the domain of the Submission use the special string `AGG:SELF`**\n\nIf nothing is specified or list is empty (default) aggregate using `aggregateOn`",
                     "items": {
@@ -1262,6 +1260,11 @@
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)(\\s+.*)*$",
                     "type": "string"
                 },
+                "mustMatchCurrent": {
+                    "default": false,
+                    "description": "When `true` the Activity being checked MUST pass the `matchThreshold` before the Rule considers any history\n\nFor use with `activityMatchThreshold`/`totalMatchThreshold` -- useful to conserve API calls",
+                    "type": "boolean"
+                },
                 "name": {
                     "description": "A descriptive name that will be used in logging and be available for templating",
                     "examples": [
@@ -1436,6 +1439,11 @@
                         }
                     ]
                 },
+                "caseSensitive": {
+                    "default": false,
+                    "description": "Should text matching be case sensitive?\n\nDefaults to false",
+                    "type": "boolean"
+                },
                 "exclude": {
                     "description": "If present, activities will be counted only if they are **NOT** found in this list of Subreddits\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`",
                     "examples": [
@@ -1526,9 +1534,14 @@
                     ],
                     "type": "string"
                 },
+                "matchScore": {
+                    "default": 85,
+                    "description": "The percentage, as a whole number, of a repost title/comment that must match the title/comment being checked in order to consider both a match\n\nNote: Setting to 0 will make every candidate considered a match -- useful if you want to match if the URL has been reposted anywhere\n\nDefaults to `85` (85%)",
+                    "type": "number"
+                },
                 "minWordCount": {
-                    "default": 1,
-                    "description": "For activities that are text-based this is the minimum number of words required for the activity to be considered for a repeat\n\nEX if `minimumWordCount=5` and a comment is `what about you` then it is ignored because `3 is less than 5`\n\n**For self-text submissions** -- title + body text\n\n**For comments* -- body text",
+                    "default": 2,
+                    "description": "The minimum number of words in the activity being checked for which this rule will run on\n\nIf the word count is below the minimum the rule fails\n\nDefaults to 2",
                     "type": "number"
                 },
                 "name": {
@@ -1544,6 +1557,13 @@
                     "description": "The number of repeat submissions that will trigger the rule",
                     "type": "string"
                 },
+                "transformations": {
+                    "description": "A set of search-and-replace operations to perform on text values before performing a match. Transformations are performed in the order they are defined.",
+                    "items": {
+                        "$ref": "#/definitions/SearchAndReplaceRegExp"
+                    },
+                    "type": "array"
+                },
                 "useSubmissionAsReference": {
                     "default": true,
                     "description": "If activity is a Submission and is a link (not self-post) then only look at Submissions that contain this link, otherwise consider all activities.",
@@ -1937,6 +1957,10 @@
                         }
                     ]
                 },
+                "isRedditMediaDomain": {
+                    "description": "Is the submission a reddit-hosted image or video?",
+                    "type": "boolean"
+                },
                 "is_self": {
                     "type": "boolean"
                 },

src/Subreddit/Manager.ts

@@ -48,7 +48,8 @@ import {CheckStructuredJson} from "../Check";
 import NotificationManager from "../Notification/NotificationManager";
 import {createHistoricalDefaults, historicalDefaults} from "../Common/defaults";
 import {ExtendedSnoowrap} from "../Utils/SnoowrapClients";
-import {isRateLimitError, isStatusError} from "../Utils/Errors";
+import {CMError, isRateLimitError, isStatusError} from "../Utils/Errors";
+import {ErrorWithCause} from "pony-cause";
 
 export interface RunningState {
     state: RunState,
@@ -269,15 +270,21 @@ export class Manager extends EventEmitter {
         })(this), 10000);
     }
 
-    protected async getModPermissions(): Promise<string[]> {
+    public async getModPermissions(): Promise<string[]> {
         if(this.modPermissions !== undefined) {
             return this.modPermissions as string[];
         }
         this.logger.debug('Retrieving mod permissions for bot');
-        const userInfo = parseRedditEntity(this.botName, 'user');
-        const mods = this.subreddit.getModerators({name: userInfo.name});
-        // @ts-ignore
-        this.modPermissions = mods[0].mod_permissions;
+        try {
+            const userInfo = parseRedditEntity(this.botName, 'user');
+            const mods = this.subreddit.getModerators({name: userInfo.name});
+            // @ts-ignore
+            this.modPermissions = mods[0].mod_permissions;
+        } catch (e) {
+            const err = new ErrorWithCause('Unable to retrieve moderator permissions', {cause: e});
+            this.logger.error(err);
+            return [];
+        }
         return this.modPermissions as string[];
     }
 
@@ -449,9 +456,6 @@ export class Manager extends EventEmitter {
                 this.logger.info(checkSummary);
             }
             this.validConfigLoaded = true;
-            if(!suppressChangeEvent) {
-                this.emit('configChange');
-            }
             if(this.eventsState.state === RUNNING) {
                 // need to update polling, potentially
                 await this.buildPolling();
@@ -462,6 +466,9 @@ export class Manager extends EventEmitter {
                     }
                 }
             }
+            if(!suppressChangeEvent) {
+                this.emit('configChange');
+            }
         } catch (err: any) {
             this.validConfigLoaded = false;
             throw err;
@@ -484,21 +491,21 @@ export class Manager extends EventEmitter {
                     if(isStatusError(err) && err.statusCode === 404) {
                         // see if we can create the page
                         if (!this.client.scope.includes('wikiedit')) {
-                            throw new Error(`Page does not exist and could not be created because Bot does not have oauth permission 'wikiedit'`);
+                            throw new ErrorWithCause(`Page does not exist and could not be created because Bot does not have oauth permission 'wikiedit'`, {cause: err});
                         }
                         const modPermissions = await this.getModPermissions();
                         if (!modPermissions.includes('all') && !modPermissions.includes('wiki')) {
-                            throw new Error(`Page does not exist and could not be created because Bot not have mod permissions for creating wiki pages. Must have 'all' or 'wiki'`);
+                            throw new ErrorWithCause(`Page does not exist and could not be created because Bot not have mod permissions for creating wiki pages. Must have 'all' or 'wiki'`, {cause: err});
                         }
                         if(!this.client.scope.includes('modwiki')) {
-                            throw new Error(`Bot COULD create wiki config page but WILL NOT because it does not have the oauth permissions 'modwiki' which is required to set page visibility and editing permissions. Safety first!`);
+                            throw new ErrorWithCause(`Bot COULD create wiki config page but WILL NOT because it does not have the oauth permissions 'modwiki' which is required to set page visibility and editing permissions. Safety first!`, {cause: err});
                         }
                         // @ts-ignore
                         wiki = await this.subreddit.getWikiPage(this.wikiLocation).edit({
                             text: '',
                             reason: 'Empty configuration created for ContextMod'
                         });
-                        this.logger.info(`Wiki page at ${this.wikiLocation} did not exist, but bot created it!`);
+                        this.logger.info(`Wiki page at ${this.wikiLocation} did not exist so bot created it!`);
 
                         // 0 = use subreddit wiki permissions
                         // 1 = only approved wiki contributors
@@ -542,11 +549,10 @@ export class Manager extends EventEmitter {
             } catch (err: any) {
                 let hint = '';
                 if(isStatusError(err) && err.statusCode === 403) {
-                    hint = `\r\nHINT: Either the page is restricted to mods only and the bot's reddit account does have the mod permission 'all' or 'wiki' OR the bot does not have the 'wikiread' oauth permission`;
+                    hint = ` -- HINT: Either the page is restricted to mods only and the bot's reddit account does have the mod permission 'all' or 'wiki' OR the bot does not have the 'wikiread' oauth permission`;
                 }
-                const msg = `Could not read wiki configuration. Please ensure the page https://reddit.com${this.subreddit.url}wiki/${this.wikiLocation} exists and is readable${hint} -- error: ${err.message}`;
-                this.logger.error(msg);
-                throw new ConfigParseError(msg);
+                const msg = `Could not read wiki configuration. Please ensure the page https://reddit.com${this.subreddit.url}wiki/${this.wikiLocation} exists and is readable${hint}`;
+                throw new ErrorWithCause(msg, {cause: err});
             }
 
             if (sourceData.replace('\r\n', '').trim() === '') {
@@ -580,8 +586,12 @@ export class Manager extends EventEmitter {
 
             return true;
         } catch (err: any) {
+            const error = new ErrorWithCause('Failed to parse subreddit configuration', {cause: err});
+            // @ts-ignore
+           //error.logged = true;
+            this.logger.error(error);
             this.validConfigLoaded = false;
-            throw err;
+            throw error;
         }
     }
 
@@ -731,7 +741,7 @@ export class Manager extends EventEmitter {
                     actionsRun = runActions.length;
 
                     if(check.notifyOnTrigger) {
-                        const ar = runActions.map(x => x.name).join(', ');
+                        const ar = runActions.filter(x => x.success).map(x => x.name).join(', ');
                         this.notificationManager.handle('eventActioned', 'Check Triggered', `Check "${check.name}" was triggered on Event: \n\n ${ePeek} \n\n with the following actions run: ${ar}`);
                     }
                     break;
@@ -906,9 +916,9 @@ export class Manager extends EventEmitter {
                     }
                     if(!this.sharedStreamCallbacks.has(source)) {
                         stream.once('listing', this.noChecksWarning(source));
-                        this.sharedStreamCallbacks.set(source, onItem);
                         this.logger.debug(`${removedOwn ? 'Stopped own polling and replace with ' : 'Set '}listener on shared polling ${source}`);
                     }
+                    this.sharedStreamCallbacks.set(source, onItem);
                 } else {
                     let ownPollingMsgParts: string[] = [];
                     let removedShared = false;
@@ -937,14 +947,9 @@ export class Manager extends EventEmitter {
 
                         this.emit('error', err);
 
-                        if (isRateLimitError(err)) {
-                            this.logger.error('Encountered rate limit while polling! Bot is all out of requests :( Stopping subreddit queue and polling.');
-                            await this.stop();
-                        }
-                        this.logger.error('Polling error occurred', err);
                         const shouldRetry = await this.pollingRetryHandler(err);
                         if (shouldRetry) {
-                            stream.startInterval(false);
+                            stream.startInterval(false, 'Within retry limits');
                         } else {
                             this.logger.warn('Stopping subreddit processing/polling due to too many errors');
                             await this.stop();
@@ -1140,7 +1145,6 @@ export class Manager extends EventEmitter {
                 s.end();
             }
             this.streams = new Map();
-            this.sharedStreamCallbacks = new Map();
             this.startedAt = undefined;
             this.logger.info(`Events STOPPED by ${causedBy}`);
             this.eventsState = {

src/Subreddit/Streams.ts

@@ -1,10 +1,11 @@
 import {Poll, SnooStormOptions} from "snoostorm"
-import Snoowrap from "snoowrap";
+import Snoowrap, {Listing} from "snoowrap";
 import {EventEmitter} from "events";
 import {PollConfiguration} from "snoostorm/out/util/Poll";
 import {DEFAULT_POLLING_INTERVAL} from "../Common/interfaces";
 import {mergeArr, parseDuration, random} from "../util";
 import { Logger } from "winston";
+import {ErrorWithCause} from "pony-cause";
 
 type Awaitable<T> = Promise<T> | T;
 
@@ -18,11 +19,12 @@ interface RCBPollingOptions<T> extends SnooStormOptions {
 }
 
 interface RCBPollConfiguration<T> extends PollConfiguration<T>,RCBPollingOptions<T> {
+    get: () => Promise<Listing<T>>
 }
 
 export class SPoll<T extends object> extends Poll<T> {
     identifier: keyof T;
-    getter: () => Awaitable<T[]>;
+    getter: () => Promise<Listing<T>>;
     frequency;
     running: boolean = false;
     // intention of newStart is to make polling behavior such that only "new" items AFTER polling has started get emitted
@@ -82,6 +84,10 @@ export class SPoll<T extends object> extends Poll<T> {
                             // @ts-ignore
                             batch = await batch.fetchMore({amount: 100});
                         }
+                        if(batch.length === 0 || batch.isFinished) {
+                            // if nothing is returned we don't want to end up in an endless loop!
+                            anyAlreadySeen = true;
+                        }
                         for (const item of batch) {
                             const id = item[self.identifier];
                             if (self.processed.has(id)) {
@@ -99,7 +105,7 @@ export class SPoll<T extends object> extends Poll<T> {
                         }
                         page++;
                     }
-                    const newItemMsg = `Found ${newItems.length} new items`;
+                    const newItemMsg = `Found ${newItems.length} new items out of ${batch.length} returned`;
                     if(self.newStart) {
                         self.logger.debug(`${newItemMsg} but will ignore all on first start.`);
                         self.emit("listing", []);
@@ -113,6 +119,8 @@ export class SPoll<T extends object> extends Poll<T> {
                     // if everything succeeded then create a new timeout
                     self.createInterval();
                 } catch (err: any) {
+                    self.running = false;
+                    self.logger.error(new ErrorWithCause('Polling Interval stopped due to error encountered', {cause: err}));
                     self.emit('error', err);
                 }
             }
@@ -120,15 +128,22 @@ export class SPoll<T extends object> extends Poll<T> {
     }
 
     // allow controlling newStart state
-    startInterval = (newStartState?: boolean) => {
+    startInterval = (newStartState?: boolean, msg?: string) => {
         this.running = true;
         if(newStartState !== undefined) {
             this.newStart = newStartState;
         }
+        const startMsg = `Polling Interval Started${msg !== undefined ? `: ${msg}` : ''}`;
+        this.logger.debug(startMsg)
         this.createInterval();
     }
 
-    end = () => {
+    end = (reason?: string) => {
+        let msg ='Stopping Polling Interval';
+        if(reason !== undefined) {
+            msg += `: ${reason}`;
+        }
+        this.logger.debug(msg);
         this.running = false;
         this.newStart = true;
         super.end();

src/Subreddit/SubredditResources.ts

@@ -12,6 +12,7 @@ import winston, {Logger} from "winston";
 import as from 'async';
 import fetch from 'node-fetch';
 import {
+    asActivity,
     asSubmission,
     buildCacheOptionsFromProvider,
     buildCachePrefix,
@@ -19,18 +20,18 @@ import {
     compareDurationValue,
     comparisonTextOp,
     createCacheManager,
-    createHistoricalStatsDisplay, FAIL,
+    createHistoricalStatsDisplay, escapeRegex, FAIL,
     fetchExternalUrl, filterCriteriaSummary,
     formatNumber,
     getActivityAuthorName,
     getActivitySubredditName,
-    isStrongSubredditState, isSubmission,
+    isStrongSubredditState, isSubmission, isUser,
     mergeArr,
     parseDurationComparison,
     parseExternalUrl,
     parseGenericValueComparison,
-    parseRedditEntity,
-    parseWikiContext, PASS,
+    parseRedditEntity, parseStringToRegex,
+    parseWikiContext, PASS, redisScanIterator,
     shouldCacheSubredditStateCriteriaResult,
     subredditStateIsNameOnly,
     toStrongSubredditState
@@ -69,6 +70,7 @@ import {check} from "tcp-port-used";
 import {ExtendedSnoowrap} from "../Utils/SnoowrapClients";
 import dayjs from "dayjs";
 import ImageData from "../Common/ImageData";
+import globrex from 'globrex';
 
 export const DEFAULT_FOOTER = '\r\n*****\r\nThis action was performed by [a bot.]({{botLink}}) Mention a moderator or [send a modmail]({{modmailLink}}) if you any ideas, questions, or concerns about this action.';
 
@@ -205,24 +207,23 @@ export class SubredditResources {
          const at = await this.cache.wrap(`${this.name}-historical-allTime`, () => createHistoricalDefaults(), {ttl: 0}) as object;
          const rehydratedAt: any = {};
          for(const [k, v] of Object.entries(at)) {
-            if(Array.isArray(v)) {
+             const t = typeof v;
+             if(t === 'number') {
+                 // simple number stat like eventsCheckedTotal
+                 rehydratedAt[k] = v;
+             } else if(Array.isArray(v)) {
+                 // a map stat that we have data for is serialized as an array of KV pairs
                 rehydratedAt[k] = new Map(v);
-            } else {
-                rehydratedAt[k] = v;
-            }
+             } else if(v === null || v === undefined || (t === 'object' && Object.keys(v).length === 0)) {
+                 // a map stat that was not serialized (for some reason) or serialized without any data
+                 rehydratedAt[k] = new Map();
+             } else {
+                 // ???? shouldn't get here
+                 this.logger.warn(`Did not recognize rehydrated historical stat "${k}" of type ${t}`);
+                 rehydratedAt[k] = v;
+             }
          }
          this.stats.historical.allTime = rehydratedAt as HistoricalStats;
-
-        // const lr = await this.cache.wrap(`${this.name}-historical-lastReload`, () => createHistoricalDefaults(), {ttl: 0}) as object;
-        // const rehydratedLr: any = {};
-        // for(const [k, v] of Object.entries(lr)) {
-        //     if(Array.isArray(v)) {
-        //         rehydratedLr[k] = new Map(v);
-        //     } else {
-        //         rehydratedLr[k] = v;
-        //     }
-        // }
-        // this.stats.historical.lastReload = rehydratedLr;
     }
 
     updateHistoricalStats(data: HistoricalStatUpdateData) {
@@ -298,6 +299,88 @@ export class SubredditResources {
         return 0;
     }
 
+    async interactWithCacheByKeyPattern(pattern: string | RegExp, action: 'get' | 'delete') {
+        let patternIsReg = pattern instanceof RegExp;
+        let regPattern: RegExp;
+        let globPattern = pattern;
+
+        const cacheDict: Record<string, any> = {};
+
+        if (typeof pattern === 'string') {
+            const possibleRegPattern = parseStringToRegex(pattern, 'ig');
+            if (possibleRegPattern !== undefined) {
+                regPattern = possibleRegPattern;
+                patternIsReg = true;
+            } else {
+                if (this.prefix !== undefined && !pattern.includes(this.prefix)) {
+                    // need to add wildcard to beginning of pattern so that the regex will still match a key with a prefix
+                    globPattern = `${this.prefix}${pattern}`;
+                }
+                // @ts-ignore
+                const result = globrex(globPattern, {flags: 'i'});
+                regPattern = result.regex;
+            }
+        } else {
+            regPattern = pattern;
+        }
+
+        if (this.cacheType === 'redis') {
+            // @ts-ignore
+            const redisClient = this.cache.store.getClient();
+            if (patternIsReg) {
+                // scan all and test key by regex
+                for await (const key of redisClient.scanIterator()) {
+                    if (regPattern.test(key) && (this.prefix === undefined || key.includes(this.prefix))) {
+                        if (action === 'delete') {
+                            await redisClient.del(key)
+                        } else {
+                            cacheDict[key] = await redisClient.get(key);
+                        }
+                    }
+                }
+            } else {
+                // not a regex means we can use glob pattern (more efficient!)
+                for await (const key of redisScanIterator(redisClient, { MATCH: globPattern })) {
+                    if (action === 'delete') {
+                        await redisClient.del(key)
+                    } else {
+                        cacheDict[key] = await redisClient.get(key);
+                    }
+                }
+            }
+        } else if (this.cache.store.keys !== undefined) {
+            for (const key of await this.cache.store.keys()) {
+                if (regPattern.test(key) && (this.prefix === undefined || key.includes(this.prefix))) {
+                    if (action === 'delete') {
+                        await this.cache.del(key)
+                    } else {
+                        cacheDict[key] = await this.cache.get(key);
+                    }
+                }
+            }
+        }
+        return cacheDict;
+    }
+
+    async deleteCacheByKeyPattern(pattern: string | RegExp) {
+        return await this.interactWithCacheByKeyPattern(pattern, 'delete');
+    }
+
+    async getCacheByKeyPattern(pattern: string | RegExp) {
+        return await this.interactWithCacheByKeyPattern(pattern, 'get');
+    }
+
+    async resetCacheForItem(item: Comment | Submission | RedditUser) {
+        if (asActivity(item)) {
+            if (this.filterCriteriaTTL !== false) {
+                await this.deleteCacheByKeyPattern(`itemCrit-${item.name}*`);
+            }
+            await this.setActivity(item, false);
+        } else if (isUser(item) && this.filterCriteriaTTL !== false) {
+            await this.deleteCacheByKeyPattern(`authorCrit-*-${getActivityAuthorName(item)}*`);
+        }
+    }
+
     async getStats() {
         const totals = Object.values(this.stats.cache).reduce((acc, curr) => ({
             miss: acc.miss + curr.miss,
@@ -379,11 +462,8 @@ export class SubredditResources {
                     this.logger.debug(`Cache Hit: Submission ${item.name}`);
                     return cachedSubmission;
                 }
-                // @ts-ignore
-                const submission = await item.fetch();
                 this.stats.cache.submission.miss++;
-                await this.cache.set(hash, submission, {ttl: this.submissionTTL});
-                return submission;
+                return await this.setActivity(item);
             } else if (this.commentTTL !== false) {
                 hash = `comm-${item.name}`;
                 await this.stats.cache.comment.identifierRequestCount.set(hash, (await this.stats.cache.comment.identifierRequestCount.wrap(hash, () => 0) as number) + 1);
@@ -394,11 +474,8 @@ export class SubredditResources {
                     this.logger.debug(`Cache Hit: Comment ${item.name}`);
                     return cachedComment;
                 }
-                // @ts-ignore
-                const comment = await item.fetch();
                 this.stats.cache.comment.miss++;
-                await this.cache.set(hash, comment, {ttl: this.commentTTL});
-                return comment;
+                return this.setActivity(item);
             } else {
                 // @ts-ignore
                 return await item.fetch();
@@ -409,6 +486,37 @@ export class SubredditResources {
         }
     }
 
+    // @ts-ignore
+    public async setActivity(item: Submission | Comment, tryToFetch = true)
+    {
+        let hash = '';
+        if(this.submissionTTL !== false && isSubmission(item)) {
+            hash = `sub-${item.name}`;
+            if(tryToFetch && item instanceof Submission) {
+                // @ts-ignore
+                const itemToCache = await item.fetch();
+                await this.cache.set(hash, itemToCache, {ttl: this.submissionTTL});
+                return itemToCache;
+            } else {
+                // @ts-ignore
+                await this.cache.set(hash, item, {ttl: this.submissionTTL});
+                return item;
+            }
+        } else if(this.commentTTL !== false){
+            hash = `comm-${item.name}`;
+            if(tryToFetch && item instanceof Comment) {
+                // @ts-ignore
+                const itemToCache = await item.fetch();
+                await this.cache.set(hash, itemToCache, {ttl: this.commentTTL});
+                return itemToCache;
+            } else {
+                // @ts-ignore
+                await this.cache.set(hash, item, {ttl: this.commentTTL});
+                return item;
+            }
+        }
+    }
+
     async hasActivity(item: Submission | Comment) {
         const hash = asSubmission(item) ? `sub-${item.name}` : `comm-${item.name}`;
         const res = await this.cache.get(hash);
@@ -978,6 +1086,20 @@ export class SubredditResources {
                                     return false
                                 }
                                 break;
+                            case 'isRedditMediaDomain':
+                                if((item instanceof Comment)) {
+                                    log.warn('`isRedditMediaDomain` is not allowed in `itemIs` criteria when the main Activity is a Comment');
+                                    continue;
+                                }
+                                // @ts-ignore
+                                const isRedditDomain = crit[k] as boolean;
+                                // @ts-ignore
+                                if (item.is_reddit_media_domain !== isRedditDomain) {
+                                    // @ts-ignore
+                                    log.debug(`Failed: Expected => ${k}:${crit[k]} | Found => ${k}:${item.is_reddit_media_domain}`)
+                                    return false
+                                }
+                                break;
                             case 'approved':
                             case 'spam':
                                 if(!item.can_mod_post) {

src/Subreddit/UserNotes.ts

@@ -14,6 +14,7 @@ import Submission from "snoowrap/dist/objects/Submission";
 import {RichContent} from "../Common/interfaces";
 import {Cache} from 'cache-manager';
 import {isScopeError} from "../Utils/Errors";
+import {ErrorWithCause} from "pony-cause";
 
 interface RawUserNotesPayload {
     ver: number,
@@ -63,6 +64,7 @@ export class UserNotes {
     identifier: string;
     cache: Cache
     cacheCB: Function;
+    mod?: RedditUser;
 
     users: Map<string, UserNote[]> = new Map();
 
@@ -110,14 +112,22 @@ export class UserNotes {
         }
     }
 
+    // @ts-ignore
+    async getMod() {
+        if(this.mod === undefined) {
+            // idgaf
+            // @ts-ignore
+            this.mod = await this.subreddit._r.getMe();
+        }
+        return this.mod as RedditUser;
+    }
+
     async addUserNote(item: (Submission|Comment), type: string | number, text: string = ''): Promise<UserNote>
     {
         const payload = await this.retrieveData();
         const userName = getActivityAuthorName(item.author);
 
-        // idgaf
-        // @ts-ignore
-        const mod = await this.subreddit._r.getMe();
+        const mod = await this.getMod();
         if(!payload.constants.users.includes(mod.name)) {
             this.logger.info(`Mod ${mod.name} does not exist in UserNote constants, adding them`);
             payload.constants.users.push(mod.name);
@@ -134,11 +144,11 @@ export class UserNotes {
         }
         payload.blob[userName].ns.push(newNote.toRaw(payload.constants));
 
+        const existingNotes = await this.getUserNotes(item.author);
         await this.saveData(payload);
         if(this.notesTTL > 0) {
-            const currNotes = this.users.get(userName) || [];
-            currNotes.push(newNote);
-            this.users.set(userName, currNotes);
+            existingNotes.push(newNote);
+            this.users.set(userName, existingNotes);
         }
         return newNote;
     }
@@ -150,7 +160,6 @@ export class UserNotes {
     }
 
     async retrieveData(): Promise<RawUserNotesPayload> {
-        let cacheMiss;
         if (this.notesTTL > 0) {
             const cachedPayload = await this.cache.get(this.identifier);
             if (cachedPayload !== undefined && cachedPayload !== null) {
@@ -158,19 +167,9 @@ export class UserNotes {
                 return cachedPayload as unknown as RawUserNotesPayload;
             }
             this.cacheCB(true);
-            cacheMiss = true;
         }
 
         try {
-            // DISABLED for now because I think its causing issues
-            // if(cacheMiss && this.debounceCB !== undefined) {
-            //     // timeout is still delayed. its our wiki data and we want it now! cm cacheworth 877 cache now
-            //     this.logger.debug(`Detected missed cache on usernotes retrieval while batch (${this.batchCount}) save is in progress, executing save immediately before retrieving new notes...`);
-            //     clearTimeout(this.saveDebounce);
-            //     await this.debounceCB();
-            //     this.debounceCB = undefined;
-            //     this.saveDebounce = undefined;
-            // }
             // @ts-ignore
             const wiki = this.client.getSubreddit(this.subreddit.display_name).getWikiPage('usernotes');
             const wikiContent = await wiki.content_md;
@@ -199,33 +198,6 @@ export class UserNotes {
         try {
             const wiki = this.client.getSubreddit(this.subreddit.display_name).getWikiPage('usernotes');
             if (this.notesTTL !== false) {
-                // DISABLED for now because if it fails throws an uncaught rejection
-                // and need to figured out how to handle this other than just logging (want to interrupt action flow too?)
-                //
-                // debounce usernote save by 5 seconds -- effectively batch usernote saves
-                //
-                // so that if we are processing a ton of checks that write user notes we aren't calling to save the wiki page on every call
-                // since we also have everything in cache (most likely...)
-                //
-                // TODO might want to increase timeout to 10 seconds
-                // if(this.saveDebounce !== undefined) {
-                //     clearTimeout(this.saveDebounce);
-                // }
-                // this.debounceCB = (async function () {
-                //     const p = wikiPayload;
-                //     // @ts-ignore
-                //     const self = this as UserNotes;
-                //     // @ts-ignore
-                //     self.wiki = await self.subreddit.getWikiPage('usernotes').edit(p);
-                //     self.logger.debug(`Batch saved ${self.batchCount} usernotes`);
-                //     self.debounceCB = undefined;
-                //     self.saveDebounce = undefined;
-                //     self.batchCount = 0;
-                // }).bind(this);
-                // this.saveDebounce = setTimeout(this.debounceCB,5000);
-                // this.batchCount++;
-                // this.logger.debug(`Saving Usernotes has been debounced for 5 seconds (${this.batchCount} batched)`)
-
                 // @ts-ignore
                 await wiki.edit(wikiPayload);
                 await this.cache.set(this.identifier, payload, {ttl: this.notesTTL});
@@ -237,15 +209,14 @@ export class UserNotes {
 
             return payload as RawUserNotesPayload;
         } catch (err: any) {
-            let msg = 'Could not edit usernotes.';
+            let msg = 'Could not edit usernotes!';
             // Make sure at least one moderator has used toolbox and usernotes before and that this account has editing permissions`;
             if(isScopeError(err)) {
                 msg = `${msg} The bot account did not have sufficient OAUTH scope to perform this action. You must re-authenticate the bot and ensure it has has 'wikiedit' permissions.`
             } else {
                 msg = `${msg} Make sure at least one moderator has used toolbox, created a usernote, and that this account has editing permissions for the wiki page.`;
             }
-            this.logger.error(msg, err);
-            throw new LoggedError(msg);
+            throw new ErrorWithCause(msg, {cause: err});
         }
     }
 }

src/Utils/Errors.ts

@@ -1,22 +1,39 @@
-import {StatusCodeError, RequestError} from "../Common/interfaces";
+import {RateLimitError, RequestError, StatusCodeError} from 'snoowrap/dist/errors';
+import ExtendableError from "es6-error";
+import {ErrorWithCause} from "pony-cause";
 
 
-export const isRateLimitError = (err: any) => {
-    return typeof err === 'object' && err.name === 'RateLimitError';
+export const isRateLimitError = (err: any): err is RateLimitError => {
+    return isRequestError(err) && err.name === 'RateLimitError';
 }
 
 export const isScopeError = (err: any): boolean => {
-    if(typeof err === 'object' && err.name === 'StatusCodeError' && err.response !== undefined) {
+    if(isStatusError(err)) {
         const authHeader = err.response.headers['www-authenticate'];
         return authHeader !== undefined && authHeader.includes('insufficient_scope');
     }
     return false;
 }
 
+export const getScopeError = (err: any): string | undefined => {
+    if(isScopeError(err)) {
+        return err.response.headers['www-authenticate'];
+    }
+    return undefined;
+}
+
 export const isStatusError = (err: any): err is StatusCodeError => {
-    return typeof err === 'object' && err.name === 'StatusCodeError' && err.response !== undefined;
+    return isRequestError(err) && err.name === 'StatusCodeError';
 }
 
 export const isRequestError = (err: any): err is RequestError => {
-    return typeof err === 'object' && err.name === 'RequestError' && err.response !== undefined;
+    return typeof err === 'object' && err.response !== undefined;
+}
+
+export class SimpleError extends ExtendableError {
+
+}
+
+export class CMError extends ErrorWithCause {
+    logged: boolean = false;
 }

src/Utils/SimpleError.ts

@@ -1,7 +0,0 @@
-import ExtendableError from "es6-error";
-
-class SimpleError extends ExtendableError {
-
-}
-
-export default SimpleError;

src/Utils/SnoowrapUtils.ts

@@ -29,10 +29,9 @@ import {
 import UserNotes from "../Subreddit/UserNotes";
 import {Logger} from "winston";
 import InvalidRegexError from "./InvalidRegexError";
-import SimpleError from "./SimpleError";
 import {AuthorCriteria} from "../Author/Author";
 import {URL} from "url";
-import {isStatusError} from "./Errors";
+import {SimpleError, isStatusError} from "./Errors";
 import {Dictionary, ElementOf, SafeDictionary} from "ts-essentials";
 
 export const BOT_LINK = 'https://www.reddit.com/r/ContextModBot/comments/otz396/introduction_to_contextmodbot';
@@ -504,7 +503,7 @@ export const testAuthorCriteria = async (item: (Comment | Submission), authorOpt
                         break;
                     case 'isMod':
                         const mods: RedditUser[] = await item.subreddit.getModerators();
-                        const isModerator = mods.some(x => x.name === authorName);
+                        const isModerator = mods.some(x => x.name === authorName) || authorName.toLowerCase() === 'automoderator';
                         const modMatch = authorOpts.isMod === isModerator;
                         propResultsMap.isMod!.found = isModerator;
                         propResultsMap.isMod!.passed = !((include && !modMatch) || (!include && modMatch));

src/Utils/loggerFactory.ts

@@ -1,19 +1,28 @@
 import {labelledFormat, logLevels} from "../util";
 import winston, {Logger} from "winston";
 import {DuplexTransport} from "winston-duplex";
+import {LoggerFactoryOptions} from "../Common/interfaces";
+import process from "process";
+import path from "path";
 
 const {transports} = winston;
 
-export const getLogger = (options: any, name = 'app'): Logger => {
+export const getLogger = (options: LoggerFactoryOptions, name = 'app'): Logger => {
     if(!winston.loggers.has(name)) {
         const {
-            path,
             level,
             additionalTransports = [],
             defaultLabel = 'App',
+            file: {
+                dirname,
+                ...fileRest
+            },
+            console,
+            stream
         } = options || {};
 
         const consoleTransport = new transports.Console({
+            ...console,
             handleExceptions: true,
             handleRejections: true,
         });
@@ -28,21 +37,39 @@ export const getLogger = (options: any, name = 'app'): Logger => {
                     objectMode: true,
                 },
                 name: 'duplex',
-                dump: false,
                 handleExceptions: true,
                 handleRejections: true,
+                ...stream,
+                dump: false,
             }),
             ...additionalTransports,
         ];
 
-        if (path !== undefined && path !== '') {
+        if (dirname !== undefined && dirname !== '' && dirname !== null) {
+
+            let realDir: string | undefined;
+            if(typeof dirname === 'boolean') {
+                if(!dirname) {
+                    realDir = undefined;
+                } else {
+                    realDir = path.resolve(__dirname, '../../logs')
+                }
+            } else if(dirname === 'true') {
+                realDir = path.resolve(__dirname, '../../logs')
+            } else if(dirname === 'false') {
+                realDir = undefined;
+            } else {
+                realDir = dirname;
+            }
+
             const rotateTransport = new winston.transports.DailyRotateFile({
-                dirname: path,
                 createSymlink: true,
                 symlinkName: 'contextBot-current.log',
                 filename: 'contextBot-%DATE%.log',
                 datePattern: 'YYYY-MM-DD',
                 maxSize: '5m',
+                dirname: realDir,
+                ...fileRest,
                 handleExceptions: true,
                 handleRejections: true,
             });

src/Web/Client/index.ts

@@ -24,7 +24,6 @@ import EventEmitter from "events";
 import stream, {Readable, Writable, Transform} from "stream";
 import winston from "winston";
 import tcpUsed from "tcp-port-used";
-import SimpleError from "../../Utils/SimpleError";
 import http from "http";
 import jwt from 'jsonwebtoken';
 import {Server as SocketServer} from "socket.io";
@@ -49,6 +48,8 @@ import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients";
 import ClientUser from "../Common/User/ClientUser";
 import {BotStatusResponse} from "../Common/interfaces";
 import {TransformableInfo} from "logform";
+import {SimpleError} from "../../Utils/Errors";
+import {ErrorWithCause} from "pony-cause";
 
 const emitter = new EventEmitter();
 
@@ -630,9 +631,7 @@ const webClient = async (options: OperatorConfig) => {
         server = await app.listen(port);
         io = new SocketServer(server);
     } catch (err: any) {
-        logger.error('Error occurred while initializing web or socket.io server', err);
-        err.logged = true;
-        throw err;
+        throw new ErrorWithCause('[Web] Error occurred while initializing web or socket.io server', {cause: err});
     }
     logger.info(`Web UI started: http://localhost:${port}`, {label: ['Web']});
 
@@ -745,6 +744,7 @@ const webClient = async (options: OperatorConfig) => {
             });
 
             if(accessibleInstance === undefined) {
+                logger.warn(`User ${user.name} is not an operator and has no subreddits in common with any *running* bot instances. If you are sure they should have common subreddits then this client may not be able to access all defined CM servers or the bot may be offline.`);
                 return res.render('noAccess');
             }
 

src/Web/Server/routes/authenticated/user/status.ts

@@ -93,6 +93,7 @@ const status = () => {
                 eventsState: m.eventsState,
                 queueState: m.queueState,
                 indicator: 'gray',
+                permissions: await m.getModPermissions(),
                 queuedActivities: m.queue.length(),
                 runningActivities: m.queue.running(),
                 maxWorkers: m.queue.concurrency,
@@ -232,12 +233,14 @@ const status = () => {
         const cacheMiss = subManagerData.reduce((acc, curr) => acc + curr.stats.cache.totalMiss, 0);
         const sharedSub = subManagerData.find(x => x.stats.cache.isShared);
         const sharedCount = sharedSub !== undefined ? sharedSub.stats.cache.currentKeyCount : 0;
+        const scopes = req.user?.isInstanceOperator(bot) ? bot.client.scope : [];
         let allManagerData: any = {
             name: 'All',
             status: bot.running ? 'RUNNING' : 'NOT RUNNING',
             indicator: bot.running ? 'green' : 'grey',
             maxWorkers,
             globalMaxWorkers,
+            scopes: scopes === null || !Array.isArray(scopes) ? [] : scopes,
             subMaxWorkers,
             runningActivities,
             queuedActivities,

src/Web/Server/server.ts

@@ -18,7 +18,6 @@ import {getLogger} from "../../Utils/loggerFactory";
 import LoggedError from "../../Utils/LoggedError";
 import {Invokee, LogInfo, OperatorConfigWithFileContext} from "../../Common/interfaces";
 import http from "http";
-import SimpleError from "../../Utils/SimpleError";
 import {heartbeat} from "./routes/authenticated/applicationRoutes";
 import logs from "./routes/authenticated/user/logs";
 import status from './routes/authenticated/user/status';
@@ -30,6 +29,8 @@ import Bot from "../../Bot";
 import addBot from "./routes/authenticated/user/addBot";
 import dayjs from "dayjs";
 import ServerUser from "../Common/User/ServerUser";
+import {SimpleError} from "../../Utils/Errors";
+import {ErrorWithCause} from "pony-cause";
 
 const server = addAsync(express());
 server.use(bodyParser.json());
@@ -116,9 +117,7 @@ const rcbServer = async function (options: OperatorConfigWithFileContext) {
         httpServer = await server.listen(port);
         io = new SocketServer(httpServer);
     } catch (err: any) {
-        logger.error('Error occurred while initializing web or socket.io server', err);
-        err.logged = true;
-        throw err;
+        throw new ErrorWithCause('[Server] Error occurred while initializing web or socket.io server', {cause: err});
     }
 
     logger.info(`API started => localhost:${port}`);

src/Web/assets/public/app.css

@@ -152,3 +152,7 @@ a {
 #saveTip .tooltip:hover {
     transition-delay: 1s;
 }
+
+#redditStatus .iconify-inline {
+    display: inline;
+}

src/Web/assets/views/partials/footer.ejs

@@ -3,3 +3,28 @@
         <a href="https://github.com/FoxxMD/context-mod">ContextMod Web</a> created by /u/FoxxMD
     </div>
 </div>
+<script type="text/javascript" src="https://cdn.statuspage.io/se-v2.js"></script>
+<script>
+    // https://www.redditstatus.com/api#status
+    var sp = new StatusPage.page({ page : '2kbc0d48tv3j' });
+    sp.status({
+        success : function(data) {
+            debugger;
+            console.log(data.status.indicator);
+            switch(data.status.indicator){
+                case 'minor':
+                    document.querySelector('#redditStatus').innerHTML = '<span class="iconify-inline yellow" data-icon="ep:warning-filled"></span>';
+                    break;
+                case 'none':
+                    document.querySelector('#redditStatus').innerHTML = '<span class="iconify-inline green" data-icon="ep:circle-check-filled"></span>';
+                    break;
+                default:
+                    document.querySelector('#redditStatus').innerHTML = '<span class="iconify-inline red" data-icon="ep:warning-filled"></span>';
+                    break;
+            }
+            // data.page.updated_at
+            // data.status.indicator => none, minor, major, or critical
+            // data.status.description
+        }
+    });
+</script>

src/Web/assets/views/partials/head.ejs

@@ -1,6 +1,6 @@
 <head>
     <link rel="stylesheet" href="/public/tailwind.min.css"/>
-    <script src="https://code.iconify.design/2/2.1.0/iconify.min.js"></script>
+    <script src="https://code.iconify.design/2/2.1.2/iconify.min.js"></script>
     <link rel="stylesheet" href="/public/themeToggle.css">
     <link rel="stylesheet" href="/public/app.css">
     <title><%= locals.title !== undefined ? title : `${locals.botName !== undefined ? `CM for ${botName}` : 'ContextMod'}`%></title>

src/Web/assets/views/partials/header.ejs

@@ -33,7 +33,12 @@
                 </ul>
                 <% } %>
             </div>
-            <div class="flex items-center flex-end text-sm">
+            <div class="flex items-center mr-8 text-sm">
+                <a href="https://redditstatus.com" target="_blank">
+                    <span>Reddit Status: <span id="redditStatus" class="ml-2"><span class="iconify-inline" data-icon="ep:question-filled"></span></span></span>
+                </a>
+            </div>
+            <div class="flex items-center text-sm">
                 <a href="logout">Logout</a>
             </div>
         </div>

src/Web/assets/views/status.ejs

@@ -160,6 +160,19 @@
                                                         </div>
                                                     </div>
                                                 </div>
+                                                <label>Mod Perms</label>
+                                                <span class="has-tooltip">
+                                                    <span class='tooltip rounded shadow-lg p-1 bg-gray-100 text-black -mt-2'>
+                                                        <span>
+                                                            <ul class="list-inside list-disc">
+                                                                <% data.permissions.forEach(function (i){ %>
+                                                                    <li class="font-mono"><%= i %></li>
+                                                                <% }) %>
+                                                            </ul>
+                                                        </span>
+                                                    </span>
+                                                    <span class="cursor-help underline" style="text-decoration-style: dotted"><%= data.permissions.length %></span>
+                                                </span>
                                                 <label>Slow Mode</label>
                                                 <span><%= data.delayBy %></span>
                                             <% } %>
@@ -223,6 +236,19 @@
                                             <% if (data.name === 'All' && isOperator) { %>
                                                 <label>Operators</label>
                                                 <span><%= operators %></span>
+                                                <label>Oauth Scopes</label>
+                                                    <span class="has-tooltip">
+                                                    <span class='tooltip rounded shadow-lg p-1 bg-gray-100 text-black -mt-2'>
+                                                        <span>
+                                                            <ul class="list-inside list-disc">
+                                                                <% data.scopes.forEach(function (i){ %>
+                                                                    <li class="font-mono"><%= i %></li>
+                                                                <% }) %>
+                                                            </ul>
+                                                        </span>
+                                                    </span>
+                                                    <span class="cursor-help underline" style="text-decoration-style: dotted"><%= data.scopes.length %></span>
+                                                </span>
                                             <% } else %>
                                         </div>
                                         <% if (data.name !== 'All') { %>

src/index.ts

@@ -205,9 +205,6 @@ const program = new Command();
     } catch (err: any) {
         if (!err.logged && !(err instanceof LoggedError)) {
             const logger = winston.loggers.has('app') ? winston.loggers.get('app') : winston.loggers.get('init');
-            if(isScopeError(err)) {
-                logger.error('Reddit responded with a 403 insufficient_scope which means the bot is lacking necessary OAUTH scopes to perform general actions.');
-            }
             logger.error(err);
         }
         process.kill(process.pid, 'SIGTERM');

src/util.ts

@@ -6,8 +6,6 @@ import deepEqual from "fast-deep-equal";
 import {Duration} from 'dayjs/plugin/duration.js';
 import Ajv from "ajv";
 import {InvalidOptionArgumentError} from "commander";
-import Submission from "snoowrap/dist/objects/Submission";
-import {Comment} from "snoowrap";
 import {inflateSync, deflateSync} from "zlib";
 import pixelmatch from 'pixelmatch';
 import os from 'os';
@@ -29,13 +27,12 @@ import {
     RedditEntityType,
     RegExResult, RepostItem, RepostItemResult,
     ResourceStats, SearchAndReplaceRegExp,
-    StatusCodeError, StringComparisonOptions,
+    StringComparisonOptions,
     StringOperator,
     StrongSubredditState,
     SubredditState
 } from "./Common/interfaces";
 import { Document as YamlDocument } from 'yaml'
-import SimpleError from "./Utils/SimpleError";
 import InvalidRegexError from "./Utils/InvalidRegexError";
 import {constants, promises} from "fs";
 import {cacheOptDefaults} from "./Common/defaults";
@@ -44,24 +41,24 @@ import redisStore from "cache-manager-redis-store";
 import crypto from "crypto";
 import Autolinker from 'autolinker';
 import {create as createMemoryStore} from './Utils/memoryStore';
-import {MESSAGE} from "triple-beam";
-import {RedditUser} from "snoowrap/dist/objects";
+import {MESSAGE, LEVEL} from "triple-beam";
+import {RedditUser,Comment,Submission} from "snoowrap/dist/objects";
 import reRegExp from '@stdlib/regexp-regexp';
 import fetch, {Response} from "node-fetch";
 import { URL } from "url";
 import ImageData from "./Common/ImageData";
 import {Sharp, SharpOptions} from "sharp";
-// @ts-ignore
-import {blockhashData, hammingDistance} from 'blockhash';
+import {ErrorWithCause, stackWithCauses} from "pony-cause";
 import {ConfigFormat, SetRandomInterval} from "./Common/types";
 import stringSimilarity from 'string-similarity';
 import calculateCosineSimilarity from "./Utils/StringMatching/CosineSimilarity";
 import levenSimilarity from "./Utils/StringMatching/levenSimilarity";
-import {isRequestError, isStatusError} from "./Utils/Errors";
+import {SimpleError, isRateLimitError, isRequestError, isScopeError, isStatusError, CMError} from "./Utils/Errors";
 import {parse} from "path";
 import JsonConfigDocument from "./Common/Config/JsonConfigDocument";
 import YamlConfigDocument from "./Common/Config/YamlConfigDocument";
 import AbstractConfigDocument, {ConfigDocumentInterface} from "./Common/Config/AbstractConfigDocument";
+import LoggedError from "./Utils/LoggedError";
 
 
 //import {ResembleSingleCallbackComparisonResult} from "resemblejs";
@@ -90,36 +87,78 @@ const CWD = process.cwd();
 //     }
 // }
 const errorAwareFormat = {
-    transform: (info: any, opts: any) => {
-        // don't need to log stack trace if we know the error is just a simple message (we handled it)
-        const stack = !(info instanceof SimpleError) && !(info.message instanceof SimpleError);
-        const {name, response, message, stack: errStack, error, statusCode} = info;
-        if(name === 'StatusCodeError' && response !== undefined && response.headers !== undefined && response.headers['content-type'].includes('html')) {
-            // reddit returns html even when we specify raw_json in the querystring (via snoowrap)
-            // which means the html gets set as the message for the error AND gets added to the stack as the message
-            // and we end up with a h u g e log statement full of noisy html >:(
+    transform: (einfo: any, {stack = true}: any = {}) => {
 
-            const errorSample = error.slice(0, 10);
-            const messageBeforeIndex = message.indexOf(errorSample);
-            let newMessage = `Status Error ${statusCode} from Reddit`;
-            if(messageBeforeIndex > 0) {
-                newMessage = `${message.slice(0, messageBeforeIndex)} - ${newMessage}`;
-            }
-            let cleanStack = errStack;
+        // because winston logger.child() re-assigns its input to an object ALWAYS the object we recieve here will never actually be of type Error
+        const includeStack = stack && (!isProbablyError(einfo, 'simpleerror') && !isProbablyError(einfo.message, 'simpleerror'));
 
-            // try to get just stacktrace by finding beginning of what we assume is the actual trace
-            if(errStack) {
-                cleanStack = `${newMessage}\n${errStack.slice(errStack.indexOf('at new StatusCodeError'))}`;
-            }
-            // now put it all together so its nice and clean
-            info.message = newMessage;
-            info.stack = cleanStack;
+        if (!isProbablyError(einfo.message) && !isProbablyError(einfo)) {
+            return einfo;
         }
-        return errors().transform(info, { stack });
+
+        let info: any = {};
+
+        if (isProbablyError(einfo)) {
+            const tinfo = transformError(einfo);
+            info = Object.assign({}, tinfo, {
+                // @ts-ignore
+                level: einfo.level,
+                // @ts-ignore
+                [LEVEL]: einfo[LEVEL] || einfo.level,
+                message: tinfo.message,
+                // @ts-ignore
+                [MESSAGE]: tinfo[MESSAGE] || tinfo.message
+            });
+            if(includeStack) {
+                // so we have to create a dummy error and re-assign all error properties from our info object to it so we can get a proper stack trace
+                const dummyErr = new ErrorWithCause('');
+                for(const k in tinfo) {
+                    if(dummyErr.hasOwnProperty(k) || k === 'cause') {
+                        // @ts-ignore
+                        dummyErr[k] = tinfo[k];
+                    }
+                }
+                // @ts-ignore
+                info.stack = stackWithCauses(dummyErr);
+            }
+        } else {
+            const err = transformError(einfo.message);
+            info = Object.assign(einfo, err);
+            // @ts-ignore
+            info.message = err.message;
+            // @ts-ignore
+            info[MESSAGE] = err.message;
+
+            if(includeStack) {
+                const dummyErr = new ErrorWithCause('');
+                for(const k in err) {
+                    if(dummyErr.hasOwnProperty(k) || k === 'cause') {
+                        // @ts-ignore
+                        dummyErr[k] = info[k];
+                    }
+                }
+                // @ts-ignore
+                info.stack = stackWithCauses(dummyErr);
+            }
+        }
+
+        // remove redundant message from stack and make stack causes easier to read
+        if(info.stack !== undefined) {
+            let cleanedStack = info.stack.replace(info.message, '');
+            cleanedStack = `${cleanedStack}`;
+            cleanedStack = cleanedStack.replaceAll('caused by:', '\ncaused by:');
+            info.stack = cleanedStack;
+        }
+
+        return info;
     }
 }
 
-export const PASS = '✔';
+const isProbablyError = (val: any, errName = 'error') => {
+    return typeof val === 'object' && val.name !== undefined && val.name.toLowerCase().includes(errName);
+}
+
+export const PASS = '✓';
 export const FAIL = '✘';
 
 export const truncateStringToLength = (length: number, truncStr = '...') => (str: string) => str.length > length ? `${str.slice(0, length - truncStr.length - 1)}${truncStr}` : str;
@@ -900,6 +939,11 @@ export const createRetryHandler = (opts: RetryOptions, logger: Logger) => {
 
         lastErrorAt = dayjs();
 
+        if(isRateLimitError(err)) {
+            logger.error('Will not retry because error was due to ratelimit exhaustion');
+            return false;
+        }
+
         const redditApiError = isRequestError(err) || isStatusError(err);
 
         if(redditApiError) {
@@ -937,6 +981,119 @@ export const createRetryHandler = (opts: RetryOptions, logger: Logger) => {
     }
 }
 
+type StringReturn = (err:any) => string;
+
+export interface LogMatch {
+    [key: string | number]: string | StringReturn
+}
+
+export interface logExceptionOptions {
+    context?: string
+    logIfNotMatched?: boolean
+    logStackTrace?: boolean
+    match?: LogMatch
+}
+
+export const parseMatchMessage = (err: any, match: LogMatch, matchTypes: (string | number)[], defaultMatch: string): [string, boolean] => {
+    for(const m of matchTypes) {
+        if(match[m] !== undefined) {
+            if(typeof match[m] === 'string') {
+                return [match[m] as string, true];
+            }
+            return [(match[m] as Function)(err), true];
+        }
+    }
+    return [defaultMatch, false];
+}
+
+export const getExceptionMessage = (err: any, match: LogMatch = {}): string | undefined => {
+
+    let matched = false,
+        matchMsg;
+
+    if (isRequestError(err)) {
+        if (isRateLimitError(err)) {
+            ([matchMsg, matched] = parseMatchMessage(err, match, ['ratelimit', err.statusCode], 'Ratelimit Exhausted'));
+        } else if (isScopeError(err)) {
+            ([matchMsg, matched] = parseMatchMessage(err, match, ['scope', err.statusCode], 'Missing OAUTH scope required for this request'));
+        } else {
+            ([matchMsg, matched] = parseMatchMessage(err, match, [err.statusCode], err.message));
+        }
+    } else {
+        ([matchMsg, matched] = parseMatchMessage(err, match, ['any'], err.message));
+    }
+
+    if (matched) {
+        return matchMsg;
+    }
+}
+
+const _transformError = (err: Error, seen: Set<Error>, matchOptions?: LogMatch) => {
+    if (!err || !isProbablyError(err)) {
+        return '';
+    }
+    if (seen.has(err)) {
+        return err;
+    }
+
+    // @ts-ignore
+    let mOpts = err.matchOptions ?? matchOptions;
+
+    if (isRequestError(err)) {
+        const errMsgParts = [`Reddit responded with a NOT OK status (${err.statusCode})`];
+
+        if (err.response.headers['content-type'].includes('html')) {
+            // reddit returns html even when we specify raw_json in the querystring (via snoowrap)
+            // which means the html gets set as the message for the error AND gets added to the stack as the message
+            // and we end up with a h u g e log statement full of noisy html >:(
+
+            const {error, statusCode, message, stack: errStack} = err;
+
+            const errorSample = (error as unknown as string).slice(0, 10);
+            const messageBeforeIndex = message.indexOf(errorSample);
+            let newMessage = `Status Error ${statusCode} from Reddit`;
+            if (messageBeforeIndex > 0) {
+                newMessage = `${message.slice(0, messageBeforeIndex)} - ${newMessage}`;
+            }
+            let cleanStack = errStack;
+
+            // try to get just stacktrace by finding beginning of what we assume is the actual trace
+            if (errStack) {
+                cleanStack = `${newMessage}\n${errStack.slice(errStack.indexOf('at new StatusCodeError'))}`;
+            }
+            // now put it all together so its nice and clean
+            err.message = newMessage;
+            err.stack = cleanStack;
+        }
+
+        const msg = getExceptionMessage(err, mOpts);
+        if (msg !== undefined) {
+            errMsgParts.push(msg);
+        }
+
+        // we don't care about stack trace for this error because we know where it came from so truncate to two lines for now...maybe remove all together later
+        if(err.stack !== undefined) {
+            err.stack = err.stack.split('\n').slice(0, 2).join('\n');
+        }
+
+        const normalizedError = new ErrorWithCause(errMsgParts.join(' => '), {cause: err});
+        normalizedError.stack = normalizedError.message;
+        return normalizedError;
+    }
+
+    // @ts-ignore
+    const cause = err.cause as unknown;
+
+    if (cause !== undefined && cause instanceof Error) {
+        // @ts-ignore
+        err.cause = _transformError(cause, seen, mOpts);
+    }
+
+    return err;
+}
+
+export const transformError = (err: Error): any => _transformError(err, new Set());
+
 const LABELS_REGEX: RegExp = /(\[.+?])*/g;
 export const parseLabels = (log: string): string[] => {
     return Array.from(log.matchAll(LABELS_REGEX), m => m[0]).map(x => x.substring(1, x.length - 1));
@@ -1000,12 +1157,14 @@ export const formatLogLineToHtml = (log: string | LogInfo, timestamp?: string) =
         .replace(/(\s*verbose\s*):/i, '<span class="error purple">$1</span>:')
         .replaceAll('\n', '<br />');
         //.replace(HYPERLINK_REGEX, '<a target="_blank" href="$&">$&</a>');
-    let line = `<div class="logLine">${logContent}</div>`
+    let line = '';
 
     if(timestamp !== undefined) {
-        line = line.replace(timestamp, (match) => {
-            return formattedTime(dayjs(match).format('HH:mm:ss z'), match);
-        });
+        const timeStampReplacement = formattedTime(dayjs(timestamp).format('HH:mm:ss z'), timestamp);
+        const splitLine = logContent.split(timestamp);
+        line = `<div class="logLine">${splitLine[0]}${timeStampReplacement}<span style="white-space: pre-wrap">${splitLine[1]}</span></div>`;
+    } else {
+        line = `<div style="white-space: pre-wrap" class="logLine">${logContent}</div>`
     }
     return line;
 }
@@ -1416,13 +1575,33 @@ export const snooLogWrapper = (logger: Logger) => {
  * Cached activities lose type information when deserialized so need to check properties as well to see if the object is the shape of a Submission
  * */
 export const isSubmission = (value: any) => {
-    return value instanceof Submission || value.domain !== undefined;
+    return value instanceof Submission || value.name.includes('t3_');
 }
 
 export const asSubmission = (value: any): value is Submission => {
     return isSubmission(value);
 }
 
+export const isComment = (value: any) => {
+    return value instanceof Comment || value.name.includes('t1_');
+}
+
+export const asComment = (value: any): value is Comment => {
+    return isComment(value);
+}
+
+export const asActivity = (value: any): value is (Submission | Comment) => {
+    return asComment(value) || asSubmission(value);
+}
+
+export const isUser = (value: any) => {
+    return value instanceof RedditUser || value.name.includes('t2_');
+}
+
+export const asUser = (value: any): value is RedditUser => {
+    return isUser(value);
+}
+
 export const isUserNoteCriteria = (value: any) => {
     return value !== null && typeof value === 'object' && value.type !== undefined;
 }
@@ -1859,3 +2038,38 @@ export const likelyJson5 = (str: string): boolean => {
     }
     return validStart;
 }
+
+const defaultScanOptions = {
+    COUNT: '100',
+    MATCH: '*'
+}
+/**
+ * Frankenstein redis scan generator
+ *
+ * Cannot use the built-in scan iterator because it is only available in > v4 of redis client but node-cache-manager-redis is using v3.x --
+ * So combining the async iterator defined in v4 from here https://github.com/redis/node-redis/blob/master/packages/client/lib/client/index.ts#L587
+ * with the scan example from v3 https://github.com/redis/node-redis/blob/8a43dea9bee11e41d33502850f6989943163020a/examples/scan.js
+ *
+ * */
+export async function* redisScanIterator(client: any, options: any = {}): AsyncIterable<string> {
+    let cursor: string = '0';
+    const scanOpts = {...defaultScanOptions, ...options};
+    do {
+        const iterScan = new Promise((resolve, reject) => {
+            client.scan(cursor, 'MATCH', scanOpts.MATCH, 'COUNT', scanOpts.COUNT, (err: any, res: any) => {
+                if(err) {
+                    return reject(err);
+                } else {
+                    const newCursor = res[0];
+                    let keys = res[1];
+                    resolve([newCursor, keys]);
+                }
+            });
+        }) as Promise<[any, string[]]>;
+        const [newCursor, keys] = await iterScan;
+        cursor = newCursor;
+        for (const key of keys) {
+            yield key;
+        }
+    } while (cursor !== '0');
+}

tsconfig.json

@@ -5,7 +5,8 @@
     "resolveJsonModule": true,
     "typeRoots": [
       "./node_modules/@types",
-      "./src/Web/types"
+      "./src/Web/types",
+      "./src/Common/typings"
     ]
   },
 //  "compilerOptions": {