Merge branch 'edge'

* Add excludeCondition to control how exclude sets are tested (and/or)
* Refactor authorIs logic from check/rule/action into standalone function (DRY)
* Simplify filter defaults -- don't need to specify automoderator since it is always a mod

docs/examples/README.md

@@ -18,6 +18,7 @@ This directory contains example of valid, ready-to-go configurations for Context
   * [Author](/docs/examples/author)
   * [Regex](/docs/examples/regex)
   * [Repost](/docs/examples/repost)
+  * [Author and post flairs](/docs/examples/onlyfansFlair)
 * [Toolbox User Notes](/docs/examples/userNotes)
 * [Advanced Concepts](/docs/examples/advancedConcepts)
   * [Rule Sets](/docs/examples/advancedConcepts/ruleSets.json5)

docs/examples/onlyfansFlair/README.md

@@ -0,0 +1,9 @@
+# Flair users and submissions
+
+Flair users and submissions based on certain keywords from submitter's profile.
+
+Consult [User Flair schema](https://json-schema.app/view/%23%2Fdefinitions%2FUserFlairActionJson?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Fcontext-mod%2Fmaster%2Fsrc%2FSchema%2FApp.json) and [Submission Flair schema](https://json-schema.app/view/%23%2Fdefinitions%2FFlairActionJson?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Fcontext-mod%2Fmaster%2Fsrc%2FSchema%2FApp.json) for a complete reference of the rule's properties.
+
+### Examples
+
+* OnlyFans submissions [YAML](/docs/examples/onlyFansFlair/onlyFansFlair.yaml) | [JSON](/docs/examples/onlyfansFlair/onlyfansFlair.json5) - Check whether submitter has typical OF keywords in their profile and flair both author + submission accordingly.

docs/examples/onlyfansFlair/onlyfansFlair.json5

@@ -0,0 +1,68 @@
+{
+  "checks": [
+    {
+      "name": "Flair OF submitters",
+      "description": "Flair submission as OF if user does not have Verified flair and has certain keywords in their profile",
+      "kind": "submission",
+      "authorIs": {
+        "exclude": [
+          {
+            "flairCssClass": ["verified"]
+          }
+        ]
+      },
+      "rules": [
+        {
+          "name": "OnlyFans strings in description",
+          "kind": "author",
+          "include": [
+            {
+              "description": [
+                "/(cashapp|allmylinks|linktr|onlyfans\\.com)/i",
+                "/(see|check|my|view) (out|of|onlyfans|kik|skype|insta|ig|profile|links)/i",
+                "my links",
+                "$"
+              ]
+            }
+          ]
+        }
+      ],
+      "actions": [
+        {
+          "name": "Set OnlyFans user flair",
+          "kind": "userflair",
+          "flair_template_id": "put-your-onlyfans-user-flair-id-here"
+        },
+        {
+          "name":"Set OF Creator SUBMISSION flair",
+          "kind": "flair",
+          "flair_template_id": "put-your-onlyfans-post-flair-id-here"
+        }
+      ]
+    },
+
+    {
+      "name": "Flair posts of OF submitters",
+      "description": "Flair submission as OnlyFans if submitter has OnlyFans userflair (override post flair set by submitter)",
+      "kind": "submission",
+      "rules": [
+        {
+          "name": "Include OF submitters",
+          "kind": "author",
+          "include": [
+            {
+              "flairCssClass": ["onlyfans"]
+            }
+          ]
+        }
+      ],
+      "actions": [
+        {
+          "name":"Set OF Creator SUBMISSION flair",
+          "kind": "flair",
+          "flair_template_id": "put-your-onlyfans-post-flair-id-here"
+        }
+      ]
+    }
+  ]
+}

docs/examples/onlyfansFlair/onlyfansFlair.yaml

@@ -0,0 +1,38 @@
+checks:
+  - name: Flair OF submitters
+    description: Flair submission as OF if user does not have Verified flair and has
+      certain keywords in their profile
+    kind: submission
+    authorIs:
+      exclude:
+        - flairCssClass:
+            - verified
+    rules:
+      - name: OnlyFans strings in description
+        kind: author
+        include:
+          - description:
+              - '/(cashapp|allmylinks|linktr|onlyfans\.com)/i'
+              - '/(see|check|my|view) (out|of|onlyfans|kik|skype|insta|ig|profile|links)/i'
+              - my links
+              - "$"
+    actions:
+      - name: Set OnlyFans user flair
+        kind: userflair
+        flair_template_id: put-your-onlyfans-user-flair-id-here
+      - name: Set OF Creator SUBMISSION flair
+        kind: flair
+        flair_template_id: put-your-onlyfans-post-flair-id-here
+  - name: Flair posts of OF submitters
+    description: Flair submission as OnlyFans if submitter has OnlyFans userflair (override post flair set by submitter)
+    kind: submission
+    rules:
+      - name: Include OF submitters
+        kind: author
+        include:
+          - flairCssClass:
+              - onlyfans
+    actions:
+      - name: Set OF Creator SUBMISSION flair
+        kind: flair
+        flair_template_id: put-your-onlyfans-post-flair-id-here

src/Action/ActionFactory.ts

@@ -10,10 +10,11 @@ import ApproveAction, {ApproveActionConfig} from "./ApproveAction";
 import BanAction, {BanActionJson} from "./BanAction";
 import {MessageAction, MessageActionJson} from "./MessageAction";
 import {SubredditResources} from "../Subreddit/SubredditResources";
-import Snoowrap from "snoowrap";
+import {UserFlairAction, UserFlairActionJson} from './UserFlairAction';
+import {ExtendedSnoowrap} from '../Utils/SnoowrapClients';
 
 export function actionFactory
-(config: ActionJson, logger: Logger, subredditName: string, resources: SubredditResources, client: Snoowrap): Action {
+(config: ActionJson, logger: Logger, subredditName: string, resources: SubredditResources, client: ExtendedSnoowrap): Action {
     switch (config.kind) {
         case 'comment':
             return new CommentAction({...config as CommentActionJson, logger, subredditName, resources, client});
@@ -25,6 +26,8 @@ export function actionFactory
             return new ReportAction({...config as ReportActionJson, logger, subredditName, resources, client});
         case 'flair':
             return new FlairAction({...config as FlairActionJson, logger, subredditName, resources, client});
+        case 'userflair':
+            return new UserFlairAction({...config as UserFlairActionJson, logger, subredditName, resources, client});
         case 'approve':
             return new ApproveAction({...config as ApproveActionConfig, logger, subredditName, resources, client});
         case 'usernote':

src/Action/ApproveAction.ts

@@ -11,6 +11,7 @@ export class ApproveAction extends Action {
 
     async process(item: Comment | Submission, ruleResults: RuleResult[], runtimeDryrun?: boolean): Promise<ActionProcessResult> {
         const dryRun = runtimeDryrun || this.dryRun;
+        const touchedEntities = [];
         //snoowrap typing issue, thinks comments can't be locked
         // @ts-ignore
         if (item.approved) {
@@ -23,11 +24,12 @@ export class ApproveAction extends Action {
         }
         if (!dryRun) {
             // @ts-ignore
-            await item.approve();
+            touchedEntities.push(await item.approve());
         }
         return {
             dryRun,
             success: true,
+            touchedEntities
         }
     }
 }

src/Action/BanAction.ts

@@ -39,6 +39,7 @@ export class BanAction extends Action {
         const renderedBody = content === undefined ? undefined : await renderContent(content, item, ruleResults, this.resources.userNotes);
         const renderedContent = renderedBody === undefined ? undefined : `${renderedBody}${await this.resources.generateFooter(item, this.footer)}`;
 
+        const touchedEntities = [];
         let banPieces = [];
         banPieces.push(`Message: ${renderedContent === undefined ? 'None' : `${renderedContent.length > 100 ? `\r\n${renderedContent}` : renderedContent}`}`);
         banPieces.push(`Reason:  ${this.reason || 'None'}`);
@@ -50,18 +51,20 @@ export class BanAction extends Action {
             // @ts-ignore
             const fetchedSub = await item.subreddit.fetch();
             const fetchedName = await item.author.name;
-            await fetchedSub.banUser({
+            const bannedUser = await fetchedSub.banUser({
                 name: fetchedName,
                 banMessage: renderedContent === undefined ? undefined : renderedContent,
                 banReason: this.reason,
                 banNote: this.note,
                 duration: this.duration
             });
+            touchedEntities.push(bannedUser);
         }
         return {
             dryRun,
             success: true,
-            result: `Banned ${item.author.name} ${durText}${this.reason !== undefined ? ` (${this.reason})` : ''}`
+            result: `Banned ${item.author.name} ${durText}${this.reason !== undefined ? ` (${this.reason})` : ''}`,
+            touchedEntities
         };
     }
 }

src/Action/CommentAction.ts

@@ -51,16 +51,19 @@ export class CommentAction extends Action {
                 result: 'Cannot comment because Item is archived'
             };
         }
+        const touchedEntities = [];
         let reply: Comment;
         if(!dryRun) {
             // @ts-ignore
            reply = await item.reply(renderedContent);
+           touchedEntities.push(reply);
         }
         if (this.lock) {
             if (!dryRun) {
                 // snoopwrap typing issue, thinks comments can't be locked
                 // @ts-ignore
                 await item.lock();
+                touchedEntities.push(item);
             }
         }
         if (this.distinguish && !dryRun) {
@@ -78,7 +81,8 @@ export class CommentAction extends Action {
         return {
             dryRun,
             success: true,
-            result: `${modifierStr}${this.lock ? ' - Locked Author\'s Activity - ' : ''}${truncateStringToLength(100)(body)}`
+            result: `${modifierStr}${this.lock ? ' - Locked Author\'s Activity - ' : ''}${truncateStringToLength(100)(body)}`,
+            touchedEntities,
         };
     }
 }

src/Action/LockAction.ts

@@ -11,6 +11,7 @@ export class LockAction extends Action {
 
     async process(item: Comment | Submission, ruleResults: RuleResult[], runtimeDryrun?: boolean): Promise<ActionProcessResult> {
         const dryRun = runtimeDryrun || this.dryRun;
+        const touchedEntities = [];
         //snoowrap typing issue, thinks comments can't be locked
         // @ts-ignore
         if (item.locked) {
@@ -25,10 +26,12 @@ export class LockAction extends Action {
             //snoowrap typing issue, thinks comments can't be locked
             // @ts-ignore
             await item.lock();
+            touchedEntities.push(item);
         }
         return {
             dryRun,
-            success: true
+            success: true,
+            touchedEntities
         }
     }
 }

src/Action/RemoveAction.ts

@@ -12,6 +12,7 @@ export class RemoveAction extends Action {
 
     async process(item: Comment | Submission, ruleResults: RuleResult[], runtimeDryrun?: boolean): Promise<ActionProcessResult> {
         const dryRun = runtimeDryrun || this.dryRun;
+        const touchedEntities = [];
         // issue with snoowrap typings, doesn't think prop exists on Submission
         // @ts-ignore
         if (activityIsRemoved(item)) {
@@ -24,11 +25,13 @@ export class RemoveAction extends Action {
         if (!dryRun) {
             // @ts-ignore
             await item.remove();
+            touchedEntities.push(item);
         }
 
         return {
             dryRun,
             success: true,
+            touchedEntities
         }
     }
 }

src/Action/ReportAction.ts

@@ -29,15 +29,20 @@ export class ReportAction extends Action {
         const renderedContent = await renderContent(content, item, ruleResults, this.resources.userNotes);
         this.logger.verbose(`Contents:\r\n${renderedContent}`);
         const truncatedContent = reportTrunc(renderedContent);
+        const touchedEntities = [];
         if(!dryRun) {
             // @ts-ignore
             await item.report({reason: truncatedContent});
+            // due to reddit not updating this in response (maybe)?? just increment stale activity
+            item.num_reports++;
+            touchedEntities.push(item);
         }
 
         return {
             dryRun,
             success: true,
-            result: truncatedContent
+            result: truncatedContent,
+            touchedEntities
         };
     }
 }

src/Action/UserFlairAction.ts

@@ -0,0 +1,109 @@
+import Action, {ActionConfig, ActionJson, ActionOptions} from './index';
+import {Comment, RedditUser, Submission} from 'snoowrap';
+import {RuleResult} from '../Rule';
+import {ActionProcessResult} from '../Common/interfaces';
+
+export class UserFlairAction extends Action {
+  text?: string;
+  css?: string;
+  flair_template_id?: string;
+
+  constructor(options: UserFlairActionOptions) {
+    super(options);
+
+    this.text = options.text === null || options.text === '' ? undefined : options.text;
+    this.css = options.css === null || options.text === '' ? undefined : options.text;
+    this.flair_template_id = options.flair_template_id === null || options.flair_template_id === '' ? undefined : options.flair_template_id;
+  }
+
+  getKind() {
+    return 'User Flair';
+  }
+
+  async process(item: Comment | Submission, ruleResults: RuleResult[], runtimeDryrun?: boolean): Promise<ActionProcessResult> {
+    const dryRun = runtimeDryrun || this.dryRun;
+    let flairParts = [];
+
+    if (this.flair_template_id !== undefined) {
+      flairParts.push(`Flair template ID: ${this.flair_template_id}`)
+      if(this.text !== undefined || this.css !== undefined) {
+        this.logger.warn('Text/CSS properties will be ignored since a flair template is specified');
+      }
+    } else {
+      if (this.text !== undefined) {
+        flairParts.push(`Text: ${this.text}`);
+      }
+      if (this.css !== undefined) {
+        flairParts.push(`CSS: ${this.css}`);
+      }
+    }
+
+    const flairSummary = flairParts.length === 0 ? 'Unflair user' : flairParts.join(' | ');
+    this.logger.verbose(flairSummary);
+
+    if (!this.dryRun) {
+      if (this.flair_template_id !== undefined) {
+        try {
+          // @ts-ignore
+          await this.client.assignUserFlairByTemplateId({
+            subredditName: item.subreddit.display_name,
+            flairTemplateId: this.flair_template_id,
+            username: item.author.name,
+          });
+        } catch (err: any) {
+          this.logger.error('Either the flair template ID is incorrect or you do not have permission to access it.');
+          throw err;
+        }
+      } else if (this.text === undefined && this.css === undefined) {
+        // @ts-ignore
+        await item.subreddit.deleteUserFlair(item.author.name);
+      } else {
+        // @ts-ignore
+        await item.author.assignFlair({
+          subredditName: item.subreddit.display_name,
+          cssClass: this.css,
+          text: this.text,
+        });
+      }
+    }
+
+    return {
+      dryRun,
+      success: true,
+      result: flairSummary,
+    }
+  }
+}
+
+/**
+ * Flair the Author of an Activity
+ *
+ * Leave all properties blank or null to remove a User's existing flair
+ * */
+export interface UserFlairActionConfig extends ActionConfig {
+  /**
+   * The text of the flair to apply
+   * */
+  text?: string,
+  /**
+   * The text of the css class of the flair to apply
+   * */
+  css?: string,
+  /**
+   * Flair template to pick.
+   *
+   * **Note:** If this template is used text/css are ignored
+   * */
+  flair_template_id?: string;
+}
+
+export interface UserFlairActionOptions extends UserFlairActionConfig, ActionOptions {
+
+}
+
+/**
+ * Flair the Submission
+ * */
+export interface UserFlairActionJson extends UserFlairActionConfig, ActionJson {
+  kind: 'userflair'
+}

src/Action/index.ts

@@ -1,17 +1,18 @@
-import Snoowrap, {Comment, Submission} from "snoowrap";
+import {Comment, Submission} from "snoowrap";
 import {Logger} from "winston";
 import {RuleResult} from "../Rule";
-import {SubredditResources} from "../Subreddit/SubredditResources";
+import {checkAuthorFilter, SubredditResources} from "../Subreddit/SubredditResources";
 import {ActionProcessResult, ActionResult, ChecksActivityState, TypedActivityStates} from "../Common/interfaces";
 import Author, {AuthorOptions} from "../Author/Author";
 import {mergeArr} from "../util";
 import LoggedError from "../Utils/LoggedError";
+import {ExtendedSnoowrap} from '../Utils/SnoowrapClients';
 
 export abstract class Action {
     name?: string;
     logger: Logger;
     resources: SubredditResources;
-    client: Snoowrap
+    client: ExtendedSnoowrap;
     authorIs: AuthorOptions;
     itemIs: TypedActivityStates;
     dryRun: boolean;
@@ -27,6 +28,7 @@ export abstract class Action {
             subredditName,
             dryRun = false,
             authorIs: {
+                excludeCondition = 'OR',
                 include = [],
                 exclude = [],
             } = {},
@@ -41,6 +43,7 @@ export abstract class Action {
         this.logger = logger.child({labels: [`Action ${this.getActionUniqueName()}`]}, mergeArr);
 
         this.authorIs = {
+            excludeCondition,
             exclude: exclude.map(x => new Author(x)),
             include: include.map(x => new Author(x)),
         }
@@ -71,27 +74,10 @@ export abstract class Action {
                 actRes.runReason = `Activity did not pass 'itemIs' test, Action not run`;
                 return actRes;
             }
-            if (this.authorIs.include !== undefined && this.authorIs.include.length > 0) {
-                for (const auth of this.authorIs.include) {
-                    if (await this.resources.testAuthorCriteria(item, auth)) {
-                        actRes.run = true;
-                        const results = await this.process(item, ruleResults, runtimeDryrun);
-                        return {...actRes, ...results};
-                    }
-                }
-                this.logger.verbose('Inclusive author criteria not matched, Action not run');
-                actRes.runReason = 'Inclusive author criteria not matched';
-                return actRes;
-            } else if (this.authorIs.exclude !== undefined && this.authorIs.exclude.length > 0) {
-                for (const auth of this.authorIs.exclude) {
-                    if (await this.resources.testAuthorCriteria(item, auth, false)) {
-                        actRes.run = true;
-                        const results = await this.process(item, ruleResults, runtimeDryrun);
-                        return {...actRes, ...results};
-                    }
-                }
-                this.logger.verbose('Exclusive author criteria not matched, Action not run');
-                actRes.runReason = 'Exclusive author criteria not matched';
+            const [authFilterResult, authFilterType] = await checkAuthorFilter(item, this.authorIs, this.resources, this.logger);
+            if(!authFilterResult) {
+                this.logger.verbose(`${authFilterType} author criteria not matched, Action not run`);
+                actRes.runReason = `${authFilterType} author criteria not matched`;
                 return actRes;
             }
 
@@ -114,8 +100,8 @@ export abstract class Action {
 export interface ActionOptions extends ActionConfig {
     logger: Logger;
     subredditName: string;
-    resources: SubredditResources
-    client: Snoowrap
+    resources: SubredditResources;
+    client: ExtendedSnoowrap;
 }
 
 export interface ActionConfig extends ChecksActivityState {
@@ -162,7 +148,7 @@ export interface ActionJson extends ActionConfig {
     /**
      * The type of action that will be performed
      */
-    kind: 'comment' | 'lock' | 'remove' | 'report' | 'approve' | 'ban' | 'flair' | 'usernote' | 'message'
+    kind: 'comment' | 'lock' | 'remove' | 'report' | 'approve' | 'ban' | 'flair' | 'usernote' | 'message' | 'userflair'
 }
 
 export const isActionJson = (obj: object): obj is ActionJson => {

src/Author/Author.ts

@@ -1,5 +1,5 @@
 import {UserNoteCriteria} from "../Rule";
-import {CompareValue, CompareValueOrPercent, DurationComparor} from "../Common/interfaces";
+import {CompareValue, CompareValueOrPercent, DurationComparor, JoinOperands} from "../Common/interfaces";
 import {parseStringToRegex} from "../util";
 
 /**
@@ -12,7 +12,17 @@ export interface AuthorOptions {
      * */
     include?: AuthorCriteria[];
     /**
-     * Only runs if `include` is not present. Will "pass" if any of set of the AuthorCriteria **does not** pass
+     * * OR => if ANY exclude condition "does not" pass then the exclude test passes
+     * * AND => if ALL exclude conditions "do not" pass then the exclude test passes
+     *
+     * Defaults to OR
+     * @default OR
+     * */
+    excludeCondition?: JoinOperands
+    /**
+     * Only runs if `include` is not present. Each AuthorCriteria is comprised of conditions that the Author being checked must "not" pass. See excludeCondition for set behavior
+     *
+     * EX: `isMod: true, name: Automoderator` => Will pass if the Author IS NOT a mod and IS NOT named Automoderator
      * */
     exclude?: AuthorCriteria[];
 }

src/Bot/index.ts

@@ -3,20 +3,30 @@ import {Logger} from "winston";
 import dayjs, {Dayjs} from "dayjs";
 import {Duration} from "dayjs/plugin/duration";
 import EventEmitter from "events";
-import {BotInstanceConfig, Invokee, PAUSED, RUNNING, STOPPED, SYSTEM, USER} from "../Common/interfaces";
+import {
+    BotInstanceConfig,
+    FilterCriteriaDefaults,
+    Invokee,
+    PAUSED,
+    PollOn,
+    RUNNING,
+    STOPPED,
+    SYSTEM,
+    USER
+} from "../Common/interfaces";
 import {
     createRetryHandler,
     formatNumber,
     mergeArr,
     parseBool,
     parseDuration,
-    parseSubredditName,
+    parseSubredditName, RetryOptions,
     sleep,
     snooLogWrapper
 } from "../util";
 import {Manager} from "../Subreddit/Manager";
 import {ExtendedSnoowrap, ProxiedSnoowrap} from "../Utils/SnoowrapClients";
-import {ModQueueStream, UnmoderatedStream} from "../Subreddit/Streams";
+import {CommentStream, ModQueueStream, SPoll, SubmissionStream, UnmoderatedStream} from "../Subreddit/Streams";
 import {BotResourcesManager} from "../Subreddit/SubredditResources";
 import LoggedError from "../Utils/LoggedError";
 import pEvent from "p-event";
@@ -33,6 +43,7 @@ class Bot {
     running: boolean = false;
     subreddits: string[];
     excludeSubreddits: string[];
+    filterCriteriaDefaults?: FilterCriteriaDefaults
     subManagers: Manager[] = [];
     heartbeatInterval: number;
     nextHeartbeat: Dayjs = dayjs();
@@ -43,6 +54,7 @@ class Bot {
     nannyMode?: 'soft' | 'hard';
     nannyRunning: boolean = false;
     nextNannyCheck: Dayjs = dayjs().add(10, 'second');
+    sharedStreamRetryHandler: Function;
     nannyRetryHandler: Function;
     managerRetryHandler: Function;
     nextExpiration: Dayjs = dayjs();
@@ -51,7 +63,7 @@ class Bot {
     botAccount?: string;
     maxWorkers: number;
     startedAt: Dayjs = dayjs();
-    sharedModqueue: boolean = false;
+    sharedStreams: PollOn[] = [];
     streamListedOnce: string[] = [];
 
     stagger: number;
@@ -78,6 +90,7 @@ class Bot {
         const {
             notifications,
             name,
+            filterCriteriaDefaults,
             subreddits: {
                 names = [],
                 exclude = [],
@@ -98,7 +111,7 @@ class Bot {
                 debug,
             },
             polling: {
-                sharedMod,
+                shared = [],
                 stagger = 2000,
             },
             queue: {
@@ -123,7 +136,8 @@ class Bot {
         this.hardLimit = hardLimit;
         this.wikiLocation = wikiConfig;
         this.heartbeatInterval = heartbeatInterval;
-        this.sharedModqueue = sharedMod;
+        this.filterCriteriaDefaults = filterCriteriaDefaults;
+        this.sharedStreams = shared;
         if(name !== undefined) {
             this.botName = name;
         }
@@ -178,7 +192,7 @@ class Bot {
             this.client = proxy === undefined ? new ExtendedSnoowrap(creds) : new ProxiedSnoowrap({...creds, proxy});
             this.client.config({
                 warnings: true,
-                maxRetryAttempts: 5,
+                maxRetryAttempts: 2,
                 debug,
                 logger: snooLogWrapper(this.logger.child({labels: ['Snoowrap']}, mergeArr)),
                 continueAfterRatelimitError: false,
@@ -190,56 +204,12 @@ class Bot {
             }
         }
 
-        const retryHandler = createRetryHandler({maxRequestRetry: 8, maxOtherRetry: 1}, this.logger);
+        this.sharedStreamRetryHandler = createRetryHandler({maxRequestRetry: 8, maxOtherRetry: 2}, this.logger);
         this.nannyRetryHandler = createRetryHandler({maxRequestRetry: 5, maxOtherRetry: 1}, this.logger);
-        this.managerRetryHandler = createRetryHandler({maxRequestRetry: 5, maxOtherRetry: 10, waitOnRetry: false, clearRetryCountAfter: 2}, this.logger);
+        this.managerRetryHandler = createRetryHandler({maxRequestRetry: 8, maxOtherRetry: 8, waitOnRetry: false, clearRetryCountAfter: 2}, this.logger);
 
         this.stagger = stagger ?? 2000;
 
-        const modStreamErrorListener = (name: string) => async (err: any) => {
-            this.logger.error('Polling error occurred', err);
-            const shouldRetry = await retryHandler(err);
-            if(shouldRetry) {
-                defaultUnmoderatedStream.startInterval();
-            } else {
-                for(const m of this.subManagers) {
-                    if(m.modStreamCallbacks.size > 0) {
-                        m.notificationManager.handle('runStateChanged', `${name.toUpperCase()} Polling Stopped`, 'Encountered too many errors from Reddit while polling. Will try to restart on next heartbeat.');
-                    }
-                }
-                this.logger.error(`Mod stream ${name.toUpperCase()} encountered too many errors while polling. Will try to restart on next heartbeat.`);
-            }
-        }
-
-        const modStreamListingListener = (name: string) => async (listing: (Comment|Submission)[]) => {
-            // dole out in order they were received
-            if(!this.streamListedOnce.includes(name)) {
-                this.streamListedOnce.push(name);
-                return;
-            }
-            for(const i of listing) {
-                const foundManager = this.subManagers.find(x => x.subreddit.display_name === i.subreddit.display_name && x.modStreamCallbacks.get(name) !== undefined);
-                if(foundManager !== undefined) {
-                    foundManager.modStreamCallbacks.get(name)(i);
-                    if(stagger !== undefined) {
-                        await sleep(stagger);
-                    }
-                }
-            }
-
-        }
-
-        const defaultUnmoderatedStream = new UnmoderatedStream(this.client, {subreddit: 'mod', limit: 100, clearProcessed: { size: 100, retain: 100 }});
-        // @ts-ignore
-        defaultUnmoderatedStream.on('error', modStreamErrorListener('unmoderated'));
-        defaultUnmoderatedStream.on('listing', modStreamListingListener('unmoderated'));
-        const defaultModqueueStream = new ModQueueStream(this.client, {subreddit: 'mod', limit: 100, clearProcessed: { size: 100, retain: 100 }});
-        // @ts-ignore
-        defaultModqueueStream.on('error', modStreamErrorListener('modqueue'));
-        defaultModqueueStream.on('listing', modStreamListingListener('modqueue'));
-        this.cacheManager.modStreams.set('unmoderated', defaultUnmoderatedStream);
-        this.cacheManager.modStreams.set('modqueue', defaultModqueueStream);
-
         process.on('uncaughtException', (e) => {
             this.error = e;
         });
@@ -263,6 +233,38 @@ class Bot {
         });
     }
 
+    createSharedStreamErrorListener = (name: string) => async (err: any) => {
+        this.logger.error(`Polling error occurred on stream ${name.toUpperCase()}`, err);
+        const shouldRetry = await this.sharedStreamRetryHandler(err);
+        if(shouldRetry) {
+            (this.cacheManager.modStreams.get(name) as SPoll<any>).startInterval(false);
+        } else {
+            for(const m of this.subManagers) {
+                if(m.sharedStreamCallbacks.size > 0) {
+                    m.notificationManager.handle('runStateChanged', `${name.toUpperCase()} Polling Stopped`, 'Encountered too many errors from Reddit while polling. Will try to restart on next heartbeat.');
+                }
+            }
+            this.logger.error(`Mod stream ${name.toUpperCase()} encountered too many errors while polling. Will try to restart on next heartbeat.`);
+        }
+    }
+
+    createSharedStreamListingListener = (name: string) => async (listing: (Comment|Submission)[]) => {
+        // dole out in order they were received
+        if(!this.streamListedOnce.includes(name)) {
+            this.streamListedOnce.push(name);
+            return;
+        }
+        for(const i of listing) {
+            const foundManager = this.subManagers.find(x => x.subreddit.display_name === i.subreddit.display_name && x.sharedStreamCallbacks.get(name) !== undefined);
+            if(foundManager !== undefined) {
+                foundManager.sharedStreamCallbacks.get(name)(i);
+                if(this.stagger !== undefined) {
+                    await sleep(this.stagger);
+                }
+            }
+        }
+    }
+
     async onTerminate(reason = 'The application was shutdown') {
         for(const m of this.subManagers) {
             await m.notificationManager.handle('runStateChanged', 'Application Shutdown', reason);
@@ -316,7 +318,7 @@ class Bot {
         while(!subListing.isFinished) {
             subListing = await subListing.fetchMore({amount: 100});
         }
-        availSubs = subListing;
+        availSubs = subListing.filter(x => x.display_name !== `u_${user.name}`);
 
         this.logger.info(`u/${user.name} is a moderator of these subreddits: ${availSubs.map(x => x.display_name_prefixed).join(', ')}`);
 
@@ -336,46 +338,173 @@ class Bot {
             }
         } else {
             if(this.excludeSubreddits.length > 0) {
-                this.logger.info(`Will run on all moderated subreddits but user-defined excluded: ${this.excludeSubreddits.join(', ')}`);
+                this.logger.info(`Will run on all moderated subreddits but own profile and user-defined excluded: ${this.excludeSubreddits.join(', ')}`);
                 const normalExcludes = this.excludeSubreddits.map(x => x.toLowerCase());
                 subsToRun = availSubs.filter(x => !normalExcludes.includes(x.display_name.toLowerCase()));
             } else {
                 this.logger.info(`No user-defined subreddit constraints detected, will run on all moderated subreddits EXCEPT own profile (${this.botAccount})`);
-                subsToRun = availSubs.filter(x => x.display_name_prefixed !== this.botAccount);
+                subsToRun = availSubs;
             }
         }
 
         // get configs for subs we want to run on and build/validate them
         for (const sub of subsToRun) {
             try {
-                this.subManagers.push(await this.createManager(sub));
+                this.subManagers.push(this.createManager(sub));
             } catch (err: any) {
 
             }
         }
+        for(const m of this.subManagers) {
+            try {
+                await this.initManager(m);
+            } catch (err: any) {
+
+            }
+        }
+
+        this.parseSharedStreams();
     }
 
-    async createManager(sub: Subreddit): Promise<Manager> {
-        const manager = new Manager(sub, this.client, this.logger, this.cacheManager, {dryRun: this.dryRun, sharedModqueue: this.sharedModqueue, wikiLocation: this.wikiLocation, botName: this.botName as string, maxWorkers: this.maxWorkers});
+    parseSharedStreams() {
+
+        const sharedCommentsSubreddits = !this.sharedStreams.includes('newComm') ? [] : this.subManagers.filter(x => x.isPollingShared('newComm')).map(x => x.subreddit.display_name);
+        if (sharedCommentsSubreddits.length > 0) {
+            const stream = this.cacheManager.modStreams.get('newComm');
+            if (stream === undefined || stream.subreddit !== sharedCommentsSubreddits.join('+')) {
+                let processed;
+                if (stream !== undefined) {
+                    this.logger.info('Restarting SHARED COMMENT STREAM due to a subreddit config change');
+                    stream.end();
+                    processed = stream.processed;
+                }
+                if (sharedCommentsSubreddits.length > 100) {
+                    this.logger.warn(`SHARED COMMENT STREAM => Reddit can only combine 100 subreddits for getting new Comments but this bot has ${sharedCommentsSubreddits.length}`);
+                }
+                const defaultCommentStream = new CommentStream(this.client, {
+                    subreddit: sharedCommentsSubreddits.join('+'),
+                    limit: 100,
+                    enforceContinuity: true,
+                    logger: this.logger,
+                    processed,
+                    label: 'Shared Polling'
+                });
+                // @ts-ignore
+                defaultCommentStream.on('error', this.createSharedStreamErrorListener('newComm'));
+                defaultCommentStream.on('listing', this.createSharedStreamListingListener('newComm'));
+                this.cacheManager.modStreams.set('newComm', defaultCommentStream);
+            }
+        } else {
+            const stream = this.cacheManager.modStreams.get('newComm');
+            if (stream !== undefined) {
+                stream.end();
+            }
+        }
+
+        const sharedSubmissionsSubreddits = !this.sharedStreams.includes('newSub') ? [] : this.subManagers.filter(x => x.isPollingShared('newSub')).map(x => x.subreddit.display_name);
+        if (sharedSubmissionsSubreddits.length > 0) {
+            const stream = this.cacheManager.modStreams.get('newSub');
+            if (stream === undefined || stream.subreddit !== sharedSubmissionsSubreddits.join('+')) {
+                let processed;
+                if (stream !== undefined) {
+                    this.logger.info('Restarting SHARED SUBMISSION STREAM due to a subreddit config change');
+                    stream.end();
+                    processed = stream.processed;
+                }
+                if (sharedSubmissionsSubreddits.length > 100) {
+                    this.logger.warn(`SHARED SUBMISSION STREAM => Reddit can only combine 100 subreddits for getting new Submissions but this bot has ${sharedSubmissionsSubreddits.length}`);
+                }
+                const defaultSubStream = new SubmissionStream(this.client, {
+                    subreddit: sharedSubmissionsSubreddits.join('+'),
+                    limit: 100,
+                    enforceContinuity: true,
+                    logger: this.logger,
+                    processed,
+                    label: 'Shared Polling'
+                });
+                // @ts-ignore
+                defaultSubStream.on('error', this.createSharedStreamErrorListener('newSub'));
+                defaultSubStream.on('listing', this.createSharedStreamListingListener('newSub'));
+                this.cacheManager.modStreams.set('newSub', defaultSubStream);
+            }
+        } else {
+            const stream = this.cacheManager.modStreams.get('newSub');
+            if (stream !== undefined) {
+                stream.end();
+            }
+        }
+
+        const isUnmoderatedShared = !this.sharedStreams.includes('unmoderated') ? false : this.subManagers.some(x => x.isPollingShared('unmoderated'));
+        const unmoderatedstream = this.cacheManager.modStreams.get('unmoderated');
+        if (isUnmoderatedShared && unmoderatedstream === undefined) {
+            const defaultUnmoderatedStream = new UnmoderatedStream(this.client, {
+                subreddit: 'mod',
+                limit: 100,
+                logger: this.logger,
+                label: 'Shared Polling'
+            });
+            // @ts-ignore
+            defaultUnmoderatedStream.on('error', this.createSharedStreamErrorListener('unmoderated'));
+            defaultUnmoderatedStream.on('listing', this.createSharedStreamListingListener('unmoderated'));
+            this.cacheManager.modStreams.set('unmoderated', defaultUnmoderatedStream);
+        } else if (!isUnmoderatedShared && unmoderatedstream !== undefined) {
+            unmoderatedstream.end();
+        }
+
+        const isModqueueShared = !this.sharedStreams.includes('modqueue') ? false : this.subManagers.some(x => x.isPollingShared('modqueue'));
+        const modqueuestream = this.cacheManager.modStreams.get('modqueue');
+        if (isModqueueShared && modqueuestream === undefined) {
+            const defaultModqueueStream = new ModQueueStream(this.client, {
+                subreddit: 'mod',
+                limit: 100,
+                logger: this.logger,
+                label: 'Shared Polling'
+            });
+            // @ts-ignore
+            defaultModqueueStream.on('error', this.createSharedStreamErrorListener('modqueue'));
+            defaultModqueueStream.on('listing', this.createSharedStreamListingListener('modqueue'));
+            this.cacheManager.modStreams.set('modqueue', defaultModqueueStream);
+        } else if (isModqueueShared && modqueuestream !== undefined) {
+            modqueuestream.end();
+        }
+    }
+
+    async initManager(manager: Manager) {
         try {
-            await manager.parseConfiguration('system', true, {suppressNotification: true});
+            await manager.parseConfiguration('system', true, {suppressNotification: true, suppressChangeEvent: true});
         } catch (err: any) {
             if (!(err instanceof LoggedError)) {
-                this.logger.error(`Config was not valid:`, {subreddit: sub.display_name_prefixed});
-                this.logger.error(err, {subreddit: sub.display_name_prefixed});
+                this.logger.error(`Config was not valid:`, {subreddit: manager.subreddit.display_name_prefixed});
+                this.logger.error(err, {subreddit: manager.subreddit.display_name_prefixed});
                 err.logged = true;
             }
         }
+    }
+
+    createManager(sub: Subreddit): Manager {
+        const manager = new Manager(sub, this.client, this.logger, this.cacheManager, {
+            dryRun: this.dryRun,
+            sharedStreams: this.sharedStreams,
+            wikiLocation: this.wikiLocation,
+            botName: this.botName as string,
+            maxWorkers: this.maxWorkers,
+            filterCriteriaDefaults: this.filterCriteriaDefaults,
+        });
         // all errors from managers will count towards bot-level retry count
         manager.on('error', async (err) => await this.panicOnRetries(err));
+        manager.on('configChange', async () => {
+           this.parseSharedStreams();
+           await this.runSharedStreams(false);
+        });
         return manager;
     }
 
     // if the cumulative errors exceeds configured threshold then stop ALL managers as there is most likely something very bad happening
     async panicOnRetries(err: any) {
-        if(!this.managerRetryHandler(err)) {
+        if(!await this.managerRetryHandler(err)) {
+            this.logger.warn('Bot detected too many errors from managers within a short time. Stopping all managers and will try to restart on next heartbeat.');
             for(const m of this.subManagers) {
-                await m.stop();
+                await m.stop('system',{reason: 'Bot detected too many errors from all managers. Stopping all manager as a failsafe.'});
             }
         }
     }
@@ -403,11 +532,12 @@ class Bot {
                 const sub = await this.client.getSubreddit(name);
                 this.logger.info(`Attempting to add manager for r/${name}`);
                 try {
-                    const manager = await this.createManager(sub);
+                    const manager = this.createManager(sub);
                     this.logger.info(`Starting manager for r/${name}`);
                     this.subManagers.push(manager);
+                    await this.initManager(manager);
                     await manager.start('system', {reason: 'Caused by creation due to moderator invite'});
-                    await this.runModStreams();
+                    await this.runSharedStreams();
                 } catch (err: any) {
                     if (!(err instanceof LoggedError)) {
                         this.logger.error(err);
@@ -425,14 +555,14 @@ class Bot {
         }
     }
 
-    async runModStreams(notify = false) {
+    async runSharedStreams(notify = false) {
         for(const [k,v] of this.cacheManager.modStreams) {
-            if(!v.running && this.subManagers.some(x => x.modStreamCallbacks.get(k) !== undefined)) {
+            if(!v.running && this.subManagers.some(x => x.sharedStreamCallbacks.get(k) !== undefined)) {
                 v.startInterval();
-                this.logger.info(`Starting default ${k.toUpperCase()} mod stream`);
+                this.logger.info(`Starting ${k.toUpperCase()} shared polling`);
                 if(notify) {
                     for(const m of this.subManagers) {
-                        if(m.modStreamCallbacks.size > 0) {
+                        if(m.sharedStreamCallbacks.size > 0) {
                             await m.notificationManager.handle('runStateChanged', `${k.toUpperCase()} Polling Started`, 'Polling was successfully restarted on heartbeat.');
                         }
                     }
@@ -443,6 +573,8 @@ class Bot {
     }
 
     async runManagers(causedBy: Invokee = 'system') {
+        this.running = true;
+
         if(this.subManagers.every(x => !x.validConfigLoaded)) {
             this.logger.warn('All managers have invalid configs!');
             this.error = 'All managers have invalid configs';
@@ -454,9 +586,8 @@ class Bot {
             }
         }
 
-        await this.runModStreams();
+        await this.runSharedStreams();
 
-        this.running = true;
         this.nextNannyCheck = dayjs().add(10, 'second');
         this.nextHeartbeat = dayjs().add(this.heartbeatInterval, 'second');
         await this.checkModInvites();
@@ -474,8 +605,8 @@ class Bot {
                     await this.runApiNanny();
                     this.nextNannyCheck = dayjs().add(10, 'second');
                 } catch (err: any) {
-                    this.logger.info('Delaying next nanny check for 2 minutes due to emitted error');
-                    this.nextNannyCheck = dayjs().add(120, 'second');
+                    this.logger.info('Delaying next nanny check for 4 minutes due to emitted error');
+                    this.nextNannyCheck = dayjs().add(240, 'second');
                 }
             }
             if(dayjs().isSameOrAfter(this.nextHeartbeat)) {
@@ -545,7 +676,7 @@ class Bot {
                 }
             }
         }
-        await this.runModStreams(true);
+        await this.runSharedStreams(true);
     }
 
     async runApiNanny() {

src/Check/index.ts

@@ -28,8 +28,9 @@ import * as RuleSchema from '../Schema/Rule.json';
 import * as RuleSetSchema from '../Schema/RuleSet.json';
 import * as ActionSchema from '../Schema/Action.json';
 import {ActionObjectJson, RuleJson, RuleObjectJson, ActionJson as ActionTypeJson} from "../Common/types";
-import {SubredditResources} from "../Subreddit/SubredditResources";
-import {Author, AuthorCriteria, AuthorOptions} from "../Author/Author";
+import {checkAuthorFilter, SubredditResources} from "../Subreddit/SubredditResources";
+import {Author, AuthorCriteria, AuthorOptions} from '..';
+import {ExtendedSnoowrap} from '../Utils/SnoowrapClients';
 
 const checkLogName = truncateStringToLength(25);
 
@@ -42,15 +43,12 @@ export abstract class Check implements ICheck {
     rules: Array<RuleSet | Rule> = [];
     logger: Logger;
     itemIs: TypedActivityStates;
-    authorIs: {
-        include: AuthorCriteria[],
-        exclude: AuthorCriteria[]
-    };
+    authorIs: AuthorOptions;
     cacheUserResult: Required<UserResultCacheOptions>;
     dryRun?: boolean;
     notifyOnTrigger: boolean;
     resources: SubredditResources;
-    client: Snoowrap;
+    client: ExtendedSnoowrap;
 
     constructor(options: CheckOptions) {
         const {
@@ -68,6 +66,7 @@ export abstract class Check implements ICheck {
             itemIs = [],
             authorIs: {
                 include = [],
+                excludeCondition = 'OR',
                 exclude = [],
             } = {},
             dryRun,
@@ -88,6 +87,7 @@ export abstract class Check implements ICheck {
         this.condition = condition;
         this.itemIs = itemIs;
         this.authorIs = {
+            excludeCondition,
             exclude: exclude.map(x => new Author(x)),
             include: include.map(x => new Author(x)),
         }
@@ -158,7 +158,7 @@ export abstract class Check implements ICheck {
         runStats.push(`${this.actions.length} Actions`);
         // not sure if this should be info or verbose
         this.logger.info(`=${this.enabled ? 'Enabled' : 'Disabled'}= ${type.toUpperCase()} (${this.condition})${this.notifyOnTrigger ? ' ||Notify on Trigger|| ' : ''} => ${runStats.join(' | ')}${this.description !== undefined ? ` => ${this.description}` : ''}`);
-        if (this.rules.length === 0 && this.itemIs.length === 0 && this.authorIs.exclude.length === 0 && this.authorIs.include.length === 0) {
+        if (this.rules.length === 0 && this.itemIs.length === 0 && this.authorIs.exclude?.length === 0 && this.authorIs.include?.length === 0) {
             this.logger.warn('No rules, item tests, or author test found -- this check will ALWAYS PASS!');
         }
         let ruleSetIndex = 1;
@@ -201,30 +201,9 @@ export abstract class Check implements ICheck {
                 this.logger.verbose(`${FAIL} => Item did not pass 'itemIs' test`);
                 return [false, allRuleResults];
             }
-            let authorPass = null;
-            if (this.authorIs.include !== undefined && this.authorIs.include.length > 0) {
-                for (const auth of this.authorIs.include) {
-                    if (await this.resources.testAuthorCriteria(item, auth)) {
-                        authorPass = true;
-                        break;
-                    }
-                }
-                if (!authorPass) {
-                    this.logger.verbose(`${FAIL} => Inclusive author criteria not matched`);
-                    return Promise.resolve([false, allRuleResults]);
-                }
-            }
-            if (authorPass === null && this.authorIs.exclude !== undefined && this.authorIs.exclude.length > 0) {
-                for (const auth of this.authorIs.exclude) {
-                    if (await this.resources.testAuthorCriteria(item, auth, false)) {
-                        authorPass = true;
-                        break;
-                    }
-                }
-                if (!authorPass) {
-                    this.logger.verbose(`${FAIL} =>  Exclusive author criteria not matched`);
-                    return Promise.resolve([false, allRuleResults]);
-                }
+            const [authFilterResult, authFilterType] = await checkAuthorFilter(item, this.authorIs, this.resources, this.logger);
+            if(!authFilterResult) {
+                return Promise.resolve([false, allRuleResults]);
             }
 
             if (this.rules.length === 0) {
@@ -345,13 +324,13 @@ export interface ICheck extends JoinCondition, ChecksActivityState {
 }
 
 export interface CheckOptions extends ICheck {
-    rules: Array<IRuleSet | IRule>
-    actions: ActionConfig[]
-    logger: Logger
-    subredditName: string
-    notifyOnTrigger?: boolean
-    resources: SubredditResources
-    client: Snoowrap
+    rules: Array<IRuleSet | IRule>;
+    actions: ActionConfig[];
+    logger: Logger;
+    subredditName: string;
+    notifyOnTrigger?: boolean;
+    resources: SubredditResources;
+    client: ExtendedSnoowrap;
     cacheUserResult?: UserResultCacheOptions;
 }
 

src/Common/defaults.ts

@@ -1,7 +1,7 @@
-import {HistoricalStats} from "./interfaces";
+import {HistoricalStats, FilterCriteriaDefaults} from "./interfaces";
 
 export const cacheOptDefaults = {ttl: 60, max: 500, checkPeriod: 600};
-export const cacheTTLDefaults = {authorTTL: 60, userNotesTTL: 300, wikiTTL: 300, submissionTTL: 60, commentTTL: 60, filterCriteriaTTL: 60, subredditTTL: 600};
+export const cacheTTLDefaults = {authorTTL: 60, userNotesTTL: 300, wikiTTL: 300, submissionTTL: 60, commentTTL: 60, filterCriteriaTTL: 60, subredditTTL: 600, selfTTL: 60};
 export const historicalDefaults: HistoricalStats = {
     eventsCheckedTotal: 0,
     eventsActionedTotal: 0,
@@ -29,3 +29,13 @@ export const createHistoricalDefaults = (): HistoricalStats => {
         actionsRun: new Map(),
     };
 }
+
+export const filterCriteriaDefault: FilterCriteriaDefaults = {
+    authorIs: {
+        exclude: [
+            {
+                isMod: true
+            }
+        ]
+    }
+}

src/Common/interfaces.ts

@@ -5,6 +5,10 @@ import Poll from "snoostorm/out/util/Poll";
 import Snoowrap from "snoowrap";
 import {RuleResult} from "../Rule";
 import {IncomingMessage} from "http";
+import Submission from "snoowrap/dist/objects/Submission";
+import Comment from "snoowrap/dist/objects/Comment";
+import RedditUser from "snoowrap/dist/objects/RedditUser";
+import {AuthorOptions} from "../Author/Author";
 
 /**
  * An ISO 8601 Duration
@@ -486,38 +490,6 @@ export type PollOn = 'unmoderated' | 'modqueue' | 'newSub' | 'newComm';
 export interface PollingOptionsStrong extends PollingOptions {
     limit: number,
     interval: number,
-    clearProcessed: ClearProcessedOptions
-}
-
-/**
- * For very long-running, high-volume subreddits clearing the list of processed activities helps manage memory bloat
- *
- * All of these options have default values based on the limit and/or interval set for polling options on each subreddit stream. They only need to modified if the defaults are not sufficient.
- *
- * If both `after` and `size` are defined whichever is hit first will trigger the list to clear. `after` will be reset after ever clear.
- * */
-export interface ClearProcessedOptions {
-    /**
-     * An interval the processed list should be cleared after.
-     *
-     * * EX `9 days`
-     * * EX `3 months`
-     * * EX `5 minutes`
-     * @pattern ^\s*(?<time>\d+)\s*(?<unit>days?|weeks?|months?|years?|hours?|minutes?|seconds?|milliseconds?)\s*$
-     * */
-    after?: string,
-    /**
-     * Number of activities found in processed list after which the list should be cleared.
-     *
-     * Defaults to the `limit` value from `PollingOptions`
-     * */
-    size?: number,
-    /**
-     * The number of activities to retain in processed list after clearing.
-     *
-     * Defaults to `limit` value from `PollingOptions`
-     * */
-    retain?: number,
 }
 
 export interface PollingDefaults {
@@ -591,8 +563,6 @@ export interface PollingOptions extends PollingDefaults {
      *
      * */
     pollOn: 'unmoderated' | 'modqueue' | 'newSub' | 'newComm'
-
-    clearProcessed?: ClearProcessedOptions
 }
 
 export interface TTLConfig {
@@ -670,6 +640,24 @@ export interface TTLConfig {
      * @default 60
      * */
     filterCriteriaTTL?: number | boolean;
+
+    /**
+     * Amount of time, in seconds, an Activity that the bot has acted on or created will be ignored if found during polling
+     *
+     * This is useful to prevent the bot from checking Activities it *just* worked on or a product of the checks. Examples:
+     *
+     * * Ignore comments created through an Action
+     * * Ignore Activity polled from modqueue that the bot just reported
+     *
+     * This value should be at least as long as the longest polling interval for modqueue/newComm
+     *
+     * * If `0` or `true` will cache indefinitely (not recommended)
+     * * If `false` will not cache
+     *
+     * @examples [50]
+     * @default 50
+     * */
+    selfTTL?: number | boolean
 }
 
 export interface CacheConfig extends TTLConfig {
@@ -834,6 +822,13 @@ export interface ManagerOptions {
     notifications?: NotificationConfig
 
     credentials?: ThirdPartyCredentialsJsonConfig
+
+    /**
+     * Set the default filter criteria for all checks. If this property is specified it will override any defaults passed from the bot's config
+     *
+     * Default behavior is to exclude all mods and automoderator from checks
+     * */
+    filterCriteriaDefaults?: FilterCriteriaDefaults
 }
 
 /**
@@ -909,6 +904,20 @@ export interface ActivityState {
     distinguished?: boolean
     approved?: boolean
     score?: CompareValue
+    /**
+     * A string containing a comparison operator and a value to compare against
+     *
+     * The syntax is `(< OR > OR <= OR >=) <number>`
+     *
+     * * EX `> 2`  => greater than 2 total reports
+     *
+     * Defaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:
+     *
+     * * EX `> 3 mod` => greater than 3 mod reports
+     * * EX `>= 1 user` => greater than 1 user report
+     *
+     * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(%?)(.*)$
+     * */
     reports?: CompareValue
     age?: DurationComparor
 }
@@ -1072,6 +1081,7 @@ export type StrongCache = {
     submissionTTL: number | boolean,
     commentTTL: number | boolean,
     subredditTTL: number | boolean,
+    selfTTL: number | boolean,
     filterCriteriaTTL: number | boolean,
     provider: CacheOptions
     actionedEventsMax?: number,
@@ -1184,6 +1194,7 @@ export interface Notifier {
 export interface ManagerStateChangeOption {
     reason?: string
     suppressNotification?: boolean
+    suppressChangeEvent?: boolean
 }
 
 /**
@@ -1289,6 +1300,53 @@ export interface WebCredentials {
     redirectUri?: string,
 }
 
+export interface SnoowrapOptions {
+    /**
+     * Proxy all requests to Reddit's API through this endpoint
+     *
+     * * ENV => `PROXY`
+     * * ARG => `--proxy <proxyEndpoint>`
+     *
+     * @examples ["http://localhost:4443"]
+     * */
+    proxy?: string,
+    /**
+     * Manually set the debug status for snoowrap
+     *
+     * When snoowrap has `debug: true` it will log the http status response of reddit api requests to at the `debug` level
+     *
+     * * Set to `true` to always output
+     * * Set to `false` to never output
+     *
+     * If not present or `null` will be set based on `logLevel`
+     *
+     * * ENV => `SNOO_DEBUG`
+     * * ARG => `--snooDebug`
+     * */
+    debug?: boolean,
+}
+
+export type FilterCriteriaDefaultBehavior = 'replace' | 'merge';
+
+export interface FilterCriteriaDefaults {
+    itemIs?: TypedActivityStates
+    /**
+     * Determine how itemIs defaults behave when itemIs is present on the check
+     *
+     * * merge => adds defaults to check's itemIs
+     * * replace => check itemIs will replace defaults (no defaults used)
+     * */
+    itemIsBehavior?: FilterCriteriaDefaultBehavior
+    /**
+     * Determine how authorIs defaults behave when authorIs is present on the check
+     *
+     * * merge => merges defaults with check's authorIs
+     * * replace => check authorIs will replace defaults (no defaults used)
+     * */
+    authorIs?: AuthorOptions
+    authorIsBehavior?: FilterCriteriaDefaultBehavior
+}
+
 /**
  * The configuration for an **individual reddit account** ContextMod will run as a bot.
  *
@@ -1309,33 +1367,20 @@ export interface BotInstanceJsonConfig {
     notifications?: NotificationConfig
 
     /**
-     * Settings to control some [Snoowrap](https://github.com/not-an-aardvark/snoowrap) behavior
+     * Settings to control some [Snoowrap](https://github.com/not-an-aardvark/snoowrap) behavior.
+     *
+     * Overrides any defaults provided at top-level operator config.
+     *
+     * Set to an empty object to "ignore" any top-level config
      * */
-    snoowrap?: {
-        /**
-         * Proxy all requests to Reddit's API through this endpoint
-         *
-         * * ENV => `PROXY`
-         * * ARG => `--proxy <proxyEndpoint>`
-         *
-         * @examples ["http://localhost:4443"]
-         * */
-        proxy?: string,
-        /**
-         * Manually set the debug status for snoowrap
-         *
-         * When snoowrap has `debug: true` it will log the http status response of reddit api requests to at the `debug` level
-         *
-         * * Set to `true` to always output
-         * * Set to `false` to never output
-         *
-         * If not present or `null` will be set based on `logLevel`
-         *
-         * * ENV => `SNOO_DEBUG`
-         * * ARG => `--snooDebug`
-         * */
-        debug?: boolean,
-    }
+    snoowrap?: SnoowrapOptions
+
+    /**
+     * Define the default behavior for all filter criteria on all checks in all subreddits
+     *
+     * Defaults to exclude mods and automoderator from checks
+     * */
+    filterCriteriaDefaults?: FilterCriteriaDefaults
 
     /**
      * Settings related to bot behavior for subreddits it is managing
@@ -1404,18 +1449,31 @@ export interface BotInstanceJsonConfig {
      * */
     polling?: PollingDefaults & {
         /**
-         * If set to `true` all subreddits polling unmoderated/modqueue with default polling settings will share a request to "r/mod"
-         * otherwise each subreddit will poll its own mod view
+         * DEPRECATED: See `shared`
+         *
+         *  Using the ENV or ARG will sett `unmoderated` and `modqueue` on `shared`
          *
          * * ENV => `SHARE_MOD`
          * * ARG => `--shareMod`
          *
          * @default false
+         * @deprecated
          * */
         sharedMod?: boolean,
 
         /**
-         * If sharing a mod stream stagger pushing relevant Activities to individual subreddits.
+         * Set which polling sources should be shared among subreddits using default polling settings for that source
+         *
+         * * For `unmoderated and `modqueue` the bot will poll on **r/mod** for new activities
+         * * For `newSub` and `newComm` all subreddits sharing the source will be combined to poll like **r/subreddit1+subreddit2/new**
+         *
+         * If set to `true` all polling sources will be shared,  otherwise specify which sourcs should be shared as a list
+         *
+         * */
+        shared?: PollOn[] | true,
+
+        /**
+         * If sharing a stream staggers pushing relevant Activities to individual subreddits.
          *
          * Useful when running many subreddits and rules are potentially cpu/memory/traffic heavy -- allows spreading out load
          * */
@@ -1558,6 +1616,11 @@ export interface OperatorJsonConfig {
      * */
     caching?: OperatorCacheConfig
 
+    /**
+     * Set global snoowrap options as well as default snoowrap config for all bots that don't specify their own
+     * */
+    snoowrap?: SnoowrapOptions
+
     bots?: BotInstanceJsonConfig[]
 
     /**
@@ -1722,7 +1785,7 @@ export interface BotInstanceConfig extends BotInstanceJsonConfig {
         heartbeatInterval: number,
     },
     polling: {
-        sharedMod: boolean,
+        shared: PollOn[],
         stagger?: number,
         limit: number,
         interval: number,
@@ -1801,20 +1864,18 @@ export interface LogInfo {
     bot?: string
 }
 
-export interface ActionResult {
+export interface ActionResult extends ActionProcessResult {
     kind: string,
     name: string,
     run: boolean,
     runReason?: string,
-    dryRun: boolean,
-    success: boolean,
-    result?: string,
 }
 
 export interface ActionProcessResult {
     success: boolean,
     dryRun: boolean,
     result?: string
+    touchedEntities?: (Submission | Comment | RedditUser | string)[]
 }
 
 export interface ActionedEvent {
@@ -1851,6 +1912,14 @@ export interface StatusCodeError extends Error {
     error: Error
 }
 
+export interface RequestError extends Error {
+    name: 'RequestError',
+    statusCode: number,
+    message: string,
+    response: IncomingMessage,
+    error: Error
+}
+
 export interface HistoricalStatsDisplay extends HistoricalStats {
     checksRunTotal: number
     checksFromCacheTotal: number

src/Common/types.ts

@@ -3,6 +3,7 @@ import {RepeatActivityJSONConfig} from "../Rule/RepeatActivityRule";
 import {AuthorRuleJSONConfig} from "../Rule/AuthorRule";
 import {AttributionJSONConfig} from "../Rule/AttributionRule";
 import {FlairActionJson} from "../Action/SubmissionAction/FlairAction";
+import {UserFlairActionJson} from "../Action/UserFlairAction";
 import {CommentActionJson} from "../Action/CommentAction";
 import {ReportActionJson} from "../Action/ReportAction";
 import {LockActionJson} from "../Action/LockAction";
@@ -18,7 +19,7 @@ import {RepostRuleJSONConfig} from "../Rule/RepostRule";
 export type RuleJson = RecentActivityRuleJSONConfig | RepeatActivityJSONConfig | AuthorRuleJSONConfig | AttributionJSONConfig | HistoryJSONConfig | RegexRuleJSONConfig | RepostRuleJSONConfig | string;
 export type RuleObjectJson = Exclude<RuleJson, string>
 
-export type ActionJson = CommentActionJson | FlairActionJson | ReportActionJson | LockActionJson | RemoveActionJson | ApproveActionJson | BanActionJson | UserNoteActionJson | MessageActionJson | string;
+export type ActionJson = CommentActionJson | FlairActionJson | ReportActionJson | LockActionJson | RemoveActionJson | ApproveActionJson | BanActionJson | UserNoteActionJson | MessageActionJson | UserFlairActionJson | string;
 export type ActionObjectJson = Exclude<ActionJson, string>;
 
 // borrowed from https://github.com/jabacchetta/set-random-interval/blob/master/src/index.ts

src/ConfigBuilder.ts

@@ -31,7 +31,11 @@ import {
     CacheOptions,
     BotInstanceJsonConfig,
     BotInstanceConfig,
-    RequiredWebRedditCredentials, RedditCredentials, BotCredentialsJsonConfig, BotCredentialsConfig
+    RequiredWebRedditCredentials,
+    RedditCredentials,
+    BotCredentialsJsonConfig,
+    BotCredentialsConfig,
+    FilterCriteriaDefaults, TypedActivityStates
 } from "./Common/interfaces";
 import {isRuleSetJSON, RuleSetJson, RuleSetObjectJson} from "./Rule/RuleSet";
 import deepEqual from "fast-deep-equal";
@@ -42,8 +46,9 @@ import {GetEnvVars} from 'env-cmd';
 import {operatorConfig} from "./Utils/CommandConfig";
 import merge from 'deepmerge';
 import * as process from "process";
-import {cacheOptDefaults, cacheTTLDefaults} from "./Common/defaults";
+import {cacheOptDefaults, cacheTTLDefaults, filterCriteriaDefault} from "./Common/defaults";
 import objectHash from "object-hash";
+import {AuthorCriteria, AuthorOptions} from "./Author/Author";
 
 export interface ConfigBuilderOptions {
     logger: Logger,
@@ -115,22 +120,45 @@ export class ConfigBuilder {
         return validConfig as JSONConfig;
     }
 
-    parseToStructured(config: JSONConfig): CheckStructuredJson[] {
+    parseToStructured(config: JSONConfig, filterCriteriaDefaultsFromBot?: FilterCriteriaDefaults): CheckStructuredJson[] {
         let namedRules: Map<string, RuleObjectJson> = new Map();
         let namedActions: Map<string, ActionObjectJson> = new Map();
-        const {checks = []} = config;
+        const {checks = [], filterCriteriaDefaults} = config;
         for (const c of checks) {
             const {rules = []} = c;
             namedRules = extractNamedRules(rules, namedRules);
             namedActions = extractNamedActions(c.actions, namedActions);
         }
 
+        const filterDefs = filterCriteriaDefaults ?? filterCriteriaDefaultsFromBot;
+        const {
+            authorIsBehavior = 'merge',
+            itemIsBehavior = 'merge',
+            authorIs: authorIsDefault = {},
+            itemIs: itemIsDefault = []
+        } = filterDefs || {};
+
         const structuredChecks: CheckStructuredJson[] = [];
         for (const c of checks) {
-            const {rules = []} = c;
+            const {rules = [], authorIs = {}, itemIs = []} = c;
             const strongRules = insertNamedRules(rules, namedRules);
             const strongActions = insertNamedActions(c.actions, namedActions);
-            const strongCheck = {...c, rules: strongRules, actions: strongActions} as CheckStructuredJson;
+
+            let derivedAuthorIs: AuthorOptions = authorIsDefault;
+            if(authorIsBehavior === 'merge') {
+                derivedAuthorIs = merge.all([authorIs, authorIsDefault], {arrayMerge: overwriteMerge});
+            } else if(Object.keys(authorIs).length > 0) {
+                derivedAuthorIs = authorIs;
+            }
+
+            let derivedItemIs: TypedActivityStates = itemIsDefault;
+            if(itemIsBehavior === 'merge') {
+                derivedItemIs = [...itemIs, ...itemIsDefault];
+            } else if(itemIs.length > 0) {
+                derivedItemIs = itemIs;
+            }
+
+            const strongCheck = {...c, authorIs: derivedAuthorIs, itemIs: derivedItemIs, rules: strongRules, actions: strongActions} as CheckStructuredJson;
             structuredChecks.push(strongCheck);
         }
 
@@ -146,10 +174,6 @@ export const buildPollingOptions = (values: (string | PollingOptions)[]): Pollin
                 pollOn: v as PollOn,
                 interval: DEFAULT_POLLING_INTERVAL,
                 limit: DEFAULT_POLLING_LIMIT,
-                clearProcessed: {
-                    size: DEFAULT_POLLING_LIMIT,
-                    retain: DEFAULT_POLLING_LIMIT,
-                }
             });
         } else {
             const {
@@ -157,14 +181,12 @@ export const buildPollingOptions = (values: (string | PollingOptions)[]): Pollin
                 interval = DEFAULT_POLLING_INTERVAL,
                 limit = DEFAULT_POLLING_LIMIT,
                 delayUntil,
-                clearProcessed = {size: limit, retain: limit},
             } = v;
             opts.push({
                 pollOn: p as PollOn,
                 interval,
                 limit,
                 delayUntil,
-                clearProcessed
             });
         }
     }
@@ -281,8 +303,6 @@ export const parseDefaultBotInstanceFromArgs = (args: any): BotInstanceJsonConfi
         heartbeat,
         hardLimit,
         authorTTL,
-        snooProxy,
-        snooDebug,
         sharedMod,
         caching,
     } = args || {};
@@ -294,10 +314,6 @@ export const parseDefaultBotInstanceFromArgs = (args: any): BotInstanceJsonConfi
             accessToken,
             refreshToken,
         },
-        snoowrap: {
-            proxy: snooProxy,
-            debug: snooDebug,
-        },
         subreddits: {
             names: subreddits,
             wikiConfig,
@@ -305,7 +321,7 @@ export const parseDefaultBotInstanceFromArgs = (args: any): BotInstanceJsonConfi
             heartbeatInterval: heartbeat,
         },
         polling: {
-            sharedMod,
+            shared: sharedMod ? ['unmoderated','modqueue'] : undefined,
         },
         nanny: {
             softLimit,
@@ -330,6 +346,8 @@ export const parseOpConfigFromArgs = (args: any): OperatorJsonConfig => {
         mode,
         caching,
         authorTTL,
+        snooProxy,
+        snooDebug,
     } = args || {};
 
     const data = {
@@ -346,6 +364,10 @@ export const parseOpConfigFromArgs = (args: any): OperatorJsonConfig => {
             provider: caching,
             authorTTL
         },
+        snoowrap: {
+            proxy: snooProxy,
+            debug: snooDebug,
+        },
         web: {
             enabled: web,
             port,
@@ -401,12 +423,8 @@ export const parseDefaultBotInstanceFromEnv = (): BotInstanceJsonConfig => {
             dryRun: parseBool(process.env.DRYRUN, undefined),
             heartbeatInterval: process.env.HEARTBEAT !== undefined ? parseInt(process.env.HEARTBEAT) : undefined,
         },
-        snoowrap: {
-            proxy: process.env.PROXY,
-            debug: parseBool(process.env.SNOO_DEBUG, undefined),
-        },
         polling: {
-            sharedMod: parseBool(process.env.SHARE_MOD),
+            shared: parseBool(process.env.SHARE_MOD) ? ['unmoderated','modqueue'] : undefined,
         },
         nanny: {
             softLimit: process.env.SOFT_LIMIT !== undefined ? parseInt(process.env.SOFT_LIMIT) : undefined,
@@ -435,6 +453,10 @@ export const parseOpConfigFromEnv = (): OperatorJsonConfig => {
             },
             authorTTL: process.env.AUTHOR_TTL !== undefined ? parseInt(process.env.AUTHOR_TTL) : undefined
         },
+        snoowrap: {
+            proxy: process.env.PROXY,
+            debug: parseBool(process.env.SNOO_DEBUG, undefined),
+        },
         web: {
             port: process.env.PORT !== undefined ? parseInt(process.env.PORT) : undefined,
             session: {
@@ -507,6 +529,16 @@ export const parseOperatorConfigFromSources = async (args: any): Promise<Operato
         }
         try {
             configFromFile = validateJson(rawConfig, operatorSchema, initLogger) as OperatorJsonConfig;
+            const {bots = []} = configFromFile || {};
+            for(const b of bots) {
+                const {polling: {
+                    sharedMod
+                } = {}} = b;
+                if(sharedMod !== undefined) {
+                    initLogger.warn(`'sharedMod' bot config property is DEPRECATED and will be removed in next minor version. Use 'shared' property instead (see docs)`);
+                    break;
+                }
+            }
         } catch (err: any) {
             initLogger.error('Cannot continue app startup because operator config file was not valid.');
             throw err;
@@ -568,6 +600,7 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
             credentials: webCredentials,
             operators,
         } = {},
+        snoowrap: snoowrapOp = {},
         api: {
             port: apiPort = 8095,
             secret: apiSecret = randomId(),
@@ -626,8 +659,10 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
     let hydratedBots: BotInstanceConfig[] = bots.map(x => {
         const {
             name: botName,
+            filterCriteriaDefaults = filterCriteriaDefault,
             polling: {
-                sharedMod = false,
+                sharedMod,
+                shared = [],
                 stagger,
                 limit = 100,
                 interval = 30,
@@ -640,7 +675,7 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
                 softLimit = 250,
                 hardLimit = 50
             } = {},
-            snoowrap = {},
+            snoowrap = snoowrapOp,
             credentials = {},
             subreddits: {
                 names = [],
@@ -742,9 +777,16 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
             botCache.provider.prefix = buildCachePrefix([botCache.provider.prefix, 'bot', (botName || objectHash.sha1(botCreds))]);
         }
 
+        let realShared = shared === true ? ['unmoderated','modqueue','newComm','newSub'] : shared;
+        if(sharedMod === true) {
+            realShared.push('unmoderated');
+            realShared.push('modqueue');
+        }
+
         return {
             name: botName,
             snoowrap,
+            filterCriteriaDefaults,
             subreddits: {
                 names,
                 exclude,
@@ -755,7 +797,7 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
             credentials: botCreds,
             caching: botCache,
             polling: {
-                sharedMod,
+                shared: [...new Set(realShared)] as PollOn[],
                 stagger,
                 limit,
                 interval,

src/Rule/RecentActivityRule.ts

@@ -1,6 +1,7 @@
 import {Rule, RuleJSONConfig, RuleOptions, RulePremise, RuleResult} from "./index";
-import {Comment, VoteableContent} from "snoowrap";
+import {VoteableContent} from "snoowrap";
 import Submission from "snoowrap/dist/objects/Submission";
+import Comment from "snoowrap/dist/objects/Comment";
 import as from 'async';
 import pMap from 'p-map';
 // @ts-ignore
@@ -23,7 +24,7 @@ import {
     parseSubredditName,
     parseUsableLinkIdentifier,
     PASS, sleep,
-    toStrongSubredditState
+    toStrongSubredditState, windowToActivityWindowCriteria
 } from "../util";
 import {
     ActivityWindow,
@@ -43,7 +44,7 @@ const parseLink = parseUsableLinkIdentifier();
 export class RecentActivityRule extends Rule {
     window: ActivityWindowType;
     thresholds: ActivityThreshold[];
-    useSubmissionAsReference: boolean;
+    useSubmissionAsReference: boolean | undefined;
     imageDetection: StrongImageDetection
     lookAt?: 'comments' | 'submissions';
 
@@ -51,7 +52,7 @@ export class RecentActivityRule extends Rule {
         super(options);
         const {
             window = 15,
-            useSubmissionAsReference = true,
+            useSubmissionAsReference,
             imageDetection,
             lookAt,
         } = options || {};
@@ -115,20 +116,53 @@ export class RecentActivityRule extends Rule {
     async process(item: Submission | Comment): Promise<[boolean, RuleResult]> {
         let activities;
 
+        // ACID is a bitch
+        // reddit may not return the activity being checked in the author's recent history due to availability/consistency issues or *something*
+        // so make sure we add it in if config is checking the same type and it isn't included
+        // TODO refactor this for SubredditState everywhere branch
+        let shouldIncludeSelf = true;
+        const strongWindow = windowToActivityWindowCriteria(this.window);
+        const {
+            subreddits: {
+                include = [],
+                exclude = []
+            } = {}
+        } = strongWindow;
+        if (include.length > 0 && !include.some(x => x.toLocaleLowerCase() === item.subreddit.display_name.toLocaleLowerCase())) {
+            shouldIncludeSelf = false;
+        } else if (exclude.length > 0 && exclude.some(x => x.toLocaleLowerCase() === item.subreddit.display_name.toLocaleLowerCase())) {
+            shouldIncludeSelf = false;
+        }
+
         switch (this.lookAt) {
             case 'comments':
                 activities = await this.resources.getAuthorComments(item.author, {window: this.window});
+                if (shouldIncludeSelf && item instanceof Comment && !activities.some(x => x.name === item.name)) {
+                    activities.unshift(item);
+                }
                 break;
             case 'submissions':
                 activities = await this.resources.getAuthorSubmissions(item.author, {window: this.window});
+                if (shouldIncludeSelf && item instanceof Submission && !activities.some(x => x.name === item.name)) {
+                    activities.unshift(item);
+                }
                 break;
             default:
                 activities = await this.resources.getAuthorActivities(item.author, {window: this.window});
+                if (shouldIncludeSelf && !activities.some(x => x.name === item.name)) {
+                    activities.unshift(item);
+                }
                 break;
         }
 
         let viableActivity = activities;
-        if (this.useSubmissionAsReference) {
+        // if config does not specify reference then we set the default based on whether the item is a submission or not
+        // -- this is essentially the same as defaulting reference to true BUT eliminates noisy "can't use comment as reference" log statement when item is a comment
+        let inferredSubmissionAsRef = this.useSubmissionAsReference;
+        if(inferredSubmissionAsRef === undefined) {
+            inferredSubmissionAsRef = isSubmission(item);
+        }
+        if (inferredSubmissionAsRef) {
             if (!asSubmission(item)) {
                 this.logger.warn('Cannot use post as reference because triggered item is not a Submission');
             } else if (item.is_self) {
@@ -310,34 +344,6 @@ export class RecentActivityRule extends Rule {
                 }
             }
 
-            for (const activity of viableActivity) {
-                if (asSubmission(activity) && submissionState !== undefined) {
-                    if (!(await this.resources.testItemCriteria(activity, [submissionState]))) {
-                        continue;
-                    }
-                } else if (commentState !== undefined) {
-                    if (!(await this.resources.testItemCriteria(activity, [commentState]))) {
-                        continue;
-                    }
-                }
-                let inSubreddits = false;
-                for (const ss of subStates) {
-                    const res = await this.resources.testSubredditCriteria(activity, ss);
-                    if (res) {
-                        inSubreddits = true;
-                        break;
-                    }
-                }
-                if (inSubreddits) {
-                    currCount++;
-                    combinedKarma += activity.score;
-                    const pSub = getActivitySubredditName(activity);
-                    if (!presentSubs.includes(pSub)) {
-                        presentSubs.push(pSub);
-                    }
-                }
-            }
-
             const {operator, value, isPercent} = parseGenericValueOrPercentComparison(threshold);
             let sum = {
                 subsWithActivity: presentSubs,
@@ -421,6 +427,7 @@ export class RecentActivityRule extends Rule {
                 threshold,
                 testValue,
                 karmaThreshold,
+                combinedKarma,
             }
         };
     }
@@ -501,6 +508,16 @@ interface RecentActivityConfig extends ActivityWindow, ReferenceSubmission {
     thresholds: ActivityThreshold[],
 
     imageDetection?: ImageDetection
+
+    /**
+     * When Activity is a submission should we only include activities that are other submissions with the same content?
+     *
+     * * When the Activity is a submission this defaults to **true**
+     * * When the Activity is a comment it is ignored (not relevant)
+     *
+     * @default true
+     * */
+    useSubmissionAsReference?: boolean
 }
 
 export interface RecentActivityRuleOptions extends RecentActivityConfig, RuleOptions {

src/Rule/RegexRule.ts

@@ -152,7 +152,8 @@ export class RegexRule extends Rule {
             }, []);
 
             // check regex
-            const reg = parseStringToRegex(regex, 'g');
+            const regexContent = await this.resources.getContent(regex);
+            const reg = parseStringToRegex(regexContent, 'g');
             if(reg === undefined) {
                 throw new SimpleError(`Value given for regex on Criteria ${name} was not valid: ${regex}`);
             }
@@ -257,7 +258,7 @@ export class RegexRule extends Rule {
             const critResults = {
                 criteria: {
                     name,
-                    regex,
+                    regex: regex !== regexContent ? `${regex} from ${regexContent}` : regex,
                     testOn,
                     matchThreshold,
                     activityMatchThreshold,

src/Rule/index.ts

@@ -2,7 +2,7 @@ import Snoowrap, {Comment} from "snoowrap";
 import Submission from "snoowrap/dist/objects/Submission";
 import {Logger} from "winston";
 import {findResultByPremise, mergeArr} from "../util";
-import {SubredditResources} from "../Subreddit/SubredditResources";
+import {checkAuthorFilter, SubredditResources} from "../Subreddit/SubredditResources";
 import {ChecksActivityState, TypedActivityStates} from "../Common/interfaces";
 import Author, {AuthorOptions} from "../Author/Author";
 
@@ -65,6 +65,7 @@ export abstract class Rule implements IRule, Triggerable {
             name = this.getKind(),
             logger,
             authorIs: {
+                excludeCondition = 'OR',
                 include = [],
                 exclude = [],
             } = {},
@@ -78,6 +79,7 @@ export abstract class Rule implements IRule, Triggerable {
         this.client = client;
 
         this.authorIs = {
+            excludeCondition,
             exclude: exclude.map(x => new Author(x)),
             include: include.map(x => new Author(x)),
         }
@@ -99,23 +101,10 @@ export abstract class Rule implements IRule, Triggerable {
                 this.logger.verbose(`(Skipped) Item did not pass 'itemIs' test`);
                 return Promise.resolve([null, this.getResult(null, {result: `Item did not pass 'itemIs' test`})]);
             }
-            if (this.authorIs.include !== undefined && this.authorIs.include.length > 0) {
-                for (const auth of this.authorIs.include) {
-                    if (await this.resources.testAuthorCriteria(item, auth)) {
-                        return this.process(item);
-                    }
-                }
-                this.logger.verbose('(Skipped) Inclusive author criteria not matched');
-                return Promise.resolve([null, this.getResult(null, {result: 'Inclusive author criteria not matched'})]);
-            }
-            if (this.authorIs.exclude !== undefined && this.authorIs.exclude.length > 0) {
-                for (const auth of this.authorIs.exclude) {
-                    if (await this.resources.testAuthorCriteria(item, auth, false)) {
-                        return this.process(item);
-                    }
-                }
-                this.logger.verbose('(Skipped) Exclusive author criteria not matched');
-                return Promise.resolve([null, this.getResult(null, {result: 'Exclusive author criteria not matched'})]);
+            const [authFilterResult, authFilterType] = await checkAuthorFilter(item, this.authorIs, this.resources, this.logger);
+            if(!authFilterResult) {
+                this.logger.verbose(`(Skipped) ${authFilterType} Author criteria not matched`);
+                return Promise.resolve([null, this.getResult(null, {result: `${authFilterType} author criteria not matched`})]);
             }
         } catch (err: any) {
             this.logger.error('Error occurred during Rule pre-process checks');

src/Schema/Action.json

@@ -131,12 +131,21 @@
             ],
             "properties": {
                 "exclude": {
-                    "description": "Only runs if `include` is not present. Will \"pass\" if any of set of the AuthorCriteria **does not** pass",
+                    "description": "Only runs if `include` is not present. Each AuthorCriteria is comprised of conditions that the Author being checked must \"not\" pass. See excludeCondition for set behavior\n\nEX: `isMod: true, name: Automoderator` => Will pass if the Author IS NOT a mod and IS NOT named Automoderator",
                     "items": {
                         "$ref": "#/definitions/AuthorCriteria"
                     },
                     "type": "array"
                 },
+                "excludeCondition": {
+                    "default": "OR",
+                    "description": "* OR => if ANY exclude condition \"does not\" pass then the exclude test passes\n* AND => if ALL exclude conditions \"do not\" pass then the exclude test passes\n\nDefaults to OR",
+                    "enum": [
+                        "AND",
+                        "OR"
+                    ],
+                    "type": "string"
+                },
                 "include": {
                     "description": "Will \"pass\" if any set of AuthorCriteria passes",
                     "items": {
@@ -189,7 +198,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
@@ -263,7 +272,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
@@ -391,6 +400,7 @@
                 "message",
                 "remove",
                 "report",
+                "userflair",
                 "usernote"
             ],
             "type": "string"

src/Schema/App.json

@@ -586,12 +586,21 @@
             ],
             "properties": {
                 "exclude": {
-                    "description": "Only runs if `include` is not present. Will \"pass\" if any of set of the AuthorCriteria **does not** pass",
+                    "description": "Only runs if `include` is not present. Each AuthorCriteria is comprised of conditions that the Author being checked must \"not\" pass. See excludeCondition for set behavior\n\nEX: `isMod: true, name: Automoderator` => Will pass if the Author IS NOT a mod and IS NOT named Automoderator",
                     "items": {
                         "$ref": "#/definitions/AuthorCriteria"
                     },
                     "type": "array"
                 },
+                "excludeCondition": {
+                    "default": "OR",
+                    "description": "* OR => if ANY exclude condition \"does not\" pass then the exclude test passes\n* AND => if ALL exclude conditions \"do not\" pass then the exclude test passes\n\nDefaults to OR",
+                    "enum": [
+                        "AND",
+                        "OR"
+                    ],
+                    "type": "string"
+                },
                 "include": {
                     "description": "Will \"pass\" if any set of AuthorCriteria passes",
                     "items": {
@@ -856,6 +865,17 @@
                     ],
                     "description": "The cache provider and, optionally, a custom configuration for that provider\n\nIf not present or `null` provider will be `memory`.\n\nTo specify another `provider` but use its default configuration set this property to a string of one of the available providers: `memory`, `redis`, or `none`"
                 },
+                "selfTTL": {
+                    "default": 50,
+                    "description": "Amount of time, in seconds, an Activity that the bot has acted on or created will be ignored if found during polling\n\nThis is useful to prevent the bot from checking Activities it *just* worked on or a product of the checks. Examples:\n\n* Ignore comments created through an Action\n* Ignore Activity polled from modqueue that the bot just reported\n\nThis value should be at least as long as the longest polling interval for modqueue/newComm\n\n* If `0` or `true` will cache indefinitely (not recommended)\n* If `false` will not cache",
+                    "examples": [
+                        50
+                    ],
+                    "type": [
+                        "number",
+                        "boolean"
+                    ]
+                },
                 "submissionTTL": {
                     "default": 60,
                     "description": "Amount of time, in seconds, a submission should be cached\n\n* If `0` or `true` will cache indefinitely (not recommended)\n* If `false` will not cache",
@@ -970,25 +990,6 @@
             ],
             "type": "string"
         },
-        "ClearProcessedOptions": {
-            "description": "For very long-running, high-volume subreddits clearing the list of processed activities helps manage memory bloat\n\nAll of these options have default values based on the limit and/or interval set for polling options on each subreddit stream. They only need to modified if the defaults are not sufficient.\n\nIf both `after` and `size` are defined whichever is hit first will trigger the list to clear. `after` will be reset after ever clear.",
-            "properties": {
-                "after": {
-                    "description": "An interval the processed list should be cleared after.\n\n* EX `9 days`\n* EX `3 months`\n* EX `5 minutes`",
-                    "pattern": "^\\s*(?<time>\\d+)\\s*(?<unit>days?|weeks?|months?|years?|hours?|minutes?|seconds?|milliseconds?)\\s*$",
-                    "type": "string"
-                },
-                "retain": {
-                    "description": "The number of activities to retain in processed list after clearing.\n\nDefaults to `limit` value from `PollingOptions`",
-                    "type": "number"
-                },
-                "size": {
-                    "description": "Number of activities found in processed list after which the list should be cleared.\n\nDefaults to the `limit` value from `PollingOptions`",
-                    "type": "number"
-                }
-            },
-            "type": "object"
-        },
         "CommentActionJson": {
             "description": "Reply to the Activity. For a submission the reply will be a top-level comment.",
             "properties": {
@@ -1118,6 +1119,9 @@
                             {
                                 "$ref": "#/definitions/FlairActionJson"
                             },
+                            {
+                                "$ref": "#/definitions/UserFlairActionJson"
+                            },
                             {
                                 "$ref": "#/definitions/CommentActionJson"
                             },
@@ -1319,7 +1323,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
@@ -1422,6 +1426,61 @@
             },
             "type": "object"
         },
+        "FilterCriteriaDefaults": {
+            "properties": {
+                "authorIs": {
+                    "$ref": "#/definitions/AuthorOptions",
+                    "description": "Determine how authorIs defaults behave when authorIs is present on the check\n\n* merge => merges defaults with check's authorIs\n* replace => check authorIs will replace defaults (no defaults used)",
+                    "examples": [
+                        {
+                            "include": [
+                                {
+                                    "flairText": [
+                                        "Contributor",
+                                        "Veteran"
+                                    ]
+                                },
+                                {
+                                    "isMod": true
+                                }
+                            ]
+                        }
+                    ]
+                },
+                "authorIsBehavior": {
+                    "enum": [
+                        "merge",
+                        "replace"
+                    ],
+                    "type": "string"
+                },
+                "itemIs": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "$ref": "#/definitions/SubmissionState"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "items": {
+                                "$ref": "#/definitions/CommentState"
+                            },
+                            "type": "array"
+                        }
+                    ]
+                },
+                "itemIsBehavior": {
+                    "description": "Determine how itemIs defaults behave when itemIs is present on the check\n\n* merge => adds defaults to check's itemIs\n* replace => check itemIs will replace defaults (no defaults used)",
+                    "enum": [
+                        "merge",
+                        "replace"
+                    ],
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
         "FlairActionJson": {
             "description": "Flair the Submission",
             "properties": {
@@ -1465,6 +1524,10 @@
                     ],
                     "type": "boolean"
                 },
+                "flair_template_id": {
+                    "description": "Flair template ID to assign",
+                    "type": "string"
+                },
                 "itemIs": {
                     "anyOf": [
                         {
@@ -2092,10 +2155,6 @@
                 }
             ],
             "properties": {
-                "clearProcessed": {
-                    "$ref": "#/definitions/ClearProcessedOptions",
-                    "description": "For very long-running, high-volume subreddits clearing the list of processed activities helps manage memory bloat\n\nAll of these options have default values based on the limit and/or interval set for polling options on each subreddit stream. They only need to modified if the defaults are not sufficient.\n\nIf both `after` and `size` are defined whichever is hit first will trigger the list to clear. `after` will be reset after ever clear."
-                },
                 "delayUntil": {
                     "description": "Delay processing Activity until it is `N` seconds old\n\nUseful if there are other bots that may process an Activity and you want this bot to run first/last/etc.\n\nIf the Activity is already `N` seconds old when it is initially retrieved no refresh of the Activity occurs (no API request is made) and it is immediately processed.",
                     "type": "number"
@@ -2215,7 +2274,7 @@
                 },
                 "useSubmissionAsReference": {
                     "default": true,
-                    "description": "If activity is a Submission and is a link (not self-post) then only look at Submissions that contain this link, otherwise consider all activities.",
+                    "description": "When Activity is a submission should we only include activities that are other submissions with the same content?\n\n* When the Activity is a submission this defaults to **true**\n* When the Activity is a comment it is ignored (not relevant)",
                     "type": "boolean"
                 },
                 "window": {
@@ -3192,6 +3251,9 @@
                             {
                                 "$ref": "#/definitions/FlairActionJson"
                             },
+                            {
+                                "$ref": "#/definitions/UserFlairActionJson"
+                            },
                             {
                                 "$ref": "#/definitions/CommentActionJson"
                             },
@@ -3400,7 +3462,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
@@ -3493,6 +3555,95 @@
             ],
             "type": "string"
         },
+        "UserFlairActionJson": {
+            "description": "Flair the Submission",
+            "properties": {
+                "authorIs": {
+                    "$ref": "#/definitions/AuthorOptions",
+                    "description": "If present then these Author criteria are checked before running the Action. If criteria fails then the Action is not run.",
+                    "examples": [
+                        {
+                            "include": [
+                                {
+                                    "flairText": [
+                                        "Contributor",
+                                        "Veteran"
+                                    ]
+                                },
+                                {
+                                    "isMod": true
+                                }
+                            ]
+                        }
+                    ]
+                },
+                "css": {
+                    "description": "The text of the css class of the flair to apply",
+                    "type": "string"
+                },
+                "dryRun": {
+                    "default": false,
+                    "description": "If `true` the Action will not make the API request to Reddit to perform its action.",
+                    "examples": [
+                        false,
+                        true
+                    ],
+                    "type": "boolean"
+                },
+                "enable": {
+                    "default": true,
+                    "description": "If set to `false` the Action will not be run",
+                    "examples": [
+                        true
+                    ],
+                    "type": "boolean"
+                },
+                "flair_template_id": {
+                    "description": "Flair template to pick.\n\n**Note:** If this template is used text/css are ignored",
+                    "type": "string"
+                },
+                "itemIs": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "$ref": "#/definitions/SubmissionState"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "items": {
+                                "$ref": "#/definitions/CommentState"
+                            },
+                            "type": "array"
+                        }
+                    ],
+                    "description": "A list of criteria to test the state of the `Activity` against before running the Action.\n\nIf any set of criteria passes the Action will be run."
+                },
+                "kind": {
+                    "description": "The type of action that will be performed",
+                    "enum": [
+                        "userflair"
+                    ],
+                    "type": "string"
+                },
+                "name": {
+                    "description": "An optional, but highly recommended, friendly name for this Action. If not present will default to `kind`.\n\nCan only contain letters, numbers, underscore, spaces, and dashes",
+                    "examples": [
+                        "myDescriptiveAction"
+                    ],
+                    "pattern": "^[a-zA-Z]([\\w -]*[\\w])?$",
+                    "type": "string"
+                },
+                "text": {
+                    "description": "The text of the flair to apply",
+                    "type": "string"
+                }
+            },
+            "required": [
+                "kind"
+            ],
+            "type": "object"
+        },
         "UserNoteActionJson": {
             "description": "Add a Toolbox User Note to the Author of this Activity",
             "properties": {
@@ -3688,6 +3839,10 @@
             ],
             "type": "boolean"
         },
+        "filterCriteriaDefaults": {
+            "$ref": "#/definitions/FilterCriteriaDefaults",
+            "description": "Set the default filter criteria for all checks. If this property is specified it will override any defaults passed from the bot's config\n\nDefault behavior is to exclude all mods and automoderator from checks"
+        },
         "footer": {
             "anyOf": [
                 {

src/Schema/OperatorConfig.json

@@ -1,6 +1,161 @@
 {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "definitions": {
+        "AuthorCriteria": {
+            "additionalProperties": false,
+            "description": "Criteria with which to test against the author of an Activity. The outcome of the test is based on:\n\n1. All present properties passing and\n2. If a property is a list then any value from the list matching",
+            "examples": [
+                {
+                    "flairText": [
+                        "Contributor",
+                        "Veteran"
+                    ],
+                    "isMod": true,
+                    "name": [
+                        "FoxxMD",
+                        "AnotherUser"
+                    ]
+                }
+            ],
+            "minProperties": 1,
+            "properties": {
+                "age": {
+                    "description": "Test the age of the Author's account (when it was created) against this comparison\n\nThe syntax is `(< OR > OR <= OR >=) <number> <unit>`\n\n* EX `> 100 days` => Passes if Author's account is older than 100 days\n* EX `<= 2 months` => Passes if Author's account is younger than or equal to 2 months\n\nUnit must be one of [DayJS Duration units](https://day.js.org/docs/en/durations/creating)\n\n[See] https://regexr.com/609n8 for example",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(days?|weeks?|months?|years?|hours?|minutes?|seconds?|milliseconds?)\\s*$",
+                    "type": "string"
+                },
+                "commentKarma": {
+                    "description": "A string containing a comparison operator and a value to compare karma against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 100`  => greater than 100 comment karma\n* EX `<= 75%` => comment karma is less than or equal to 75% of **all karma**",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "description": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "An (array of) string/regular expression to test contents of an Author's profile description against\n\nIf no flags are specified then the **insensitive** flag is used by default\n\nIf using an array then if **any** value in the array passes the description test passes",
+                    "examples": [
+                        [
+                            "/test$/i",
+                            "look for this string literal"
+                        ]
+                    ]
+                },
+                "flairCssClass": {
+                    "description": "A list of (user) flair css class values from the subreddit to match against",
+                    "examples": [
+                        "red"
+                    ],
+                    "items": {
+                        "type": "string"
+                    },
+                    "type": "array"
+                },
+                "flairText": {
+                    "description": "A list of (user) flair text values from the subreddit to match against",
+                    "examples": [
+                        "Approved"
+                    ],
+                    "items": {
+                        "type": "string"
+                    },
+                    "type": "array"
+                },
+                "isMod": {
+                    "description": "Is the author a moderator?",
+                    "type": "boolean"
+                },
+                "linkKarma": {
+                    "description": "A string containing a comparison operator and a value to compare link karma against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 100`  => greater than 100 link karma\n* EX `<= 75%` => link karma is less than or equal to 75% of **all karma**",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "name": {
+                    "description": "A list of reddit usernames (case-insensitive) to match against. Do not include the \"u/\" prefix\n\n EX to match against /u/FoxxMD and /u/AnotherUser use [\"FoxxMD\",\"AnotherUser\"]",
+                    "examples": [
+                        "FoxxMD",
+                        "AnotherUser"
+                    ],
+                    "items": {
+                        "type": "string"
+                    },
+                    "type": "array"
+                },
+                "shadowBanned": {
+                    "description": "Is the author shadowbanned?\n\nThis is determined by trying to retrieve the author's profile. If a 404 is returned it is likely they are shadowbanned",
+                    "type": "boolean"
+                },
+                "totalKarma": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "userNotes": {
+                    "description": "A list of UserNote properties to check against the User Notes attached to this Author in this Subreddit (must have Toolbox enabled and used User Notes at least once)",
+                    "items": {
+                        "$ref": "#/definitions/UserNoteCriteria"
+                    },
+                    "type": "array"
+                },
+                "verified": {
+                    "description": "Does Author's account have a verified email?",
+                    "type": "boolean"
+                }
+            },
+            "type": "object"
+        },
+        "AuthorOptions": {
+            "description": "If present then these Author criteria are checked before running the rule. If criteria fails then the rule is skipped.",
+            "examples": [
+                {
+                    "include": [
+                        {
+                            "flairText": [
+                                "Contributor",
+                                "Veteran"
+                            ]
+                        },
+                        {
+                            "isMod": true
+                        }
+                    ]
+                }
+            ],
+            "properties": {
+                "exclude": {
+                    "description": "Only runs if `include` is not present. Each AuthorCriteria is comprised of conditions that the Author being checked must \"not\" pass. See excludeCondition for set behavior\n\nEX: `isMod: true, name: Automoderator` => Will pass if the Author IS NOT a mod and IS NOT named Automoderator",
+                    "items": {
+                        "$ref": "#/definitions/AuthorCriteria"
+                    },
+                    "type": "array"
+                },
+                "excludeCondition": {
+                    "default": "OR",
+                    "description": "* OR => if ANY exclude condition \"does not\" pass then the exclude test passes\n* AND => if ALL exclude conditions \"do not\" pass then the exclude test passes\n\nDefaults to OR",
+                    "enum": [
+                        "AND",
+                        "OR"
+                    ],
+                    "type": "string"
+                },
+                "include": {
+                    "description": "Will \"pass\" if any set of AuthorCriteria passes",
+                    "items": {
+                        "$ref": "#/definitions/AuthorCriteria"
+                    },
+                    "type": "array"
+                }
+            },
+            "type": "object"
+        },
         "BotConnection": {
             "description": "Configuration required to connect to a CM Server",
             "properties": {
@@ -58,6 +213,10 @@
                         }
                     ]
                 },
+                "filterCriteriaDefaults": {
+                    "$ref": "#/definitions/FilterCriteriaDefaults",
+                    "description": "Define the default behavior for all filter criteria on all checks in all subreddits\n\nDefaults to exclude mods and automoderator from checks"
+                },
                 "name": {
                     "type": "string"
                 },
@@ -94,13 +253,36 @@
                         },
                         {
                             "properties": {
+                                "shared": {
+                                    "anyOf": [
+                                        {
+                                            "items": {
+                                                "enum": [
+                                                    "modqueue",
+                                                    "newComm",
+                                                    "newSub",
+                                                    "unmoderated"
+                                                ],
+                                                "type": "string"
+                                            },
+                                            "type": "array"
+                                        },
+                                        {
+                                            "enum": [
+                                                true
+                                            ],
+                                            "type": "boolean"
+                                        }
+                                    ],
+                                    "description": "Set which polling sources should be shared among subreddits using default polling settings for that source\n\n* For `unmoderated and `modqueue` the bot will poll on **r/mod** for new activities\n* For `newSub` and `newComm` all subreddits sharing the source will be combined to poll like **r/subreddit1+subreddit2/new**\n\nIf set to `true` all polling sources will be shared,  otherwise specify which sourcs should be shared as a list"
+                                },
                                 "sharedMod": {
                                     "default": false,
-                                    "description": "If set to `true` all subreddits polling unmoderated/modqueue with default polling settings will share a request to \"r/mod\"\notherwise each subreddit will poll its own mod view\n\n* ENV => `SHARE_MOD`\n* ARG => `--shareMod`",
+                                    "description": "DEPRECATED: See `shared`\n\n Using the ENV or ARG will sett `unmoderated` and `modqueue` on `shared`\n\n* ENV => `SHARE_MOD`\n* ARG => `--shareMod`",
                                     "type": "boolean"
                                 },
                                 "stagger": {
-                                    "description": "If sharing a mod stream stagger pushing relevant Activities to individual subreddits.\n\nUseful when running many subreddits and rules are potentially cpu/memory/traffic heavy -- allows spreading out load",
+                                    "description": "If sharing a stream staggers pushing relevant Activities to individual subreddits.\n\nUseful when running many subreddits and rules are potentially cpu/memory/traffic heavy -- allows spreading out load",
                                     "type": "number"
                                 }
                             },
@@ -124,21 +306,8 @@
                     "type": "object"
                 },
                 "snoowrap": {
-                    "description": "Settings to control some [Snoowrap](https://github.com/not-an-aardvark/snoowrap) behavior",
-                    "properties": {
-                        "debug": {
-                            "description": "Manually set the debug status for snoowrap\n\nWhen snoowrap has `debug: true` it will log the http status response of reddit api requests to at the `debug` level\n\n* Set to `true` to always output\n* Set to `false` to never output\n\nIf not present or `null` will be set based on `logLevel`\n\n* ENV => `SNOO_DEBUG`\n* ARG => `--snooDebug`",
-                            "type": "boolean"
-                        },
-                        "proxy": {
-                            "description": "Proxy all requests to Reddit's API through this endpoint\n\n* ENV => `PROXY`\n* ARG => `--proxy <proxyEndpoint>`",
-                            "examples": [
-                                "http://localhost:4443"
-                            ],
-                            "type": "string"
-                        }
-                    },
-                    "type": "object"
+                    "$ref": "#/definitions/SnoowrapOptions",
+                    "description": "Settings to control some [Snoowrap](https://github.com/not-an-aardvark/snoowrap) behavior.\n\nOverrides any defaults provided at top-level operator config.\n\nSet to an empty object to \"ignore\" any top-level config"
                 },
                 "subreddits": {
                     "description": "Settings related to bot behavior for subreddits it is managing",
@@ -266,6 +435,73 @@
             ],
             "type": "string"
         },
+        "CommentState": {
+            "description": "Different attributes a `Comment` can be in. Only include a property if you want to check it.",
+            "examples": [
+                {
+                    "op": true,
+                    "removed": false
+                }
+            ],
+            "properties": {
+                "age": {
+                    "description": "A duration and how to compare it against a value\n\nThe syntax is `(< OR > OR <= OR >=) <number> <unit>` EX `> 100 days`, `<= 2 months`\n\n* EX `> 100 days` => Passes if the date being compared is before 100 days ago\n* EX `<= 2 months` => Passes if the date being compared is after or equal to 2 months\n\nUnit must be one of [DayJS Duration units](https://day.js.org/docs/en/durations/creating)\n\n[See] https://regexr.com/609n8 for example",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(days|weeks|months|years|hours|minutes|seconds|milliseconds)\\s*$",
+                    "type": "string"
+                },
+                "approved": {
+                    "type": "boolean"
+                },
+                "deleted": {
+                    "type": "boolean"
+                },
+                "depth": {
+                    "description": "The (nested) level of a comment.\n\n* 0 mean the comment is at top-level (replying to submission)\n* non-zero, Nth value means the comment has N parent comments",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(days|weeks|months|years|hours|minutes|seconds|milliseconds)\\s*$",
+                    "type": "string"
+                },
+                "distinguished": {
+                    "type": "boolean"
+                },
+                "filtered": {
+                    "type": "boolean"
+                },
+                "locked": {
+                    "type": "boolean"
+                },
+                "op": {
+                    "description": "Is this Comment Author also the Author of the Submission this comment is in?",
+                    "type": "boolean"
+                },
+                "removed": {
+                    "type": "boolean"
+                },
+                "reports": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "score": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "spam": {
+                    "type": "boolean"
+                },
+                "stickied": {
+                    "type": "boolean"
+                },
+                "submissionState": {
+                    "description": "A list of SubmissionState attributes to test the Submission this comment is in",
+                    "items": {
+                        "$ref": "#/definitions/SubmissionState"
+                    },
+                    "type": "array"
+                }
+            },
+            "type": "object"
+        },
         "DiscordProviderConfig": {
             "properties": {
                 "name": {
@@ -288,6 +524,61 @@
             ],
             "type": "object"
         },
+        "FilterCriteriaDefaults": {
+            "properties": {
+                "authorIs": {
+                    "$ref": "#/definitions/AuthorOptions",
+                    "description": "Determine how authorIs defaults behave when authorIs is present on the check\n\n* merge => merges defaults with check's authorIs\n* replace => check authorIs will replace defaults (no defaults used)",
+                    "examples": [
+                        {
+                            "include": [
+                                {
+                                    "flairText": [
+                                        "Contributor",
+                                        "Veteran"
+                                    ]
+                                },
+                                {
+                                    "isMod": true
+                                }
+                            ]
+                        }
+                    ]
+                },
+                "authorIsBehavior": {
+                    "enum": [
+                        "merge",
+                        "replace"
+                    ],
+                    "type": "string"
+                },
+                "itemIs": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "$ref": "#/definitions/SubmissionState"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "items": {
+                                "$ref": "#/definitions/CommentState"
+                            },
+                            "type": "array"
+                        }
+                    ]
+                },
+                "itemIsBehavior": {
+                    "description": "Determine how itemIs defaults behave when itemIs is present on the check\n\n* merge => adds defaults to check's itemIs\n* replace => check itemIs will replace defaults (no defaults used)",
+                    "enum": [
+                        "merge",
+                        "replace"
+                    ],
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
         "NotificationConfig": {
             "properties": {
                 "events": {
@@ -414,6 +705,17 @@
                     ],
                     "description": "The cache provider and, optionally, a custom configuration for that provider\n\nIf not present or `null` provider will be `memory`.\n\nTo specify another `provider` but use its default configuration set this property to a string of one of the available providers: `memory`, `redis`, or `none`"
                 },
+                "selfTTL": {
+                    "default": 50,
+                    "description": "Amount of time, in seconds, an Activity that the bot has acted on or created will be ignored if found during polling\n\nThis is useful to prevent the bot from checking Activities it *just* worked on or a product of the checks. Examples:\n\n* Ignore comments created through an Action\n* Ignore Activity polled from modqueue that the bot just reported\n\nThis value should be at least as long as the longest polling interval for modqueue/newComm\n\n* If `0` or `true` will cache indefinitely (not recommended)\n* If `false` will not cache",
+                    "examples": [
+                        50
+                    ],
+                    "type": [
+                        "number",
+                        "boolean"
+                    ]
+                },
                 "submissionTTL": {
                     "default": 60,
                     "description": "Amount of time, in seconds, a submission should be cached\n\n* If `0` or `true` will cache indefinitely (not recommended)\n* If `false` will not cache",
@@ -529,6 +831,96 @@
             },
             "type": "object"
         },
+        "SnoowrapOptions": {
+            "properties": {
+                "debug": {
+                    "description": "Manually set the debug status for snoowrap\n\nWhen snoowrap has `debug: true` it will log the http status response of reddit api requests to at the `debug` level\n\n* Set to `true` to always output\n* Set to `false` to never output\n\nIf not present or `null` will be set based on `logLevel`\n\n* ENV => `SNOO_DEBUG`\n* ARG => `--snooDebug`",
+                    "type": "boolean"
+                },
+                "proxy": {
+                    "description": "Proxy all requests to Reddit's API through this endpoint\n\n* ENV => `PROXY`\n* ARG => `--proxy <proxyEndpoint>`",
+                    "examples": [
+                        "http://localhost:4443"
+                    ],
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
+        "SubmissionState": {
+            "description": "Different attributes a `Submission` can be in. Only include a property if you want to check it.",
+            "examples": [
+                {
+                    "over_18": true,
+                    "removed": false
+                }
+            ],
+            "properties": {
+                "age": {
+                    "description": "A duration and how to compare it against a value\n\nThe syntax is `(< OR > OR <= OR >=) <number> <unit>` EX `> 100 days`, `<= 2 months`\n\n* EX `> 100 days` => Passes if the date being compared is before 100 days ago\n* EX `<= 2 months` => Passes if the date being compared is after or equal to 2 months\n\nUnit must be one of [DayJS Duration units](https://day.js.org/docs/en/durations/creating)\n\n[See] https://regexr.com/609n8 for example",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(days|weeks|months|years|hours|minutes|seconds|milliseconds)\\s*$",
+                    "type": "string"
+                },
+                "approved": {
+                    "type": "boolean"
+                },
+                "deleted": {
+                    "type": "boolean"
+                },
+                "distinguished": {
+                    "type": "boolean"
+                },
+                "filtered": {
+                    "type": "boolean"
+                },
+                "is_self": {
+                    "type": "boolean"
+                },
+                "link_flair_css_class": {
+                    "type": "string"
+                },
+                "link_flair_text": {
+                    "type": "string"
+                },
+                "locked": {
+                    "type": "boolean"
+                },
+                "over_18": {
+                    "description": "NSFW",
+                    "type": "boolean"
+                },
+                "pinned": {
+                    "type": "boolean"
+                },
+                "removed": {
+                    "type": "boolean"
+                },
+                "reports": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "score": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "spam": {
+                    "type": "boolean"
+                },
+                "spoiler": {
+                    "type": "boolean"
+                },
+                "stickied": {
+                    "type": "boolean"
+                },
+                "title": {
+                    "description": "A valid regular expression to match against the title of the submission",
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
         "ThirdPartyCredentialsJsonConfig": {
             "additionalProperties": {
             },
@@ -547,6 +939,43 @@
             },
             "type": "object"
         },
+        "UserNoteCriteria": {
+            "properties": {
+                "count": {
+                    "default": ">= 1",
+                    "description": "Number of occurrences of this type. Ignored if `search` is `current`\n\nA string containing a comparison operator and/or a value to compare number of occurrences against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign] [ascending|descending]`",
+                    "examples": [
+                        ">= 1"
+                    ],
+                    "pattern": "^\\s*(?<opStr>>|>=|<|<=)\\s*(?<value>\\d+)\\s*(?<percent>%?)\\s*(?<extra>asc.*|desc.*)*$",
+                    "type": "string"
+                },
+                "search": {
+                    "default": "current",
+                    "description": "How to test the notes for this Author:\n\n### current\n\nOnly the most recent note is checked for `type`\n\n### total\n\nThe `count` comparison of `type` must be found within all notes\n\n* EX `count: > 3`   => Must have more than 3 notes of `type`, total\n* EX `count: <= 25%` => Must have 25% or less of notes of `type`, total\n\n### consecutive\n\nThe `count` **number** of `type` notes must be found in a row.\n\nYou may also specify the time-based order in which to search the notes by specifying `ascending (asc)` or `descending (desc)` in the `count` value. Default is `descending`\n\n* EX `count: >= 3` => Must have 3 or more notes of `type` consecutively, in descending order\n* EX `count: < 2`  => Must have less than 2 notes of `type` consecutively, in descending order\n* EX `count: > 4 asc` => Must have greater than 4 notes of `type` consecutively, in ascending order",
+                    "enum": [
+                        "consecutive",
+                        "current",
+                        "total"
+                    ],
+                    "examples": [
+                        "current"
+                    ],
+                    "type": "string"
+                },
+                "type": {
+                    "description": "User Note type key to search for",
+                    "examples": [
+                        "spamwarn"
+                    ],
+                    "type": "string"
+                }
+            },
+            "required": [
+                "type"
+            ],
+            "type": "object"
+        },
         "WebCredentials": {
             "description": "Separate credentials for the web interface can be provided when also running the api.\n\nAll properties not specified will default to values given in ENV/ARG credential properties\n\nRefer to the [required credentials table](https://github.com/FoxxMD/context-mod/blob/master/docs/operatorConfiguration.md#minimum-required-configuration) to see what is necessary for the web interface.",
             "examples": [
@@ -694,6 +1123,10 @@
             },
             "type": "object"
         },
+        "snoowrap": {
+            "$ref": "#/definitions/SnoowrapOptions",
+            "description": "Set global snoowrap options as well as default snoowrap config for all bots that don't specify their own"
+        },
         "web": {
             "description": "Settings for the web interface",
             "properties": {

src/Schema/Rule.json

@@ -535,12 +535,21 @@
             ],
             "properties": {
                 "exclude": {
-                    "description": "Only runs if `include` is not present. Will \"pass\" if any of set of the AuthorCriteria **does not** pass",
+                    "description": "Only runs if `include` is not present. Each AuthorCriteria is comprised of conditions that the Author being checked must \"not\" pass. See excludeCondition for set behavior\n\nEX: `isMod: true, name: Automoderator` => Will pass if the Author IS NOT a mod and IS NOT named Automoderator",
                     "items": {
                         "$ref": "#/definitions/AuthorCriteria"
                     },
                     "type": "array"
                 },
+                "excludeCondition": {
+                    "default": "OR",
+                    "description": "* OR => if ANY exclude condition \"does not\" pass then the exclude test passes\n* AND => if ALL exclude conditions \"do not\" pass then the exclude test passes\n\nDefaults to OR",
+                    "enum": [
+                        "AND",
+                        "OR"
+                    ],
+                    "type": "string"
+                },
                 "include": {
                     "description": "Will \"pass\" if any set of AuthorCriteria passes",
                     "items": {
@@ -666,7 +675,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
@@ -1148,7 +1157,7 @@
                 },
                 "useSubmissionAsReference": {
                     "default": true,
-                    "description": "If activity is a Submission and is a link (not self-post) then only look at Submissions that contain this link, otherwise consider all activities.",
+                    "description": "When Activity is a submission should we only include activities that are other submissions with the same content?\n\n* When the Activity is a submission this defaults to **true**\n* When the Activity is a comment it is ignored (not relevant)",
                     "type": "boolean"
                 },
                 "window": {
@@ -1936,7 +1945,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },

src/Schema/RuleSet.json

@@ -509,12 +509,21 @@
             ],
             "properties": {
                 "exclude": {
-                    "description": "Only runs if `include` is not present. Will \"pass\" if any of set of the AuthorCriteria **does not** pass",
+                    "description": "Only runs if `include` is not present. Each AuthorCriteria is comprised of conditions that the Author being checked must \"not\" pass. See excludeCondition for set behavior\n\nEX: `isMod: true, name: Automoderator` => Will pass if the Author IS NOT a mod and IS NOT named Automoderator",
                     "items": {
                         "$ref": "#/definitions/AuthorCriteria"
                     },
                     "type": "array"
                 },
+                "excludeCondition": {
+                    "default": "OR",
+                    "description": "* OR => if ANY exclude condition \"does not\" pass then the exclude test passes\n* AND => if ALL exclude conditions \"do not\" pass then the exclude test passes\n\nDefaults to OR",
+                    "enum": [
+                        "AND",
+                        "OR"
+                    ],
+                    "type": "string"
+                },
                 "include": {
                     "description": "Will \"pass\" if any set of AuthorCriteria passes",
                     "items": {
@@ -640,7 +649,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
@@ -1122,7 +1131,7 @@
                 },
                 "useSubmissionAsReference": {
                     "default": true,
-                    "description": "If activity is a Submission and is a link (not self-post) then only look at Submissions that contain this link, otherwise consider all activities.",
+                    "description": "When Activity is a submission should we only include activities that are other submissions with the same content?\n\n* When the Activity is a submission this defaults to **true**\n* When the Activity is a comment it is ignored (not relevant)",
                     "type": "boolean"
                 },
                 "window": {
@@ -1910,7 +1919,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },

src/Subreddit/Manager.ts

@@ -18,17 +18,15 @@ import {
     totalFromMapStats,
     triggeredIndicator,
 } from "../util";
-import {Poll} from "snoostorm";
-import pEvent from "p-event";
 import {RuleResult} from "../Rule";
 import {ConfigBuilder, buildPollingOptions} from "../ConfigBuilder";
 import {
     ActionedEvent,
     ActionResult,
     DEFAULT_POLLING_INTERVAL,
-    DEFAULT_POLLING_LIMIT, Invokee,
+    DEFAULT_POLLING_LIMIT, FilterCriteriaDefaults, Invokee,
     ManagerOptions, ManagerStateChangeOption, ManagerStats, PAUSED,
-    PollingOptionsStrong, ResourceStats, RUNNING, RunState, STOPPED, SYSTEM, USER
+    PollingOptionsStrong, PollOn, RUNNING, RunState, STOPPED, SYSTEM, USER
 } from "../Common/interfaces";
 import Submission from "snoowrap/dist/objects/Submission";
 import {activityIsRemoved, itemContentPeek} from "../Utils/SnoowrapUtils";
@@ -48,7 +46,6 @@ import {queue, QueueObject} from 'async';
 import {JSONConfig} from "../JsonConfig";
 import {CheckStructuredJson} from "../Check";
 import NotificationManager from "../Notification/NotificationManager";
-import action from "../Web/Server/routes/authenticated/user/action";
 import {createHistoricalDefaults, historicalDefaults} from "../Common/defaults";
 import {ExtendedSnoowrap} from "../Utils/SnoowrapClients";
 import {isRateLimitError, isStatusError} from "../Utils/Errors";
@@ -63,6 +60,7 @@ export interface runCheckOptions {
     delayUntil?: number,
     dryRun?: boolean,
     refresh?: boolean,
+    force?: boolean,
 }
 
 export interface CheckTask {
@@ -72,7 +70,7 @@ export interface CheckTask {
 }
 
 export interface RuntimeManagerOptions extends ManagerOptions {
-    sharedModqueue?: boolean;
+    sharedStreams?: PollOn[];
     wikiLocation?: string;
     botName: string;
     maxWorkers: number;
@@ -97,13 +95,14 @@ export class Manager extends EventEmitter {
     lastWikiRevision?: DayjsObj
     lastWikiCheck: DayjsObj = dayjs();
     wikiFormat: ('yaml' | 'json') = 'yaml';
+    filterCriteriaDefaults?: FilterCriteriaDefaults
     //wikiUpdateRunning: boolean = false;
 
-    streamListedOnce: string[] = [];
-    streams: SPoll<Snoowrap.Submission | Snoowrap.Comment>[] = [];
-    modStreamCallbacks: Map<string, any> = new Map();
+    streams: Map<string, SPoll<Snoowrap.Submission | Snoowrap.Comment>> = new Map();
+    sharedStreamCallbacks: Map<string, any> = new Map();
+    pollingRetryHandler: Function;
     dryRun?: boolean;
-    sharedModqueue: boolean;
+    sharedStreams: PollOn[];
     cacheManager: BotResourcesManager;
     globalDryRun?: boolean;
     queue: QueueObject<CheckTask>;
@@ -197,7 +196,7 @@ export class Manager extends EventEmitter {
     constructor(sub: Subreddit, client: ExtendedSnoowrap, logger: Logger, cacheManager: BotResourcesManager, opts: RuntimeManagerOptions = {botName: 'ContextMod', maxWorkers: 1}) {
         super();
 
-        const {dryRun, sharedModqueue = false, wikiLocation = 'botconfig/contextbot', botName, maxWorkers} = opts;
+        const {dryRun, sharedStreams = [], wikiLocation = 'botconfig/contextbot', botName, maxWorkers, filterCriteriaDefaults} = opts;
         this.displayLabel = opts.nickname || `${sub.display_name_prefixed}`;
         const getLabels = this.getCurrentLabels;
         const getDisplay = this.getDisplay;
@@ -213,7 +212,9 @@ export class Manager extends EventEmitter {
         }, mergeArr);
         this.globalDryRun = dryRun;
         this.wikiLocation = wikiLocation;
-        this.sharedModqueue = sharedModqueue;
+        this.filterCriteriaDefaults = filterCriteriaDefaults;
+        this.sharedStreams = sharedStreams;
+        this.pollingRetryHandler = createRetryHandler({maxRequestRetry: 3, maxOtherRetry: 2}, this.logger);
         this.subreddit = sub;
         this.client = client;
         this.botName = botName;
@@ -272,6 +273,7 @@ export class Manager extends EventEmitter {
         if(this.modPermissions !== undefined) {
             return this.modPermissions as string[];
         }
+        this.logger.debug('Retrieving mod permissions for bot');
         const userInfo = parseRedditEntity(this.botName, 'user');
         const mods = this.subreddit.getModerators({name: userInfo.name});
         // @ts-ignore
@@ -357,8 +359,7 @@ export class Manager extends EventEmitter {
         return q;
     }
 
-    protected async parseConfigurationFromObject(configObj: object) {
-        await this.getModPermissions();
+    protected async parseConfigurationFromObject(configObj: object, suppressChangeEvent: boolean = false) {
         try {
             const configBuilder = new ConfigBuilder({logger: this.logger});
             const validJson = configBuilder.validateJson(configObj);
@@ -418,7 +419,7 @@ export class Manager extends EventEmitter {
 
             const commentChecks: Array<CommentCheck> = [];
             const subChecks: Array<SubmissionCheck> = [];
-            const structuredChecks = configBuilder.parseToStructured(validJson);
+            const structuredChecks = configBuilder.parseToStructured(validJson, this.filterCriteriaDefaults);
 
             // TODO check that bot has permissions for subreddit for all specified actions
             // can find permissions in this.subreddit.mod_permissions
@@ -448,6 +449,19 @@ export class Manager extends EventEmitter {
                 this.logger.info(checkSummary);
             }
             this.validConfigLoaded = true;
+            if(!suppressChangeEvent) {
+                this.emit('configChange');
+            }
+            if(this.eventsState.state === RUNNING) {
+                // need to update polling, potentially
+                await this.buildPolling();
+                for(const stream of this.streams.values()) {
+                    if(!stream.running) {
+                        this.logger.debug(`Starting Polling for ${stream.name.toUpperCase()} ${stream.frequency / 1000}s interval`);
+                        stream.startInterval();
+                    }
+                }
+            }
         } catch (err: any) {
             this.validConfigLoaded = false;
             throw err;
@@ -455,7 +469,7 @@ export class Manager extends EventEmitter {
     }
 
     async parseConfiguration(causedBy: Invokee = 'system', force: boolean = false, options?: ManagerStateChangeOption) {
-        const {reason, suppressNotification = false} = options || {};
+        const {reason, suppressNotification = false, suppressChangeEvent = false} = options || {};
         //this.wikiUpdateRunning = true;
         this.lastWikiCheck = dayjs();
 
@@ -563,7 +577,7 @@ export class Manager extends EventEmitter {
                 throw new ConfigParseError('Could not parse wiki page contents as JSON or YAML')
             }
 
-            await this.parseConfigurationFromObject(configObj);
+            await this.parseConfigurationFromObject(configObj, suppressChangeEvent);
             this.logger.info('Checks updated');
 
             if(!suppressNotification) {
@@ -581,6 +595,18 @@ export class Manager extends EventEmitter {
         const checks = checkType === 'Comment' ? this.commentChecks : this.submissionChecks;
         let item = activity;
         const itemId = await item.id;
+
+        if(await this.resources.hasRecentSelf(item)) {
+            const {force = false} = options || {};
+            let recentMsg = `Found in Activities recently (last ${this.resources.selfTTL} seconds) modified/created by this bot`;
+            if(force) {
+                this.logger.debug(`${recentMsg} but will run anyway because "force" option was true.`);
+            } else {
+                this.logger.debug(`${recentMsg} so will skip running.`);
+                return;
+            }
+        }
+
         let allRuleResults: RuleResult[] = [];
         const itemIdentifier = `${checkType === 'Submission' ? 'SUB' : 'COM'} ${itemId}`;
         this.currentLabels = [itemIdentifier];
@@ -690,6 +716,7 @@ export class Manager extends EventEmitter {
                     if (e.logged !== true) {
                         this.logger.warn(`Running rules for Check ${check.name} failed due to uncaught exception`, e);
                     }
+                    this.emit('error', e);
                 }
 
                 if (triggered) {
@@ -702,6 +729,11 @@ export class Manager extends EventEmitter {
                         actionedEvent.ruleSummary = resultsSummary(currentResults, check.condition);
                     }
                     runActions = await check.runActions(item, currentResults.filter(x => x.triggered), dryRun);
+                    // we only can about report and comment actions since those can produce items for newComm and modqueue
+                    const recentCandidates = runActions.filter(x => ['report','comment'].includes(x.kind.toLocaleLowerCase())).map(x => x.touchedEntities === undefined ? [] : x.touchedEntities).flat();
+                    for(const recent of recentCandidates) {
+                        await this.resources.setRecentSelf(recent as (Submission|Comment));
+                    }
                     actionsRun = runActions.length;
 
                     if(check.notifyOnTrigger) {
@@ -749,132 +781,197 @@ export class Manager extends EventEmitter {
         }
     }
 
-    async buildPolling() {
-        // give current handle() time to stop
-        //await sleep(1000);
+    isPollingShared(streamName: string): boolean {
+        const pollOption = this.pollOptions.find(x => x.pollOn === streamName);
+        return pollOption !== undefined && pollOption.limit === DEFAULT_POLLING_LIMIT && pollOption.interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(streamName as PollOn);
+    }
 
-        const retryHandler = createRetryHandler({maxRequestRetry: 3, maxOtherRetry: 1}, this.logger);
+    async buildPolling() {
+
+        const sources: PollOn[] = ['unmoderated', 'modqueue', 'newComm', 'newSub'];
 
         const subName = this.subreddit.display_name;
 
-        for (const pollOpt of this.pollOptions) {
-            const {
-                pollOn,
-                limit,
-                interval,
-                delayUntil,
-                clearProcessed,
-            } = pollOpt;
-            let stream: SPoll<Snoowrap.Submission | Snoowrap.Comment>;
-            let modStreamType: string | undefined;
+        for (const source of sources) {
 
-            switch (pollOn) {
-                case 'unmoderated':
-                    if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedModqueue) {
-                        modStreamType = 'unmoderated';
-                        // use default mod stream from resources
-                        stream = this.cacheManager.modStreams.get('unmoderated') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
-                    } else {
-                        stream = new UnmoderatedStream(this.client, {
-                            subreddit: this.subreddit.display_name,
-                            limit: limit,
-                            pollTime: interval * 1000,
-                            clearProcessed,
-                        });
-                    }
-                    break;
-                case 'modqueue':
-                    if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL) {
-                        modStreamType = 'modqueue';
-                        // use default mod stream from resources
-                        stream = this.cacheManager.modStreams.get('modqueue') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
-                    } else {
-                        stream = new ModQueueStream(this.client, {
-                            subreddit: this.subreddit.display_name,
-                            limit: limit,
-                            pollTime: interval * 1000,
-                            clearProcessed
-                        });
-                    }
-                    break;
-                case 'newSub':
-                    stream = new SubmissionStream(this.client, {
-                        subreddit: this.subreddit.display_name,
-                        limit: limit,
-                        pollTime: interval * 1000,
-                        clearProcessed
-                    });
-                    break;
-                case 'newComm':
-                    stream = new CommentStream(this.client, {
-                        subreddit: this.subreddit.display_name,
-                        limit: limit,
-                        pollTime: interval * 1000,
-                        clearProcessed
-                    });
-                    break;
+            if (!sources.includes(source)) {
+                this.logger.error(`'${source}' is not a valid polling source. Valid sources: unmoderated | modqueue | newComm | newSub`);
+                continue;
             }
 
-            stream.once('listing', async (listing) => {
-                if (!this.streamListedOnce.includes(pollOn)) {
-                    // warning if poll event could potentially miss activities
-                    if (this.commentChecks.length === 0 && ['unmoderated', 'modqueue', 'newComm'].some(x => x === pollOn)) {
-                        this.logger.warn(`Polling '${pollOn}' may return Comments but no comments checks were configured.`);
-                    }
-                    if (this.submissionChecks.length === 0 && ['unmoderated', 'modqueue', 'newSub'].some(x => x === pollOn)) {
-                        this.logger.warn(`Polling '${pollOn}' may return Submissions but no submission checks were configured.`);
-                    }
-                    this.streamListedOnce.push(pollOn);
+            const pollOpt = this.pollOptions.find(x => x.pollOn.toLowerCase() === source.toLowerCase());
+            if (pollOpt === undefined) {
+                if(this.sharedStreamCallbacks.has(source)) {
+                    this.logger.debug(`Removing listener for shared polling on ${source.toUpperCase()} because it no longer exists in config`);
+                    this.sharedStreamCallbacks.delete(source);
                 }
-            });
-
-            const onItem = async (item: Comment | Submission) => {
-                if (!this.streamListedOnce.includes(pollOn)) {
-                    return;
+                const existingStream = this.streams.get(source);
+                if (existingStream !== undefined) {
+                    this.logger.debug(`Stopping polling on ${source.toUpperCase()} because it no longer exists in config`);
+                    existingStream.end();
+                    this.streams.delete(source);
                 }
-                if (item.subreddit.display_name !== subName || this.eventsState.state !== RUNNING) {
-                    return;
-                }
-                let checkType: 'Submission' | 'Comment' | undefined;
-                if (item instanceof Submission) {
-                    if (this.submissionChecks.length > 0) {
-                        checkType = 'Submission';
-                    }
-                } else if (this.commentChecks.length > 0) {
-                    checkType = 'Comment';
-                }
-                if (checkType !== undefined) {
-                    this.firehose.push({checkType, activity: item, options: {delayUntil}})
-                }
-            };
-
-            if (modStreamType !== undefined) {
-                this.modStreamCallbacks.set(pollOn, onItem);
             } else {
-                stream.on('item', onItem);
-                // @ts-ignore
-                stream.on('error', async (err: any) => {
 
-                    this.emit('error', err);
+                const {
+                    limit,
+                    interval,
+                    delayUntil,
+                } = pollOpt;
+                let stream: SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                let modStreamType: string | undefined;
 
-                    if(isRateLimitError(err)) {
-                        this.logger.error('Encountered rate limit while polling! Bot is all out of requests :( Stopping subreddit queue and polling.');
-                        await this.stop();
+                switch (source) {
+                    case 'unmoderated':
+                        if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
+                            modStreamType = 'unmoderated';
+                            // use default mod stream from resources
+                            stream = this.cacheManager.modStreams.get('unmoderated') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                        } else {
+                            stream = new UnmoderatedStream(this.client, {
+                                subreddit: this.subreddit.display_name,
+                                limit: limit,
+                                pollTime: interval * 1000,
+                                logger: this.logger,
+                            });
+                        }
+                        break;
+                    case 'modqueue':
+                        if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
+                            modStreamType = 'modqueue';
+                            // use default mod stream from resources
+                            stream = this.cacheManager.modStreams.get('modqueue') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                        } else {
+                            stream = new ModQueueStream(this.client, {
+                                subreddit: this.subreddit.display_name,
+                                limit: limit,
+                                pollTime: interval * 1000,
+                                logger: this.logger,
+                            });
+                        }
+                        break;
+                    case 'newSub':
+                        if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
+                            modStreamType = 'newSub';
+                            // use default mod stream from resources
+                            stream = this.cacheManager.modStreams.get('newSub') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                        } else {
+                            stream = new SubmissionStream(this.client, {
+                                subreddit: this.subreddit.display_name,
+                                limit: limit,
+                                pollTime: interval * 1000,
+                                logger: this.logger,
+                            });
+                        }
+                        break;
+                    case 'newComm':
+                        if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
+                            modStreamType = 'newComm';
+                            // use default mod stream from resources
+                            stream = this.cacheManager.modStreams.get('newComm') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                        } else {
+                            stream = new CommentStream(this.client, {
+                                subreddit: this.subreddit.display_name,
+                                limit: limit,
+                                pollTime: interval * 1000,
+                                logger: this.logger,
+                            });
+                        }
+                        break;
+                }
+
+                if (stream === undefined) {
+                    this.logger.error(`Should have found polling source for '${source}' but it did not exist for some reason!`);
+                    continue;
+                }
+
+                const onItem = async (item: Comment | Submission) => {
+                    if (item.subreddit.display_name !== subName || this.eventsState.state !== RUNNING) {
+                        return;
                     }
-                    this.logger.error('Polling error occurred', err);
-                    const shouldRetry = await retryHandler(err);
-                    if (shouldRetry) {
-                        stream.startInterval();
+                    let checkType: 'Submission' | 'Comment' | undefined;
+                    if (item instanceof Submission) {
+                        if (this.submissionChecks.length > 0) {
+                            checkType = 'Submission';
+                        }
+                    } else if (this.commentChecks.length > 0) {
+                        checkType = 'Comment';
+                    }
+                    if (checkType !== undefined) {
+                        this.firehose.push({checkType, activity: item, options: {delayUntil}})
+                    }
+                };
+
+                if (modStreamType !== undefined) {
+                    let removedOwn = false;
+                    const existingStream = this.streams.get(source);
+                    if(existingStream !== undefined) {
+                        existingStream.end();
+                        this.streams.delete(source);
+                        removedOwn = true;
+                    }
+                    if(!this.sharedStreamCallbacks.has(source)) {
+                        stream.once('listing', this.noChecksWarning(source));
+                        this.sharedStreamCallbacks.set(source, onItem);
+                        this.logger.debug(`${removedOwn ? 'Stopped own polling and replace with ' : 'Set '}listener on shared polling ${source}`);
+                    }
+                } else {
+                    let ownPollingMsgParts: string[] = [];
+                    let removedShared = false;
+                    if(this.sharedStreamCallbacks.has(source)) {
+                        removedShared = true;
+                        this.sharedStreamCallbacks.delete(source);
+                        ownPollingMsgParts.push('removed shared polling listener');
+                    }
+
+                    const existingStream = this.streams.get(source);
+                    let processed;
+                    if (existingStream !== undefined) {
+                        ownPollingMsgParts.push('replaced existing');
+                        processed = existingStream.processed;
+                        existingStream.end();
                     } else {
-                        this.logger.warn('Stopping subreddit processing/polling due to too many errors');
-                        await this.stop();
+                        ownPollingMsgParts.push('create new');
+                        stream.once('listing', this.noChecksWarning(source));
                     }
-                });
-                this.streams.push(stream);
+
+                    this.logger.debug(`Polling ${source.toUpperCase()} => ${ownPollingMsgParts.join('and')} dedicated stream`);
+
+                    stream.on('item', onItem);
+                    // @ts-ignore
+                    stream.on('error', async (err: any) => {
+
+                        this.emit('error', err);
+
+                        if (isRateLimitError(err)) {
+                            this.logger.error('Encountered rate limit while polling! Bot is all out of requests :( Stopping subreddit queue and polling.');
+                            await this.stop();
+                        }
+                        this.logger.error('Polling error occurred', err);
+                        const shouldRetry = await this.pollingRetryHandler(err);
+                        if (shouldRetry) {
+                            stream.startInterval(false);
+                        } else {
+                            this.logger.warn('Stopping subreddit processing/polling due to too many errors');
+                            await this.stop();
+                        }
+                    });
+
+                    this.streams.set(source, stream);
+                }
             }
         }
     }
 
+    noChecksWarning = (source: PollOn) => (listing: any) => {
+        if (this.commentChecks.length === 0 && ['modqueue', 'newComm'].some(x => x === source)) {
+            this.logger.warn(`Polling '${source.toUpperCase()}' may return Comments but no comments checks were configured.`);
+        }
+        if (this.submissionChecks.length === 0 && ['unmoderated', 'modqueue', 'newSub'].some(x => x === source)) {
+            this.logger.warn(`Polling '${source.toUpperCase()}' may return Submissions but no submission checks were configured.`);
+        }
+    }
+
     startQueue(causedBy: Invokee = 'system', options?: ManagerStateChangeOption) {
         const {reason, suppressNotification = false} = options || {};
         if(this.queueState.state === RUNNING) {
@@ -1000,7 +1097,10 @@ export class Manager extends EventEmitter {
                 this.logger.warn('No submission or comment checks found!');
             }
 
-            for (const s of this.streams) {
+            if (this.streams.size > 0) {
+                this.logger.debug(`Starting own streams => ${[...this.streams.values()].map(x => `${x.name.toUpperCase()} ${x.frequency / 1000}s interval`).join(' | ')}`)
+            }
+            for (const s of this.streams.values()) {
                 s.startInterval();
             }
             this.startedAt = dayjs();
@@ -1025,7 +1125,7 @@ export class Manager extends EventEmitter {
                 state: PAUSED,
                 causedBy
             };
-            for(const s of this.streams) {
+            for(const s of this.streams.values()) {
                 s.end();
             }
             if(causedBy === USER) {
@@ -1042,15 +1142,11 @@ export class Manager extends EventEmitter {
     stopEvents(causedBy: Invokee = 'system', options?: ManagerStateChangeOption) {
         const {reason, suppressNotification = false} = options || {};
         if(this.eventsState.state !== STOPPED) {
-            for (const s of this.streams) {
+            for (const s of this.streams.values()) {
                 s.end();
             }
-            this.streams = [];
-            // for (const [k, v] of this.modStreamCallbacks) {
-            //     const stream = this.cacheManager.modStreams.get(k) as Poll<Snoowrap.Submission | Snoowrap.Comment>;
-            //     stream.removeListener('item', v);
-            // }
-            this.modStreamCallbacks = new Map();
+            this.streams = new Map();
+            this.sharedStreamCallbacks = new Map();
             this.startedAt = undefined;
             this.logger.info(`Events STOPPED by ${causedBy}`);
             this.eventsState = {

src/Subreddit/Streams.ts

@@ -2,20 +2,22 @@ import {Poll, SnooStormOptions} from "snoostorm"
 import Snoowrap from "snoowrap";
 import {EventEmitter} from "events";
 import {PollConfiguration} from "snoostorm/out/util/Poll";
-import {ClearProcessedOptions, DEFAULT_POLLING_INTERVAL} from "../Common/interfaces";
-import dayjs, {Dayjs} from "dayjs";
-import { Duration } from "dayjs/plugin/duration";
-import {parseDuration, setRandomInterval, sleep} from "../util";
+import {DEFAULT_POLLING_INTERVAL} from "../Common/interfaces";
+import {mergeArr, parseDuration, random} from "../util";
+import { Logger } from "winston";
 
 type Awaitable<T> = Promise<T> | T;
 
-interface RCBPollingOptions extends SnooStormOptions {
+interface RCBPollingOptions<T> extends SnooStormOptions {
     subreddit: string,
-    clearProcessed?: ClearProcessedOptions
+    enforceContinuity?: boolean
+    logger: Logger
+    name?: string,
+    processed?: Set<T[keyof T]>
+    label?: string
 }
 
-interface RCBPollConfiguration<T> extends PollConfiguration<T> {
-    clearProcessed?: ClearProcessedOptions
+interface RCBPollConfiguration<T> extends PollConfiguration<T>,RCBPollingOptions<T> {
 }
 
 export class SPoll<T extends object> extends Poll<T> {
@@ -23,89 +25,112 @@ export class SPoll<T extends object> extends Poll<T> {
     getter: () => Awaitable<T[]>;
     frequency;
     running: boolean = false;
-    clearProcessedDuration?: Duration;
-    clearProcessedSize?: number;
-    clearProcessedAfter?: Dayjs;
-    retainProcessed: number = 0;
+    // intention of newStart is to make polling behavior such that only "new" items AFTER polling has started get emitted
+    // -- that is, we don't want to emit the items we immediately fetch on a fresh poll start since they existed "before" polling started
+    newStart: boolean = true;
+    enforceContinuity: boolean;
     randInterval?: { clear: () => void };
+    name: string = 'Reddit Stream';
+    logger: Logger;
+    subreddit: string;
 
     constructor(options: RCBPollConfiguration<T>) {
         super(options);
-        this.identifier = options.identifier;
-        this.getter = options.get;
-        this.frequency = options.frequency;
         const {
-            after,
-            size,
-            retain = 0,
-        } = options.clearProcessed || {};
-        if(after !== undefined) {
-            this.clearProcessedDuration = parseDuration(after);
-        }
-        this.clearProcessedSize = size;
-        this.retainProcessed = retain;
-        if (this.clearProcessedDuration !== undefined) {
-            this.clearProcessedAfter = dayjs().add(this.clearProcessedDuration.asSeconds(), 's');
+            identifier,
+            get,
+            frequency,
+            enforceContinuity = false,
+            logger,
+            name,
+            subreddit,
+            label = 'Polling',
+            processed
+        } = options;
+        this.subreddit = subreddit;
+        this.name = name !== undefined ? name : this.name;
+        this.logger = logger.child({labels: [label, this.name]}, mergeArr)
+        this.identifier = identifier;
+        this.getter = get;
+        this.frequency = frequency;
+        this.enforceContinuity = enforceContinuity;
+
+        // if we pass in processed on init the intention is to "continue" from where the previous stream left off
+        // WITHOUT new start behavior
+        if (processed !== undefined) {
+            this.processed = processed;
+            this.newStart = false;
         }
+
         clearInterval(this.interval);
     }
 
-    startInterval = () => {
-        this.running = true;
-        this.randInterval = setRandomInterval((function (self) {
+    createInterval = () => {
+        this.interval = setTimeout((function (self) {
             return async () => {
                 try {
-                    // DEBUGGING
-                    //
-                    // Removing processed clearing to see if it fixes weird, duplicate/delayed comment processing behavior
-                    //
-                    // clear the tracked, processed activity ids after a set period or number of activities have been processed
-                    // because when RCB is long-running and has streams from high-volume subreddits this list never gets smaller...
-
-                    // so clear if after time period
-                    // if ((self.clearProcessedAfter !== undefined && dayjs().isSameOrAfter(self.clearProcessedAfter))
-                    //     // or clear if processed list is larger than defined max allowable size (default setting, 2 * polling option limit)
-                    //     || (self.clearProcessedSize !== undefined && self.processed.size >= self.clearProcessedSize)) {
-                    //     if (self.retainProcessed === 0) {
-                    //         self.processed = new Set();
-                    //     } else {
-                    //         // retain some processed so we have continuity between processed list resets -- this is default behavior and retains polling option limit # of activities
-                    //         // we can slice from the set here because ID order is guaranteed for Set object so list is oldest -> newest
-                    //         // -- retain last LIMIT number of activities (or all if retain # is larger than list due to user config error)
-                    //         self.processed = new Set(Array.from(self.processed).slice(Math.max(0, self.processed.size - self.retainProcessed)));
-                    //     }
-                    //     // reset time interval if there is one
-                    //     if (self.clearProcessedAfter !== undefined && self.clearProcessedDuration !== undefined) {
-                    //         self.clearProcessedAfter = dayjs().add(self.clearProcessedDuration.asSeconds(), 's');
-                    //     }
-                    // }
-                    const batch = await self.getter();
+                    self.logger.debug('Polling...');
+                    let batch = await self.getter();
                     const newItems: T[] = [];
-                    for (const item of batch) {
-                        const id = item[self.identifier];
-                        if (self.processed.has(id)) continue;
+                    let anyAlreadySeen = false;
+                    let page = 1;
+                    // initial iteration should always run
+                    // but only continue iterating if stream enforces continuity and we've only seen new items so far
+                    while(page === 1 || (self.enforceContinuity && !self.newStart && !anyAlreadySeen)) {
+                        if(page !== 1) {
+                            self.logger.debug(`Did not find any already seen activities and continuity is enforced. This probably means there were more new items than 1 api call can return. Fetching next page (${page})...`);
+                            // @ts-ignore
+                            batch = await batch.fetchMore({amount: 100});
+                        }
+                        for (const item of batch) {
+                            const id = item[self.identifier];
+                            if (self.processed.has(id)) {
+                                anyAlreadySeen = true;
+                                continue;
+                            }
 
-                        // Emit for new items and add it to the list
-                        newItems.push(item);
-                        self.processed.add(id);
-                        self.emit("item", item);
+                            // Emit for new items and add it to the list
+                            newItems.push(item);
+                            self.processed.add(id);
+                            // but don't emit on new start since we are "buffering" already existing activities
+                            if(!self.newStart) {
+                                self.emit("item", item);
+                            }
+                        }
+                        page++;
                     }
-
-                    // Emit the new listing of all new items
-                    self.emit("listing", newItems);
+                    const newItemMsg = `Found ${newItems.length} new items`;
+                    if(self.newStart) {
+                        self.logger.debug(`${newItemMsg} but will ignore all on first start.`);
+                        self.emit("listing", []);
+                    } else {
+                        self.logger.debug(newItemMsg);
+                        // Emit the new listing of all new items
+                        self.emit("listing", newItems);
+                    }
+                    // no longer new start on n+1 interval
+                    self.newStart = false;
+                    // if everything succeeded then create a new timeout
+                    self.createInterval();
                 } catch (err: any) {
                     self.emit('error', err);
-                    self.end();
                 }
             }
-        })(this), this.frequency - 1, this.frequency + 1);
+        })(this), random(this.frequency - 1, this.frequency + 1));
+    }
+
+    // allow controlling newStart state
+    startInterval = (newStartState?: boolean) => {
+        this.running = true;
+        if(newStartState !== undefined) {
+            this.newStart = newStartState;
+        }
+        this.createInterval();
     }
 
     end = () => {
         this.running = false;
-        if(this.randInterval !== undefined) {
-            this.randInterval.clear();
-        }
+        this.newStart = true;
         super.end();
     }
 }
@@ -113,12 +138,13 @@ export class SPoll<T extends object> extends Poll<T> {
 export class UnmoderatedStream extends SPoll<Snoowrap.Submission | Snoowrap.Comment> {
     constructor(
         client: Snoowrap,
-        options: RCBPollingOptions) {
+        options: RCBPollingOptions<Snoowrap.Submission | Snoowrap.Comment>) {
         super({
             frequency: options.pollTime || DEFAULT_POLLING_INTERVAL * 1000,
             get: async () => client.getSubreddit(options.subreddit).getUnmoderated(options),
             identifier: "id",
-            clearProcessed: options.clearProcessed
+            name: 'Unmoderated',
+            ...options,
         });
     }
 }
@@ -126,12 +152,13 @@ export class UnmoderatedStream extends SPoll<Snoowrap.Submission | Snoowrap.Comm
 export class ModQueueStream extends SPoll<Snoowrap.Submission | Snoowrap.Comment> {
     constructor(
         client: Snoowrap,
-        options: RCBPollingOptions) {
+        options: RCBPollingOptions<Snoowrap.Submission | Snoowrap.Comment>) {
         super({
             frequency: options.pollTime || DEFAULT_POLLING_INTERVAL * 1000,
             get: async () => client.getSubreddit(options.subreddit).getModqueue(options),
             identifier: "id",
-            clearProcessed: options.clearProcessed
+            name: 'Modqueue',
+            ...options,
         });
     }
 }
@@ -139,12 +166,13 @@ export class ModQueueStream extends SPoll<Snoowrap.Submission | Snoowrap.Comment
 export class SubmissionStream extends SPoll<Snoowrap.Submission | Snoowrap.Comment> {
     constructor(
         client: Snoowrap,
-        options: RCBPollingOptions) {
+        options: RCBPollingOptions<Snoowrap.Submission | Snoowrap.Comment>) {
         super({
             frequency: options.pollTime || DEFAULT_POLLING_INTERVAL * 1000,
             get: async () => client.getNew(options.subreddit, options),
             identifier: "id",
-            clearProcessed: options.clearProcessed
+            name: 'Submission',
+            ...options,
         });
     }
 }
@@ -152,12 +180,13 @@ export class SubmissionStream extends SPoll<Snoowrap.Submission | Snoowrap.Comme
 export class CommentStream extends SPoll<Snoowrap.Submission | Snoowrap.Comment> {
     constructor(
         client: Snoowrap,
-        options: RCBPollingOptions) {
+        options: RCBPollingOptions<Snoowrap.Submission | Snoowrap.Comment>) {
         super({
             frequency: options.pollTime || DEFAULT_POLLING_INTERVAL * 1000,
             get: async () => client.getNewComments(options.subreddit, options),
             identifier: "id",
-            clearProcessed: options.clearProcessed
+            name: 'Comment',
+            ...options,
         });
     }
 }

src/Subreddit/SubredditResources.ts

@@ -13,12 +13,27 @@ import as from 'async';
 import fetch from 'node-fetch';
 import {
     asSubmission,
-    buildCacheOptionsFromProvider, buildCachePrefix,
-    cacheStats, compareDurationValue, comparisonTextOp, createCacheManager, createHistoricalStatsDisplay,
-    formatNumber, getActivityAuthorName, getActivitySubredditName, isStrongSubredditState,
-    mergeArr, parseDurationComparison,
-    parseExternalUrl, parseGenericValueComparison, parseRedditEntity,
-    parseWikiContext, shouldCacheSubredditStateCriteriaResult, subredditStateIsNameOnly, toStrongSubredditState
+    buildCacheOptionsFromProvider,
+    buildCachePrefix,
+    cacheStats,
+    compareDurationValue,
+    comparisonTextOp,
+    createCacheManager,
+    createHistoricalStatsDisplay, FAIL,
+    fetchExternalUrl,
+    formatNumber,
+    getActivityAuthorName,
+    getActivitySubredditName,
+    isStrongSubredditState,
+    mergeArr,
+    parseDurationComparison,
+    parseExternalUrl,
+    parseGenericValueComparison,
+    parseRedditEntity,
+    parseWikiContext, PASS,
+    shouldCacheSubredditStateCriteriaResult,
+    subredditStateIsNameOnly,
+    toStrongSubredditState
 } from "../util";
 import LoggedError from "../Utils/LoggedError";
 import {
@@ -45,7 +60,7 @@ import {
 import UserNotes from "./UserNotes";
 import Mustache from "mustache";
 import he from "he";
-import {AuthorCriteria} from "../Author/Author";
+import {AuthorCriteria, AuthorOptions} from "../Author/Author";
 import {SPoll} from "./Streams";
 import {Cache} from 'cache-manager';
 import {Submission, Comment, Subreddit} from "snoowrap/dist/objects";
@@ -90,6 +105,7 @@ export class SubredditResources {
     protected submissionTTL: number | false = cacheTTLDefaults.submissionTTL;
     protected commentTTL: number | false = cacheTTLDefaults.commentTTL;
     protected filterCriteriaTTL: number | false = cacheTTLDefaults.filterCriteriaTTL;
+    public selfTTL: number | false = cacheTTLDefaults.selfTTL;
     name: string;
     protected logger: Logger;
     userNotes: UserNotes;
@@ -119,6 +135,7 @@ export class SubredditResources {
                 authorTTL,
                 wikiTTL,
                 filterCriteriaTTL,
+                selfTTL,
                 submissionTTL,
                 commentTTL,
                 subredditTTL,
@@ -144,6 +161,7 @@ export class SubredditResources {
         this.subredditTTL = subredditTTL === true ? 0 : subredditTTL;
         this.wikiTTL = wikiTTL === true ? 0 : wikiTTL;
         this.filterCriteriaTTL = filterCriteriaTTL === true ? 0 : filterCriteriaTTL;
+        this.selfTTL = selfTTL === true ? 0 : selfTTL;
         this.subreddit = subreddit;
         this.thirdPartyCredentials = thirdPartyCredentials;
         this.name = name;
@@ -391,6 +409,50 @@ export class SubredditResources {
         }
     }
 
+    async hasActivity(item: Submission | Comment) {
+        const hash = asSubmission(item) ? `sub-${item.name}` : `comm-${item.name}`;
+        const res = await this.cache.get(hash);
+        return res !== undefined && res !== null;
+    }
+
+    // @ts-ignore
+    async getRecentSelf(item: Submission | Comment): Promise<(Submission | Comment | undefined)> {
+        const hash = asSubmission(item) ? `sub-recentSelf-${item.name}` : `comm-recentSelf-${item.name}`;
+        const res = await this.cache.get(hash);
+        if(res === null) {
+            return undefined;
+        }
+        return res as (Submission | Comment | undefined);
+    }
+
+    async setRecentSelf(item: Submission | Comment) {
+        if(this.selfTTL !== false) {
+            const hash = asSubmission(item) ? `sub-recentSelf-${item.name}` : `comm-recentSelf-${item.name}`;
+            // @ts-ignore
+            await this.cache.set(hash, item, {ttl: this.selfTTL});
+        }
+        return;
+    }
+    /**
+    * Returns true if the activity being checked was recently acted on/created by the bot and has not changed since that time
+    * */
+    async hasRecentSelf(item: Submission | Comment) {
+        const recent = await this.getRecentSelf(item) as (Submission | Comment | undefined);
+        if (recent !== undefined) {
+            return item.num_reports === recent.num_reports;
+
+            // can't really used edited since its only ever updated once with no timestamp
+            // if(item.num_reports !== recent.num_reports) {
+            //     return false;
+            // }
+            // if(!asSubmission(item)) {
+            //     return item.edited === recent.edited;
+            // }
+            // return true;
+        }
+        return false;
+    }
+
     // @ts-ignore
     async getSubreddit(item: Submission | Comment) {
         try {
@@ -546,8 +608,7 @@ export class SubredditResources {
             }
         } else {
             try {
-                const response = await fetch(extUrl as string);
-                wikiContent = await response.text();
+                wikiContent = await fetchExternalUrl(extUrl as string, this.logger);
             } catch (err: any) {
                 const msg = `Error occurred while trying to fetch the url ${extUrl}`;
                 this.logger.error(msg, err);
@@ -734,8 +795,8 @@ export class SubredditResources {
         return await this.isItem(i, activityStates, this.logger);
     }
 
-    async isSubreddit (subreddit: Subreddit, stateCriteria: SubredditState | StrongSubredditState, logger: Logger) {
-        delete stateCriteria.stateDescription;
+    async isSubreddit (subreddit: Subreddit, stateCriteriaRaw: SubredditState | StrongSubredditState, logger: Logger) {
+        const {stateDescription, ...stateCriteria} = stateCriteriaRaw;
 
         if (Object.keys(stateCriteria).length === 0) {
             return true;
@@ -836,13 +897,30 @@ export class SubredditResources {
                                 break;
                             case 'reports':
                                 if (!item.can_mod_post) {
-                                    log.debug(`Cannot test for reports on Activity in a subreddit bot account is not a moderato Activist. Skipping criteria...`);
+                                    log.debug(`Cannot test for reports on Activity in a subreddit bot account is not a moderator of. Skipping criteria...`);
                                     break;
                                 }
                                 const reportCompare = parseGenericValueComparison(crit[k] as string);
-                                if(!comparisonTextOp(item.num_reports, reportCompare.operator, reportCompare.value)) {
+                                let reportType = 'total';
+                                if(reportCompare.extra !== undefined && reportCompare.extra.trim() !== '') {
+                                    const requestedType = reportCompare.extra.toLocaleLowerCase().trim();
+                                    if(requestedType.includes('mod')) {
+                                        reportType = 'mod';
+                                    } else if(requestedType.includes('user')) {
+                                        reportType = 'user';
+                                    } else {
+                                        log.warn(`Did not recognize the report type "${requestedType}" -- can only use "mod" or "user". Will default to TOTAL reports`);
+                                    }
+                                }
+                                let reportNum = item.num_reports;
+                                if(reportType === 'user') {
+                                    reportNum = item.user_reports.length;
+                                } else {
+                                    reportNum = item.mod_reports.length;
+                                }
+                                if(!comparisonTextOp(reportNum, reportCompare.operator, reportCompare.value)) {
                                     // @ts-ignore
-                                    log.debug(`Failed: Expected => ${k}:${crit[k]} | Found => ${k}:${item.num_reports}`)
+                                    log.debug(`Failed: Expected => ${k}:${crit[k]} ${reportType} reports | Found => ${k}:${reportNum} ${reportType} reports`)
                                     return false
                                 }
                                 break;
@@ -1064,6 +1142,7 @@ export class BotResourcesManager {
                 submissionTTL,
                 subredditTTL,
                 filterCriteriaTTL,
+                selfTTL,
                 provider,
                 actionedEventsMax,
                 actionedEventsDefault,
@@ -1080,7 +1159,7 @@ export class BotResourcesManager {
         this.cacheHash = objectHash.sha1(relevantCacheSettings);
         this.defaultCacheConfig = caching;
         this.defaultThirdPartyCredentials = thirdParty;
-        this.ttlDefaults = {authorTTL, userNotesTTL, wikiTTL, commentTTL, submissionTTL, filterCriteriaTTL, subredditTTL};
+        this.ttlDefaults = {authorTTL, userNotesTTL, wikiTTL, commentTTL, submissionTTL, filterCriteriaTTL, subredditTTL, selfTTL};
 
         const options = provider;
         this.cacheType = options.store;
@@ -1212,3 +1291,50 @@ export class BotResourcesManager {
         return;
     }
 }
+
+export const checkAuthorFilter = async (item: (Submission | Comment), filter: AuthorOptions, resources: SubredditResources, logger: Logger): Promise<[boolean, ('inclusive' | 'exclusive' | undefined)]> => {
+    const authLogger = logger.child({labels: ['Author Filter']}, mergeArr);
+    const {
+        include = [],
+        excludeCondition = 'OR',
+        exclude = [],
+    } = filter;
+    let authorPass = null;
+    if (include.length > 0) {
+        for (const auth of include) {
+            if (await resources.testAuthorCriteria(item, auth)) {
+                authLogger.verbose(`${PASS} => Inclusive author criteria matched`);
+                authLogger.debug(`Inclusive is always OR => At least one of ${include.length} matched`);
+                return [true, 'inclusive'];
+            }
+        }
+        authLogger.verbose(`${FAIL} => Inclusive author criteria not matched`);
+        authLogger.debug(`Inclusive is always OR => None of ${include.length} criteria matched`);
+        return [false, 'inclusive'];
+    }
+    if (exclude.length > 0) {
+        for (const auth of exclude) {
+            const excludePass = await resources.testAuthorCriteria(item, auth, false);
+            if (excludePass && excludeCondition === 'OR') {
+                authorPass = true;
+                break;
+            } else if (!excludePass && excludeCondition === 'AND') {
+                authorPass = false;
+                break;
+            }
+        }
+        if (authorPass !== true) {
+            authLogger.verbose(`${FAIL} => Exclusive author criteria not matched`);
+            if(exclude.length > 1) {
+                authLogger.debug(excludeCondition === 'OR' ? `Exclusive OR => No criteria from set of ${exclude.length} matched` : `Exclusive AND => At least one of ${exclude.length} criteria did not match`)
+            }
+            return [false, 'exclusive']
+        }
+        authLogger.verbose(`${PASS} => Exclusive author criteria matched`);
+        if(exclude.length > 1) {
+            authLogger.debug(excludeCondition === 'OR' ? `Exclusive OR => At least 1 in set of ${exclude.length} matched` : `Exclusive AND => All ${exclude.length} matched`)
+        }
+        return [true, 'exclusive'];
+    }
+    return [true, undefined];
+}

src/Utils/Errors.ts

@@ -1,4 +1,4 @@
-import {StatusCodeError} from "../Common/interfaces";
+import {StatusCodeError, RequestError} from "../Common/interfaces";
 
 
 export const isRateLimitError = (err: any) => {
@@ -16,3 +16,7 @@ export const isScopeError = (err: any): boolean => {
 export const isStatusError = (err: any): err is StatusCodeError => {
     return typeof err === 'object' && err.name === 'StatusCodeError' && err.response !== undefined;
 }
+
+export const isRequestError = (err: any): err is RequestError => {
+    return typeof err === 'object' && err.name === 'RequestError' && err.response !== undefined;
+}

src/Web/Client/index.ts

@@ -46,6 +46,7 @@ import {MESSAGE} from "triple-beam";
 import Autolinker from "autolinker";
 import path from "path";
 import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients";
+import ClientUser from "../Common/User/ClientUser";
 
 const emitter = new EventEmitter();
 
@@ -89,19 +90,6 @@ declare module 'express-session' {
     }
 }
 
-// declare global {
-//     namespace Express {
-//         interface User {
-//             name: string
-//             subreddits: string[]
-//             machine?: boolean
-//             isOperator?: boolean
-//             realManagers?: string[]
-//             moderatedManagers?: string[]
-//         }
-//     }
-// }
-
 interface ConnectedUserInfo {
     level?: string,
     user?: string,
@@ -202,8 +190,9 @@ const webClient = async (options: OperatorConfig) => {
         done(null, { subreddits: subreddits.map((x: Subreddit) => x.display_name), isOperator: webOps.includes(user.toLowerCase()), name: user, scope, token, tokenExpiresAt: dayjs().unix() + (60 * 60) });
     });
 
-    passport.deserializeUser(async function (obj, done) {
-        done(null, obj as Express.User);
+    passport.deserializeUser(async function (obj: any, done) {
+        const user = new ClientUser(obj.name, obj.subreddits, {token: obj.token, scope: obj.scope, webOperator: obj.isOperator, tokenExpiresAt: obj.tokenExpiresAt});
+        done(null, user);
         // const data = await webCache.get(`userSession-${obj}`) as object;
         // if (data === undefined) {
         //     done('Not Found');
@@ -236,7 +225,10 @@ const webClient = async (options: OperatorConfig) => {
                 code: code as string,
             });
             const user = await client.getMe().name as string;
-            const subs = await client.getModeratedSubreddits();
+            let subs = await client.getModeratedSubreddits({count: 100});
+            while(!subs.isFinished) {
+                subs = await subs.fetchMore({amount: 100});
+            }
             io.to(req.session.id).emit('authStatus', {canSaveWiki: req.session.scope?.includes('wikiedit')});
             return done(null, {user, subreddits: subs, scope: req.session.scope, token: client.accessToken});
         }
@@ -352,6 +344,9 @@ const webClient = async (options: OperatorConfig) => {
                         msg = `${msg}. ${botAddResult.stored === false ? 'Additionally, the bot was not stored in config so the operator will need to add it manually to persist after a restart.' : ''}`;
                     }
                     data.addResult = msg;
+                    // @ts-ignore
+                    req.session.destroy();
+                    req.logout();
                 }
             }
             return res.render('callback', data);
@@ -414,7 +409,7 @@ const webClient = async (options: OperatorConfig) => {
                     '<div>or as an argument: <span class="font-mono">--operator YourRedditUsername</span></div>'});
         }
         // or if there is an operator and current user is operator
-        if(req.user.isOperator) {
+        if(req.user?.clientData?.webOperator) {
             return next();
         } else {
             return res.render('error', {error: 'You must be an <b>Operator</b> to access this route.'});
@@ -426,7 +421,7 @@ const webClient = async (options: OperatorConfig) => {
             redirectUri,
             clientId,
             clientSecret,
-            token: req.isAuthenticated() && req.user.isOperator ? token : undefined
+            token: req.isAuthenticated() && req.user?.clientData?.webOperator ? token : undefined
         });
     });
 
@@ -623,20 +618,16 @@ const webClient = async (options: OperatorConfig) => {
             return res.status(404).render('error', {error: msg});
         }
 
-        const user = req.user as Express.User;
-
-        const isOperator = instance.operators.includes(user.name);
-        const canAccessBot = isOperator || intersect(user.subreddits, instance.subreddits).length > 0;
-        if (!user.isOperator && !canAccessBot) {
+        if (!req.user?.clientData?.webOperator && !req.user?.canAccessInstance(instance)) {
             return res.status(404).render('error', {error: msg});
         }
 
-        if (req.params.subreddit !== undefined && !isOperator && !user.subreddits.includes(req.params.subreddit)) {
+        if (req.params.subreddit !== undefined && !req.user?.isInstanceOperator(instance) && !req.user?.subreddits.includes(req.params.subreddit)) {
             return res.status(404).render('error', {error: msg});
         }
         req.instance = instance;
         req.session.botId = instance.friendly;
-        if(canAccessBot) {
+        if(req.user?.canAccessInstance(instance)) {
             req.session.authBotId = instance.friendly;
         }
         return next();
@@ -661,15 +652,11 @@ const webClient = async (options: OperatorConfig) => {
             return res.status(404).render('error', {error: msg});
         }
 
-        const user = req.user as Express.User;
-
-        const isOperator = instance.operators.includes(user.name);
-        const canAccessBot = isOperator || intersect(user.subreddits, botInstance.subreddits).length > 0;
-        if (!user.isOperator && !canAccessBot) {
+        if (!req.user?.clientData?.webOperator && !req.user?.canAccessBot(botInstance)) {
             return res.status(404).render('error', {error: msg});
         }
 
-        if (req.params.subreddit !== undefined && !isOperator && !user.subreddits.includes(req.params.subreddit)) {
+        if (req.params.subreddit !== undefined && !req.user?.isInstanceOperator(instance) && !req.user?.subreddits.includes(req.params.subreddit)) {
             return res.status(404).render('error', {error: msg});
         }
         req.bot = botInstance;
@@ -769,12 +756,12 @@ const webClient = async (options: OperatorConfig) => {
         const level = req.session.level;
 
         const shownInstances = cmInstances.reduce((acc: CMInstance[], curr) => {
-            const isBotOperator = curr.operators.map(x => x.toLowerCase()).includes(user.name.toLowerCase());
-            if(user.isOperator) {
+            const isBotOperator = req.user?.isInstanceOperator(curr);
+            if(user?.clientData?.webOperator) {
                 // @ts-ignore
                 return acc.concat({...curr, canAccessLocation: true, isOperator: isBotOperator});
             }
-            if(!isBotOperator && intersect(user.subreddits, curr.subreddits).length === 0) {
+            if(!isBotOperator && !req.user?.canAccessInstance(curr)) {
                 return acc;
             }
             // @ts-ignore
@@ -801,7 +788,7 @@ const webClient = async (options: OperatorConfig) => {
             return res.render('offline', {
                 instances: shownInstances,
                 instanceId: (req.instance as CMInstance).friendly,
-                isOperator: instance.operators.includes((req.user as Express.User).name),
+                isOperator: req.user?.isInstanceOperator(instance),
                 // @ts-ignore
                 logs: filterLogBySubreddit(instanceLogMap, [instance.friendly], {limit, sort, level, allLogName: 'web', allLogsParser: parseInstanceLogInfoName }).get(instance.friendly),
                 logSettings: {
@@ -834,7 +821,7 @@ const webClient = async (options: OperatorConfig) => {
             bots: resp.bots,
             botId: (req.instance as CMInstance).friendly,
             instanceId: (req.instance as CMInstance).friendly,
-            isOperator: instance.operators.includes((req.user as Express.User).name),
+            isOperator: req.user?.isInstanceOperator(instance),
             operators: instance.operators.join(', '),
             operatorDisplay: instance.operatorDisplay,
             logSettings: {
@@ -856,7 +843,7 @@ const webClient = async (options: OperatorConfig) => {
         res.render('config', {
             title: `Configuration Editor`,
             format,
-            canSave: req.user?.scope?.includes('wikiedit') && req.user?.tokenExpiresAt !== undefined && dayjs.unix(req.user?.tokenExpiresAt).isAfter(dayjs())
+            canSave: req.user?.clientData?.scope?.includes('wikiedit') && req.user?.clientData?.tokenExpiresAt !== undefined && dayjs.unix(req.user?.clientData.tokenExpiresAt).isAfter(dayjs())
         });
     });
 
@@ -868,7 +855,7 @@ const webClient = async (options: OperatorConfig) => {
             userAgent,
             clientId,
             clientSecret,
-            accessToken: req.user?.token
+            accessToken: req.user?.clientData?.token
         });
 
         try {
@@ -1021,7 +1008,7 @@ const webClient = async (options: OperatorConfig) => {
             // setup general web log event
             const webLogListener = (log: string) => {
                 const subName = parseSubredditLogName(log);
-                if((subName === undefined || user.isOperator) && isLogLineMinLevel(log, session.level as string)) {
+                if((subName === undefined || user.clientData?.webOperator === true) && isLogLineMinLevel(log, session.level as string)) {
                     io.to(session.id).emit('webLog', formatLogLineToHtml(log));
                 }
             }
@@ -1117,7 +1104,7 @@ const webClient = async (options: OperatorConfig) => {
                 if(lastCheck > 15) {
                     shouldCheck = true;
                 }
-            } else if(lastCheck > 300) {
+            } else if(lastCheck > 60) {
                 shouldCheck = true;
             }
         }
@@ -1152,7 +1139,9 @@ const webClient = async (options: OperatorConfig) => {
                     }
                 }).json() as CMInstance;
 
-                botStat = {...botStat, ...resp, online: true};
+                const {bots, ...restResp} = resp;
+
+                botStat = {...botStat, ...restResp, bots: bots.map(x => ({...x, instance: botStat})), online: true};
                 const sameNameIndex = cmInstances.findIndex(x => x.friendly === botStat.friendly);
                 if(sameNameIndex > -1 && sameNameIndex !== existingClientIndex) {
                     logger.warn(`Client returned a friendly name that is not unique (${botStat.friendly}), will fallback to host as friendly (${botStat.normalUrl})`);

src/Web/Common/User/CMUser.ts

@@ -0,0 +1,23 @@
+import {IUser} from "../interfaces";
+
+export interface ClientUserData {
+    token?: string
+    tokenExpiresAt?: number
+    scope?: string[]
+    webOperator?: boolean
+}
+
+abstract class CMUser<Instance, Bot, SubredditEntity> implements IUser {
+    constructor(public name: string, public subreddits: string[], public clientData: ClientUserData = {}) {
+
+    }
+
+    public abstract isInstanceOperator(val: Instance): boolean;
+    public abstract canAccessInstance(val: Instance): boolean;
+    public abstract canAccessBot(val: Bot): boolean;
+    public abstract accessibleBots(bots: Bot[]): Bot[]
+    public abstract canAccessSubreddit(val: Bot, name: string): boolean;
+    public abstract accessibleSubreddits(bot: Bot): SubredditEntity[]
+}
+
+export default CMUser;

src/Web/Common/User/ClientUser.ts

@@ -0,0 +1,41 @@
+import {BotInstance, CMInstance} from "../../interfaces";
+import CMUser from "./CMUser";
+import {intersect, parseRedditEntity} from "../../../util";
+
+class ClientUser extends CMUser<CMInstance, BotInstance, string> {
+
+    isInstanceOperator(val: CMInstance): boolean {
+        return val.operators.map(x=> x.toLowerCase()).includes(this.name.toLowerCase());
+    }
+
+    canAccessInstance(val: CMInstance): boolean {
+        return this.isInstanceOperator(val) || intersect(this.subreddits, val.subreddits.map(x => parseRedditEntity(x).name)).length > 0;
+    }
+
+    canAccessBot(val: BotInstance): boolean {
+        return this.isInstanceOperator(val.instance) || intersect(this.subreddits, val.subreddits.map(x => parseRedditEntity(x).name)).length > 0;
+    }
+
+    canAccessSubreddit(val: BotInstance, name: string): boolean {
+        return this.isInstanceOperator(val.instance) || this.subreddits.map(x => x.toLowerCase()).includes(parseRedditEntity(name).name.toLowerCase());
+    }
+
+    accessibleBots(bots: BotInstance[]): BotInstance[] {
+        if (bots.length === 0) {
+            return bots;
+        }
+        return bots.filter(x => {
+            if (this.isInstanceOperator(x.instance)) {
+                return true;
+            }
+            return intersect(this.subreddits, x.subreddits.map(y => parseRedditEntity(y).name)).length > 0
+        });
+    }
+
+    accessibleSubreddits(bot: BotInstance): string[] {
+        return this.isInstanceOperator(bot.instance) ? bot.subreddits.map(x => parseRedditEntity(x).name) : intersect(this.subreddits, bot.subreddits.map(x => parseRedditEntity(x).name));
+    }
+
+}
+
+export default ClientUser;

src/Web/Common/User/ServerUser.ts

@@ -0,0 +1,39 @@
+import {BotInstance, CMInstance} from "../../interfaces";
+import CMUser from "./CMUser";
+import {intersect, parseRedditEntity} from "../../../util";
+import {App} from "../../../App";
+import Bot from "../../../Bot";
+import {Manager} from "../../../Subreddit/Manager";
+
+class ServerUser extends CMUser<App, Bot, Manager> {
+
+    constructor(public name: string, public subreddits: string[], public machine: boolean, public isOperator: boolean) {
+        super(name, subreddits);
+    }
+
+    isInstanceOperator(): boolean {
+        return this.isOperator;
+    }
+
+    canAccessInstance(val: App): boolean {
+        return this.isOperator || val.bots.filter(x => intersect(this.subreddits, x.subManagers.map(y => y.subreddit.display_name))).length > 0;
+    }
+
+    canAccessBot(val: Bot): boolean {
+        return this.isOperator || intersect(this.subreddits, val.subManagers.map(y => y.subreddit.display_name)).length > 0;
+    }
+
+    accessibleBots(bots: Bot[]): Bot[] {
+        return this.isOperator ? bots : bots.filter(x => intersect(this.subreddits, x.subManagers.map(y => y.subreddit.display_name)).length > 0);
+    }
+
+    canAccessSubreddit(val: Bot, name: string): boolean {
+        return this.isOperator || this.subreddits.includes(parseRedditEntity(name).name) && val.subManagers.some(y => y.subreddit.display_name.toLowerCase() === parseRedditEntity(name).name.toLowerCase());
+    }
+
+    accessibleSubreddits(bot: Bot): Manager[] {
+        return this.isOperator ? bot.subManagers : bot.subManagers.filter(x => intersect(this.subreddits, [x.subreddit.display_name]).length > 0);
+    }
+}
+
+export default ServerUser;

src/Web/Common/interfaces.ts

@@ -59,3 +59,17 @@ export interface BotStatusResponse {
     }
     subreddits: SubredditDataResponse[]
 }
+
+export interface IUser {
+    name: string
+    subreddits: string[]
+    machine?: boolean
+    isOperator?: boolean
+    realManagers?: string[]
+    moderatedManagers?: string[]
+    realBots?: string[]
+    moderatedBots?: string[]
+    scope?: string[]
+    token?: string
+    tokenExpiresAt?: number
+}

src/Web/Server/interfaces.ts

@@ -1,27 +0,0 @@
-import { Request } from "express";
-import {App} from "../../App";
-import Bot from "../../Bot";
-
-// export interface ServerRequest extends Request {
-//     botApp: App
-//     bot?: Bot
-//     //user?: AuthenticatedUser
-// }
-//
-// export interface ServerRequestRedditor extends ServerRequest {
-//     user?: AuthenticatedRedditUser
-// }
-//
-// export interface AuthenticatedUser extends Express.User {
-//     machine: boolean
-// }
-//
-// export interface AuthenticatedRedditUser extends AuthenticatedUser {
-//     name: string
-//     subreddits: string[]
-//     isOperator: boolean
-//     realManagers: string[]
-//     moderatedManagers: string[]
-//     realBots: string[]
-//     moderatedBots: string[]
-// }

src/Web/Server/middleware.ts

@@ -1,9 +1,10 @@
-import {Request, Response} from "express";
+import {Request, Response, NextFunction} from "express";
 import Bot from "../../Bot";
+import ServerUser from "../Common/User/ServerUser";
 
 export const authUserCheck = (userRequired: boolean = true) => async (req: Request, res: Response, next: Function) => {
     if (req.isAuthenticated()) {
-        if (userRequired && req.user.machine) {
+        if (userRequired && (req.user as ServerUser).machine) {
             return res.status(403).send('Must be authenticated as a user to access this route');
         }
         return next();
@@ -23,10 +24,15 @@ export const botRoute = (required = true) => async (req: Request, res: Response,
     const botStr = botVal as string;
 
     if(req.user !== undefined) {
-        if (req.user.realBots === undefined || !req.user.realBots.map(x => x.toLowerCase()).includes(botStr.toLowerCase())) {
+        const serverBot = req.botApp.bots.find(x => x.botName === botStr) as Bot;
+
+        if(serverBot === undefined) {
             return res.status(404).send(`Bot named ${botStr} does not exist or you do not have permission to access it.`);
         }
-        req.serverBot = req.botApp.bots.find(x => x.botName === botStr) as Bot;
+        if (!req.user?.canAccessBot(serverBot)) {
+            return res.status(404).send(`Bot named ${botStr} does not exist or you do not have permission to access it.`);
+        }
+        req.serverBot = serverBot;
         return next();
     }
     return next();
@@ -37,18 +43,20 @@ export const subredditRoute = (required = true) => async (req: Request, res: Res
     const bot = req.serverBot;
 
     const {subreddit} = req.query as any;
-    if(subreddit === undefined && required === false) {
+    if(subreddit === undefined && !required) {
         next();
     } else {
-        const {name: userName, realManagers = [], isOperator} = req.user as Express.User;
-        if (!isOperator && !realManagers.includes(subreddit)) {
-            return res.status(400).send('Cannot access route for subreddit you do not manage or is not run by the bot')
-        }
+        //const {name: userName} = req.user as Express.User;
+
         const manager = bot.subManagers.find(x => x.displayLabel === subreddit);
         if (manager === undefined) {
             return res.status(400).send('Cannot access route for subreddit you do not manage or is not run by the bot')
         }
 
+        if (!req.user?.canAccessSubreddit(bot, subreddit)) {
+            return res.status(400).send('Cannot access route for subreddit you do not manage or is not run by the bot')
+        }
+
         req.manager = manager;
 
         next();

src/Web/Server/routes/authenticated/user/action.ts

@@ -1,29 +1,23 @@
-import express, {Request, Response} from 'express';
+import {Request, Response} from 'express';
 import {RUNNING, USER} from "../../../../../Common/interfaces";
 import Submission from "snoowrap/dist/objects/Submission";
 import LoggedError from "../../../../../Utils/LoggedError";
-import winston from "winston";
 import {authUserCheck, botRoute} from "../../../middleware";
 import {booleanMiddle} from "../../../../Common/middleware";
+import {Manager} from "../../../../../Subreddit/Manager";
+import {parseRedditEntity} from "../../../../../util";
 
-const action = async (req: express.Request, res: express.Response) => {
+const action = async (req: Request, res: Response) => {
     const bot = req.serverBot;
 
     const {type, action, subreddit, force = false} = req.query as any;
-    const {name: userName, realManagers = [], isOperator} = req.user as Express.User;
-    let subreddits: string[] = [];
-    if (subreddit === 'All') {
-        subreddits = realManagers;
-    } else if (realManagers.includes(subreddit)) {
-        subreddits = [subreddit];
+    const userName = req.user?.name;
+    let subreddits: Manager[] = req.user?.accessibleSubreddits(bot) as Manager[];
+    if (subreddit !== 'All') {
+        subreddits = subreddits.filter(x => x.subreddit.display_name === parseRedditEntity(subreddit).name);
     }
 
-    for (const s of subreddits) {
-        const manager = bot.subManagers.find(x => x.displayLabel === s);
-        if (manager === undefined) {
-            winston.loggers.get('app').warn(`Manager for ${s} does not exist`, {subreddit: `/u/${userName}`});
-            continue;
-        }
+    for (const manager of subreddits) {
         const mLogger = manager.logger;
         mLogger.info(`/u/${userName} invoked '${action}' action for ${type} on ${manager.displayLabel}`);
         try {
@@ -70,6 +64,9 @@ const action = async (req: express.Request, res: express.Response) => {
                             await manager.firehose.push({
                                 checkType: a instanceof Submission ? 'Submission' : 'Comment',
                                 activity: a,
+                                options: {
+                                    force: true,
+                                }
                             });
                         }
                     } else {
@@ -78,6 +75,9 @@ const action = async (req: express.Request, res: express.Response) => {
                             await manager.firehose.push({
                                 checkType: a instanceof Submission ? 'Submission' : 'Comment',
                                 activity: a,
+                                options: {
+                                    force: true
+                                }
                             });
                         }
                     }

src/Web/Server/routes/authenticated/user/addBot.ts

@@ -12,7 +12,7 @@ const addBot = () => {
 
     const response = async (req: Request, res: Response) => {
 
-        if (!(req.user as Express.User).isOperator) {
+        if (!req.user?.isInstanceOperator(req.app)) {
             return res.status(401).send("Must be an Operator to use this route");
         }
 

src/Web/Server/routes/authenticated/user/index.ts

@@ -66,7 +66,7 @@ const actionedEvents = async (req: Request, res: Response) => {
         managers.push(manager);
     } else {
         for(const manager of req.serverBot.subManagers) {
-            if((req.user?.realManagers as string[]).includes(manager.displayLabel)) {
+            if(req.user?.canAccessSubreddit(req.serverBot, manager.subreddit.display_name)) {
                 managers.push(manager);
             }
         }
@@ -89,7 +89,7 @@ const action = async (req: Request, res: Response) => {
     const bot = req.serverBot;
 
     const {url, dryRun = false, subreddit} = req.query as any;
-    const {name: userName, realManagers = [], isOperator} = req.user as Express.User;
+    const {name: userName} = req.user as Express.User;
 
     let a;
     const commentId = commentReg(url);
@@ -115,7 +115,7 @@ const action = async (req: Request, res: Response) => {
 
         let manager = subreddit === 'All' ? bot.subManagers.find(x => x.subreddit.display_name === sub) : bot.subManagers.find(x => x.displayLabel === subreddit);
 
-        if (manager === undefined || (!realManagers.includes(manager.displayLabel))) {
+        if (manager === undefined || !req.user?.canAccessSubreddit(req.serverBot, manager.subreddit.display_name)) {
             let msg = 'Activity does not belong to a subreddit you moderate or the bot runs on.';
             if (subreddit === 'All') {
                 msg = `${msg} If you want to test an Activity against a Subreddit\'s config it does not belong to then switch to that Subreddit's tab first.`
@@ -127,7 +127,7 @@ const action = async (req: Request, res: Response) => {
         // will run dryrun if specified or if running activity on subreddit it does not belong to
         const dr: boolean | undefined = (dryRun || manager.subreddit.display_name !== sub) ? true : undefined;
         manager.logger.info(`/u/${userName} running${dr === true ? ' DRY RUN ' : ' '}check on${manager.subreddit.display_name !== sub ? ' FOREIGN ACTIVITY ' : ' '}${url}`);
-        await manager.runChecks(activity instanceof Submission ? 'Submission' : 'Comment', activity, {dryRun: dr})
+        await manager.runChecks(activity instanceof Submission ? 'Submission' : 'Comment', activity, {dryRun: dr, force: true})
     }
     res.send('OK');
 };

src/Web/Server/routes/authenticated/user/logs.ts

@@ -24,7 +24,9 @@ const logs = (subLogMap: Map<string, LogEntry[]>) => {
 
         const logger = winston.loggers.get('app');
 
-        const {name: userName, realManagers = [], isOperator} = req.user as Express.User;
+        const userName = req.user?.name as string;
+        const isOperator = req.user?.isInstanceOperator(req.botApp);
+        const realManagers = req.botApp.bots.map(x => req.user?.accessibleSubreddits(x).map(x => x.displayLabel)).flat() as string[];
         const {level = 'verbose', stream, limit = 200, sort = 'descending', streamObjects = false} = req.query;
         if (stream) {
             const origin = req.header('X-Forwarded-For') ?? req.header('host');

src/Web/Server/routes/authenticated/user/status.ts

@@ -41,14 +41,14 @@ const status = () => {
         if(req.serverBot !== undefined) {
             bots = [req.serverBot];
         } else {
-            bots = (req.user as Express.User).isOperator ? req.botApp.bots : req.botApp.bots.filter(x => intersect(req.user?.subreddits as string[], x.subManagers.map(y => y.subreddit.display_name)));
+            bots = req.user?.accessibleBots(req.botApp.bots) as Bot[];
         }
         const botResponses: BotStatusResponse[] = [];
         for(const b of bots) {
             botResponses.push(await botStatResponse(b, req, botLogMap));
         }
         const system: any = {};
-        if((req.user as Express.User).isOperator) {
+        if(req.user?.isInstanceOperator(req.botApp)) {
             // @ts-ignore
             system.logs = filterLogBySubreddit(new Map([['app', systemLogs]]), [], {level, sort, limit, operator: true}).get('app');
         }
@@ -69,14 +69,11 @@ const status = () => {
             lastCheck
         } = req.query;
 
-        const {name: userName, realManagers = [], isOperator} = req.user as Express.User;
-        const user = userName as string;
-        const subreddits = realManagers;
-        //const isOperator = opNames.includes(user.toLowerCase())
+        const user = req.user?.name as string;
 
-        const logs = filterLogBySubreddit(botLogMap.get(bot.botName as string) || new Map(), realManagers, {
+        const logs = filterLogBySubreddit(botLogMap.get(bot.botName as string) || new Map(), req.user?.accessibleSubreddits(bot).map(x => x.displayLabel) as string[], {
             level: (level as string),
-            operator: isOperator,
+            operator: req.user?.isInstanceOperator(req.botApp),
             user,
             // @ts-ignore
             sort,
@@ -84,15 +81,11 @@ const status = () => {
         });
 
         const subManagerData = [];
-        for (const s of subreddits) {
-            const m = bot.subManagers.find(x => x.displayLabel === s) as Manager;
-            if(m === undefined) {
-                continue;
-            }
+        for (const m of req.user?.accessibleSubreddits(bot) as Manager[]) {
             const sd = {
-                name: s,
+                name: m.displayLabel,
                 //linkName: s.replace(/\W/g, ''),
-                logs: logs.get(s) || [], // provide a default empty value in case we truly have not logged anything for this subreddit yet
+                logs: logs.get(m.displayLabel) || [], // provide a default empty value in case we truly have not logged anything for this subreddit yet
                 botState: m.botState,
                 eventsState: m.eventsState,
                 queueState: m.queueState,

src/Web/Server/server.ts

@@ -1,5 +1,5 @@
 import {addAsync, Router} from '@awaitjs/express';
-import express, {Request, Response} from 'express';
+import express, {Request, Response, NextFunction, RequestHandler} from 'express';
 import bodyParser from 'body-parser';
 import {App} from "../../App";
 import {Transform} from "stream";
@@ -29,6 +29,7 @@ import {opStats} from "../Common/util";
 import Bot from "../../Bot";
 import addBot from "./routes/authenticated/user/addBot";
 import dayjs from "dayjs";
+import ServerUser from "../Common/User/ServerUser";
 
 const server = addAsync(express());
 server.use(bodyParser.json());
@@ -87,7 +88,7 @@ const rcbServer = async function (options: OperatorConfig) {
                 botLog.set('app', appLogs.slice(0, 200 + 1));
             } else {
                 let botSubs = botSubreddits.get(botName) || [];
-                if(botSubs.length === 0 && app !== undefined) {
+                if(app !== undefined && (botSubs.length === 0 || !botSubs.includes(subName))) {
                     const b = app.bots.find(x => x.botName === botName);
                     if(b !== undefined) {
                         botSubs = b.subManagers.map(x => x.displayLabel);
@@ -128,31 +129,35 @@ const rcbServer = async function (options: OperatorConfig) {
     }, function (jwtPayload, done) {
         const {name, subreddits = [], machine = true} = jwtPayload.data;
         if (machine) {
-            return done(null, {machine});
+            const user = new ServerUser(name, subreddits, true, false);
+            return done(null, user);
+            //return done(null, {machine});
         }
         const isOperator = opNames.includes(name.toLowerCase());
-        let moderatedBots: string[] = [];
-        let moderatedManagers: string[] = [];
-        let realBots: string[] = [];
-        let realManagers: string[] = [];
-        if(app !== undefined) {
-            const modBots =  app.bots.filter(x => intersect(subreddits, x.subManagers.map(y => y.subreddit.display_name)));
-            moderatedBots = modBots.map(x => x.botName as string);
-            moderatedManagers = [...new Set(modBots.map(x => x.subManagers.map(y => y.displayLabel)).flat())];
-            realBots = isOperator ? app.bots.map(x => x.botName as string) : moderatedBots;
-            realManagers = isOperator ? [...new Set(app.bots.map(x => x.subManagers.map(y => y.displayLabel)).flat())] : moderatedManagers
-        }
+        // let moderatedBots: string[] = [];
+        // let moderatedManagers: string[] = [];
+        // let realBots: string[] = [];
+        // let realManagers: string[] = [];
+        // if(app !== undefined) {
+        //     const modBots =  app.bots.filter(x => intersect(subreddits, x.subManagers.map(y => y.subreddit.display_name)).length > 0);
+        //     moderatedBots = modBots.map(x => x.botName as string);
+        //     moderatedManagers = [...new Set(modBots.map(x => x.subManagers).flat().filter(x => subreddits.includes(x.subreddit.display_name)).map(x => x.displayLabel))];
+        //     realBots = isOperator ? app.bots.map(x => x.botName as string) : moderatedBots;
+        //     realManagers = isOperator ? [...new Set(app.bots.map(x => x.subManagers.map(y => y.displayLabel)).flat())] : moderatedManagers
+        // }
 
-        return done(null, {
-            name,
-            subreddits,
-            isOperator,
-            machine: false,
-            moderatedManagers,
-            realManagers,
-            moderatedBots,
-            realBots,
-        });
+        const user = new ServerUser(name, subreddits, false, isOperator);
+        return done(null, user);
+        // return done(null, {
+        //     name,
+        //     subreddits,
+        //     isOperator,
+        //     machine: false,
+        //     moderatedManagers,
+        //     realManagers,
+        //     moderatedBots,
+        //     realBots,
+        // });
     }));
 
     server.use(passport.authenticate('jwt', {session: false}));
@@ -169,8 +174,8 @@ const rcbServer = async function (options: OperatorConfig) {
         let bots: Bot[] = [];
         if(req.serverBot !== undefined) {
             bots = [req.serverBot];
-        } else {
-            bots = (req.user as Express.User).isOperator ? req.botApp.bots : req.botApp.bots.filter(x => intersect(req.user?.subreddits as string[], x.subManagers.map(y => y.subreddit.display_name)));
+        } else if(req.user !== undefined) {
+            bots = req.user.accessibleBots(req.botApp.bots);
         }
         const resp = [];
         for(const b of bots) {
@@ -206,24 +211,20 @@ const rcbServer = async function (options: OperatorConfig) {
     server.deleteAsync('/bot/invite', ...deleteInviteRoute);
 
     const initBot = async (causedBy: Invokee = 'system') => {
-        if(app !== undefined) {
+        if (app !== undefined) {
             logger.info('A bot instance already exists. Attempting to stop event/queue processing first before building new bot.');
             await app.destroy(causedBy);
         }
         const newApp = new App(options);
-        if(newApp.error === undefined) {
-            try {
-                await newApp.initBots(causedBy);
-            } catch (err: any) {
-                if(newApp.error === undefined) {
-                    newApp.error = err.message;
-                }
-                logger.error('Server is still ONLINE but bot cannot recover from this error and must be re-built');
-                if(!err.logged || !(err instanceof LoggedError)) {
-                    logger.error(err);
-                }
+        newApp.initBots(causedBy).catch((err: any) => {
+            if (newApp.error === undefined) {
+                newApp.error = err.message;
             }
-        }
+            logger.error('Server is still ONLINE but bot cannot recover from this error and must be re-built');
+            if (!err.logged || !(err instanceof LoggedError)) {
+                logger.error(err);
+            }
+        });
         return newApp;
     }
 

src/Web/assets/views/helper.ejs

@@ -128,6 +128,12 @@
                                             class="font-mono font-semibold">modcontributors</span> for the bot to
                                     ban/mute users in the subreddits it moderates</label>
                             </div>
+                            <div>
+                                <input class="permissionToggle" type="checkbox" id="flair" name="flair"
+                                       checked>
+                                <label for="flair"><span class="font-mono font-semibold">flair</span> for the bot
+                                    to select subreddit flairs.</label>
+                            </div>
                             <div>
                                 <input class="permissionToggle" type="checkbox" id="modposts" name="modposts"
                                        checked>

src/Web/assets/views/invite.ejs

@@ -86,6 +86,11 @@
                                             class="font-mono font-semibold">modcontributors</span> for the bot to
                                     ban/mute users in the subreddits it moderates</label>
                             </div>
+                            <div>
+                                <input class="permissionToggle" type="checkbox" id="flair" name="flair" disabled>
+                                <label for="flair"><span class="font-mono font-semibold">flair</span> for the bot
+                                    to select subreddit flairs.</label>
+                            </div>
                             <div>
                                 <input class="permissionToggle" type="checkbox" id="modposts" name="modposts" disabled>
                                 <label for="modposts"><span class="font-mono font-semibold">modposts</span> for the bot

src/Web/assets/views/status.ejs

@@ -611,12 +611,9 @@
                                     <input data-subreddit="<%= data.name %>" style="min-width: 420px;"
                                            class="border-gray-50 placeholder-gray-500 rounded mt-2 mb-3 p-2 text-black checkUrl"
                                            placeholder="<%= data.name === 'All' ? 'Run Bot on a permalink from any moderated Subreddit' : `Run Bot on a permalink using this Subreddit's config` %>"/>
-                                    <span class="mx-2">
-                                        <input type="checkbox" class="dryrunCheck" data-subreddit="<%= data.name %>"
-                                               name="dryrunCheck">
-                                        <label for="dryrunCheck">Dry Run?</label>
-                                    </span>
-                                    <a class="runCheck" data-subreddit="<%= data.name %>" href="">Run</a>
+
+                                    <a class="hover:bg-gray-700 pointer-events-none opacity-20 no-underline rounded-md mx-4 py-2 px-3 border checkActions dryRunCheck" data-subreddit="<%= data.name %>" href="">Dry Run</a>
+                                    <a class="hover:bg-gray-700 pointer-events-none opacity-20 no-underline rounded-md py-2 px-3 border checkActions runCheck" data-subreddit="<%= data.name %>" href="">Run</a>
                                 </div>
                                 <%- include('partials/logSettings') %>
                             </div>
@@ -658,23 +655,42 @@
         });
     })
 
-    document.querySelectorAll(".runCheck").forEach(el => {
+    document.querySelectorAll(".checkUrl").forEach(el => {
+        const toggleButtons = (e) => {
+            const subFilter = `.sub[data-subreddit="${e.target.dataset.subreddit}"]`;
+            const inputVal = document.querySelector(`${subFilter} .checkUrl`).value;
+            if (inputVal.length > 0) {
+                document.querySelectorAll(`${subFilter} .checkActions`).forEach(el => {
+                    el.classList.remove('pointer-events-none', 'opacity-20');
+                });
+            } else {
+                document.querySelectorAll(`${subFilter} .checkActions`).forEach(el => {
+                    el.classList.add('pointer-events-none', 'opacity-20');
+                });
+            }
+        }
+        el.addEventListener('keyup', toggleButtons, false);
+        el.addEventListener('change', toggleButtons, false);
+    });
+
+    document.querySelectorAll(".checkActions").forEach(el => {
         el.addEventListener('click', e => {
             e.preventDefault();
             const subreddit = e.target.dataset.subreddit;
-            const urlInput = document.querySelector(`[data-subreddit="${subreddit}"].checkUrl`);
-            const dryRunCheck = document.querySelector(`[data-subreddit="${subreddit}"].dryrunCheck`);
+            const subFilter = `.sub[data-subreddit="${subreddit}"]`;
+            const urlInput = document.querySelector(`${subFilter} .checkUrl`);
+
+            const isDryun = e.target.classList.contains('dryRunCheck');
 
             const subSection = e.target.closest('div.sub');
             bot = subSection.dataset.bot;
             const url = urlInput.value;
-            const dryRun = dryRunCheck.checked ? 1 : 0;
 
-            const fetchUrl = `/api/check?instance=<%= instanceId %>&bot=${bot}&url=${url}&dryRun=${dryRun}&subreddit=${subreddit}`;
+            const fetchUrl = `/api/check?instance=<%= instanceId %>&bot=${bot}&url=${url}&dryRun=${isDryun ? 1 : 0}&subreddit=${subreddit}`;
             fetch(fetchUrl);
 
             urlInput.value = '';
-            dryRunCheck.checked = false;
+            urlInput.dispatchEvent(new Event('change'));
         });
     });
 

src/Web/interfaces.ts

@@ -8,6 +8,7 @@ export interface BotInstance {
     subreddits: string[]
     nanny?: string
     running: boolean
+    instance: CMInstance
 }
 
 export interface CMInstance extends BotConnection {

src/Web/types/express/index.d.ts

@@ -2,6 +2,7 @@ import {App} from "../../../App";
 import Bot from "../../../Bot";
 import {BotInstance, CMInstance} from "../../interfaces";
 import {Manager} from "../../../Subreddit/Manager";
+import CMUser from "../../Common/User/CMUser";
 
 declare global {
     declare namespace Express {
@@ -13,18 +14,7 @@ declare global {
             serverBot: Bot,
             manager?: Manager,
         }
-        interface User {
-            name: string
-            subreddits: string[]
-            machine?: boolean
-            isOperator?: boolean
-            realManagers?: string[]
-            moderatedManagers?: string[]
-            realBots?: string[]
-            moderatedBots?: string[]
-            scope?: string[]
-            token?: string
-            tokenExpiresAt?: number
+        class User extends CMUser<any, any, any> {
         }
     }
 }

src/util.ts

@@ -58,6 +58,7 @@ import {SetRandomInterval} from "./Common/types";
 import stringSimilarity from 'string-similarity';
 import calculateCosineSimilarity from "./Utils/StringMatching/CosineSimilarity";
 import levenSimilarity from "./Utils/StringMatching/levenSimilarity";
+import {isRequestError, isStatusError} from "./Utils/Errors";
 //import {ResembleSingleCallbackComparisonResult} from "resemblejs";
 
 // want to guess how many concurrent image comparisons we should be doing
@@ -661,6 +662,126 @@ export const parseExternalUrl = (val: string) => {
     return (matches.groups as any).url as string;
 }
 
+export const dummyLogger = {
+    debug: (v: any) => null,
+    error: (v: any) => null,
+    warn: (v: any) => null,
+    info: (v: any) => null
+}
+
+const GIST_REGEX = new RegExp(/.*gist\.github\.com\/.+\/(.+)/i)
+const GH_BLOB_REGEX = new RegExp(/.*github\.com\/(.+)\/(.+)\/blob\/(.+)/i);
+const REGEXR_REGEX = new RegExp(/^.*((regexr\.com)\/[\w\d]+).*$/i);
+const REGEXR_PAGE_REGEX = new RegExp(/(.|[\n\r])+"expression":"(.+)","text"/g);
+export const fetchExternalUrl = async (url: string, logger: (any) = dummyLogger): Promise<string> => {
+    let hadError = false;
+    logger.debug(`Attempting to detect resolvable URL for ${url}`);
+    let match = url.match(GIST_REGEX);
+    if (match !== null) {
+        const gistApiUrl = `https://api.github.com/gists/${match[1]}`;
+        logger.debug(`Looks like a non-raw gist URL! Trying to resolve ${gistApiUrl}`);
+
+        try {
+            const response = await fetch(gistApiUrl);
+            if (!response.ok) {
+                logger.error(`Response was not OK from Gist API (${response.statusText}) -- will return response from original URL instead`);
+                if (response.size > 0) {
+                    logger.error(await response.text())
+                }
+                hadError = true;
+            } else {
+                const data = await response.json();
+                // get first found file
+                const fileKeys = Object.keys(data.files);
+                if (fileKeys.length === 0) {
+                    logger.error(`No files found in gist!`);
+                } else {
+                    if (fileKeys.length > 1) {
+                        logger.warn(`More than one file found in gist! Using first found: ${fileKeys[0]}`);
+                    } else {
+                        logger.debug(`Using file ${fileKeys[0]}`);
+                    }
+                    const file = data.files[fileKeys[0]];
+                    if (file.truncated === false) {
+                        return file.content;
+                    }
+                    const rawUrl = file.raw_url;
+                    logger.debug(`File contents was truncated, retrieving full contents from ${rawUrl}`);
+                    try {
+                        const rawUrlResponse = await fetch(rawUrl);
+                        return await rawUrlResponse.text();
+                    } catch (err: any) {
+                        logger.error('Gist Raw URL Response returned an error, will return response from original URL instead');
+                        logger.error(err);
+                    }
+                }
+            }
+        } catch (err: any) {
+            logger.error('Response returned an error, will return response from original URL instead');
+            logger.error(err);
+        }
+    }
+    match = url.match(GH_BLOB_REGEX);
+
+    if (match !== null) {
+        const rawUrl = `https://raw.githubusercontent.com/${match[1]}/${match[2]}/${match[3]}`
+        logger.debug(`Looks like a single file github URL! Resolving to ${rawUrl}`);
+        try {
+            const response = await fetch(rawUrl);
+            if (!response.ok) {
+                logger.error(`Response was not OK (${response.statusText}) -- will return response from original URL instead`);
+                if (response.size > 0) {
+                    logger.error(await response.text())
+                }
+                hadError = true;
+            } else {
+                return await response.text();
+            }
+        } catch (err: any) {
+            logger.error('Response returned an error, will return response from original URL instead');
+            logger.error(err);
+        }
+    }
+
+    match = url.match(REGEXR_REGEX);
+    if(match !== null) {
+        logger.debug(`Looks like a Regexr URL! Trying to get expression from page HTML`);
+        try {
+            const response = await fetch(url);
+            if (!response.ok) {
+                if (response.size > 0) {
+                    logger.error(await response.text())
+                }
+                throw new Error(`Response was not OK: ${response.statusText}`);
+            } else {
+                const page = await response.text();
+                const pageMatch = [...page.matchAll(REGEXR_PAGE_REGEX)];
+                if(pageMatch.length > 0) {
+                    const unescaped = JSON.parse(`{"value": "${pageMatch[0][2]}"}`)
+                    return unescaped.value;
+                } else {
+                    throw new Error('Could not parse regex expression from page HTML');
+                }
+            }
+        } catch (err: any) {
+            logger.error('Response returned an error');
+            throw err;
+        }
+    }
+
+    if(!hadError) {
+        logger.debug('URL was not special (gist, github blob, etc...) so will retrieve plain contents');
+    }
+    const response = await fetch(url);
+    if(!response.ok) {
+        if (response.size > 0) {
+            logger.error(await response.text())
+        }
+        throw new Error(`Response was not OK: ${response.statusText}`);
+    }
+    return await response.text();
+}
+
 export interface RetryOptions {
     maxRequestRetry: number,
     maxOtherRetry: number,
@@ -684,37 +805,40 @@ export const createRetryHandler = (opts: RetryOptions, logger: Logger) => {
 
         lastErrorAt = dayjs();
 
-        if(err.name === 'RequestError' || err.name === 'StatusCodeError') {
+        const redditApiError = isRequestError(err) || isStatusError(err);
+
+        if(redditApiError) {
             if (err.statusCode === undefined || ([401, 500, 503, 502, 504, 522].includes(err.statusCode))) {
                 timeoutCount++;
+                let msg = `Error occurred while making a request to Reddit (${timeoutCount}/${maxRequestRetry+1} in ${clearRetryCountAfter} minutes).`;
                 if (timeoutCount > maxRequestRetry) {
-                    logger.error(`Reddit request error retries (${timeoutCount}) exceeded max allowed (${maxRequestRetry})`);
+                    logger.error(`${msg} Exceeded max allowed.`);
                     return false;
                 }
                 if(waitOnRetry) {
                     // exponential backoff
                     const ms = (Math.pow(2, timeoutCount - 1) + (Math.random() - 0.3) + 1) * 1000;
-                    logger.warn(`Error occurred while making a request to Reddit (${timeoutCount} in 3 minutes). Will wait ${formatNumber(ms / 1000)} seconds before retrying`);
+                    logger.warn(`${msg} Will wait ${formatNumber(ms / 1000)} seconds before retrying.`);
                     await sleep(ms);
                 }
                 return true;
-
-            } else {
-                return false;
             }
-        } else {
-            // linear backoff
-            otherRetryCount++;
-            if (maxOtherRetry < otherRetryCount) {
-                return false;
-            }
-            if(waitOnRetry) {
-                const ms = (4 * 1000) * otherRetryCount;
-                logger.warn(`Non-request error occurred. Will wait ${formatNumber(ms / 1000)} seconds before retrying`);
-                await sleep(ms);
-            }
-            return true;
+            // if it's a request error but not a known "oh probably just a reddit blip" status code treat it as other, which should usually have a lower retry max
         }
+
+        // linear backoff
+        otherRetryCount++;
+        let msg = redditApiError ? `Error occurred while making a request to Reddit (${otherRetryCount}/${maxOtherRetry} in ${clearRetryCountAfter} minutes) but it was NOT a well-known "reddit blip" error.` : `Non-request error occurred (${otherRetryCount}/${maxOtherRetry} in ${clearRetryCountAfter} minutes).`;
+        if (maxOtherRetry < otherRetryCount) {
+            logger.warn(`${msg} Exceeded max allowed.`);
+            return false;
+        }
+        if(waitOnRetry) {
+            const ms = (4 * 1000) * otherRetryCount;
+            logger.warn(`${msg} Will wait ${formatNumber(ms / 1000)} seconds before retrying`);
+            await sleep(ms);
+        }
+        return true;
     }
 }
 
@@ -992,6 +1116,8 @@ export const toStrongSubredditState = (s: SubredditState, opts?: StrongSubreddit
 
     if (generateDescription && stateDescription === undefined) {
         strongState.stateDescription = objectToStringSummary(strongState);
+    } else {
+        strongState.stateDescription = stateDescription;
     }
 
     return strongState;
@@ -1559,40 +1685,6 @@ export const random = (min: number, max: number): number => (
     Math.floor(Math.random() * (max - min + 1)) + min
 );
 
-
-// copy-pasting interval function from this project because bad tooling on the author's side means A WHOLE OTHER typescript lib is always downloaded just for this one function
-
-/**
- * Repeatedly calls a function with a random time delay between each call.
- *
- * @param intervalFunction - A function to be executed at random times between `minDelay` and
- * `maxDelay`. The function is not passed any arguments, and no return value is expected.
- * @param minDelay - The minimum amount of time, in milliseconds (thousandths of a second), the
- * timer should delay in between executions of `intervalFunction`.
- * @param maxDelay - The maximum amount of time, in milliseconds (thousandths of a second), the
- * timer should delay in between executions of `intervalFunction`.
- *
- * Borrowed from https://github.com/jabacchetta/set-random-interval/blob/master/src/index.ts
- */
-export const setRandomInterval: SetRandomInterval = (intervalFunction, minDelay = 0, maxDelay = 0) => {
-    let timeout: ReturnType<typeof setTimeout>;
-
-    const runInterval = (): void => {
-        timeout = globalThis.setTimeout(() => {
-            intervalFunction();
-            runInterval();
-        }, random(minDelay, maxDelay));
-    };
-
-    runInterval();
-
-    return {
-        clear(): void {
-            clearTimeout(timeout);
-        },
-    };
-};
-
 /**
  * Naively detect if a string is most likely json5
  *

tsconfig.json

@@ -3,7 +3,10 @@
   "compilerOptions": {
     "sourceMap": true,
     "resolveJsonModule": true,
-    "typeRoots": ["./src/Web/types"]
+    "typeRoots": [
+      "./node_modules/@types",
+      "./src/Web/types"
+    ]
   },
 //  "compilerOptions": {
 //    "module": "es6",