Merge branch 'edge'

* Moving into cache means stats will persist after restart (yay!)
* Refactored stats structure to be simpler

Dockerfile

@@ -4,6 +4,12 @@ ENV TZ=Etc/GMT
 
 RUN apk update
 
+# required dependencies in order to compile linux-musl (node-canvas) on alpine
+# https://github.com/node-gfx/node-canvas-prebuilt/issues/77#issuecomment-884365161
+RUN apk add --no-cache build-base g++ cairo-dev jpeg-dev pango-dev giflib-dev
+# required dependencies in order to compile linux-musl (node-canvas) on alpine
+RUN apk add --update --repository http://dl-3.alpinelinux.org/alpine/edge/testing libmount ttf-dejavu ttf-droid ttf-freefont ttf-liberation ttf-ubuntu-font-family fontconfig
+
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 WORKDIR /usr/app
@@ -11,7 +17,9 @@ WORKDIR /usr/app
 COPY package*.json ./
 COPY tsconfig.json .
 
-RUN npm install
+# no prebuild support for node-canvas on alpine so need to compile
+# https://github.com/Automattic/node-canvas#compiling
+RUN npm install --build-from-source
 
 ADD . /usr/app
 

cliff.toml

@@ -0,0 +1,67 @@
+# configuration file for git-cliff (0.1.0)
+
+[changelog]
+# changelog header
+header = """
+# Changelog
+All notable changes to this project will be documented in this file.\n
+"""
+# template for the changelog body
+# https://tera.netlify.app/docs/#introduction
+body = """
+{% if version %}\
+    ## [{{ version | replace(from="v", to="") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
+{% else %}\
+    ## [unreleased]
+{% endif %}\
+{% for group, commits in commits | group_by(attribute="group") %}
+    ### {{ group | upper_first }}
+    {% for commit in commits
+    | filter(attribute="scope")
+    | sort(attribute="scope") %}
+        - *({{commit.scope}})* {{ commit.message | upper_first }}
+        {%- if commit.breaking %}
+        {% raw %}  {% endraw %}- **BREAKING**: {{commit.breaking_description}}
+        {%- endif -%}
+    {%- endfor -%}
+    {%- for commit in commits %}
+        {%- if commit.scope -%}
+        {% else -%}
+            - *(No Category)* {{ commit.message | upper_first }}
+            {% if commit.breaking -%}
+            {% raw %}  {% endraw %}- **BREAKING**: {{commit.breaking_description}}
+            {% endif -%}
+        {% endif -%}
+    {% endfor -%}
+{% endfor %}
+"""
+# remove the leading and trailing whitespaces from the template
+trim = true
+# changelog footer
+footer = """
+<!-- generated by git-cliff -->
+"""
+
+[git]
+# allow only conventional commits
+# https://www.conventionalcommits.org
+conventional_commits = true
+# regex for parsing and grouping commits
+commit_parsers = [
+    { message = "^feat", group = "Features"},
+    { message = "^fix", group = "Bug Fixes"},
+    { message = "^doc", group = "Documentation"},
+    { message = "^perf", group = "Performance"},
+    { message = "^refactor", group = "Refactor"},
+    { message = "^style", group = "Styling"},
+    { message = "^test", group = "Testing"},
+    { message = "^chore\\(release\\): prepare for", skip = true},
+    { message = "^chore", group = "Miscellaneous Tasks"},
+    { body = ".*security", group = "Security"},
+]
+# filter out the commits that are not matched by commit parsers
+filter_commits = false
+# glob pattern for matching git tags
+tag_pattern = "[0-9]*"
+# regex for skipping tags
+skip_tags = "v0.1.0-beta.1"

package.json

@@ -26,6 +26,7 @@
   "license": "ISC",
   "dependencies": {
     "@awaitjs/express": "^0.8.0",
+    "@stdlib/regexp-regexp": "^0.0.6",
     "ajv": "^7.2.4",
     "async": "^3.2.0",
     "autolinker": "^3.14.3",
@@ -49,6 +50,7 @@
     "got": "^11.8.2",
     "he": "^1.2.0",
     "http-proxy": "^1.18.1",
+    "image-size": "^1.0.0",
     "js-yaml": "^4.1.0",
     "json5": "^2.2.0",
     "jsonwebtoken": "^8.5.1",
@@ -64,6 +66,7 @@
     "passport-custom": "^1.1.1",
     "passport-jwt": "^4.0.0",
     "pretty-print-json": "^1.0.3",
+    "resemblejs": "^4.0.0",
     "safe-stable-stringify": "^1.1.1",
     "snoostorm": "^1.5.2",
     "snoowrap": "^1.23.0",
@@ -100,10 +103,15 @@
     "@types/object-hash": "^2.1.0",
     "@types/passport": "^1.0.7",
     "@types/passport-jwt": "^3.0.6",
+    "@types/pixelmatch": "^5.2.4",
+    "@types/resemblejs": "^3.2.1",
     "@types/tcp-port-used": "^1.0.0",
     "@types/triple-beam": "^1.3.2",
     "ts-auto-guard": "*",
     "ts-json-schema-generator": "^0.93.0",
     "typescript-json-schema": "^0.50.1"
+  },
+  "optionalDependencies": {
+    "node-canvas": "^2.7.0"
   }
 }

src/Action/CommentAction.ts

@@ -4,6 +4,7 @@ import Submission from "snoowrap/dist/objects/Submission";
 import {renderContent} from "../Utils/SnoowrapUtils";
 import {ActionProcessResult, Footer, RequiredRichContent, RichContent} from "../Common/interfaces";
 import {RuleResult} from "../Rule";
+import {truncateStringToLength} from "../util";
 
 export class CommentAction extends Action {
     content: string;
@@ -66,9 +67,18 @@ export class CommentAction extends Action {
             // @ts-ignore
             await reply.distinguish({sticky: this.sticky});
         }
+        let modifiers = [];
+        if(this.distinguish) {
+            modifiers.push('Distinguished');
+        }
+        if(this.sticky) {
+            modifiers.push('Stickied');
+        }
+        const modifierStr = modifiers.length === 0 ? '' : `[${modifiers.join(' | ')}]`;
         return {
             dryRun,
             success: true,
+            result: `${modifierStr}${this.lock ? ' - Locked Author\'s Activity - ' : ''}${truncateStringToLength(100)(body)}`
         };
     }
 }

src/Action/MessageAction.ts

@@ -4,7 +4,15 @@ import Submission from "snoowrap/dist/objects/Submission";
 import {renderContent} from "../Utils/SnoowrapUtils";
 import {ActionProcessResult, Footer, RequiredRichContent, RichContent} from "../Common/interfaces";
 import {RuleResult} from "../Rule";
-import {asSubmission, boolToString, isSubmission} from "../util";
+import {
+    asSubmission,
+    boolToString,
+    isSubmission,
+    parseRedditEntity,
+    REDDIT_ENTITY_REGEX_URL,
+    truncateStringToLength
+} from "../util";
+import SimpleError from "../Utils/SimpleError";
 
 export class MessageAction extends Action {
     content: string;
@@ -14,6 +22,7 @@ export class MessageAction extends Action {
     footer?: false | string;
 
     title?: string;
+    to?: string;
     asSubreddit: boolean;
 
     constructor(options: MessageActionOptions) {
@@ -23,7 +32,9 @@ export class MessageAction extends Action {
             asSubreddit,
             title,
             footer,
+            to,
         } = options;
+        this.to = to;
         this.footer = footer;
         this.content = content;
         this.asSubreddit = asSubreddit;
@@ -42,11 +53,30 @@ export class MessageAction extends Action {
         const footer = await this.resources.generateFooter(item, this.footer);
 
         const renderedContent = `${body}${footer}`;
-        // @ts-ignore
-        const author = await item.author.fetch() as RedditUser;
+
+        let recipient = item.author.name;
+        if(this.to !== undefined) {
+            // parse to value
+            try {
+                const entityData = parseRedditEntity(this.to);
+                if(entityData.type === 'user') {
+                    recipient = entityData.name;
+                } else {
+                    recipient = `/r/${entityData.name}`;
+                }
+            } catch (err) {
+                this.logger.error(`'to' field for message was not in a valid format. See ${REDDIT_ENTITY_REGEX_URL} for valid examples`);
+                this.logger.error(err);
+                err.logged = true;
+                throw err;
+            }
+            if(recipient.includes('/r/') && this.asSubreddit) {
+                throw new SimpleError(`Cannot send a message as a subreddit to another subreddit. Requested recipient: ${recipient}`);
+            }
+        }
 
         const msgOpts: ComposeMessageParams = {
-            to: author,
+            to: recipient,
             text: renderedContent,
             // @ts-ignore
             fromSubreddit: this.asSubreddit ? await item.subreddit.fetch() : undefined,
@@ -54,7 +84,7 @@ export class MessageAction extends Action {
         };
 
         const msgPreview = `\r\n
-        TO: ${author.name}\r\n
+        TO: ${recipient}\r\n
         Subject: ${msgOpts.subject}\r\n
         Sent As Modmail: ${boolToString(this.asSubreddit)}\r\n\r\n
         ${renderedContent}`;
@@ -67,6 +97,7 @@ export class MessageAction extends Action {
         return {
             dryRun,
             success: true,
+            result: truncateStringToLength(200)(msgPreview)
         }
     }
 }
@@ -77,6 +108,24 @@ export interface MessageActionConfig extends RequiredRichContent, Footer {
      * */
     asSubreddit: boolean
 
+    /**
+     * Entity to send message to.
+     *
+     * If not present Message be will sent to the Author of the Activity being checked.
+     *
+     * Valid formats:
+     *
+     * * `aUserName` -- send to /u/aUserName
+     * * `u/aUserName` -- send to /u/aUserName
+     * * `r/aSubreddit` -- sent to modmail of /r/aSubreddit
+     *
+     * **Note:** Reddit does not support sending a message AS a subreddit TO another subreddit
+     *
+     * @pattern ^\s*(\/[ru]\/|[ru]\/)*(\w+)*\s*$
+     * @examples ["aUserName","u/aUserName","r/aSubreddit"]
+     * */
+    to?: string
+
     /**
      * The title of the message
      *

src/Action/ReportAction.ts

@@ -36,7 +36,8 @@ export class ReportAction extends Action {
 
         return {
             dryRun,
-            success: true
+            success: true,
+            result: truncatedContent
         };
     }
 }

src/Action/SubmissionAction/FlairAction.ts

@@ -23,6 +23,15 @@ export class FlairAction extends Action {
 
     async process(item: Comment | Submission, ruleResults: RuleResult[], runtimeDryrun?: boolean): Promise<ActionProcessResult> {
         const dryRun = runtimeDryrun || this.dryRun;
+        let flairParts = [];
+        if(this.text !== '') {
+            flairParts.push(`Text: ${this.text}`);
+        }
+        if(this.css !== '') {
+            flairParts.push(`CSS: ${this.css}`);
+        }
+        const flairSummary = flairParts.length === 0 ? 'No flair (unflaired)' : flairParts.join(' | ');
+        this.logger.verbose(flairSummary);
         if (item instanceof Submission) {
             if(!this.dryRun) {
                 // @ts-ignore
@@ -39,6 +48,7 @@ export class FlairAction extends Action {
         return {
             dryRun,
             success: true,
+            result: flairSummary
         }
     }
 }

src/Action/UserNoteAction.ts

@@ -50,7 +50,8 @@ export class UserNoteAction extends Action {
         }
         return {
             success: true,
-            dryRun
+            dryRun,
+            result: `(${this.type}) ${renderedContent}`
         }
     }
 }

src/Author/Author.ts

@@ -99,6 +99,13 @@ export interface AuthorCriteria {
      * Does Author's account have a verified email?
      * */
     verified?: boolean
+
+    /**
+     * Is the author shadowbanned?
+     *
+     * This is determined by trying to retrieve the author's profile. If a 404 is returned it is likely they are shadowbanned
+     * */
+    shadowBanned?: boolean
 }
 
 export class Author implements AuthorCriteria {
@@ -112,6 +119,7 @@ export class Author implements AuthorCriteria {
     linkKarma?: string;
     totalKarma?: string;
     verified?: boolean;
+    shadowBanned?: boolean;
 
     constructor(options: AuthorCriteria) {
         this.name = options.name;
@@ -123,6 +131,7 @@ export class Author implements AuthorCriteria {
         this.commentKarma = options.commentKarma;
         this.linkKarma = options.linkKarma;
         this.totalKarma = options.totalKarma;
+        this.shadowBanned = options.shadowBanned;
     }
 }
 

src/Common/defaults.ts

@@ -1,2 +1,31 @@
+import {HistoricalStats} from "./interfaces";
+
 export const cacheOptDefaults = {ttl: 60, max: 500, checkPeriod: 600};
-export const cacheTTLDefaults = {authorTTL: 60, userNotesTTL: 300, wikiTTL: 300, submissionTTL: 60, commentTTL: 60, filterCriteriaTTL: 60};
+export const cacheTTLDefaults = {authorTTL: 60, userNotesTTL: 300, wikiTTL: 300, submissionTTL: 60, commentTTL: 60, filterCriteriaTTL: 60, subredditTTL: 600};
+export const historicalDefaults: HistoricalStats = {
+    eventsCheckedTotal: 0,
+    eventsActionedTotal: 0,
+    checksRun: new Map(),
+    checksFromCache: new Map(),
+    checksTriggered: new Map(),
+    rulesRun: new Map(),
+    //rulesCached: new Map(),
+    rulesCachedTotal: 0,
+    rulesTriggered: new Map(),
+    actionsRun: new Map(),
+}
+
+export const createHistoricalDefaults = (): HistoricalStats => {
+    return {
+        eventsCheckedTotal: 0,
+        eventsActionedTotal: 0,
+        checksRun: new Map(),
+        checksFromCache: new Map(),
+        checksTriggered: new Map(),
+        rulesRun: new Map(),
+        //rulesCached: new Map(),
+        rulesCachedTotal: 0,
+        rulesTriggered: new Map(),
+        actionsRun: new Map(),
+    };
+}

src/Common/interfaces.ts

@@ -4,6 +4,8 @@ import {MESSAGE} from 'triple-beam';
 import Poll from "snoostorm/out/util/Poll";
 import Snoowrap from "snoowrap";
 import {RuleResult} from "../Rule";
+import {IncomingMessage} from "http";
+import {ResembleSingleCallbackComparisonResult} from "resemblejs";
 
 /**
  * An ISO 8601 Duration
@@ -223,6 +225,59 @@ export interface ReferenceSubmission {
     useSubmissionAsReference?: boolean,
 }
 
+/**
+ * When comparing submissions detect if the reference submission is an image and do a pixel-comparison to other detected image submissions.
+ *
+ * **Note:** This is an **experimental feature**
+ * */
+export interface ImageDetection {
+    /**
+     * Is image detection enabled?
+     * */
+    enable?: boolean
+    /**
+     * Determines how and when to check if a URL is an image
+     *
+     * **Note:** After fetching a URL the **Content-Type** is validated to contain `image` before detection occurs
+     *
+     * **When `extension`:** (default)
+     *
+     * * Only URLs that end in known image extensions (.png, .jpg, etc...) are fetched
+     *
+     * **When `unknown`:**
+     *
+     * * URLs that end in known image extensions (.png, .jpg, etc...) are fetched
+     * * URLs with no extension or unknown (IE non-video, non-doc, etc...) are fetched
+     *
+     * **When `all`:**
+     *
+     * * All submissions that have URLs (non-self) will be fetched, regardless of extension
+     * * **Note:** This can be bandwidth/CPU intensive if history window is large so use with care
+     *
+     * @default "extension"
+     * */
+    fetchBehavior?: 'extension' | 'unknown' | 'all',
+    /**
+     * The percentage, as a whole number, of pixels that are **different** between the two images at which point the images are not considered the same.
+     *
+     * Default is `5`
+     *
+     * @default 5
+     * */
+    threshold?: number
+}
+
+export interface ImageData {
+    data: Buffer,
+    width: number,
+    height: number
+    pixels: number
+}
+
+export interface ResembleResult extends ResembleSingleCallbackComparisonResult {
+    rawMisMatchPercentage: number
+}
+
 export interface RichContent {
     /**
      * The Content to submit for this Action. Content is interpreted as reddit-flavored Markdown.
@@ -433,6 +488,16 @@ export interface TTLConfig {
      * @default 60
      * */
     commentTTL?: number | boolean;
+    /**
+     * Amount of time, in seconds, a subreddit (attributes) should be cached
+     *
+     * * If `0` or `true` will cache indefinitely (not recommended)
+     * * If `false` will not cache
+     *
+     * @examples [600]
+     * @default 600
+     * */
+    subredditTTL?: number | boolean;
     /**
      * Amount of time, in seconds, to cache filter criteria results (`authorIs` and `itemIs` results)
      *
@@ -665,6 +730,8 @@ export interface ActivityState {
     stickied?: boolean
     distinguished?: boolean
     approved?: boolean
+    score?: CompareValue
+    reports?: CompareValue
 }
 
 /**
@@ -703,6 +770,41 @@ export interface CommentState extends ActivityState {
     submissionState?: SubmissionState[]
 }
 
+/**
+ * Different attributes a `Subreddit` can be in. Only include a property if you want to check it.
+ * @examples [{"over18": true}]
+ * */
+export interface SubredditState {
+    /**
+     * Is subreddit quarantined?
+     * */
+    quarantine?: boolean
+    /**
+     * Is subreddit NSFW/over 18?
+     *
+     * **Note**: This is **mod-controlled flag** so it is up to the mods of the subreddit to correctly mark their subreddit as NSFW
+     * */
+    over18?: boolean
+    /**
+     * The name the subreddit.
+     *
+     * Can be a normal string (will check case-insensitive) or a regular expression
+     *
+     * EX `["mealtimevideos", "/onlyfans*\/i"]`
+     *
+     * @examples ["mealtimevideos", "/onlyfans*\/i"]
+     * */
+    name?: string | RegExp
+    /**
+     * A friendly description of what this State is trying to parse
+     * */
+    stateDescription?: string
+}
+
+export interface StrongSubredditState extends SubredditState {
+    name?: RegExp
+}
+
 export type TypedActivityStates = SubmissionState[] | CommentState[];
 
 export interface DomainInfo {
@@ -755,6 +857,7 @@ export type StrongCache = {
     wikiTTL: number | boolean,
     submissionTTL: number | boolean,
     commentTTL: number | boolean,
+    subredditTTL: number | boolean,
     filterCriteriaTTL: number | boolean,
     provider: CacheOptions
     actionedEventsMax?: number,
@@ -1492,3 +1595,98 @@ export interface UserResultCache {
     result: boolean,
     ruleResults: RuleResult[]
 }
+
+export type RedditEntityType = 'user' | 'subreddit';
+
+export interface RedditEntity {
+    name: string
+    type: RedditEntityType
+}
+
+export interface StatusCodeError extends Error {
+    name: 'StatusCodeError',
+    statusCode: number,
+    message: string,
+    response: IncomingMessage,
+    error: Error
+}
+
+export interface HistoricalStatsDisplay extends HistoricalStats {
+    checksRunTotal: number
+    checksFromCacheTotal: number
+    checksTriggeredTotal: number
+    rulesRunTotal: number
+    rulesCachedTotal: number
+    rulesTriggeredTotal: number
+    actionsRunTotal: number
+}
+
+export interface HistoricalStats {
+    eventsCheckedTotal: number
+    eventsActionedTotal: number
+    checksRun: Map<string, number>
+    checksFromCache: Map<string, number>
+    checksTriggered: Map<string, number>
+    rulesRun: Map<string, number>
+    //rulesCached: Map<string, number>
+    rulesCachedTotal: number
+    rulesTriggered: Map<string, number>
+    actionsRun: Map<string, number>
+    [index: string]: any
+}
+
+export interface SubredditHistoricalStats {
+    allTime: HistoricalStats
+    lastReload: HistoricalStats
+}
+
+export interface SubredditHistoricalStatsDisplay {
+    allTime: HistoricalStatsDisplay
+    lastReload: HistoricalStatsDisplay
+}
+
+export interface ManagerStats {
+    // eventsCheckedTotal: number
+    // eventsCheckedSinceStartTotal: number
+    eventsAvg: number
+    // checksRunTotal: number
+    // checksRunSinceStartTotal: number
+    // checksTriggered: number
+    // checksTriggeredTotal: number
+    // checksTriggeredSinceStart: number
+    // checksTriggeredSinceStartTotal: number
+    // rulesRunTotal: number
+    // rulesRunSinceStartTotal: number
+    // rulesCachedTotal: number
+    // rulesCachedSinceStartTotal: number
+    // rulesTriggeredTotal: number
+    // rulesTriggeredSinceStartTotal: number
+    rulesAvg: number
+    // actionsRun: number
+    // actionsRunTotal: number
+    // actionsRunSinceStart: number,
+    // actionsRunSinceStartTotal: number
+    historical: SubredditHistoricalStatsDisplay
+    cache: {
+        provider: string,
+        currentKeyCount: number,
+        isShared: boolean,
+        totalRequests: number,
+        totalMiss: number,
+        missPercent: string,
+        requestRate: number,
+        types: ResourceStats
+    },
+}
+
+export interface HistoricalStatUpdateData {
+    eventsCheckedTotal?: number
+    eventsActionedTotal?: number
+    checksRun: string[] | string
+    checksTriggered: string[] | string
+    checksFromCache: string[] | string
+    actionsRun: string[] | string
+    rulesRun: string[] | string
+    rulesCachedTotal: number
+    rulesTriggered: string[] | string
+}

src/Rule/AttributionRule.ts

@@ -53,8 +53,6 @@ export interface AttributionCriteria {
     /**
      * A list of domains whose Activities will be tested against `threshold`.
      *
-     * If this is present then `aggregateOn` is ignored.
-     *
      * The values are tested as partial strings so you do not need to include full URLs, just the part that matters.
      *
      * EX `["youtube"]` will match submissions with the domain `https://youtube.com/c/aChannel`
@@ -98,18 +96,19 @@ export interface AttributionCriteria {
     exclude?: string[],
 
     /**
-     * If `domains` is not specified this list determines which categories of domains should be aggregated on. All aggregated domains will be tested against `threshold`
+     * This list determines which categories of domains should be aggregated on. All aggregated domains will be tested against `threshold`
      *
      * * If `media` is included then aggregate author's submission history which reddit recognizes as media (youtube, vimeo, etc.)
-     * * If `self` is included then aggregate on author's submission history which are self-post (`self.[subreddit]`) or reddit image/video (i.redd.it / v.redd.it)
-     * * If `link` is included then aggregate author's submission history which is external links but not media
+     * * If `redditMedia` is included then aggregate on author's submissions history which are media hosted on reddit: galleries, videos, and images (i.redd.it / v.redd.it)
+     * * If `self` is included then aggregate on author's submission history which are self-post (`self.[subreddit]`) or domain is `reddit.com`
+     * * If `link` is included then aggregate author's submission history which is external links and not recognized as `media` by reddit
      *
-     * If nothing is specified or list is empty (default) all domains are aggregated
+     * If nothing is specified or list is empty (default) rule will only aggregate on `link` and `media` (ignores reddit-hosted content and self-posts)
      *
      *  @default undefined
      *  @examples [[]]
      * */
-    aggregateOn?: ('media' | 'self' | 'link')[],
+    aggregateOn?: ('media' | 'redditMedia' | 'self' | 'link')[],
 
     /**
      * Should the criteria consolidate recognized media domains into the parent domain?
@@ -175,7 +174,7 @@ export class AttributionRule extends Rule {
                 window,
                 thresholdOn = 'all',
                 minActivityCount = 10,
-                aggregateOn = [],
+                aggregateOn = ['link','media'],
                 consolidateMediaDomains = false,
                 domains = [],
                 domainsCombined = false,
@@ -234,16 +233,23 @@ export class AttributionRule extends Rule {
                 const domainInfo = getAttributionIdentifier(sub, consolidateMediaDomains)
 
                 let domainType = 'link';
-                if(sub.secure_media !== undefined && sub.secure_media !== null) {
-                    domainType = 'media';
-                } else if((sub.is_self || sub.is_video || sub.domain === 'i.redd.it')) {
+                if(sub.is_video || ['i.redd.it','v.redd.it'].includes(sub.domain)
+                    // @ts-ignore
+                    || sub.gallery_data !== undefined) {
+                    domainType = 'redditMedia';
+                } else if(sub.is_self || sub.domain === 'reddit.com') {
                     domainType = 'self';
+                } else if(sub.secure_media !== undefined && sub.secure_media !== null) {
+                    domainType = 'media';
                 }
 
-                if(realDomains.length === 0 && aggregateOn.length !== 0) {
+                if(aggregateOn.length !== 0) {
                     if(domainType === 'media' && !aggregateOn.includes('media')) {
                         return acc;
                     }
+                    if(domainType === 'redditMedia' && !aggregateOn.includes('redditMedia')) {
+                        return acc;
+                    }
                     if(domainType === 'self' && !aggregateOn.includes('self')) {
                         return acc;
                     }
@@ -386,7 +392,7 @@ export class AttributionRule extends Rule {
 
 }
 
-interface AttributionConfig extends ReferenceSubmission {
+interface AttributionConfig {
 
     /**
      * A list threshold-window values to test attribution against

src/Rule/HistoryRule.ts

@@ -1,5 +1,5 @@
 
-import {ActivityWindowType, CompareValueOrPercent, ThresholdCriteria} from "../Common/interfaces";
+import {ActivityWindowType, CompareValueOrPercent, SubredditState, ThresholdCriteria} from "../Common/interfaces";
 import {Rule, RuleJSONConfig, RuleOptions, RuleResult} from "./index";
 import Submission from "snoowrap/dist/objects/Submission";
 import {getAuthorActivities} from "../Utils/SnoowrapUtils";
@@ -11,8 +11,9 @@ import {
     formatNumber, getActivitySubredditName, isSubmission,
     parseGenericValueOrPercentComparison, parseSubredditName,
     PASS,
-    percentFromString
+    percentFromString, toStrongSubredditState
 } from "../util";
+import {Comment} from "snoowrap";
 
 export interface CommentThresholdCriteria extends ThresholdCriteria {
     /**
@@ -24,42 +25,56 @@ export interface CommentThresholdCriteria extends ThresholdCriteria {
     asOp?: boolean
 }
 /**
- * If both `submission` and `comment` are defined then criteria will only trigger if BOTH thresholds are met
+ * Criteria will only trigger if ALL present thresholds (comment, submission, total) are met
  * */
 export interface HistoryCriteria {
 
     /**
-     * A string containing a comparison operator and a value to compare submissions against
+     * A string containing a comparison operator and a value to compare **filtered** (using `include` or `exclude`, if present) submissions against
      *
      * The syntax is `(< OR > OR <= OR >=) <number>[percent sign]`
      *
-     * * EX `> 100`  => greater than 100 submissions
-     * * EX `<= 75%` => submissions are equal to or less than 75% of all Activities
+     * * EX `> 100`  => greater than 100 filtered submissions
+     * * EX `<= 75%` => filtered submissions are equal to or less than 75% of unfiltered Activities
      *
      * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(%?)(.*)$
      * */
     submission?: CompareValueOrPercent
     /**
-     * A string containing a comparison operator and a value to compare comments against
+     * A string containing a comparison operator and a value to compare **filtered** (using `include` or `exclude`, if present) comments against
      *
      * The syntax is `(< OR > OR <= OR >=) <number>[percent sign] [OP]`
      *
      * * EX `> 100`  => greater than 100 comments
-     * * EX `<= 75%` => comments are equal to or less than 75% of all Activities
+     * * EX `<= 75%` => comments are equal to or less than 75% of unfiltered Activities
      *
      * If your string also contains the text `OP` somewhere **after** `<number>[percent sign]`...:
      *
-     * * EX `> 100 OP`  => greater than 100 comments as OP
-     * * EX `<= 25% as OP` => Comments as OP were less then or equal to 25% of **all Comments**
+     * * EX `> 100 OP`  => greater than 100 filtered comments as OP
+     * * EX `<= 25% as OP` => **Filtered** comments as OP were less then or equal to 25% of **unfiltered Comments**
      *
      * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(%?)(.*)$
      * */
     comment?: CompareValueOrPercent
 
+    /**
+     * A string containing a comparison operator and a value to compare **filtered** (using `include` or `exclude`) activities against
+     *
+     * **Note:** This is only useful if using `include` or `exclude` otherwise percent will always be 100% and total === activityTotal
+     *
+     * The syntax is `(< OR > OR <= OR >=) <number>[percent sign] [OP]`
+     *
+     * * EX `> 100`  => greater than 100 filtered activities
+     * * EX `<= 75%` => filtered activities are equal to or less than 75% of all Activities
+     *
+     * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(%?)(.*)$
+     * */
+    total?: CompareValueOrPercent
+
     window: ActivityWindowType
 
     /**
-     * The minimum number of activities that must exist from the `window` results for this criteria to run
+     * The minimum number of **filtered** activities that must exist from the `window` results for this criteria to run
      * @default 5
      * */
     minActivityCount?: number
@@ -69,8 +84,9 @@ export interface HistoryCriteria {
 export class HistoryRule extends Rule {
     criteria: HistoryCriteria[];
     condition: 'AND' | 'OR';
-    include: string[];
-    exclude: string[];
+    include: (string | SubredditState)[];
+    exclude: (string | SubredditState)[];
+    activityFilterFunc: (x: Submission|Comment) => Promise<boolean> = async (x) => true;
 
     constructor(options: HistoryOptions) {
         super(options);
@@ -86,8 +102,41 @@ export class HistoryRule extends Rule {
         if (this.criteria.length === 0) {
             throw new Error('Must provide at least one HistoryCriteria');
         }
-        this.include = include.map(x => parseSubredditName(x).toLowerCase());
-        this.exclude = exclude.map(x => parseSubredditName(x).toLowerCase());
+
+        this.include = include;
+        this.exclude = exclude;
+
+        if(this.include.length > 0) {
+            const subStates = include.map((x) => {
+                if(typeof x === 'string') {
+                    return toStrongSubredditState({name: x, stateDescription: x}, {defaultFlags: 'i', generateDescription: true});
+                }
+                return toStrongSubredditState(x, {defaultFlags: 'i', generateDescription: true});
+            });
+            this.activityFilterFunc = async (x: Submission|Comment) => {
+                for(const ss of subStates) {
+                    if(await this.resources.testSubredditCriteria(x, ss)) {
+                        return true;
+                    }
+                }
+                return false;
+            };
+        } else if(this.exclude.length > 0) {
+            const subStates = exclude.map((x) => {
+                if(typeof x === 'string') {
+                    return toStrongSubredditState({name: x, stateDescription: x}, {defaultFlags: 'i', generateDescription: true});
+                }
+                return toStrongSubredditState(x, {defaultFlags: 'i', generateDescription: true});
+            });
+            this.activityFilterFunc = async (x: Submission|Comment) => {
+                for(const ss of subStates) {
+                    if(await this.resources.testSubredditCriteria(x, ss)) {
+                        return false;
+                    }
+                }
+                return true;
+            };
+        }
     }
 
     getKind(): string {
@@ -108,19 +157,17 @@ export class HistoryRule extends Rule {
 
         for (const criteria of this.criteria) {
 
-            const {comment, window, submission, minActivityCount = 5} = criteria;
+            const {comment, window, submission, total, minActivityCount = 5} = criteria;
 
             let activities = await this.resources.getAuthorActivities(item.author, {window: window});
-            activities = activities.filter(act => {
-                if (this.include.length > 0) {
-                    return this.include.some(x => x === getActivitySubredditName(act).toLowerCase());
-                } else if (this.exclude.length > 0) {
-                    return !this.exclude.some(x => x === getActivitySubredditName(act).toLowerCase())
+            const filteredActivities = [];
+            for(const a of activities) {
+                if(await this.activityFilterFunc(a)) {
+                    filteredActivities.push(a);
                 }
-                return true;
-            });
+            }
 
-            if (activities.length < minActivityCount) {
+            if (filteredActivities.length < minActivityCount) {
                 continue;
             }
 
@@ -135,6 +182,24 @@ export class HistoryRule extends Rule {
                 }
                 return a;
             },{submissionTotal: 0, commentTotal: 0, opTotal: 0});
+            let fSubmissionTotal = submissionTotal;
+            let fCommentTotal = commentTotal;
+            let fOpTotal = opTotal;
+            if(activities.length !== filteredActivities.length) {
+                const filteredCounts = filteredActivities.reduce((acc, act) => {
+                    if(asSubmission(act)) {
+                        return {...acc, submissionTotal: acc.submissionTotal + 1};
+                    }
+                    let a = {...acc, commentTotal: acc.commentTotal + 1};
+                    if(act.is_submitter) {
+                        a.opTotal = a.opTotal + 1;
+                    }
+                    return a;
+                },{submissionTotal: 0, commentTotal: 0, opTotal: 0});
+                fSubmissionTotal = filteredCounts.submissionTotal;
+                fCommentTotal = filteredCounts.commentTotal;
+                fOpTotal = filteredCounts.opTotal;
+            }
 
             let commentTrigger = undefined;
             if(comment !== undefined) {
@@ -143,15 +208,15 @@ export class HistoryRule extends Rule {
                 if(isPercent) {
                     const per = value / 100;
                     if(asOp) {
-                        commentTrigger = comparisonTextOp(opTotal / commentTotal, operator, per);
+                        commentTrigger = comparisonTextOp(fOpTotal / commentTotal, operator, per);
                     } else {
-                        commentTrigger = comparisonTextOp(commentTotal / activityTotal, operator, per);
+                        commentTrigger = comparisonTextOp(fCommentTotal / activityTotal, operator, per);
                     }
                 } else {
                     if(asOp) {
-                        commentTrigger = comparisonTextOp(opTotal, operator, value);
+                        commentTrigger = comparisonTextOp(fOpTotal, operator, value);
                     } else {
-                        commentTrigger = comparisonTextOp(commentTotal, operator, value);
+                        commentTrigger = comparisonTextOp(fCommentTotal, operator, value);
                     }
                 }
             }
@@ -161,9 +226,20 @@ export class HistoryRule extends Rule {
                 const {operator, value, isPercent} = parseGenericValueOrPercentComparison(submission);
                 if(isPercent) {
                     const per = value / 100;
-                    submissionTrigger = comparisonTextOp(submissionTotal / activityTotal, operator, per);
+                    submissionTrigger = comparisonTextOp(fSubmissionTotal / activityTotal, operator, per);
                 } else {
-                    submissionTrigger = comparisonTextOp(submissionTotal, operator, value);
+                    submissionTrigger = comparisonTextOp(fSubmissionTotal, operator, value);
+                }
+            }
+
+            let totalTrigger = undefined;
+            if(total !== undefined) {
+                const {operator, value, isPercent} = parseGenericValueOrPercentComparison(total);
+                if(isPercent) {
+                    const per = value / 100;
+                    totalTrigger = comparisonTextOp(filteredActivities.length / activityTotal, operator, per);
+                } else {
+                    totalTrigger = comparisonTextOp(filteredActivities.length, operator, value);
                 }
             }
 
@@ -176,12 +252,14 @@ export class HistoryRule extends Rule {
                 criteria,
                 activityTotal,
                 activityTotalWindow,
-                submissionTotal,
-                commentTotal,
-                opTotal,
+                submissionTotal: fSubmissionTotal,
+                commentTotal: fCommentTotal,
+                opTotal: fOpTotal,
+                filteredTotal: filteredActivities.length,
                 submissionTrigger,
                 commentTrigger,
-                triggered: (submissionTrigger === undefined || submissionTrigger === true) && (commentTrigger === undefined || commentTrigger === true)
+                totalTrigger,
+                triggered: (submissionTrigger === undefined || submissionTrigger === true) && (commentTrigger === undefined || commentTrigger === true) && (totalTrigger === undefined || totalTrigger === true)
             });
         }
 
@@ -224,36 +302,50 @@ export class HistoryRule extends Rule {
             activityTotalWindow,
             submissionTotal,
             commentTotal,
+            filteredTotal,
             opTotal,
             criteria: {
                 comment,
                 submission,
+                total,
                 window,
             },
             criteria,
             triggered,
             submissionTrigger,
             commentTrigger,
+            totalTrigger,
         } = results;
 
         const data: any = {
             activityTotal,
             submissionTotal,
             commentTotal,
+            filteredTotal,
             opTotal,
             commentPercent: formatNumber((commentTotal/activityTotal)*100),
             submissionPercent: formatNumber((submissionTotal/activityTotal)*100),
             opPercent: formatNumber((opTotal/commentTotal)*100),
+            filteredPercent: formatNumber((filteredTotal/activityTotal)*100),
             criteria,
             window: typeof window === 'number' || activityTotal === 0 ? `${activityTotal} Items` : activityTotalWindow.humanize(true),
             triggered,
             submissionTrigger,
             commentTrigger,
+            totalTrigger,
         };
 
         let thresholdSummary = [];
+        let totalSummary;
         let submissionSummary;
         let commentSummary;
+        if(total !== undefined) {
+            const {operator, value, isPercent, displayText} = parseGenericValueOrPercentComparison(total);
+            const suffix = !isPercent ? 'Items' : `(${formatNumber((filteredTotal/activityTotal)*100)}%) of ${activityTotal} Total`;
+            totalSummary = `${includePassFailSymbols ? `${submissionTrigger ? PASS : FAIL} ` : ''}Filtered Activities (${filteredTotal}) were${totalTrigger ? '' : ' not'} ${displayText} ${suffix}`;
+            data.totalSummary = totalSummary;
+            thresholdSummary.push(totalSummary);
+        }
         if(submission !== undefined) {
             const {operator, value, isPercent, displayText} = parseGenericValueOrPercentComparison(submission);
             const suffix = !isPercent ? 'Items' : `(${formatNumber((submissionTotal/activityTotal)*100)}%) of ${activityTotal} Total`;
@@ -299,21 +391,45 @@ interface HistoryConfig  {
     condition?: 'AND' | 'OR'
 
     /**
-     * Only include Submissions from this list of Subreddits (by name, case-insensitive)
+     * If present, activities will be counted only if they are found in this list of Subreddits.
      *
-     * EX `["mealtimevideos","askscience"]`
-     * @examples ["mealtimevideos","askscience"]
-     * @minItems 1
+     * Each value in the list can be either:
+     *
+     *  * string (name of subreddit)
+     *  * regular expression to run on the subreddit name
+     *  * `SubredditState`
+     *
+     * EX `["mealtimevideos","askscience", "/onlyfans*\/i", {"over18": true}]`
+     *
+     *  **Note:** This affects **post-window retrieval** activities. So that:
+     *
+     * * `activityTotal` is number of activities retrieved from `window` -- NOT post-filtering
+     * * all comparisons using **percentages** will compare **post-filtering** results against **activity count from window**
+     * * -- to run this rule where all activities are only from include/exclude filtering instead use include/exclude in `window`
+     *
+     * @examples [["mealtimevideos","askscience", "/onlyfans*\/i", {"over18": true}]]
      * */
-    include?: string[],
+    include?: (string | SubredditState)[],
     /**
-     * Do not include Submissions from this list of Subreddits (by name, case-insensitive)
+     * If present, activities will be counted only if they are **NOT** found in this list of Subreddits
      *
-     * EX `["mealtimevideos","askscience"]`
-     * @examples ["mealtimevideos","askscience"]
-     * @minItems 1
+     * Each value in the list can be either:
+     *
+     *  * string (name of subreddit)
+     *  * regular expression to run on the subreddit name
+     *  * `SubredditState`
+     *
+     * EX `["mealtimevideos","askscience", "/onlyfans*\/i", {"over18": true}]`
+     *
+     * **Note:** This affects **post-window retrieval** activities. So that:
+     *
+     * * `activityTotal` is number of activities retrieved from `window` -- NOT post-filtering
+     * * all comparisons using **percentages** will compare **post-filtering** results against **activity count from window**
+     * * -- to run this rule where all activities are only from include/exclude filtering instead use include/exclude in `window`
+     *
+     * @examples [["mealtimevideos","askscience", "/onlyfans*\/i", {"over18": true}]]
      * */
-    exclude?: string[],
+    exclude?: (string | SubredditState)[],
 }
 
 export interface HistoryOptions extends HistoryConfig, RuleOptions {

src/Rule/RecentActivityRule.ts

@@ -1,27 +1,40 @@
 import {Rule, RuleJSONConfig, RuleOptions, RulePremise, RuleResult} from "./index";
 import {Comment, VoteableContent} from "snoowrap";
 import Submission from "snoowrap/dist/objects/Submission";
+// @ts-ignore
+import subImageMatch from 'matches-subimage';
 import {
-    activityWindowText, asSubmission,
-    comparisonTextOp, FAIL, formatNumber, getActivitySubredditName, isSubmission,
-    parseGenericValueOrPercentComparison, parseSubredditName,
+    activityWindowText,
+    asSubmission, compareImages,
+    comparisonTextOp,
+    FAIL,
+    formatNumber,
+    getActivitySubredditName, getImageDataFromUrl,
+    isSubmission,
+    isValidImageURL,
+    objectToStringSummary,
+    parseGenericValueOrPercentComparison,
+    parseStringToRegex,
+    parseSubredditName,
     parseUsableLinkIdentifier,
-    PASS
+    PASS,
+    toStrongSubredditState
 } from "../util";
 import {
     ActivityWindow,
     ActivityWindowCriteria,
-    ActivityWindowType,
-    ReferenceSubmission,
-    SubredditCriteria
+    ActivityWindowType, CommentState, ImageDetection,
+    ReferenceSubmission, StrongSubredditState, SubmissionState,
+    SubredditCriteria, SubredditState
 } from "../Common/interfaces";
 
 const parseLink = parseUsableLinkIdentifier();
 
 export class RecentActivityRule extends Rule {
     window: ActivityWindowType;
-    thresholds: SubThreshold[];
+    thresholds: ActivityThreshold[];
     useSubmissionAsReference: boolean;
+    imageDetection: Required<ImageDetection>
     lookAt?: 'comments' | 'submissions';
 
     constructor(options: RecentActivityRuleOptions) {
@@ -29,8 +42,21 @@ export class RecentActivityRule extends Rule {
         const {
             window = 15,
             useSubmissionAsReference = true,
+            imageDetection,
             lookAt,
         } = options || {};
+
+        const {
+            enable = false,
+            fetchBehavior = 'extension',
+            threshold = 5
+        } = imageDetection || {};
+
+        this.imageDetection = {
+            enable,
+            fetchBehavior,
+            threshold
+        };
         this.lookAt = lookAt;
         this.useSubmissionAsReference = useSubmissionAsReference;
         this.window = window;
@@ -72,45 +98,111 @@ export class RecentActivityRule extends Rule {
             } else if (item.is_self) {
                 this.logger.warn('Cannot use post as reference because triggered Submission is not a link type');
             } else {
-                const usableUrl = parseLink(await item.url);
-                viableActivity = viableActivity.filter((x) => {
-                    if (!asSubmission(x)) {
-                        return false;
+                const itemId = item.id;
+                const referenceUrl = await item.url;
+                const usableUrl = parseLink(referenceUrl);
+                const filteredActivity = [];
+                let referenceImage;
+                if(this.imageDetection.enable) {
+                    const [response, imgData, reason] = await getImageDataFromUrl(referenceUrl);
+                    if(reason !== undefined) {
+                        this.logger.verbose(reason);
+                    } else {
+                        referenceImage = imgData
+                    }
+                }
+                let longRun;
+                if(referenceImage !== undefined) {
+                    const l = this.logger;
+                    longRun = setTimeout(() => {
+                        l.verbose('FYI: Image processing is causing rule to take longer than normal');
+                    }, 2500);
+                }
+                for(const x of viableActivity) {
+                    if (!asSubmission(x) || x.id === itemId) {
+                        continue;
                     }
                     if (x.url === undefined) {
-                        return false;
+                        continue;
                     }
-                    return parseLink(x.url) === usableUrl;
-                });
+                    if(parseLink(x.url) === usableUrl) {
+                        filteredActivity.push(x);
+                    }
+                    // only do image detection if regular URL comparison and other conditions fail first
+                    // to reduce CPU/bandwidth usage
+                    if(referenceImage !== undefined) {
+                        const [response, imgData, reason] = await getImageDataFromUrl(x.url);
+                        if(imgData !== undefined) {
+                            try {
+                                const [compareResult, sameImage] = await compareImages(referenceImage, imgData, this.imageDetection.threshold);
+                                if(sameImage) {
+                                    filteredActivity.push(x);
+                                }
+                            } catch (err) {
+                                this.logger.warn(`Unexpected error encountered while comparing images, will skip comparison: ${err.message}`);
+                            }
+                        }
+                    }
+                }
+                if(longRun !== undefined) {
+                    clearTimeout(longRun);
+                }
+                viableActivity = filteredActivity;
             }
         }
-        const groupedActivity = viableActivity.reduce((grouped, activity) => {
-            const s = getActivitySubredditName(activity).toLowerCase();
-            grouped[s] = (grouped[s] || []).concat(activity);
-            return grouped;
-        }, {} as Record<string, (Submission | Comment)[]>);
-
 
         const summaries = [];
         let totalTriggeredOn;
         for (const triggerSet of this.thresholds) {
             let currCount = 0;
-            const presentSubs = [];
+            const presentSubs: string[] = [];
             let combinedKarma = 0;
-            const {threshold = '>= 1', subreddits = [], karma: karmaThreshold} = triggerSet;
-            for (const sub of subreddits.map(x => parseSubredditName(x))) {
-                const isub = sub.toLowerCase();
-                const {[isub]: tSub = []} = groupedActivity;
-                if (tSub.length > 0) {
-                    currCount += tSub.length;
-                    presentSubs.push(sub);
-                    for(const a of tSub) {
-                        combinedKarma += a.score;
+            const {
+                threshold = '>= 1',
+                subreddits = [],
+                karma: karmaThreshold,
+                commentState,
+                submissionState,
+            } = triggerSet;
+
+            // convert subreddits array into entirely StrongSubredditState
+            const subStates: StrongSubredditState[] = subreddits.map((x) => {
+                if(typeof x === 'string') {
+                    return toStrongSubredditState({name: x, stateDescription: x}, {defaultFlags: 'i', generateDescription: true});
+                }
+                return toStrongSubredditState(x, {defaultFlags: 'i', generateDescription: true});
+            });
+
+            for(const activity of viableActivity) {
+                if(asSubmission(activity) && submissionState !== undefined) {
+                    if(!(await this.resources.testItemCriteria(activity, [submissionState]))) {
+                        continue;
+                    }
+                } else if(commentState !== undefined) {
+                    if(!(await this.resources.testItemCriteria(activity, [commentState]))) {
+                        continue;
+                    }
+                }
+                let inSubreddits = false;
+                for(const ss of subStates) {
+                    const res = await this.resources.testSubredditCriteria(activity, ss);
+                    if(res) {
+                        inSubreddits = true;
+                        break;
+                    }
+                }
+                if(inSubreddits) {
+                    currCount++;
+                    combinedKarma += activity.score;
+                    const pSub = getActivitySubredditName(activity);
+                    if(!presentSubs.includes(pSub)) {
+                        presentSubs.push(pSub);
                     }
                 }
             }
+
             const {operator, value, isPercent} = parseGenericValueOrPercentComparison(threshold);
-            let sum = {subsWithActivity: presentSubs, combinedKarma, karmaThreshold, subreddits, count: currCount, threshold, triggered: false, testValue: currCount.toString()};
+            let sum = {subsWithActivity: presentSubs, combinedKarma, karmaThreshold, subreddits: subStates.map(x => x.stateDescription), count: currCount, threshold, triggered: false, testValue: currCount.toString()};
             if (isPercent) {
                 sum.testValue = `${formatNumber((currCount / viableActivity.length) * 100)}%`;
                 if (comparisonTextOp(currCount / viableActivity.length, operator, value / 100)) {
@@ -194,7 +286,16 @@ export class RecentActivityRule extends Rule {
  * @minProperties 1
  * @additionalProperties false
  * */
-export interface SubThreshold extends SubredditCriteria {
+export interface ActivityThreshold {
+    /**
+    * When present, a Submission will only be counted if it meets this criteria
+    * */
+    submissionState?: SubmissionState
+    /**
+    * When present, a Comment will only be counted if it meets this criteria
+    * */
+    commentState?: CommentState
+
     /**
      * A string containing a comparison operator and a value to compare recent activities against
      *
@@ -225,6 +326,20 @@ export interface SubThreshold extends SubredditCriteria {
      * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(%?)(.*)$
      * */
     karma?: string
+
+    /**
+     * Activities will be counted if they are found in this list of Subreddits
+     *
+     * Each value in the list can be either:
+     *
+     *  * string (name of subreddit)
+     *  * regular expression to run on the subreddit name
+     *  * `SubredditState`
+     *
+     * EX `["mealtimevideos","askscience", "/onlyfans*\/i", {"over18": true}]`
+     * @examples [["mealtimevideos","askscience", "/onlyfans*\/i", {"over18": true}]]
+     * */
+    subreddits?: (string | SubredditState)[]
 }
 
 interface RecentActivityConfig extends ActivityWindow, ReferenceSubmission {
@@ -237,7 +352,9 @@ interface RecentActivityConfig extends ActivityWindow, ReferenceSubmission {
      * A list of subreddits/count criteria that may trigger this rule. ANY SubThreshold will trigger this rule.
      * @minItems 1
      * */
-    thresholds: SubThreshold[],
+    thresholds: ActivityThreshold[],
+
+    imageDetection?: ImageDetection
 }
 
 export interface RecentActivityRuleOptions extends RecentActivityConfig, RuleOptions {

src/Rule/RegexRule.ts

@@ -4,13 +4,14 @@ import Submission from "snoowrap/dist/objects/Submission";
 import {
     asSubmission,
     comparisonTextOp, FAIL, isExternalUrlSubmission, isSubmission, parseGenericValueComparison,
-    parseGenericValueOrPercentComparison, parseRegex,
+    parseGenericValueOrPercentComparison, parseRegex, parseStringToRegex,
     PASS, triggeredIndicator
 } from "../util";
 import {
     ActivityWindowType, JoinOperands,
 } from "../Common/interfaces";
 import dayjs from 'dayjs';
+import SimpleError from "../Utils/SimpleError";
 
 export interface RegexCriteria {
     /**
@@ -22,17 +23,11 @@ export interface RegexCriteria {
     /**
      * A valid Regular Expression to test content against
      *
-     * Do not wrap expression in forward slashes
+     * If no flags are specified then the **global** flag is used by default
      *
-     * EX For the expression `/reddit|FoxxMD/` use the value should be `reddit|FoxxMD`
-     *
-     * @examples ["reddit|FoxxMD"]
+     * @examples ["/reddit|FoxxMD/ig"]
      * */
     regex: string,
-    /**
-     * Regex flags to use
-     * */
-    regexFlags?: string,
 
     /**
      * Which content from an Activity to test the regex against
@@ -135,12 +130,11 @@ export class RegexRule extends Rule {
 
         let criteriaResults = [];
 
-        for (const criteria of this.criteria) {
+        for (const [index, criteria] of this.criteria.entries()) {
 
             const {
-                name,
+                name = (index + 1),
                 regex,
-                regexFlags = 'g',
                 testOn: testOnVals = ['title', 'body'],
                 lookAt = 'all',
                 matchThreshold = '> 0',
@@ -158,7 +152,10 @@ export class RegexRule extends Rule {
             }, []);
 
             // check regex
-            const reg = new RegExp(regex, regexFlags);
+            const reg = parseStringToRegex(regex, 'g');
+            if(reg === undefined) {
+                throw new SimpleError(`Value given for regex on Criteria ${name} was not valid: ${regex}`);
+            }
             // ok cool its a valid regex
 
             const matchComparison = parseGenericValueComparison(matchThreshold);
@@ -177,7 +174,7 @@ export class RegexRule extends Rule {
 
             // first lets see if the activity we are checking satisfies thresholds
             // since we may be able to avoid api calls to get history
-            let actMatches = this.getMatchesFromActivity(item, testOn, reg, regexFlags);
+            let actMatches = this.getMatchesFromActivity(item, testOn, reg);
             matches = matches.concat(actMatches).slice(0, 100);
             matchCount += actMatches.length;
 
@@ -227,7 +224,7 @@ export class RegexRule extends Rule {
 
                 for (const h of history) {
                     activitiesTested++;
-                    const aMatches = this.getMatchesFromActivity(h, testOn, reg, regexFlags);
+                    const aMatches = this.getMatchesFromActivity(h, testOn, reg);
                     matches = matches.concat(aMatches).slice(0, 100);
                     matchCount += aMatches.length;
                     const matched = comparisonTextOp(aMatches.length, matchComparison.operator, matchComparison.value);
@@ -325,7 +322,7 @@ export class RegexRule extends Rule {
         return Promise.resolve([criteriaMet, this.getResult(criteriaMet, {result, data: criteriaResults})]);
     }
 
-    protected getMatchesFromActivity(a: (Submission | Comment), testOn: string[], reg: RegExp, flags?: string): string[] {
+    protected getMatchesFromActivity(a: (Submission | Comment), testOn: string[], reg: RegExp): string[] {
         let m: string[] = [];
         // determine what content we are testing
         let contents: string[] = [];
@@ -352,7 +349,7 @@ export class RegexRule extends Rule {
         }
 
         for (const c of contents) {
-            const results = parseRegex(reg, c, flags);
+            const results = parseRegex(reg, c);
             if (results.matched) {
                 m = m.concat(results.matches);
             }

src/Rule/RepeatActivityRule.ts

@@ -4,9 +4,15 @@ import {
     activityWindowText, asSubmission,
     comparisonTextOp, FAIL, getActivitySubredditName, isExternalUrlSubmission, isRedditMedia,
     parseGenericValueComparison, parseSubredditName,
-    parseUsableLinkIdentifier as linkParser, PASS
+    parseUsableLinkIdentifier as linkParser, PASS, toStrongSubredditState
 } from "../util";
-import {ActivityWindow, ActivityWindowType, ReferenceSubmission} from "../Common/interfaces";
+import {
+    ActivityWindow,
+    ActivityWindowType,
+    ReferenceSubmission,
+    StrongSubredditState,
+    SubredditState
+} from "../Common/interfaces";
 import Submission from "snoowrap/dist/objects/Submission";
 import dayjs from "dayjs";
 import Fuse from 'fuse.js'
@@ -50,8 +56,9 @@ export class RepeatActivityRule extends Rule {
     gapAllowance?: number;
     useSubmissionAsReference: boolean;
     lookAt: 'submissions' | 'all';
-    include: string[];
-    exclude: string[];
+    include: (string | SubredditState)[];
+    exclude: (string | SubredditState)[];
+    activityFilterFunc: (x: Submission|Comment) => Promise<boolean> = async (x) => true;
     keepRemoved: boolean;
     minWordCount: number;
 
@@ -74,8 +81,40 @@ export class RepeatActivityRule extends Rule {
         this.window = window;
         this.gapAllowance = gapAllowance;
         this.useSubmissionAsReference = useSubmissionAsReference;
-        this.include = include.map(x => parseSubredditName(x).toLowerCase());
-        this.exclude = exclude.map(x => parseSubredditName(x).toLowerCase());
+        this.include = include;
+        this.exclude = exclude;
+
+        if(this.include.length > 0) {
+            const subStates = include.map((x) => {
+                if(typeof x === 'string') {
+                    return toStrongSubredditState({name: x, stateDescription: x}, {defaultFlags: 'i', generateDescription: true});
+                }
+                return toStrongSubredditState(x, {defaultFlags: 'i', generateDescription: true});
+            });
+            this.activityFilterFunc = async (x: Submission|Comment) => {
+                for(const ss of subStates) {
+                    if(await this.resources.testSubredditCriteria(x, ss)) {
+                        return true;
+                    }
+                }
+                return false;
+            };
+        } else if(this.exclude.length > 0) {
+            const subStates = exclude.map((x) => {
+                if(typeof x === 'string') {
+                    return toStrongSubredditState({name: x, stateDescription: x}, {defaultFlags: 'i', generateDescription: true});
+                }
+                return toStrongSubredditState(x, {defaultFlags: 'i', generateDescription: true});
+            });
+            this.activityFilterFunc = async (x: Submission|Comment) => {
+                for(const ss of subStates) {
+                    if(await this.resources.testSubredditCriteria(x, ss)) {
+                        return false;
+                    }
+                }
+                return true;
+            };
+        }
         this.lookAt = lookAt;
     }
 
@@ -100,13 +139,6 @@ export class RepeatActivityRule extends Rule {
             referenceUrl = await item.url;
         }
 
-        let filterFunc = (x: any) => true;
-        if(this.include.length > 0) {
-            filterFunc = (x: Submission|Comment) => this.include.includes(getActivitySubredditName(x).toLowerCase());
-        } else if(this.exclude.length > 0) {
-            filterFunc = (x: Submission|Comment) => !this.exclude.includes(getActivitySubredditName(x).toLowerCase());
-        }
-
         let activities: (Submission | Comment)[] = [];
         switch (this.lookAt) {
             case 'submissions':
@@ -117,13 +149,14 @@ export class RepeatActivityRule extends Rule {
                 break;
         }
 
-        const condensedActivities = activities.reduce((acc: RepeatActivityReducer, activity: (Submission | Comment), index: number) => {
+        const condensedActivities = await activities.reduce(async (accProm: Promise<RepeatActivityReducer>, activity: (Submission | Comment), index: number) => {
+            const acc = await accProm;
             const {openSets = [], allSets = []} = acc;
 
             let identifier = getActivityIdentifier(activity);
             const isUrl = isExternalUrlSubmission(activity);
             let fu = new Fuse([identifier], !isUrl ? fuzzyOptions : {...fuzzyOptions, distance: 5});
-            const validSub = filterFunc(activity);
+            const validSub = await this.activityFilterFunc(activity);
             let minMet = identifier.length >= this.minWordCount;
 
             let updatedAllSets = [...allSets];
@@ -174,7 +207,7 @@ export class RepeatActivityRule extends Rule {
 
             return {openSets: updatedOpenSets, allSets: updatedAllSets};
 
-        }, {openSets: [], allSets: []});
+        }, Promise.resolve({openSets: [], allSets: []}));
 
         const allRepeatSets = [...condensedActivities.allSets, ...condensedActivities.openSets];
 
@@ -294,21 +327,31 @@ interface RepeatActivityConfig extends ActivityWindow, ReferenceSubmission {
      * */
     gapAllowance?: number,
     /**
-     * Only include Submissions from this list of Subreddits (by name, case-insensitive)
+     * If present, activities will be counted only if they are found in this list of Subreddits
      *
-     * EX `["mealtimevideos","askscience"]`
-     * @examples ["mealtimevideos","askscience"]
-     * @minItems 1
+     * Each value in the list can be either:
+     *
+     *  * string (name of subreddit)
+     *  * regular expression to run on the subreddit name
+     *  * `SubredditState`
+     *
+     * EX `["mealtimevideos","askscience", "/onlyfans*\/i", {"over18": true}]`
+     * @examples [["mealtimevideos","askscience", "/onlyfans*\/i", {"over18": true}]]
      * */
-    include?: string[],
+    include?: (string | SubredditState)[],
     /**
-     * Do not include Submissions from this list of Subreddits (by name, case-insensitive)
+     * If present, activities will be counted only if they are **NOT** found in this list of Subreddits
      *
-     * EX `["mealtimevideos","askscience"]`
-     * @examples ["mealtimevideos","askscience"]
-     * @minItems 1
+     * Each value in the list can be either:
+     *
+     *  * string (name of subreddit)
+     *  * regular expression to run on the subreddit name
+     *  * `SubredditState`
+     *
+     * EX `["mealtimevideos","askscience", "/onlyfans*\/i", {"over18": true}]`
+     * @examples [["mealtimevideos","askscience", "/onlyfans*\/i", {"over18": true}]]
      * */
-    exclude?: string[],
+    exclude?: (string | SubredditState)[],
 
     /**
      * If present determines which activities to consider for gapAllowance.

src/Rule/index.ts

@@ -28,10 +28,16 @@ interface ResultContext {
 
 export interface RuleResult extends ResultContext {
     premise: RulePremise
+    kind: string
     name: string
     triggered: (boolean | null)
 }
 
+export type FormattedRuleResult = RuleResult & {
+    triggered: string
+    result: string
+}
+
 export interface RuleSetResult {
     results: RuleResult[],
     condition: 'OR' | 'AND',
@@ -148,6 +154,7 @@ export abstract class Rule implements IRule, Triggerable {
     protected getResult(triggered: (boolean | null) = null, context: ResultContext = {}): RuleResult {
         return {
             premise: this.getPremise(),
+            kind: this.getKind(),
             name: this.name,
             triggered,
             ...context,

src/Schema/Action.json

@@ -69,6 +69,10 @@
                     },
                     "type": "array"
                 },
+                "shadowBanned": {
+                    "description": "Is the author shadowbanned?\n\nThis is determined by trying to retrieve the author's profile. If a 404 is returned it is likely they are shadowbanned",
+                    "type": "boolean"
+                },
                 "totalKarma": {
                     "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
@@ -154,6 +158,16 @@
                 "removed": {
                     "type": "boolean"
                 },
+                "reports": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "score": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
                 "spam": {
                     "type": "boolean"
                 },
@@ -213,6 +227,16 @@
                 "removed": {
                     "type": "boolean"
                 },
+                "reports": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "score": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
                 "spam": {
                     "type": "boolean"
                 },

src/Schema/App.json

@@ -1,6 +1,72 @@
 {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "definitions": {
+        "ActivityThreshold": {
+            "additionalProperties": false,
+            "description": "At least one count property must be present. If both are present then either can trigger the rule",
+            "minProperties": 1,
+            "properties": {
+                "commentState": {
+                    "$ref": "#/definitions/CommentState",
+                    "description": "When present, a Comment will only be counted if it meets this criteria",
+                    "examples": [
+                        {
+                            "op": true,
+                            "removed": false
+                        }
+                    ]
+                },
+                "karma": {
+                    "description": "Test the **combined karma** from Activities found in the specified subreddits\n\nValue is a string containing a comparison operator and a number of **combined karma** to compare against\n\nIf specified then both `threshold` and `karma` must be met for this `SubThreshold` to be satisfied\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 50`  => greater than 50 combined karma for all found Activities in specified subreddits",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "submissionState": {
+                    "$ref": "#/definitions/SubmissionState",
+                    "description": "When present, a Submission will only be counted if it meets this criteria",
+                    "examples": [
+                        {
+                            "over_18": true,
+                            "removed": false
+                        }
+                    ]
+                },
+                "subreddits": {
+                    "description": "Activities will be counted if they are found in this list of Subreddits\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`",
+                    "examples": [
+                        [
+                            "mealtimevideos",
+                            "askscience",
+                            "/onlyfans*/i",
+                            {
+                                "over18": true
+                            }
+                        ]
+                    ],
+                    "items": {
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/SubredditState"
+                            },
+                            {
+                                "type": "string"
+                            }
+                        ]
+                    },
+                    "type": "array"
+                },
+                "threshold": {
+                    "default": ">= 1",
+                    "description": "A string containing a comparison operator and a value to compare recent activities against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 3`  => greater than 3 activities found in the listed subreddits\n* EX `<= 75%` => number of Activities in the subreddits listed are equal to or less than 75% of all Activities\n\n**Note:** If you use percentage comparison here as well as `useSubmissionAsReference` then \"all Activities\" is only pertains to Activities that had the Link of the Submission, rather than all Activities from this window.",
+                    "examples": [
+                        ">= 1"
+                    ],
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
         "ActivityWindowCriteria": {
             "additionalProperties": false,
             "description": "Multiple properties that may be used to define what range of Activity to retrieve.\n\nMay specify one, or both properties along with the `satisfyOn` property, to affect the retrieval behavior.",
@@ -167,7 +233,7 @@
             "properties": {
                 "aggregateOn": {
                     "default": "undefined",
-                    "description": "If `domains` is not specified this list determines which categories of domains should be aggregated on. All aggregated domains will be tested against `threshold`\n\n* If `media` is included then aggregate author's submission history which reddit recognizes as media (youtube, vimeo, etc.)\n* If `self` is included then aggregate on author's submission history which are self-post (`self.[subreddit]`) or reddit image/video (i.redd.it / v.redd.it)\n* If `link` is included then aggregate author's submission history which is external links but not media\n\nIf nothing is specified or list is empty (default) all domains are aggregated",
+                    "description": "This list determines which categories of domains should be aggregated on. All aggregated domains will be tested against `threshold`\n\n* If `media` is included then aggregate author's submission history which reddit recognizes as media (youtube, vimeo, etc.)\n* If `redditMedia` is included then aggregate on author's submissions history which are media hosted on reddit: galleries, videos, and images (i.redd.it / v.redd.it)\n* If `self` is included then aggregate on author's submission history which are self-post (`self.[subreddit]`) or domain is `reddit.com`\n* If `link` is included then aggregate author's submission history which is external links and not recognized as `media` by reddit\n\nIf nothing is specified or list is empty (default) rule will only aggregate on `link` and `media` (ignores reddit-hosted content and self-posts)",
                     "examples": [
                         [
                         ]
@@ -176,6 +242,7 @@
                         "enum": [
                             "link",
                             "media",
+                            "redditMedia",
                             "self"
                         ],
                         "type": "string"
@@ -195,7 +262,7 @@
                         [
                         ]
                     ],
-                    "description": "A list of domains whose Activities will be tested against `threshold`.\n\nIf this is present then `aggregateOn` is ignored.\n\nThe values are tested as partial strings so you do not need to include full URLs, just the part that matters.\n\nEX `[\"youtube\"]` will match submissions with the domain `https://youtube.com/c/aChannel`\nEX `[\"youtube.com/c/bChannel\"]` will NOT match submissions with the domain `https://youtube.com/c/aChannel`\n\nIf you wish to aggregate on self-posts for a subreddit use the syntax `self.[subreddit]` EX `self.AskReddit`\n\n**If this Rule is part of a Check for a Submission and you wish to aggregate on the domain of the Submission use the special string `AGG:SELF`**\n\nIf nothing is specified or list is empty (default) aggregate using `aggregateOn`",
+                    "description": "A list of domains whose Activities will be tested against `threshold`.\n\nThe values are tested as partial strings so you do not need to include full URLs, just the part that matters.\n\nEX `[\"youtube\"]` will match submissions with the domain `https://youtube.com/c/aChannel`\nEX `[\"youtube.com/c/bChannel\"]` will NOT match submissions with the domain `https://youtube.com/c/aChannel`\n\nIf you wish to aggregate on self-posts for a subreddit use the syntax `self.[subreddit]` EX `self.AskReddit`\n\n**If this Rule is part of a Check for a Submission and you wish to aggregate on the domain of the Submission use the special string `AGG:SELF`**\n\nIf nothing is specified or list is empty (default) aggregate using `aggregateOn`",
                     "items": {
                         "type": "string"
                     },
@@ -352,11 +419,6 @@
                     ],
                     "pattern": "^[a-zA-Z]([\\w -]*[\\w])?$",
                     "type": "string"
-                },
-                "useSubmissionAsReference": {
-                    "default": true,
-                    "description": "If activity is a Submission and is a link (not self-post) then only look at Submissions that contain this link, otherwise consider all activities.",
-                    "type": "boolean"
                 }
             },
             "required": [
@@ -432,6 +494,10 @@
                     },
                     "type": "array"
                 },
+                "shadowBanned": {
+                    "description": "Is the author shadowbanned?\n\nThis is determined by trying to retrieve the author's profile. If a 404 is returned it is likely they are shadowbanned",
+                    "type": "boolean"
+                },
                 "totalKarma": {
                     "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
@@ -753,6 +819,17 @@
                         "boolean"
                     ]
                 },
+                "subredditTTL": {
+                    "default": 600,
+                    "description": "Amount of time, in seconds, a subreddit (attributes) should be cached\n\n* If `0` or `true` will cache indefinitely (not recommended)\n* If `false` will not cache",
+                    "examples": [
+                        600
+                    ],
+                    "type": [
+                        "number",
+                        "boolean"
+                    ]
+                },
                 "userNotesTTL": {
                     "default": 300,
                     "description": "Amount of time, in seconds, [Toolbox User Notes](https://www.reddit.com/r/toolbox/wiki/docs/usernotes) should be cached\n\n* If `0` or `true` will cache indefinitely (not recommended)\n* If `false` will not cache",
@@ -1161,6 +1238,16 @@
                 "removed": {
                     "type": "boolean"
                 },
+                "reports": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "score": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
                 "spam": {
                     "type": "boolean"
                 },
@@ -1341,23 +1428,28 @@
             "type": "object"
         },
         "HistoryCriteria": {
-            "description": "If both `submission` and `comment` are defined then criteria will only trigger if BOTH thresholds are met",
+            "description": "Criteria will only trigger if ALL present thresholds (comment, submission, total) are met",
             "properties": {
                 "comment": {
-                    "description": "A string containing a comparison operator and a value to compare comments against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign] [OP]`\n\n* EX `> 100`  => greater than 100 comments\n* EX `<= 75%` => comments are equal to or less than 75% of all Activities\n\nIf your string also contains the text `OP` somewhere **after** `<number>[percent sign]`...:\n\n* EX `> 100 OP`  => greater than 100 comments as OP\n* EX `<= 25% as OP` => Comments as OP were less then or equal to 25% of **all Comments**",
+                    "description": "A string containing a comparison operator and a value to compare **filtered** (using `include` or `exclude`, if present) comments against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign] [OP]`\n\n* EX `> 100`  => greater than 100 comments\n* EX `<= 75%` => comments are equal to or less than 75% of unfiltered Activities\n\nIf your string also contains the text `OP` somewhere **after** `<number>[percent sign]`...:\n\n* EX `> 100 OP`  => greater than 100 filtered comments as OP\n* EX `<= 25% as OP` => **Filtered** comments as OP were less then or equal to 25% of **unfiltered Comments**",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
                 "minActivityCount": {
                     "default": 5,
-                    "description": "The minimum number of activities that must exist from the `window` results for this criteria to run",
+                    "description": "The minimum number of **filtered** activities that must exist from the `window` results for this criteria to run",
                     "type": "number"
                 },
                 "name": {
                     "type": "string"
                 },
                 "submission": {
-                    "description": "A string containing a comparison operator and a value to compare submissions against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 100`  => greater than 100 submissions\n* EX `<= 75%` => submissions are equal to or less than 75% of all Activities",
+                    "description": "A string containing a comparison operator and a value to compare **filtered** (using `include` or `exclude`, if present) submissions against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 100`  => greater than 100 filtered submissions\n* EX `<= 75%` => filtered submissions are equal to or less than 75% of unfiltered Activities",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "total": {
+                    "description": "A string containing a comparison operator and a value to compare **filtered** (using `include` or `exclude`) activities against\n\n**Note:** This is only useful if using `include` or `exclude` otherwise percent will always be 100% and total === activityTotal\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign] [OP]`\n\n* EX `> 100`  => greater than 100 filtered activities\n* EX `<= 75%` => filtered activities are equal to or less than 75% of all Activities",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
@@ -1426,27 +1518,51 @@
                     "type": "array"
                 },
                 "exclude": {
-                    "description": "Do not include Submissions from this list of Subreddits (by name, case-insensitive)\n\nEX `[\"mealtimevideos\",\"askscience\"]`",
+                    "description": "If present, activities will be counted only if they are **NOT** found in this list of Subreddits\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`\n\n**Note:** This affects **post-window retrieval** activities. So that:\n\n* `activityTotal` is number of activities retrieved from `window` -- NOT post-filtering\n* all comparisons using **percentages** will compare **post-filtering** results against **activity count from window**\n* -- to run this rule where all activities are only from include/exclude filtering instead use include/exclude in `window`",
                     "examples": [
-                        "mealtimevideos",
-                        "askscience"
+                        [
+                            "mealtimevideos",
+                            "askscience",
+                            "/onlyfans*/i",
+                            {
+                                "over18": true
+                            }
+                        ]
                     ],
                     "items": {
-                        "type": "string"
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/SubredditState"
+                            },
+                            {
+                                "type": "string"
+                            }
+                        ]
                     },
-                    "minItems": 1,
                     "type": "array"
                 },
                 "include": {
-                    "description": "Only include Submissions from this list of Subreddits (by name, case-insensitive)\n\nEX `[\"mealtimevideos\",\"askscience\"]`",
+                    "description": "If present, activities will be counted only if they are found in this list of Subreddits.\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`\n\n **Note:** This affects **post-window retrieval** activities. So that:\n\n* `activityTotal` is number of activities retrieved from `window` -- NOT post-filtering\n* all comparisons using **percentages** will compare **post-filtering** results against **activity count from window**\n* -- to run this rule where all activities are only from include/exclude filtering instead use include/exclude in `window`",
                     "examples": [
-                        "mealtimevideos",
-                        "askscience"
+                        [
+                            "mealtimevideos",
+                            "askscience",
+                            "/onlyfans*/i",
+                            {
+                                "over18": true
+                            }
+                        ]
                     ],
                     "items": {
-                        "type": "string"
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/SubredditState"
+                            },
+                            {
+                                "type": "string"
+                            }
+                        ]
                     },
-                    "minItems": 1,
                     "type": "array"
                 },
                 "itemIs": {
@@ -1488,6 +1604,31 @@
             ],
             "type": "object"
         },
+        "ImageDetection": {
+            "description": "When comparing submissions detect if the reference submission is an image and do a pixel-comparison to other detected image submissions.\n\n**Note:** This is an **experimental feature**",
+            "properties": {
+                "enable": {
+                    "description": "Is image detection enabled?",
+                    "type": "boolean"
+                },
+                "fetchBehavior": {
+                    "default": "extension",
+                    "description": "Determines how and when to check if a URL is an image\n\n**Note:** After fetching a URL the **Content-Type** is validated to contain `image` before detection occurs\n\n**When `extension`:** (default)\n\n* Only URLs that end in known image extensions (.png, .jpg, etc...) are fetched\n\n**When `unknown`:**\n\n* URLs that end in known image extensions (.png, .jpg, etc...) are fetched\n* URLs with no extension or unknown (IE non-video, non-doc, etc...) are fetched\n\n**When `all`:**\n\n* All submissions that have URLs (non-self) will be fetched, regardless of extension\n* **Note:** This can be bandwidth/CPU intensive if history window is large so use with care",
+                    "enum": [
+                        "all",
+                        "extension",
+                        "unknown"
+                    ],
+                    "type": "string"
+                },
+                "threshold": {
+                    "default": 5,
+                    "description": "The percentage, as a whole number, of pixels that are **different** between the two images at which point the images are not considered the same.\n\nDefault is `5`",
+                    "type": "number"
+                }
+            },
+            "type": "object"
+        },
         "LockActionJson": {
             "description": "Lock the Activity",
             "properties": {
@@ -1661,6 +1802,16 @@
                 "title": {
                     "description": "The title of the message\n\nIf not specified will be defaulted to `Concerning your [Submission/Comment]`",
                     "type": "string"
+                },
+                "to": {
+                    "description": "Entity to send message to.\n\nIf not present Message be will sent to the Author of the Activity being checked.\n\nValid formats:\n\n* `aUserName` -- send to /u/aUserName\n* `u/aUserName` -- send to /u/aUserName\n* `r/aSubreddit` -- sent to modmail of /r/aSubreddit\n\n**Note:** Reddit does not support sending a message AS a subreddit TO another subreddit",
+                    "examples": [
+                        "aUserName",
+                        "u/aUserName",
+                        "r/aSubreddit"
+                    ],
+                    "pattern": "^\\s*(\\/[ru]\\/|[ru]\\/)*(\\w+)*\\s*$",
+                    "type": "string"
                 }
             },
             "required": [
@@ -1803,6 +1954,10 @@
                         }
                     ]
                 },
+                "imageDetection": {
+                    "$ref": "#/definitions/ImageDetection",
+                    "description": "When comparing submissions detect if the reference submission is an image and do a pixel-comparison to other detected image submissions.\n\n**Note:** This is an **experimental feature**"
+                },
                 "itemIs": {
                     "anyOf": [
                         {
@@ -1853,7 +2008,7 @@
                 "thresholds": {
                     "description": "A list of subreddits/count criteria that may trigger this rule. ANY SubThreshold will trigger this rule.",
                     "items": {
-                        "$ref": "#/definitions/SubThreshold"
+                        "$ref": "#/definitions/ActivityThreshold"
                     },
                     "minItems": 1,
                     "type": "array"
@@ -1890,6 +2045,49 @@
             ],
             "type": "object"
         },
+        "RegExp": {
+            "properties": {
+                "dotAll": {
+                    "type": "boolean"
+                },
+                "flags": {
+                    "type": "string"
+                },
+                "global": {
+                    "type": "boolean"
+                },
+                "ignoreCase": {
+                    "type": "boolean"
+                },
+                "lastIndex": {
+                    "type": "number"
+                },
+                "multiline": {
+                    "type": "boolean"
+                },
+                "source": {
+                    "type": "string"
+                },
+                "sticky": {
+                    "type": "boolean"
+                },
+                "unicode": {
+                    "type": "boolean"
+                }
+            },
+            "required": [
+                "dotAll",
+                "flags",
+                "global",
+                "ignoreCase",
+                "lastIndex",
+                "multiline",
+                "source",
+                "sticky",
+                "unicode"
+            ],
+            "type": "object"
+        },
         "RegexCriteria": {
             "properties": {
                 "activityMatchThreshold": {
@@ -1928,16 +2126,12 @@
                     "type": "string"
                 },
                 "regex": {
-                    "description": "A valid Regular Expression to test content against\n\nDo not wrap expression in forward slashes\n\nEX For the expression `/reddit|FoxxMD/` use the value should be `reddit|FoxxMD`",
+                    "description": "A valid Regular Expression to test content against\n\nIf no flags are specified then the **global** flag is used by default",
                     "examples": [
-                        "reddit|FoxxMD"
+                        "/reddit|FoxxMD/ig"
                     ],
                     "type": "string"
                 },
-                "regexFlags": {
-                    "description": "Regex flags to use",
-                    "type": "string"
-                },
                 "testOn": {
                     "default": [
                         "title",
@@ -2176,15 +2370,27 @@
                     ]
                 },
                 "exclude": {
-                    "description": "Do not include Submissions from this list of Subreddits (by name, case-insensitive)\n\nEX `[\"mealtimevideos\",\"askscience\"]`",
+                    "description": "If present, activities will be counted only if they are **NOT** found in this list of Subreddits\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`",
                     "examples": [
-                        "mealtimevideos",
-                        "askscience"
+                        [
+                            "mealtimevideos",
+                            "askscience",
+                            "/onlyfans*/i",
+                            {
+                                "over18": true
+                            }
+                        ]
                     ],
                     "items": {
-                        "type": "string"
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/SubredditState"
+                            },
+                            {
+                                "type": "string"
+                            }
+                        ]
                     },
-                    "minItems": 1,
                     "type": "array"
                 },
                 "gapAllowance": {
@@ -2192,15 +2398,27 @@
                     "type": "number"
                 },
                 "include": {
-                    "description": "Only include Submissions from this list of Subreddits (by name, case-insensitive)\n\nEX `[\"mealtimevideos\",\"askscience\"]`",
+                    "description": "If present, activities will be counted only if they are found in this list of Subreddits\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`",
                     "examples": [
-                        "mealtimevideos",
-                        "askscience"
+                        [
+                            "mealtimevideos",
+                            "askscience",
+                            "/onlyfans*/i",
+                            {
+                                "over18": true
+                            }
+                        ]
                     ],
                     "items": {
-                        "type": "string"
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/SubredditState"
+                            },
+                            {
+                                "type": "string"
+                            }
+                        ]
                     },
-                    "minItems": 1,
                     "type": "array"
                 },
                 "itemIs": {
@@ -2427,45 +2645,6 @@
             ],
             "type": "object"
         },
-        "SubThreshold": {
-            "additionalProperties": false,
-            "description": "At least one count property must be present. If both are present then either can trigger the rule",
-            "minProperties": 1,
-            "properties": {
-                "karma": {
-                    "description": "Test the **combined karma** from Activities found in the specified subreddits\n\nValue is a string containing a comparison operator and a number of **combined karma** to compare against\n\nIf specified then both `threshold` and `karma` must be met for this `SubThreshold` to be satisfied\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 50`  => greater than 50 combined karma for all found Activities in specified subreddits",
-                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
-                    "type": "string"
-                },
-                "subreddits": {
-                    "description": "A list of Subreddits (by name, case-insensitive) to look for.\n\nEX [\"mealtimevideos\",\"askscience\"]",
-                    "examples": [
-                        [
-                            "mealtimevideos",
-                            "askscience"
-                        ]
-                    ],
-                    "items": {
-                        "type": "string"
-                    },
-                    "minItems": 1,
-                    "type": "array"
-                },
-                "threshold": {
-                    "default": ">= 1",
-                    "description": "A string containing a comparison operator and a value to compare recent activities against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 3`  => greater than 3 activities found in the listed subreddits\n* EX `<= 75%` => number of Activities in the subreddits listed are equal to or less than 75% of all Activities\n\n**Note:** If you use percentage comparison here as well as `useSubmissionAsReference` then \"all Activities\" is only pertains to Activities that had the Link of the Submission, rather than all Activities from this window.",
-                    "examples": [
-                        ">= 1"
-                    ],
-                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
-                    "type": "string"
-                }
-            },
-            "required": [
-                "subreddits"
-            ],
-            "type": "object"
-        },
         "SubmissionCheckJson": {
             "properties": {
                 "actions": {
@@ -2686,6 +2865,16 @@
                 "removed": {
                     "type": "boolean"
                 },
+                "reports": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "score": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
                 "spam": {
                     "type": "boolean"
                 },
@@ -2702,6 +2891,44 @@
             },
             "type": "object"
         },
+        "SubredditState": {
+            "description": "Different attributes a `Subreddit` can be in. Only include a property if you want to check it.",
+            "examples": [
+                {
+                    "over18": true
+                }
+            ],
+            "properties": {
+                "name": {
+                    "anyOf": [
+                        {
+                            "$ref": "#/definitions/RegExp"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "The name the subreddit.\n\nCan be a normal string (will check case-insensitive) or a regular expression\n\nEX `[\"mealtimevideos\", \"/onlyfans*\\/i\"]`",
+                    "examples": [
+                        "mealtimevideos",
+                        "/onlyfans*/i"
+                    ]
+                },
+                "over18": {
+                    "description": "Is subreddit NSFW/over 18?\n\n**Note**: This is **mod-controlled flag** so it is up to the mods of the subreddit to correctly mark their subreddit as NSFW",
+                    "type": "boolean"
+                },
+                "quarantine": {
+                    "description": "Is subreddit quarantined?",
+                    "type": "boolean"
+                },
+                "stateDescription": {
+                    "description": "A friendly description of what this State is trying to parse",
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
         "UserNoteActionJson": {
             "description": "Add a Toolbox User Note to the Author of this Activity",
             "properties": {

src/Schema/OperatorConfig.json

@@ -402,6 +402,17 @@
                         "boolean"
                     ]
                 },
+                "subredditTTL": {
+                    "default": 600,
+                    "description": "Amount of time, in seconds, a subreddit (attributes) should be cached\n\n* If `0` or `true` will cache indefinitely (not recommended)\n* If `false` will not cache",
+                    "examples": [
+                        600
+                    ],
+                    "type": [
+                        "number",
+                        "boolean"
+                    ]
+                },
                 "userNotesTTL": {
                     "default": 300,
                     "description": "Amount of time, in seconds, [Toolbox User Notes](https://www.reddit.com/r/toolbox/wiki/docs/usernotes) should be cached\n\n* If `0` or `true` will cache indefinitely (not recommended)\n* If `false` will not cache",

src/Schema/Rule.json

@@ -24,6 +24,72 @@
         }
     ],
     "definitions": {
+        "ActivityThreshold": {
+            "additionalProperties": false,
+            "description": "At least one count property must be present. If both are present then either can trigger the rule",
+            "minProperties": 1,
+            "properties": {
+                "commentState": {
+                    "$ref": "#/definitions/CommentState",
+                    "description": "When present, a Comment will only be counted if it meets this criteria",
+                    "examples": [
+                        {
+                            "op": true,
+                            "removed": false
+                        }
+                    ]
+                },
+                "karma": {
+                    "description": "Test the **combined karma** from Activities found in the specified subreddits\n\nValue is a string containing a comparison operator and a number of **combined karma** to compare against\n\nIf specified then both `threshold` and `karma` must be met for this `SubThreshold` to be satisfied\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 50`  => greater than 50 combined karma for all found Activities in specified subreddits",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "submissionState": {
+                    "$ref": "#/definitions/SubmissionState",
+                    "description": "When present, a Submission will only be counted if it meets this criteria",
+                    "examples": [
+                        {
+                            "over_18": true,
+                            "removed": false
+                        }
+                    ]
+                },
+                "subreddits": {
+                    "description": "Activities will be counted if they are found in this list of Subreddits\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`",
+                    "examples": [
+                        [
+                            "mealtimevideos",
+                            "askscience",
+                            "/onlyfans*/i",
+                            {
+                                "over18": true
+                            }
+                        ]
+                    ],
+                    "items": {
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/SubredditState"
+                            },
+                            {
+                                "type": "string"
+                            }
+                        ]
+                    },
+                    "type": "array"
+                },
+                "threshold": {
+                    "default": ">= 1",
+                    "description": "A string containing a comparison operator and a value to compare recent activities against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 3`  => greater than 3 activities found in the listed subreddits\n* EX `<= 75%` => number of Activities in the subreddits listed are equal to or less than 75% of all Activities\n\n**Note:** If you use percentage comparison here as well as `useSubmissionAsReference` then \"all Activities\" is only pertains to Activities that had the Link of the Submission, rather than all Activities from this window.",
+                    "examples": [
+                        ">= 1"
+                    ],
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
         "ActivityWindowCriteria": {
             "additionalProperties": false,
             "description": "Multiple properties that may be used to define what range of Activity to retrieve.\n\nMay specify one, or both properties along with the `satisfyOn` property, to affect the retrieval behavior.",
@@ -113,7 +179,7 @@
             "properties": {
                 "aggregateOn": {
                     "default": "undefined",
-                    "description": "If `domains` is not specified this list determines which categories of domains should be aggregated on. All aggregated domains will be tested against `threshold`\n\n* If `media` is included then aggregate author's submission history which reddit recognizes as media (youtube, vimeo, etc.)\n* If `self` is included then aggregate on author's submission history which are self-post (`self.[subreddit]`) or reddit image/video (i.redd.it / v.redd.it)\n* If `link` is included then aggregate author's submission history which is external links but not media\n\nIf nothing is specified or list is empty (default) all domains are aggregated",
+                    "description": "This list determines which categories of domains should be aggregated on. All aggregated domains will be tested against `threshold`\n\n* If `media` is included then aggregate author's submission history which reddit recognizes as media (youtube, vimeo, etc.)\n* If `redditMedia` is included then aggregate on author's submissions history which are media hosted on reddit: galleries, videos, and images (i.redd.it / v.redd.it)\n* If `self` is included then aggregate on author's submission history which are self-post (`self.[subreddit]`) or domain is `reddit.com`\n* If `link` is included then aggregate author's submission history which is external links and not recognized as `media` by reddit\n\nIf nothing is specified or list is empty (default) rule will only aggregate on `link` and `media` (ignores reddit-hosted content and self-posts)",
                     "examples": [
                         [
                         ]
@@ -122,6 +188,7 @@
                         "enum": [
                             "link",
                             "media",
+                            "redditMedia",
                             "self"
                         ],
                         "type": "string"
@@ -141,7 +208,7 @@
                         [
                         ]
                     ],
-                    "description": "A list of domains whose Activities will be tested against `threshold`.\n\nIf this is present then `aggregateOn` is ignored.\n\nThe values are tested as partial strings so you do not need to include full URLs, just the part that matters.\n\nEX `[\"youtube\"]` will match submissions with the domain `https://youtube.com/c/aChannel`\nEX `[\"youtube.com/c/bChannel\"]` will NOT match submissions with the domain `https://youtube.com/c/aChannel`\n\nIf you wish to aggregate on self-posts for a subreddit use the syntax `self.[subreddit]` EX `self.AskReddit`\n\n**If this Rule is part of a Check for a Submission and you wish to aggregate on the domain of the Submission use the special string `AGG:SELF`**\n\nIf nothing is specified or list is empty (default) aggregate using `aggregateOn`",
+                    "description": "A list of domains whose Activities will be tested against `threshold`.\n\nThe values are tested as partial strings so you do not need to include full URLs, just the part that matters.\n\nEX `[\"youtube\"]` will match submissions with the domain `https://youtube.com/c/aChannel`\nEX `[\"youtube.com/c/bChannel\"]` will NOT match submissions with the domain `https://youtube.com/c/aChannel`\n\nIf you wish to aggregate on self-posts for a subreddit use the syntax `self.[subreddit]` EX `self.AskReddit`\n\n**If this Rule is part of a Check for a Submission and you wish to aggregate on the domain of the Submission use the special string `AGG:SELF`**\n\nIf nothing is specified or list is empty (default) aggregate using `aggregateOn`",
                     "items": {
                         "type": "string"
                     },
@@ -298,11 +365,6 @@
                     ],
                     "pattern": "^[a-zA-Z]([\\w -]*[\\w])?$",
                     "type": "string"
-                },
-                "useSubmissionAsReference": {
-                    "default": true,
-                    "description": "If activity is a Submission and is a link (not self-post) then only look at Submissions that contain this link, otherwise consider all activities.",
-                    "type": "boolean"
                 }
             },
             "required": [
@@ -378,6 +440,10 @@
                     },
                     "type": "array"
                 },
+                "shadowBanned": {
+                    "description": "Is the author shadowbanned?\n\nThis is determined by trying to retrieve the author's profile. If a 404 is returned it is likely they are shadowbanned",
+                    "type": "boolean"
+                },
                 "totalKarma": {
                     "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
@@ -538,6 +604,16 @@
                 "removed": {
                     "type": "boolean"
                 },
+                "reports": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "score": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
                 "spam": {
                     "type": "boolean"
                 },
@@ -611,23 +687,28 @@
             "type": "object"
         },
         "HistoryCriteria": {
-            "description": "If both `submission` and `comment` are defined then criteria will only trigger if BOTH thresholds are met",
+            "description": "Criteria will only trigger if ALL present thresholds (comment, submission, total) are met",
             "properties": {
                 "comment": {
-                    "description": "A string containing a comparison operator and a value to compare comments against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign] [OP]`\n\n* EX `> 100`  => greater than 100 comments\n* EX `<= 75%` => comments are equal to or less than 75% of all Activities\n\nIf your string also contains the text `OP` somewhere **after** `<number>[percent sign]`...:\n\n* EX `> 100 OP`  => greater than 100 comments as OP\n* EX `<= 25% as OP` => Comments as OP were less then or equal to 25% of **all Comments**",
+                    "description": "A string containing a comparison operator and a value to compare **filtered** (using `include` or `exclude`, if present) comments against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign] [OP]`\n\n* EX `> 100`  => greater than 100 comments\n* EX `<= 75%` => comments are equal to or less than 75% of unfiltered Activities\n\nIf your string also contains the text `OP` somewhere **after** `<number>[percent sign]`...:\n\n* EX `> 100 OP`  => greater than 100 filtered comments as OP\n* EX `<= 25% as OP` => **Filtered** comments as OP were less then or equal to 25% of **unfiltered Comments**",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
                 "minActivityCount": {
                     "default": 5,
-                    "description": "The minimum number of activities that must exist from the `window` results for this criteria to run",
+                    "description": "The minimum number of **filtered** activities that must exist from the `window` results for this criteria to run",
                     "type": "number"
                 },
                 "name": {
                     "type": "string"
                 },
                 "submission": {
-                    "description": "A string containing a comparison operator and a value to compare submissions against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 100`  => greater than 100 submissions\n* EX `<= 75%` => submissions are equal to or less than 75% of all Activities",
+                    "description": "A string containing a comparison operator and a value to compare **filtered** (using `include` or `exclude`, if present) submissions against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 100`  => greater than 100 filtered submissions\n* EX `<= 75%` => filtered submissions are equal to or less than 75% of unfiltered Activities",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "total": {
+                    "description": "A string containing a comparison operator and a value to compare **filtered** (using `include` or `exclude`) activities against\n\n**Note:** This is only useful if using `include` or `exclude` otherwise percent will always be 100% and total === activityTotal\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign] [OP]`\n\n* EX `> 100`  => greater than 100 filtered activities\n* EX `<= 75%` => filtered activities are equal to or less than 75% of all Activities",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
@@ -696,27 +777,51 @@
                     "type": "array"
                 },
                 "exclude": {
-                    "description": "Do not include Submissions from this list of Subreddits (by name, case-insensitive)\n\nEX `[\"mealtimevideos\",\"askscience\"]`",
+                    "description": "If present, activities will be counted only if they are **NOT** found in this list of Subreddits\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`\n\n**Note:** This affects **post-window retrieval** activities. So that:\n\n* `activityTotal` is number of activities retrieved from `window` -- NOT post-filtering\n* all comparisons using **percentages** will compare **post-filtering** results against **activity count from window**\n* -- to run this rule where all activities are only from include/exclude filtering instead use include/exclude in `window`",
                     "examples": [
-                        "mealtimevideos",
-                        "askscience"
+                        [
+                            "mealtimevideos",
+                            "askscience",
+                            "/onlyfans*/i",
+                            {
+                                "over18": true
+                            }
+                        ]
                     ],
                     "items": {
-                        "type": "string"
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/SubredditState"
+                            },
+                            {
+                                "type": "string"
+                            }
+                        ]
                     },
-                    "minItems": 1,
                     "type": "array"
                 },
                 "include": {
-                    "description": "Only include Submissions from this list of Subreddits (by name, case-insensitive)\n\nEX `[\"mealtimevideos\",\"askscience\"]`",
+                    "description": "If present, activities will be counted only if they are found in this list of Subreddits.\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`\n\n **Note:** This affects **post-window retrieval** activities. So that:\n\n* `activityTotal` is number of activities retrieved from `window` -- NOT post-filtering\n* all comparisons using **percentages** will compare **post-filtering** results against **activity count from window**\n* -- to run this rule where all activities are only from include/exclude filtering instead use include/exclude in `window`",
                     "examples": [
-                        "mealtimevideos",
-                        "askscience"
+                        [
+                            "mealtimevideos",
+                            "askscience",
+                            "/onlyfans*/i",
+                            {
+                                "over18": true
+                            }
+                        ]
                     ],
                     "items": {
-                        "type": "string"
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/SubredditState"
+                            },
+                            {
+                                "type": "string"
+                            }
+                        ]
                     },
-                    "minItems": 1,
                     "type": "array"
                 },
                 "itemIs": {
@@ -758,6 +863,31 @@
             ],
             "type": "object"
         },
+        "ImageDetection": {
+            "description": "When comparing submissions detect if the reference submission is an image and do a pixel-comparison to other detected image submissions.\n\n**Note:** This is an **experimental feature**",
+            "properties": {
+                "enable": {
+                    "description": "Is image detection enabled?",
+                    "type": "boolean"
+                },
+                "fetchBehavior": {
+                    "default": "extension",
+                    "description": "Determines how and when to check if a URL is an image\n\n**Note:** After fetching a URL the **Content-Type** is validated to contain `image` before detection occurs\n\n**When `extension`:** (default)\n\n* Only URLs that end in known image extensions (.png, .jpg, etc...) are fetched\n\n**When `unknown`:**\n\n* URLs that end in known image extensions (.png, .jpg, etc...) are fetched\n* URLs with no extension or unknown (IE non-video, non-doc, etc...) are fetched\n\n**When `all`:**\n\n* All submissions that have URLs (non-self) will be fetched, regardless of extension\n* **Note:** This can be bandwidth/CPU intensive if history window is large so use with care",
+                    "enum": [
+                        "all",
+                        "extension",
+                        "unknown"
+                    ],
+                    "type": "string"
+                },
+                "threshold": {
+                    "default": 5,
+                    "description": "The percentage, as a whole number, of pixels that are **different** between the two images at which point the images are not considered the same.\n\nDefault is `5`",
+                    "type": "number"
+                }
+            },
+            "type": "object"
+        },
         "RecentActivityRuleJSONConfig": {
             "description": "Checks a user's history for any Activity (Submission/Comment) in the subreddits specified in thresholds\n\nAvailable data for [Action templating](https://github.com/FoxxMD/context-mod#action-templating):\n\n```\nsummary    => comma-deliminated list of subreddits that hit the threshold and their count EX subredditA(1), subredditB(4),...\nsubCount   => Total number of subreddits that hit the threshold\ntotalCount => Total number of all activity occurrences in subreddits\n```",
             "properties": {
@@ -780,6 +910,10 @@
                         }
                     ]
                 },
+                "imageDetection": {
+                    "$ref": "#/definitions/ImageDetection",
+                    "description": "When comparing submissions detect if the reference submission is an image and do a pixel-comparison to other detected image submissions.\n\n**Note:** This is an **experimental feature**"
+                },
                 "itemIs": {
                     "anyOf": [
                         {
@@ -830,7 +964,7 @@
                 "thresholds": {
                     "description": "A list of subreddits/count criteria that may trigger this rule. ANY SubThreshold will trigger this rule.",
                     "items": {
-                        "$ref": "#/definitions/SubThreshold"
+                        "$ref": "#/definitions/ActivityThreshold"
                     },
                     "minItems": 1,
                     "type": "array"
@@ -867,6 +1001,49 @@
             ],
             "type": "object"
         },
+        "RegExp": {
+            "properties": {
+                "dotAll": {
+                    "type": "boolean"
+                },
+                "flags": {
+                    "type": "string"
+                },
+                "global": {
+                    "type": "boolean"
+                },
+                "ignoreCase": {
+                    "type": "boolean"
+                },
+                "lastIndex": {
+                    "type": "number"
+                },
+                "multiline": {
+                    "type": "boolean"
+                },
+                "source": {
+                    "type": "string"
+                },
+                "sticky": {
+                    "type": "boolean"
+                },
+                "unicode": {
+                    "type": "boolean"
+                }
+            },
+            "required": [
+                "dotAll",
+                "flags",
+                "global",
+                "ignoreCase",
+                "lastIndex",
+                "multiline",
+                "source",
+                "sticky",
+                "unicode"
+            ],
+            "type": "object"
+        },
         "RegexCriteria": {
             "properties": {
                 "activityMatchThreshold": {
@@ -905,16 +1082,12 @@
                     "type": "string"
                 },
                 "regex": {
-                    "description": "A valid Regular Expression to test content against\n\nDo not wrap expression in forward slashes\n\nEX For the expression `/reddit|FoxxMD/` use the value should be `reddit|FoxxMD`",
+                    "description": "A valid Regular Expression to test content against\n\nIf no flags are specified then the **global** flag is used by default",
                     "examples": [
-                        "reddit|FoxxMD"
+                        "/reddit|FoxxMD/ig"
                     ],
                     "type": "string"
                 },
-                "regexFlags": {
-                    "description": "Regex flags to use",
-                    "type": "string"
-                },
                 "testOn": {
                     "default": [
                         "title",
@@ -1076,15 +1249,27 @@
                     ]
                 },
                 "exclude": {
-                    "description": "Do not include Submissions from this list of Subreddits (by name, case-insensitive)\n\nEX `[\"mealtimevideos\",\"askscience\"]`",
+                    "description": "If present, activities will be counted only if they are **NOT** found in this list of Subreddits\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`",
                     "examples": [
-                        "mealtimevideos",
-                        "askscience"
+                        [
+                            "mealtimevideos",
+                            "askscience",
+                            "/onlyfans*/i",
+                            {
+                                "over18": true
+                            }
+                        ]
                     ],
                     "items": {
-                        "type": "string"
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/SubredditState"
+                            },
+                            {
+                                "type": "string"
+                            }
+                        ]
                     },
-                    "minItems": 1,
                     "type": "array"
                 },
                 "gapAllowance": {
@@ -1092,15 +1277,27 @@
                     "type": "number"
                 },
                 "include": {
-                    "description": "Only include Submissions from this list of Subreddits (by name, case-insensitive)\n\nEX `[\"mealtimevideos\",\"askscience\"]`",
+                    "description": "If present, activities will be counted only if they are found in this list of Subreddits\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`",
                     "examples": [
-                        "mealtimevideos",
-                        "askscience"
+                        [
+                            "mealtimevideos",
+                            "askscience",
+                            "/onlyfans*/i",
+                            {
+                                "over18": true
+                            }
+                        ]
                     ],
                     "items": {
-                        "type": "string"
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/SubredditState"
+                            },
+                            {
+                                "type": "string"
+                            }
+                        ]
                     },
-                    "minItems": 1,
                     "type": "array"
                 },
                 "itemIs": {
@@ -1190,45 +1387,6 @@
             ],
             "type": "object"
         },
-        "SubThreshold": {
-            "additionalProperties": false,
-            "description": "At least one count property must be present. If both are present then either can trigger the rule",
-            "minProperties": 1,
-            "properties": {
-                "karma": {
-                    "description": "Test the **combined karma** from Activities found in the specified subreddits\n\nValue is a string containing a comparison operator and a number of **combined karma** to compare against\n\nIf specified then both `threshold` and `karma` must be met for this `SubThreshold` to be satisfied\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 50`  => greater than 50 combined karma for all found Activities in specified subreddits",
-                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
-                    "type": "string"
-                },
-                "subreddits": {
-                    "description": "A list of Subreddits (by name, case-insensitive) to look for.\n\nEX [\"mealtimevideos\",\"askscience\"]",
-                    "examples": [
-                        [
-                            "mealtimevideos",
-                            "askscience"
-                        ]
-                    ],
-                    "items": {
-                        "type": "string"
-                    },
-                    "minItems": 1,
-                    "type": "array"
-                },
-                "threshold": {
-                    "default": ">= 1",
-                    "description": "A string containing a comparison operator and a value to compare recent activities against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 3`  => greater than 3 activities found in the listed subreddits\n* EX `<= 75%` => number of Activities in the subreddits listed are equal to or less than 75% of all Activities\n\n**Note:** If you use percentage comparison here as well as `useSubmissionAsReference` then \"all Activities\" is only pertains to Activities that had the Link of the Submission, rather than all Activities from this window.",
-                    "examples": [
-                        ">= 1"
-                    ],
-                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
-                    "type": "string"
-                }
-            },
-            "required": [
-                "subreddits"
-            ],
-            "type": "object"
-        },
         "SubmissionState": {
             "description": "Different attributes a `Submission` can be in. Only include a property if you want to check it.",
             "examples": [
@@ -1272,6 +1430,16 @@
                 "removed": {
                     "type": "boolean"
                 },
+                "reports": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "score": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
                 "spam": {
                     "type": "boolean"
                 },
@@ -1288,6 +1456,44 @@
             },
             "type": "object"
         },
+        "SubredditState": {
+            "description": "Different attributes a `Subreddit` can be in. Only include a property if you want to check it.",
+            "examples": [
+                {
+                    "over18": true
+                }
+            ],
+            "properties": {
+                "name": {
+                    "anyOf": [
+                        {
+                            "$ref": "#/definitions/RegExp"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "The name the subreddit.\n\nCan be a normal string (will check case-insensitive) or a regular expression\n\nEX `[\"mealtimevideos\", \"/onlyfans*\\/i\"]`",
+                    "examples": [
+                        "mealtimevideos",
+                        "/onlyfans*/i"
+                    ]
+                },
+                "over18": {
+                    "description": "Is subreddit NSFW/over 18?\n\n**Note**: This is **mod-controlled flag** so it is up to the mods of the subreddit to correctly mark their subreddit as NSFW",
+                    "type": "boolean"
+                },
+                "quarantine": {
+                    "description": "Is subreddit quarantined?",
+                    "type": "boolean"
+                },
+                "stateDescription": {
+                    "description": "A friendly description of what this State is trying to parse",
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
         "UserNoteCriteria": {
             "properties": {
                 "count": {

src/Schema/RuleSet.json

@@ -1,6 +1,72 @@
 {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "definitions": {
+        "ActivityThreshold": {
+            "additionalProperties": false,
+            "description": "At least one count property must be present. If both are present then either can trigger the rule",
+            "minProperties": 1,
+            "properties": {
+                "commentState": {
+                    "$ref": "#/definitions/CommentState",
+                    "description": "When present, a Comment will only be counted if it meets this criteria",
+                    "examples": [
+                        {
+                            "op": true,
+                            "removed": false
+                        }
+                    ]
+                },
+                "karma": {
+                    "description": "Test the **combined karma** from Activities found in the specified subreddits\n\nValue is a string containing a comparison operator and a number of **combined karma** to compare against\n\nIf specified then both `threshold` and `karma` must be met for this `SubThreshold` to be satisfied\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 50`  => greater than 50 combined karma for all found Activities in specified subreddits",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "submissionState": {
+                    "$ref": "#/definitions/SubmissionState",
+                    "description": "When present, a Submission will only be counted if it meets this criteria",
+                    "examples": [
+                        {
+                            "over_18": true,
+                            "removed": false
+                        }
+                    ]
+                },
+                "subreddits": {
+                    "description": "Activities will be counted if they are found in this list of Subreddits\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`",
+                    "examples": [
+                        [
+                            "mealtimevideos",
+                            "askscience",
+                            "/onlyfans*/i",
+                            {
+                                "over18": true
+                            }
+                        ]
+                    ],
+                    "items": {
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/SubredditState"
+                            },
+                            {
+                                "type": "string"
+                            }
+                        ]
+                    },
+                    "type": "array"
+                },
+                "threshold": {
+                    "default": ">= 1",
+                    "description": "A string containing a comparison operator and a value to compare recent activities against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 3`  => greater than 3 activities found in the listed subreddits\n* EX `<= 75%` => number of Activities in the subreddits listed are equal to or less than 75% of all Activities\n\n**Note:** If you use percentage comparison here as well as `useSubmissionAsReference` then \"all Activities\" is only pertains to Activities that had the Link of the Submission, rather than all Activities from this window.",
+                    "examples": [
+                        ">= 1"
+                    ],
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
         "ActivityWindowCriteria": {
             "additionalProperties": false,
             "description": "Multiple properties that may be used to define what range of Activity to retrieve.\n\nMay specify one, or both properties along with the `satisfyOn` property, to affect the retrieval behavior.",
@@ -90,7 +156,7 @@
             "properties": {
                 "aggregateOn": {
                     "default": "undefined",
-                    "description": "If `domains` is not specified this list determines which categories of domains should be aggregated on. All aggregated domains will be tested against `threshold`\n\n* If `media` is included then aggregate author's submission history which reddit recognizes as media (youtube, vimeo, etc.)\n* If `self` is included then aggregate on author's submission history which are self-post (`self.[subreddit]`) or reddit image/video (i.redd.it / v.redd.it)\n* If `link` is included then aggregate author's submission history which is external links but not media\n\nIf nothing is specified or list is empty (default) all domains are aggregated",
+                    "description": "This list determines which categories of domains should be aggregated on. All aggregated domains will be tested against `threshold`\n\n* If `media` is included then aggregate author's submission history which reddit recognizes as media (youtube, vimeo, etc.)\n* If `redditMedia` is included then aggregate on author's submissions history which are media hosted on reddit: galleries, videos, and images (i.redd.it / v.redd.it)\n* If `self` is included then aggregate on author's submission history which are self-post (`self.[subreddit]`) or domain is `reddit.com`\n* If `link` is included then aggregate author's submission history which is external links and not recognized as `media` by reddit\n\nIf nothing is specified or list is empty (default) rule will only aggregate on `link` and `media` (ignores reddit-hosted content and self-posts)",
                     "examples": [
                         [
                         ]
@@ -99,6 +165,7 @@
                         "enum": [
                             "link",
                             "media",
+                            "redditMedia",
                             "self"
                         ],
                         "type": "string"
@@ -118,7 +185,7 @@
                         [
                         ]
                     ],
-                    "description": "A list of domains whose Activities will be tested against `threshold`.\n\nIf this is present then `aggregateOn` is ignored.\n\nThe values are tested as partial strings so you do not need to include full URLs, just the part that matters.\n\nEX `[\"youtube\"]` will match submissions with the domain `https://youtube.com/c/aChannel`\nEX `[\"youtube.com/c/bChannel\"]` will NOT match submissions with the domain `https://youtube.com/c/aChannel`\n\nIf you wish to aggregate on self-posts for a subreddit use the syntax `self.[subreddit]` EX `self.AskReddit`\n\n**If this Rule is part of a Check for a Submission and you wish to aggregate on the domain of the Submission use the special string `AGG:SELF`**\n\nIf nothing is specified or list is empty (default) aggregate using `aggregateOn`",
+                    "description": "A list of domains whose Activities will be tested against `threshold`.\n\nThe values are tested as partial strings so you do not need to include full URLs, just the part that matters.\n\nEX `[\"youtube\"]` will match submissions with the domain `https://youtube.com/c/aChannel`\nEX `[\"youtube.com/c/bChannel\"]` will NOT match submissions with the domain `https://youtube.com/c/aChannel`\n\nIf you wish to aggregate on self-posts for a subreddit use the syntax `self.[subreddit]` EX `self.AskReddit`\n\n**If this Rule is part of a Check for a Submission and you wish to aggregate on the domain of the Submission use the special string `AGG:SELF`**\n\nIf nothing is specified or list is empty (default) aggregate using `aggregateOn`",
                     "items": {
                         "type": "string"
                     },
@@ -275,11 +342,6 @@
                     ],
                     "pattern": "^[a-zA-Z]([\\w -]*[\\w])?$",
                     "type": "string"
-                },
-                "useSubmissionAsReference": {
-                    "default": true,
-                    "description": "If activity is a Submission and is a link (not self-post) then only look at Submissions that contain this link, otherwise consider all activities.",
-                    "type": "boolean"
                 }
             },
             "required": [
@@ -355,6 +417,10 @@
                     },
                     "type": "array"
                 },
+                "shadowBanned": {
+                    "description": "Is the author shadowbanned?\n\nThis is determined by trying to retrieve the author's profile. If a 404 is returned it is likely they are shadowbanned",
+                    "type": "boolean"
+                },
                 "totalKarma": {
                     "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
@@ -515,6 +581,16 @@
                 "removed": {
                     "type": "boolean"
                 },
+                "reports": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "score": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
                 "spam": {
                     "type": "boolean"
                 },
@@ -588,23 +664,28 @@
             "type": "object"
         },
         "HistoryCriteria": {
-            "description": "If both `submission` and `comment` are defined then criteria will only trigger if BOTH thresholds are met",
+            "description": "Criteria will only trigger if ALL present thresholds (comment, submission, total) are met",
             "properties": {
                 "comment": {
-                    "description": "A string containing a comparison operator and a value to compare comments against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign] [OP]`\n\n* EX `> 100`  => greater than 100 comments\n* EX `<= 75%` => comments are equal to or less than 75% of all Activities\n\nIf your string also contains the text `OP` somewhere **after** `<number>[percent sign]`...:\n\n* EX `> 100 OP`  => greater than 100 comments as OP\n* EX `<= 25% as OP` => Comments as OP were less then or equal to 25% of **all Comments**",
+                    "description": "A string containing a comparison operator and a value to compare **filtered** (using `include` or `exclude`, if present) comments against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign] [OP]`\n\n* EX `> 100`  => greater than 100 comments\n* EX `<= 75%` => comments are equal to or less than 75% of unfiltered Activities\n\nIf your string also contains the text `OP` somewhere **after** `<number>[percent sign]`...:\n\n* EX `> 100 OP`  => greater than 100 filtered comments as OP\n* EX `<= 25% as OP` => **Filtered** comments as OP were less then or equal to 25% of **unfiltered Comments**",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
                 "minActivityCount": {
                     "default": 5,
-                    "description": "The minimum number of activities that must exist from the `window` results for this criteria to run",
+                    "description": "The minimum number of **filtered** activities that must exist from the `window` results for this criteria to run",
                     "type": "number"
                 },
                 "name": {
                     "type": "string"
                 },
                 "submission": {
-                    "description": "A string containing a comparison operator and a value to compare submissions against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 100`  => greater than 100 submissions\n* EX `<= 75%` => submissions are equal to or less than 75% of all Activities",
+                    "description": "A string containing a comparison operator and a value to compare **filtered** (using `include` or `exclude`, if present) submissions against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 100`  => greater than 100 filtered submissions\n* EX `<= 75%` => filtered submissions are equal to or less than 75% of unfiltered Activities",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "total": {
+                    "description": "A string containing a comparison operator and a value to compare **filtered** (using `include` or `exclude`) activities against\n\n**Note:** This is only useful if using `include` or `exclude` otherwise percent will always be 100% and total === activityTotal\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign] [OP]`\n\n* EX `> 100`  => greater than 100 filtered activities\n* EX `<= 75%` => filtered activities are equal to or less than 75% of all Activities",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
@@ -673,27 +754,51 @@
                     "type": "array"
                 },
                 "exclude": {
-                    "description": "Do not include Submissions from this list of Subreddits (by name, case-insensitive)\n\nEX `[\"mealtimevideos\",\"askscience\"]`",
+                    "description": "If present, activities will be counted only if they are **NOT** found in this list of Subreddits\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`\n\n**Note:** This affects **post-window retrieval** activities. So that:\n\n* `activityTotal` is number of activities retrieved from `window` -- NOT post-filtering\n* all comparisons using **percentages** will compare **post-filtering** results against **activity count from window**\n* -- to run this rule where all activities are only from include/exclude filtering instead use include/exclude in `window`",
                     "examples": [
-                        "mealtimevideos",
-                        "askscience"
+                        [
+                            "mealtimevideos",
+                            "askscience",
+                            "/onlyfans*/i",
+                            {
+                                "over18": true
+                            }
+                        ]
                     ],
                     "items": {
-                        "type": "string"
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/SubredditState"
+                            },
+                            {
+                                "type": "string"
+                            }
+                        ]
                     },
-                    "minItems": 1,
                     "type": "array"
                 },
                 "include": {
-                    "description": "Only include Submissions from this list of Subreddits (by name, case-insensitive)\n\nEX `[\"mealtimevideos\",\"askscience\"]`",
+                    "description": "If present, activities will be counted only if they are found in this list of Subreddits.\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`\n\n **Note:** This affects **post-window retrieval** activities. So that:\n\n* `activityTotal` is number of activities retrieved from `window` -- NOT post-filtering\n* all comparisons using **percentages** will compare **post-filtering** results against **activity count from window**\n* -- to run this rule where all activities are only from include/exclude filtering instead use include/exclude in `window`",
                     "examples": [
-                        "mealtimevideos",
-                        "askscience"
+                        [
+                            "mealtimevideos",
+                            "askscience",
+                            "/onlyfans*/i",
+                            {
+                                "over18": true
+                            }
+                        ]
                     ],
                     "items": {
-                        "type": "string"
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/SubredditState"
+                            },
+                            {
+                                "type": "string"
+                            }
+                        ]
                     },
-                    "minItems": 1,
                     "type": "array"
                 },
                 "itemIs": {
@@ -735,6 +840,31 @@
             ],
             "type": "object"
         },
+        "ImageDetection": {
+            "description": "When comparing submissions detect if the reference submission is an image and do a pixel-comparison to other detected image submissions.\n\n**Note:** This is an **experimental feature**",
+            "properties": {
+                "enable": {
+                    "description": "Is image detection enabled?",
+                    "type": "boolean"
+                },
+                "fetchBehavior": {
+                    "default": "extension",
+                    "description": "Determines how and when to check if a URL is an image\n\n**Note:** After fetching a URL the **Content-Type** is validated to contain `image` before detection occurs\n\n**When `extension`:** (default)\n\n* Only URLs that end in known image extensions (.png, .jpg, etc...) are fetched\n\n**When `unknown`:**\n\n* URLs that end in known image extensions (.png, .jpg, etc...) are fetched\n* URLs with no extension or unknown (IE non-video, non-doc, etc...) are fetched\n\n**When `all`:**\n\n* All submissions that have URLs (non-self) will be fetched, regardless of extension\n* **Note:** This can be bandwidth/CPU intensive if history window is large so use with care",
+                    "enum": [
+                        "all",
+                        "extension",
+                        "unknown"
+                    ],
+                    "type": "string"
+                },
+                "threshold": {
+                    "default": 5,
+                    "description": "The percentage, as a whole number, of pixels that are **different** between the two images at which point the images are not considered the same.\n\nDefault is `5`",
+                    "type": "number"
+                }
+            },
+            "type": "object"
+        },
         "RecentActivityRuleJSONConfig": {
             "description": "Checks a user's history for any Activity (Submission/Comment) in the subreddits specified in thresholds\n\nAvailable data for [Action templating](https://github.com/FoxxMD/context-mod#action-templating):\n\n```\nsummary    => comma-deliminated list of subreddits that hit the threshold and their count EX subredditA(1), subredditB(4),...\nsubCount   => Total number of subreddits that hit the threshold\ntotalCount => Total number of all activity occurrences in subreddits\n```",
             "properties": {
@@ -757,6 +887,10 @@
                         }
                     ]
                 },
+                "imageDetection": {
+                    "$ref": "#/definitions/ImageDetection",
+                    "description": "When comparing submissions detect if the reference submission is an image and do a pixel-comparison to other detected image submissions.\n\n**Note:** This is an **experimental feature**"
+                },
                 "itemIs": {
                     "anyOf": [
                         {
@@ -807,7 +941,7 @@
                 "thresholds": {
                     "description": "A list of subreddits/count criteria that may trigger this rule. ANY SubThreshold will trigger this rule.",
                     "items": {
-                        "$ref": "#/definitions/SubThreshold"
+                        "$ref": "#/definitions/ActivityThreshold"
                     },
                     "minItems": 1,
                     "type": "array"
@@ -844,6 +978,49 @@
             ],
             "type": "object"
         },
+        "RegExp": {
+            "properties": {
+                "dotAll": {
+                    "type": "boolean"
+                },
+                "flags": {
+                    "type": "string"
+                },
+                "global": {
+                    "type": "boolean"
+                },
+                "ignoreCase": {
+                    "type": "boolean"
+                },
+                "lastIndex": {
+                    "type": "number"
+                },
+                "multiline": {
+                    "type": "boolean"
+                },
+                "source": {
+                    "type": "string"
+                },
+                "sticky": {
+                    "type": "boolean"
+                },
+                "unicode": {
+                    "type": "boolean"
+                }
+            },
+            "required": [
+                "dotAll",
+                "flags",
+                "global",
+                "ignoreCase",
+                "lastIndex",
+                "multiline",
+                "source",
+                "sticky",
+                "unicode"
+            ],
+            "type": "object"
+        },
         "RegexCriteria": {
             "properties": {
                 "activityMatchThreshold": {
@@ -882,16 +1059,12 @@
                     "type": "string"
                 },
                 "regex": {
-                    "description": "A valid Regular Expression to test content against\n\nDo not wrap expression in forward slashes\n\nEX For the expression `/reddit|FoxxMD/` use the value should be `reddit|FoxxMD`",
+                    "description": "A valid Regular Expression to test content against\n\nIf no flags are specified then the **global** flag is used by default",
                     "examples": [
-                        "reddit|FoxxMD"
+                        "/reddit|FoxxMD/ig"
                     ],
                     "type": "string"
                 },
-                "regexFlags": {
-                    "description": "Regex flags to use",
-                    "type": "string"
-                },
                 "testOn": {
                     "default": [
                         "title",
@@ -1053,15 +1226,27 @@
                     ]
                 },
                 "exclude": {
-                    "description": "Do not include Submissions from this list of Subreddits (by name, case-insensitive)\n\nEX `[\"mealtimevideos\",\"askscience\"]`",
+                    "description": "If present, activities will be counted only if they are **NOT** found in this list of Subreddits\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`",
                     "examples": [
-                        "mealtimevideos",
-                        "askscience"
+                        [
+                            "mealtimevideos",
+                            "askscience",
+                            "/onlyfans*/i",
+                            {
+                                "over18": true
+                            }
+                        ]
                     ],
                     "items": {
-                        "type": "string"
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/SubredditState"
+                            },
+                            {
+                                "type": "string"
+                            }
+                        ]
                     },
-                    "minItems": 1,
                     "type": "array"
                 },
                 "gapAllowance": {
@@ -1069,15 +1254,27 @@
                     "type": "number"
                 },
                 "include": {
-                    "description": "Only include Submissions from this list of Subreddits (by name, case-insensitive)\n\nEX `[\"mealtimevideos\",\"askscience\"]`",
+                    "description": "If present, activities will be counted only if they are found in this list of Subreddits\n\nEach value in the list can be either:\n\n * string (name of subreddit)\n * regular expression to run on the subreddit name\n * `SubredditState`\n\nEX `[\"mealtimevideos\",\"askscience\", \"/onlyfans*\\/i\", {\"over18\": true}]`",
                     "examples": [
-                        "mealtimevideos",
-                        "askscience"
+                        [
+                            "mealtimevideos",
+                            "askscience",
+                            "/onlyfans*/i",
+                            {
+                                "over18": true
+                            }
+                        ]
                     ],
                     "items": {
-                        "type": "string"
+                        "anyOf": [
+                            {
+                                "$ref": "#/definitions/SubredditState"
+                            },
+                            {
+                                "type": "string"
+                            }
+                        ]
                     },
-                    "minItems": 1,
                     "type": "array"
                 },
                 "itemIs": {
@@ -1167,45 +1364,6 @@
             ],
             "type": "object"
         },
-        "SubThreshold": {
-            "additionalProperties": false,
-            "description": "At least one count property must be present. If both are present then either can trigger the rule",
-            "minProperties": 1,
-            "properties": {
-                "karma": {
-                    "description": "Test the **combined karma** from Activities found in the specified subreddits\n\nValue is a string containing a comparison operator and a number of **combined karma** to compare against\n\nIf specified then both `threshold` and `karma` must be met for this `SubThreshold` to be satisfied\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 50`  => greater than 50 combined karma for all found Activities in specified subreddits",
-                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
-                    "type": "string"
-                },
-                "subreddits": {
-                    "description": "A list of Subreddits (by name, case-insensitive) to look for.\n\nEX [\"mealtimevideos\",\"askscience\"]",
-                    "examples": [
-                        [
-                            "mealtimevideos",
-                            "askscience"
-                        ]
-                    ],
-                    "items": {
-                        "type": "string"
-                    },
-                    "minItems": 1,
-                    "type": "array"
-                },
-                "threshold": {
-                    "default": ">= 1",
-                    "description": "A string containing a comparison operator and a value to compare recent activities against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 3`  => greater than 3 activities found in the listed subreddits\n* EX `<= 75%` => number of Activities in the subreddits listed are equal to or less than 75% of all Activities\n\n**Note:** If you use percentage comparison here as well as `useSubmissionAsReference` then \"all Activities\" is only pertains to Activities that had the Link of the Submission, rather than all Activities from this window.",
-                    "examples": [
-                        ">= 1"
-                    ],
-                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
-                    "type": "string"
-                }
-            },
-            "required": [
-                "subreddits"
-            ],
-            "type": "object"
-        },
         "SubmissionState": {
             "description": "Different attributes a `Submission` can be in. Only include a property if you want to check it.",
             "examples": [
@@ -1249,6 +1407,16 @@
                 "removed": {
                     "type": "boolean"
                 },
+                "reports": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "score": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
                 "spam": {
                     "type": "boolean"
                 },
@@ -1265,6 +1433,44 @@
             },
             "type": "object"
         },
+        "SubredditState": {
+            "description": "Different attributes a `Subreddit` can be in. Only include a property if you want to check it.",
+            "examples": [
+                {
+                    "over18": true
+                }
+            ],
+            "properties": {
+                "name": {
+                    "anyOf": [
+                        {
+                            "$ref": "#/definitions/RegExp"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "The name the subreddit.\n\nCan be a normal string (will check case-insensitive) or a regular expression\n\nEX `[\"mealtimevideos\", \"/onlyfans*\\/i\"]`",
+                    "examples": [
+                        "mealtimevideos",
+                        "/onlyfans*/i"
+                    ]
+                },
+                "over18": {
+                    "description": "Is subreddit NSFW/over 18?\n\n**Note**: This is **mod-controlled flag** so it is up to the mods of the subreddit to correctly mark their subreddit as NSFW",
+                    "type": "boolean"
+                },
+                "quarantine": {
+                    "description": "Is subreddit quarantined?",
+                    "type": "boolean"
+                },
+                "stateDescription": {
+                    "description": "A friendly description of what this State is trying to parse",
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
         "UserNoteCriteria": {
             "properties": {
                 "count": {

src/Subreddit/Manager.ts

@@ -3,9 +3,9 @@ import {Logger} from "winston";
 import {SubmissionCheck} from "../Check/SubmissionCheck";
 import {CommentCheck} from "../Check/CommentCheck";
 import {
-    cacheStats,
+    cacheStats, createHistoricalStatsDisplay,
     createRetryHandler,
-    determineNewResults, formatNumber,
+    determineNewResults, findLastIndex, formatNumber,
     mergeArr, parseFromJsonOrYamlToObject, pollingInfo, resultsSummary, sleep, totalFromMapStats, triggeredIndicator,
 } from "../util";
 import {Poll} from "snoostorm";
@@ -17,7 +17,7 @@ import {
     ActionResult,
     DEFAULT_POLLING_INTERVAL,
     DEFAULT_POLLING_LIMIT, Invokee,
-    ManagerOptions, ManagerStateChangeOption, PAUSED,
+    ManagerOptions, ManagerStateChangeOption, ManagerStats, PAUSED,
     PollingOptionsStrong, ResourceStats, RUNNING, RunState, STOPPED, SYSTEM, USER
 } from "../Common/interfaces";
 import Submission from "snoowrap/dist/objects/Submission";
@@ -39,6 +39,7 @@ import {JSONConfig} from "../JsonConfig";
 import {CheckStructuredJson} from "../Check";
 import NotificationManager from "../Notification/NotificationManager";
 import action from "../Web/Server/routes/authenticated/user/action";
+import {createHistoricalDefaults, historicalDefaults} from "../Common/defaults";
 
 export interface RunningState {
     state: RunState,
@@ -49,6 +50,7 @@ export interface runCheckOptions {
     checkNames?: string[],
     delayUntil?: number,
     dryRun?: boolean,
+    refresh?: boolean,
 }
 
 export interface CheckTask {
@@ -64,37 +66,10 @@ export interface RuntimeManagerOptions extends ManagerOptions {
     maxWorkers: number;
 }
 
-export interface ManagerStats {
-    eventsCheckedTotal: number
-    eventsCheckedSinceStartTotal: number
-    eventsAvg: number
-    checksRunTotal: number
-    checksRunSinceStartTotal: number
-    checksTriggered: number
-    checksTriggeredTotal: number
-    checksTriggeredSinceStart: number
-    checksTriggeredSinceStartTotal: number
-    rulesRunTotal: number
-    rulesRunSinceStartTotal: number
-    rulesCachedTotal: number
-    rulesCachedSinceStartTotal: number
-    rulesTriggeredTotal: number
-    rulesTriggeredSinceStartTotal: number
-    rulesAvg: number
-    actionsRun: number
-    actionsRunTotal: number
-    actionsRunSinceStart: number,
-    actionsRunSinceStartTotal: number
-    cache: {
-        provider: string,
-        currentKeyCount: number,
-        isShared: boolean,
-        totalRequests: number,
-        totalMiss: number,
-        missPercent: string,
-        requestRate: number,
-        types: ResourceStats
-    },
+interface QueuedIdentifier {
+    id: string,
+    shouldRefresh: boolean
+    state: 'queued' | 'processing'
 }
 
 export class Manager {
@@ -120,6 +95,15 @@ export class Manager {
     globalDryRun?: boolean;
     emitter: EventEmitter = new EventEmitter();
     queue: QueueObject<CheckTask>;
+    // firehose is used to ensure all activities from different polling streams are unique
+    // that is -- if the same activities is in both modqueue and unmoderated we don't want to process the activity twice or use stale data
+    //
+    // so all activities get queued to firehose, it keeps track of items by id (using queuedItemsMeta)
+    // and ensures that if any activities are ingested while they are ALSO currently queued or working then they are properly handled by either
+    // 1) if queued, do not re-queue but instead tell worker to refresh before processing
+    // 2) if currently processing then re-queue but also refresh before processing
+    firehose: QueueObject<CheckTask>;
+    queuedItemsMeta: QueuedIdentifier[] = [];
     globalMaxWorkers: number;
     subMaxWorkers?: number;
 
@@ -148,50 +132,22 @@ export class Manager {
     // use by api nanny to slow event consumption
     delayBy?: number;
 
-    eventsCheckedTotal: number = 0;
-    eventsCheckedSinceStartTotal: number = 0;
     eventsSample: number[] = [];
     eventsSampleInterval: any;
     eventsRollingAvg: number = 0;
-    checksRunTotal: number = 0;
-    checksRunSinceStartTotal: number = 0;
-    checksTriggered: Map<string, number> = new Map();
-    checksTriggeredSinceStart: Map<string, number> = new Map();
-    rulesRunTotal: number = 0;
-    rulesRunSinceStartTotal: number = 0;
-    rulesCachedTotal: number = 0;
-    rulesCachedSinceStartTotal: number = 0;
-    rulesTriggeredTotal: number = 0;
-    rulesTriggeredSinceStartTotal: number = 0;
     rulesUniqueSample: number[] = [];
     rulesUniqueSampleInterval: any;
     rulesUniqueRollingAvg: number = 0;
-    actionsRun: Map<string, number> = new Map();
-    actionsRunSinceStart: Map<string, number> = new Map();
     actionedEvents: ActionedEvent[] = [];
 
     getStats = async (): Promise<ManagerStats> => {
         const data: any = {
-            eventsCheckedTotal: this.eventsCheckedTotal,
-            eventsCheckedSinceStartTotal: this.eventsCheckedSinceStartTotal,
             eventsAvg: formatNumber(this.eventsRollingAvg),
-            checksRunTotal: this.checksRunTotal,
-            checksRunSinceStartTotal: this.checksRunSinceStartTotal,
-            checksTriggered: this.checksTriggered,
-            checksTriggeredTotal: totalFromMapStats(this.checksTriggered),
-            checksTriggeredSinceStart: this.checksTriggeredSinceStart,
-            checksTriggeredSinceStartTotal: totalFromMapStats(this.checksTriggeredSinceStart),
-            rulesRunTotal: this.rulesRunTotal,
-            rulesRunSinceStartTotal: this.rulesRunSinceStartTotal,
-            rulesCachedTotal: this.rulesCachedTotal,
-            rulesCachedSinceStartTotal: this.rulesCachedSinceStartTotal,
-            rulesTriggeredTotal: this.rulesTriggeredTotal,
-            rulesTriggeredSinceStartTotal: this.rulesTriggeredSinceStartTotal,
             rulesAvg: formatNumber(this.rulesUniqueRollingAvg),
-            actionsRun: this.actionsRun,
-            actionsRunTotal: totalFromMapStats(this.actionsRun),
-            actionsRunSinceStart: this.actionsRunSinceStart,
-            actionsRunSinceStartTotal: totalFromMapStats(this.actionsRunSinceStart),
+            historical: {
+                lastReload: createHistoricalStatsDisplay(createHistoricalDefaults()),
+                allTime: createHistoricalStatsDisplay(createHistoricalDefaults()),
+            },
             cache: {
                 provider: 'none',
                 currentKeyCount: 0,
@@ -207,6 +163,7 @@ export class Manager {
         if (this.resources !== undefined) {
             const resStats = await this.resources.getStats();
 
+            data.historical = this.resources.getHistoricalDisplayStats();
             data.cache = resStats.cache;
             data.cache.currentKeyCount = await this.resources.getCacheKeyCount();
             data.cache.isShared = this.resources.cacheSettingsHash === 'default';
@@ -250,11 +207,13 @@ export class Manager {
 
         this.queue = this.generateQueue(this.getMaxWorkers(this.globalMaxWorkers));
         this.queue.pause();
+        this.firehose = this.generateFirehose();
 
         this.eventsSampleInterval = setInterval((function(self) {
             return function() {
+                const et = self.resources !== undefined ? self.resources.stats.historical.allTime.eventsCheckedTotal : 0;
                 const rollingSample = self.eventsSample.slice(0, 7)
-                rollingSample.unshift(self.eventsCheckedTotal)
+                rollingSample.unshift(et)
                 self.eventsSample = rollingSample;
                 const diff = self.eventsSample.reduceRight((acc: number[], curr, index) => {
                     if(self.eventsSample[index + 1] !== undefined) {
@@ -274,7 +233,8 @@ export class Manager {
         this.rulesUniqueSampleInterval = setInterval((function(self) {
             return function() {
                 const rollingSample = self.rulesUniqueSample.slice(0, 7)
-                rollingSample.unshift(self.rulesRunTotal - self.rulesCachedTotal);
+                const rt = self.resources !== undefined ? self.resources.stats.historical.allTime.rulesRunTotal - self.resources.stats.historical.allTime.rulesCachedTotal : 0;
+                rollingSample.unshift(rt);
                 self.rulesUniqueSample = rollingSample;
                 const diff = self.rulesUniqueSample.reduceRight((acc: number[], curr, index) => {
                     if(self.rulesUniqueSample[index + 1] !== undefined) {
@@ -310,6 +270,32 @@ export class Manager {
         return maxWorkers;
     }
 
+    protected generateFirehose() {
+        return queue(async (task: CheckTask, cb) => {
+            // items in queuedItemsMeta will be processing FIFO so earlier elements (by index) are older
+            //
+            // if we insert the same item again because it is currently being processed AND THEN we get the item AGAIN we only want to update the newest meta
+            // so search the array backwards to get the neweset only
+            const queuedItemIndex = findLastIndex(this.queuedItemsMeta, x => x.id === task.activity.id);
+            if(queuedItemIndex !== -1) {
+                const itemMeta = this.queuedItemsMeta[queuedItemIndex];
+                let msg = `Item ${itemMeta.id} is already ${itemMeta.state}.`;
+                if(itemMeta.state === 'queued') {
+                    this.logger.debug(`${msg} Flagging to refresh data before processing.`);
+                    this.queuedItemsMeta.splice(queuedItemIndex, 1, {...itemMeta, shouldRefresh: true});
+                } else {
+                    this.logger.debug(`${msg} Re-queuing item but will also refresh data before processing.`);
+                    this.queuedItemsMeta.push({id: task.activity.id, shouldRefresh: true, state: 'queued'});
+                    this.queue.push(task);
+                }
+            } else {
+                this.queuedItemsMeta.push({id: task.activity.id, shouldRefresh: false, state: 'queued'});
+                this.queue.push(task);
+            }
+        }
+        , 1);
+    }
+
     protected generateQueue(maxWorkers: number) {
         if (maxWorkers > 1) {
             this.logger.warn(`Setting max queue workers above 1 (specified: ${maxWorkers}) may have detrimental effects to log readability and api usage. Consult the documentation before using this advanced/experimental feature.`);
@@ -320,7 +306,16 @@ export class Manager {
                     this.logger.debug(`SOFT API LIMIT MODE: Delaying Event run by ${this.delayBy} seconds`);
                     await sleep(this.delayBy * 1000);
                 }
-                await this.runChecks(task.checkType, task.activity, task.options);
+
+                const queuedItemIndex = this.queuedItemsMeta.findIndex(x => x.id === task.activity.id);
+                try {
+                    const itemMeta = this.queuedItemsMeta[queuedItemIndex];
+                    this.queuedItemsMeta.splice(queuedItemIndex, 1, {...itemMeta, state: 'processing'});
+                    await this.runChecks(task.checkType, task.activity, {...task.options, refresh: itemMeta.shouldRefresh});
+                } finally {
+                    // always remove item meta regardless of success or failure since we are done with it meow
+                    this.queuedItemsMeta.splice(queuedItemIndex, 1);
+                }
             }
             , maxWorkers);
         q.error((err, task) => {
@@ -335,7 +330,7 @@ export class Manager {
         return q;
     }
 
-    protected parseConfigurationFromObject(configObj: object) {
+    protected async parseConfigurationFromObject(configObj: object) {
         try {
             const configBuilder = new ConfigBuilder({logger: this.logger});
             const validJson = configBuilder.validateJson(configObj);
@@ -385,7 +380,7 @@ export class Manager {
                 caching,
                 client: this.client,
             };
-            this.resources = this.cacheManager.set(this.subreddit.display_name, resourceConfig);
+            this.resources = await this.cacheManager.set(this.subreddit.display_name, resourceConfig);
             this.resources.setLogger(this.logger);
 
             this.logger.info('Subreddit-specific options updated');
@@ -480,7 +475,7 @@ export class Manager {
                 throw new ConfigParseError('Could not parse wiki page contents as JSON or YAML')
             }
 
-            this.parseConfigurationFromObject(configObj);
+            await this.parseConfigurationFromObject(configObj);
             this.logger.info('Checks updated');
 
             if(!suppressNotification) {
@@ -497,8 +492,6 @@ export class Manager {
     async runChecks(checkType: ('Comment' | 'Submission'), activity: (Submission | Comment), options?: runCheckOptions): Promise<void> {
         const checks = checkType === 'Comment' ? this.commentChecks : this.submissionChecks;
         let item = activity;
-        this.eventsCheckedTotal++;
-        this.eventsCheckedSinceStartTotal++;
         const itemId = await item.id;
         let allRuleResults: RuleResult[] = [];
         const itemIdentifier = `${checkType === 'Submission' ? 'SUB' : 'COM'} ${itemId}`;
@@ -516,8 +509,11 @@ export class Manager {
             checkNames = [],
             delayUntil,
             dryRun,
+            refresh = false,
         } = options || {};
 
+        let wasRefreshed = false;
+
         if (delayUntil !== undefined) {
             const created = dayjs.unix(item.created_utc);
             const diff = dayjs().diff(created, 's');
@@ -526,8 +522,16 @@ export class Manager {
                 await sleep(delayUntil - diff);
                 // @ts-ignore
                 item = await activity.refresh();
+                wasRefreshed = true;
             }
         }
+        // refresh signal from firehose if activity was ingested multiple times before processing or re-queued while processing
+        // want to make sure we have the most recent data
+        if(!wasRefreshed && refresh === true) {
+            this.logger.verbose('Refreshed data (probably due to signal from firehose)');
+            // @ts-ignore
+            item = await activity.refresh();
+        }
 
         const startingApiLimit = this.client.ratelimitRemaining;
 
@@ -559,7 +563,9 @@ export class Manager {
             actionResults: [],
         }
         let triggered = false;
-
+        let triggeredCheckName;
+        const checksRunNames = [];
+        const cachedCheckNames = [];
         try {
             for (const check of checks) {
                 if (checkNames.length > 0 && !checkNames.map(x => x.toLowerCase()).some(x => x === check.name.toLowerCase())) {
@@ -570,6 +576,7 @@ export class Manager {
                     this.logger.info(`Check ${check.name} not run because it is not enabled, skipping...`);
                     continue;
                 }
+                checksRunNames.push(check.name);
                 checksRun++;
                 triggered = false;
                 let isFromCache = false;
@@ -579,6 +586,8 @@ export class Manager {
                     isFromCache = fromCache;
                     if(!fromCache) {
                         await check.setCacheResult(item, {result: checkTriggered, ruleResults: checkResults});
+                    } else {
+                        cachedCheckNames.push(check.name);
                     }
                     currentResults = checkResults;
                     totalRulesRun += checkResults.length;
@@ -595,6 +604,7 @@ export class Manager {
                 }
 
                 if (triggered) {
+                    triggeredCheckName = check.name;
                     actionedEvent.check = check.name;
                     actionedEvent.ruleResults = currentResults;
                     if(isFromCache) {
@@ -602,8 +612,6 @@ export class Manager {
                     } else {
                         actionedEvent.ruleSummary = resultsSummary(currentResults, check.condition);
                     }
-                    this.checksTriggered.set(check.name, (this.checksTriggered.get(check.name) || 0) + 1);
-                    this.checksTriggeredSinceStart.set(check.name, (this.checksTriggeredSinceStart.get(check.name) || 0) + 1);
                     runActions = await check.runActions(item, currentResults.filter(x => x.triggered), dryRun);
                     actionsRun = runActions.length;
 
@@ -625,23 +633,6 @@ export class Manager {
             }
         } finally {
             try {
-                const cachedTotal = totalRulesRun - allRuleResults.length;
-                const triggeredRulesTotal = allRuleResults.filter(x => x.triggered).length;
-
-                this.checksRunTotal += checksRun;
-                this.checksRunSinceStartTotal += checksRun;
-                this.rulesRunTotal += totalRulesRun;
-                this.rulesRunSinceStartTotal += totalRulesRun;
-                this.rulesCachedTotal += cachedTotal;
-                this.rulesCachedSinceStartTotal += cachedTotal;
-                this.rulesTriggeredTotal += triggeredRulesTotal;
-                this.rulesTriggeredSinceStartTotal += triggeredRulesTotal;
-
-                for (const a of runActions) {
-                    const name = a.name;
-                    this.actionsRun.set(name, (this.actionsRun.get(name) || 0) + 1);
-                    this.actionsRunSinceStart.set(name, (this.actionsRunSinceStart.get(name) || 0) + 1);
-                }
                 actionedEvent.actionResults = runActions;
                 if(triggered) {
                     await this.resources.addActionedEvent(actionedEvent);
@@ -652,6 +643,18 @@ export class Manager {
                 this.currentLabels = [];
             } catch (err) {
                 this.logger.error('Error occurred while cleaning up Activity check and generating stats', err);
+            } finally {
+                this.resources.updateHistoricalStats({
+                    eventsCheckedTotal: 1,
+                    eventsActionedTotal: triggered ? 1 : 0,
+                    checksTriggered: triggeredCheckName !== undefined ? [triggeredCheckName] : [],
+                    checksRun: checksRunNames,
+                    checksFromCache: cachedCheckNames,
+                    actionsRun: runActions.map(x => x.name),
+                    rulesRun: allRuleResults.map(x => x.name),
+                    rulesTriggered: allRuleResults.filter(x => x.triggered).map(x => x.name),
+                    rulesCachedTotal: totalRulesRun - allRuleResults.length,
+                });
             }
         }
     }
@@ -746,7 +749,7 @@ export class Manager {
                     checkType = 'Comment';
                 }
                 if (checkType !== undefined) {
-                    this.queue.push({checkType, activity: item, options: {delayUntil}})
+                    this.firehose.push({checkType, activity: item, options: {delayUntil}})
                 }
             };
 
@@ -779,14 +782,19 @@ export class Manager {
         } else if (!this.validConfigLoaded) {
             this.logger.warn('Cannot start activity processing queue while manager has an invalid configuration');
         } else {
+            if(this.queueState.state === STOPPED) {
+                // extra precaution to make sure queue meta is cleared before starting queue
+                this.queuedItemsMeta = [];
+            }
             this.queue.resume();
+            this.firehose.resume();
             this.logger.info(`Activity processing queue started RUNNING with ${this.queue.length()} queued activities`);
             this.queueState = {
                 state: RUNNING,
                 causedBy
             }
             if(!suppressNotification) {
-                this.notificationManager.handle('runStateChanged', 'Queue Started', reason, causedBy)
+                this.notificationManager.handle('runStateChanged', 'Queue Started', reason, causedBy);
             }
         }
     }
@@ -848,7 +856,9 @@ export class Manager {
                     this.logger.verbose(`Activity processing queue is stopping...waiting for ${this.queue.running()} activities to finish processing`);
                 }
                 this.logger.info(`Activity processing queue stopped by ${causedBy} and ${this.queue.length()} queued activities cleared (waited ${dayjs().diff(pauseWaitStart, 's')} seconds while activity processing finished)`);
+                this.firehose.kill();
                 this.queue.kill();
+                this.queuedItemsMeta = [];
             }
 
             this.queueState = {
@@ -932,13 +942,6 @@ export class Manager {
                 stream.removeListener('item', v);
             }
             this.startedAt = undefined;
-            this.eventsCheckedSinceStartTotal = 0;
-            this.checksRunSinceStartTotal = 0;
-            this.rulesRunSinceStartTotal = 0;
-            this.rulesCachedSinceStartTotal = 0;
-            this.rulesTriggeredSinceStartTotal = 0;
-            this.checksTriggeredSinceStart = new Map();
-            this.actionsRunSinceStart = new Map();
             this.logger.info(`Events STOPPED by ${causedBy}`);
             this.eventsState = {
                 state: STOPPED,

src/Subreddit/SubredditResources.ts

@@ -1,4 +1,4 @@
-import Snoowrap, {RedditUser} from "snoowrap";
+import Snoowrap, {RedditUser, Subreddit} from "snoowrap";
 import objectHash from 'object-hash';
 import {
     activityIsDeleted, activityIsFiltered,
@@ -8,24 +8,35 @@ import {
     getAuthorActivities,
     testAuthorCriteria
 } from "../Utils/SnoowrapUtils";
-import Subreddit from 'snoowrap/dist/objects/Subreddit';
 import winston, {Logger} from "winston";
 import fetch from 'node-fetch';
 import {
     asSubmission,
     buildCacheOptionsFromProvider, buildCachePrefix,
-    cacheStats, createCacheManager,
-    formatNumber, getActivityAuthorName,
+    cacheStats, comparisonTextOp, createCacheManager, createHistoricalStatsDisplay,
+    formatNumber, getActivityAuthorName, getActivitySubredditName, isStrongSubredditState,
     mergeArr,
-    parseExternalUrl,
-    parseWikiContext
+    parseExternalUrl, parseGenericValueComparison,
+    parseWikiContext, toStrongSubredditState
 } from "../util";
 import LoggedError from "../Utils/LoggedError";
 import {
     BotInstanceConfig,
-    CacheOptions, CommentState,
-    Footer, OperatorConfig, ResourceStats, StrongCache, SubmissionState,
-    CacheConfig, TTLConfig, TypedActivityStates, UserResultCache, ActionedEvent
+    CacheOptions,
+    CommentState,
+    Footer,
+    OperatorConfig,
+    ResourceStats,
+    StrongCache,
+    SubmissionState,
+    CacheConfig,
+    TTLConfig,
+    TypedActivityStates,
+    UserResultCache,
+    ActionedEvent,
+    SubredditState,
+    StrongSubredditState,
+    HistoricalStats, HistoricalStatUpdateData, SubredditHistoricalStats, SubredditHistoricalStatsDisplay
 } from "../Common/interfaces";
 import UserNotes from "./UserNotes";
 import Mustache from "mustache";
@@ -34,7 +45,7 @@ import {AuthorCriteria} from "../Author/Author";
 import {SPoll} from "./Streams";
 import {Cache} from 'cache-manager';
 import {Submission, Comment} from "snoowrap/dist/objects";
-import {cacheTTLDefaults} from "../Common/defaults";
+import {cacheTTLDefaults, createHistoricalDefaults, historicalDefaults} from "../Common/defaults";
 import {check} from "tcp-port-used";
 
 export const DEFAULT_FOOTER = '\r\n*****\r\nThis action was performed by [a bot.]({{botLink}}) Mention a moderator or [send a modmail]({{modmailLink}}) if you any ideas, questions, or concerns about this action.';
@@ -65,6 +76,7 @@ export class SubredditResources {
     //enabled!: boolean;
     protected useSubredditAuthorCache!: boolean;
     protected authorTTL: number | false = cacheTTLDefaults.authorTTL;
+    protected subredditTTL: number | false = cacheTTLDefaults.subredditTTL;
     protected wikiTTL: number | false = cacheTTLDefaults.wikiTTL;
     protected submissionTTL: number | false = cacheTTLDefaults.submissionTTL;
     protected commentTTL: number | false = cacheTTLDefaults.commentTTL;
@@ -79,10 +91,14 @@ export class SubredditResources {
     cacheType: string
     cacheSettingsHash?: string;
     pruneInterval?: any;
+    historicalSaveInterval?: any;
     prefix?: string
     actionedEventsMax: number;
 
-    stats: { cache: ResourceStats };
+    stats: {
+        cache: ResourceStats
+        historical: SubredditHistoricalStats
+    };
 
     constructor(name: string, options: SubredditResourceOptions) {
         const {
@@ -95,6 +111,7 @@ export class SubredditResources {
                 filterCriteriaTTL,
                 submissionTTL,
                 commentTTL,
+                subredditTTL,
             },
             cache,
             prefix,
@@ -113,6 +130,7 @@ export class SubredditResources {
         this.authorTTL = authorTTL === true ? 0 : authorTTL;
         this.submissionTTL = submissionTTL === true ? 0 : submissionTTL;
         this.commentTTL = commentTTL === true ? 0 : commentTTL;
+        this.subredditTTL = subredditTTL === true ? 0 : subredditTTL;
         this.wikiTTL = wikiTTL === true ? 0 : wikiTTL;
         this.filterCriteriaTTL = filterCriteriaTTL === true ? 0 : filterCriteriaTTL;
         this.subreddit = subreddit;
@@ -125,7 +143,11 @@ export class SubredditResources {
         }
 
         this.stats = {
-            cache: cacheStats()
+            cache: cacheStats(),
+            historical: {
+                allTime: createHistoricalDefaults(),
+                lastReload: createHistoricalDefaults()
+            }
         };
 
         const cacheUseCB = (miss: boolean) => {
@@ -149,6 +171,92 @@ export class SubredditResources {
         }
     }
 
+    async initHistoricalStats() {
+         const at = await this.cache.wrap(`${this.name}-historical-allTime`, () => createHistoricalDefaults(), {ttl: 0}) as object;
+         const rehydratedAt: any = {};
+         for(const [k, v] of Object.entries(at)) {
+            if(Array.isArray(v)) {
+                rehydratedAt[k] = new Map(v);
+            } else {
+                rehydratedAt[k] = v;
+            }
+         }
+         this.stats.historical.allTime = rehydratedAt as HistoricalStats;
+
+        // const lr = await this.cache.wrap(`${this.name}-historical-lastReload`, () => createHistoricalDefaults(), {ttl: 0}) as object;
+        // const rehydratedLr: any = {};
+        // for(const [k, v] of Object.entries(lr)) {
+        //     if(Array.isArray(v)) {
+        //         rehydratedLr[k] = new Map(v);
+        //     } else {
+        //         rehydratedLr[k] = v;
+        //     }
+        // }
+        // this.stats.historical.lastReload = rehydratedLr;
+    }
+
+    updateHistoricalStats(data: HistoricalStatUpdateData) {
+        for(const [k, v] of Object.entries(data)) {
+            if(this.stats.historical.lastReload[k] !== undefined) {
+                if(typeof v === 'number') {
+                    this.stats.historical.lastReload[k] += v;
+                } else if(this.stats.historical.lastReload[k] instanceof Map) {
+                    const keys = Array.isArray(v) ? v : [v];
+                    for(const key of keys) {
+                        this.stats.historical.lastReload[k].set(key, (this.stats.historical.lastReload[k].get(key) || 0) + 1);
+                    }
+                }
+            }
+            if(this.stats.historical.allTime[k] !== undefined) {
+                if(typeof v === 'number') {
+                    this.stats.historical.allTime[k] += v;
+                } else if(this.stats.historical.allTime[k] instanceof Map) {
+                    const keys = Array.isArray(v) ? v : [v];
+                    for(const key of keys) {
+                        this.stats.historical.allTime[k].set(key, (this.stats.historical.allTime[k].get(key) || 0) + 1);
+                    }
+                }
+            }
+        }
+    }
+
+    getHistoricalDisplayStats(): SubredditHistoricalStatsDisplay {
+        return {
+            allTime: createHistoricalStatsDisplay(this.stats.historical.allTime),
+            lastReload: createHistoricalStatsDisplay(this.stats.historical.lastReload)
+        }
+    }
+
+    async saveHistoricalStats() {
+        const atSerializable: any = {};
+        for(const [k, v] of Object.entries(this.stats.historical.allTime)) {
+            if(v instanceof Map) {
+                atSerializable[k] = Array.from(v.entries());
+            } else {
+                atSerializable[k] = v;
+            }
+        }
+        await this.cache.set(`${this.name}-historical-allTime`, atSerializable, {ttl: 0});
+
+        // const lrSerializable: any = {};
+        // for(const [k, v] of Object.entries(this.stats.historical.lastReload)) {
+        //     if(v instanceof Map) {
+        //         lrSerializable[k] = Array.from(v.entries());
+        //     } else {
+        //         lrSerializable[k] = v;
+        //     }
+        // }
+        // await this.cache.set(`${this.name}-historical-lastReload`, lrSerializable, {ttl: 0});
+    }
+
+    setHistoricalSaveInterval() {
+        this.historicalSaveInterval = setInterval((function(self) {
+            return async () => {
+                await self.saveHistoricalStats();
+            }
+        })(this),10000);
+    }
+
     async getCacheKeyCount() {
         if (this.cache.store.keys !== undefined) {
             if(this.cacheType === 'redis') {
@@ -270,6 +378,40 @@ export class SubredditResources {
         }
     }
 
+    // @ts-ignore
+    async getSubreddit(item: Submission | Comment) {
+        try {
+            let hash = '';
+            if (this.subredditTTL !== false) {
+                hash = `sub-${getActivitySubredditName(item)}`;
+                await this.stats.cache.subreddit.identifierRequestCount.set(hash, (await this.stats.cache.subreddit.identifierRequestCount.wrap(hash, () => 0) as number) + 1);
+                this.stats.cache.subreddit.requestTimestamps.push(Date.now());
+                this.stats.cache.subreddit.requests++;
+                const cachedSubreddit = await this.cache.get(hash);
+                if (cachedSubreddit !== undefined && cachedSubreddit !== null) {
+                    this.logger.debug(`Cache Hit: Subreddit ${item.subreddit.display_name}`);
+                    // @ts-ignore
+                    return cachedSubreddit as Subreddit;
+                }
+                // @ts-ignore
+                const subreddit = await this.client.getSubreddit(getActivitySubredditName(item)).fetch() as Subreddit;
+                this.stats.cache.subreddit.miss++;
+                // @ts-ignore
+                await this.cache.set(hash, subreddit, {ttl: this.subredditTTL});
+                // @ts-ignore
+                return subreddit as Subreddit;
+            } else {
+                // @ts-ignore
+                let subreddit = await this.client.getSubreddit(getActivitySubredditName(item));
+
+                return subreddit as Subreddit;
+            }
+        } catch (err) {
+            this.logger.error('Error while trying to fetch a cached activity', err);
+            throw err.logged;
+        }
+    }
+
     async getAuthorActivities(user: RedditUser, options: AuthorTypedActivitiesOptions): Promise<Array<Submission | Comment>> {
         const userName = getActivityAuthorName(user);
         if (this.authorTTL !== false) {
@@ -391,6 +533,46 @@ export class SubredditResources {
         return wikiContent;
     }
 
+    async testSubredditCriteria(item: (Comment | Submission), state: SubredditState | StrongSubredditState) {
+        if(Object.keys(state).length === 0) {
+            return true;
+        }
+        // optimize for name-only criteria checks
+        // -- we don't need to store cache results for this since we know subreddit name is always available from item (no request required)
+        const critCount = Object.entries(state).filter(([key, val]) => {
+            return val !== undefined && !['name','stateDescription'].includes(key);
+        }).length;
+        if(critCount === 0) {
+            const subName = getActivitySubredditName(item);
+            return await this.isSubreddit({display_name: subName} as Subreddit, state, this.logger);
+        }
+
+        if (this.filterCriteriaTTL !== false) {
+            try {
+                const hash = `subredditCrit-${getActivitySubredditName(item)}-${objectHash.sha1(state)}`;
+                await this.stats.cache.subredditCrit.identifierRequestCount.set(hash, (await this.stats.cache.subredditCrit.identifierRequestCount.wrap(hash, () => 0) as number) + 1);
+                this.stats.cache.subredditCrit.requestTimestamps.push(Date.now());
+                this.stats.cache.subredditCrit.requests++;
+                const cachedItem = await this.cache.get(hash);
+                if (cachedItem !== undefined && cachedItem !== null) {
+                    this.logger.debug(`Cache Hit: Subreddit Check on ${getActivitySubredditName(item)} (Hash ${hash})`);
+                    return cachedItem as boolean;
+                }
+                const itemResult = await this.isSubreddit(await this.getSubreddit(item), state, this.logger);
+                this.stats.cache.subredditCrit.miss++;
+                await this.cache.set(hash, itemResult, {ttl: this.filterCriteriaTTL});
+                return itemResult;
+            } catch (err) {
+                if (err.logged !== true) {
+                    this.logger.error('Error occurred while testing subreddit criteria', err);
+                }
+                throw err;
+            }
+        }
+
+        return await this.isSubreddit(await this.getSubreddit(item), state, this.logger);
+    }
+
     async testAuthorCriteria(item: (Comment | Submission), authorOpts: AuthorCriteria, include = true) {
         if (this.filterCriteriaTTL !== false) {
             // in the criteria check we only actually use the `item` to get the author flair
@@ -420,10 +602,10 @@ export class SubredditResources {
         return await testAuthorCriteria(item, authorOpts, include, this.userNotes);
     }
 
-    async testItemCriteria(i: (Comment | Submission), s: TypedActivityStates) {
+    async testItemCriteria(i: (Comment | Submission), activityStates: TypedActivityStates) {
         if (this.filterCriteriaTTL !== false) {
             let item = i;
-            let states = s;
+            let states = activityStates;
             // optimize for submission only checks on comment item
             if (item instanceof Comment && states.length === 1 && Object.keys(states[0]).length === 1 && (states[0] as CommentState).submissionState !== undefined) {
                 // @ts-ignore
@@ -454,7 +636,50 @@ export class SubredditResources {
             }
         }
 
-        return await this.isItem(i, s, this.logger);
+        return await this.isItem(i, activityStates, this.logger);
+    }
+
+    async isSubreddit (subreddit: Subreddit, stateCriteria: SubredditState | StrongSubredditState, logger: Logger) {
+        delete stateCriteria.stateDescription;
+
+        if (Object.keys(stateCriteria).length === 0) {
+            return true;
+        }
+
+        const crit = isStrongSubredditState(stateCriteria) ? stateCriteria : toStrongSubredditState(stateCriteria, {defaultFlags: 'i'});
+
+        const log = logger.child({leaf: 'Subreddit Check'}, mergeArr);
+
+        return await (async () => {
+            for (const k of Object.keys(crit)) {
+                // @ts-ignore
+                if (crit[k] !== undefined) {
+                    switch (k) {
+                        case 'name':
+                            const nameReg = crit[k] as RegExp;
+                            if(!nameReg.test(subreddit.display_name)) {
+                                return false;
+                            }
+                            break;
+                        default:
+                            // @ts-ignore
+                            if (crit[k] !== undefined) {
+                                // @ts-ignore
+                                if (crit[k] !== subreddit[k]) {
+                                    // @ts-ignore
+                                    log.debug(`Failed: Expected => ${k}:${crit[k]} | Found => ${k}:${subreddit[k]}`)
+                                    return false
+                                }
+                            } else {
+                                log.warn(`Tried to test for Subreddit property '${k}' but it did not exist`);
+                            }
+                            break;
+                    }
+                }
+            }
+            log.debug(`Passed: ${JSON.stringify(stateCriteria)}`);
+            return true;
+        })() as boolean;
     }
 
     async isItem (item: Submission | Comment, stateCriteria: TypedActivityStates, logger: Logger) {
@@ -486,6 +711,22 @@ export class SubredditResources {
                                     return false;
                                 }
                                 break;
+                            case 'score':
+                                const scoreCompare = parseGenericValueComparison(crit[k] as string);
+                                if(!comparisonTextOp(item.score, scoreCompare.operator, scoreCompare.value)) {
+                                    // @ts-ignore
+                                    log.debug(`Failed: Expected => ${k}:${crit[k]} | Found => ${k}:${item.score}`)
+                                    return false
+                                }
+                                break;
+                            case 'reports':
+                                const reportCompare = parseGenericValueComparison(crit[k] as string);
+                                if(!comparisonTextOp(item.num_reports, reportCompare.operator, reportCompare.value)) {
+                                    // @ts-ignore
+                                    log.debug(`Failed: Expected => ${k}:${crit[k]} | Found => ${k}:${item.num_reports}`)
+                                    return false
+                                }
+                                break;
                             case 'removed':
                                 const removed = activityIsRemoved(item);
                                 if (removed !== crit['removed']) {
@@ -612,6 +853,7 @@ export class BotResourcesManager {
                 wikiTTL,
                 commentTTL,
                 submissionTTL,
+                subredditTTL,
                 filterCriteriaTTL,
                 provider,
                 actionedEventsMax,
@@ -625,7 +867,7 @@ export class BotResourcesManager {
         const {actionedEventsMax: eMax, actionedEventsDefault: eDef, ...relevantCacheSettings} = caching;
         this.cacheHash = objectHash.sha1(relevantCacheSettings);
         this.defaultCacheConfig = caching;
-        this.ttlDefaults = {authorTTL, userNotesTTL, wikiTTL, commentTTL, submissionTTL, filterCriteriaTTL};
+        this.ttlDefaults = {authorTTL, userNotesTTL, wikiTTL, commentTTL, submissionTTL, filterCriteriaTTL, subredditTTL};
 
         const options = provider;
         this.cacheType = options.store;
@@ -655,7 +897,7 @@ export class BotResourcesManager {
         return undefined;
     }
 
-    set(subName: string, initOptions: SubredditResourceConfig): SubredditResources {
+    async set(subName: string, initOptions: SubredditResourceConfig): Promise<SubredditResources> {
         let hash = 'default';
         const { caching, ...init } = initOptions;
 
@@ -705,6 +947,8 @@ export class BotResourcesManager {
                 res.cache.reset();
             }
             resource = new SubredditResources(subName, opts);
+            await resource.initHistoricalStats();
+            resource.setHistoricalSaveInterval();
             this.resources.set(subName, resource);
         } else {
             // just set non-cache related settings
@@ -715,6 +959,7 @@ export class BotResourcesManager {
             // reset cache stats when configuration is reloaded
             resource.stats.cache = cacheStats();
         }
+        resource.stats.historical.lastReload = createHistoricalDefaults();
 
         return resource;
     }

src/Utils/SnoowrapUtils.ts

@@ -13,10 +13,16 @@ import {
     TypedActivityStates
 } from "../Common/interfaces";
 import {
-    compareDurationValue, comparisonTextOp,
-    isActivityWindowCriteria,
-    normalizeName, parseDuration,
-    parseDurationComparison, parseGenericValueComparison, parseGenericValueOrPercentComparison, parseSubredditName,
+    compareDurationValue,
+    comparisonTextOp, getActivityAuthorName,
+    isActivityWindowCriteria, isStatusError,
+    normalizeName,
+    parseDuration,
+    parseDurationComparison,
+    parseGenericValueComparison,
+    parseGenericValueOrPercentComparison,
+    parseRuleResultsToMarkdownSummary,
+    parseSubredditName,
     truncateStringToLength
 } from "../util";
 import UserNotes from "../Subreddit/UserNotes";
@@ -119,17 +125,25 @@ export async function getAuthorActivities(user: RedditUser, options: AuthorTyped
 
     let items: Array<Submission | Comment> = [];
     //let count = 1;
-    let listing;
-    switch (options.type) {
-        case 'comment':
-            listing = await user.getComments({limit: chunkSize});
-            break;
-        case 'submission':
-            listing = await user.getSubmissions({limit: chunkSize});
-            break;
-        default:
-            listing = await user.getOverview({limit: chunkSize});
-            break;
+    let listing = [];
+    try {
+        switch (options.type) {
+            case 'comment':
+                listing = await user.getComments({limit: chunkSize});
+                break;
+            case 'submission':
+                listing = await user.getSubmissions({limit: chunkSize});
+                break;
+            default:
+                listing = await user.getOverview({limit: chunkSize});
+                break;
+        }
+    } catch (err) {
+        if(isStatusError(err) && err.statusCode === 404) {
+            throw new SimpleError('Reddit returned a 404 for user history. Likely this user is shadowbanned.');
+        } else {
+            throw err;
+        }
     }
     let hitEnd = false;
     let offset = chunkSize;
@@ -305,185 +319,217 @@ export const renderContent = async (template: string, data: (Submission | Commen
         };
     }, {});
 
-    const view = {item: templateData, rules: normalizedRuleResults};
+    const view = {item: templateData, ruleSummary: parseRuleResultsToMarkdownSummary(ruleResults), rules: normalizedRuleResults};
     const rendered = Mustache.render(template, view) as string;
     return he.decode(rendered);
 }
 
 export const testAuthorCriteria = async (item: (Comment | Submission), authorOpts: AuthorCriteria, include = true, userNotes: UserNotes) => {
-    // @ts-ignore
-    const author: RedditUser = await item.author;
-    for (const k of Object.keys(authorOpts)) {
-        // @ts-ignore
-        if (authorOpts[k] !== undefined) {
-            switch (k) {
-                case 'name':
-                    const authPass = () => {
-                        // @ts-ignore
-                        for (const n of authorOpts[k]) {
-                            if (n.toLowerCase() === author.name.toLowerCase()) {
-                                return true;
-                            }
-                        }
-                        return false;
-                    }
-                    const authResult = authPass();
-                    if ((include && !authResult) || (!include && authResult)) {
-                        return false;
-                    }
-                    break;
-                case 'flairCssClass':
-                    const css = await item.author_flair_css_class;
-                    const cssPass = () => {
-                        // @ts-ignore
-                        for (const c of authorOpts[k]) {
-                            if (c === css) {
-                                return;
-                            }
-                        }
-                        return false;
-                    }
-                    const cssResult = cssPass();
-                    if ((include && !cssResult) || (!include && cssResult)) {
-                        return false;
-                    }
-                    break;
-                case 'flairText':
-                    const text = await item.author_flair_text;
-                    const textPass = () => {
-                        // @ts-ignore
-                        for (const c of authorOpts[k]) {
-                            if (c === text) {
-                                return
-                            }
-                        }
-                        return false;
-                    };
-                    const textResult = textPass();
-                    if ((include && !textResult) || (!include && textResult)) {
-                        return false;
-                    }
-                    break;
-                case 'isMod':
-                    const mods: RedditUser[] = await item.subreddit.getModerators();
-                    const isModerator = mods.some(x => x.name === item.author.name);
-                    const modMatch = authorOpts.isMod === isModerator;
-                    if ((include && !modMatch) || (!include && modMatch)) {
-                        return false;
-                    }
-                    break;
-                case 'age':
-                    const ageTest = compareDurationValue(parseDurationComparison(await authorOpts.age as string), dayjs.unix(await item.author.created));
-                    if ((include && !ageTest) || (!include && ageTest)) {
-                        return false;
-                    }
-                    break;
-                case 'linkKarma':
-                    const lkCompare = parseGenericValueOrPercentComparison(await authorOpts.linkKarma as string);
-                    let lkMatch;
-                    if (lkCompare.isPercent) {
-                        // @ts-ignore
-                        const tk = author.total_karma as number;
-                        lkMatch = comparisonTextOp(author.link_karma / tk, lkCompare.operator, lkCompare.value / 100);
-                    } else {
-                        lkMatch = comparisonTextOp(author.link_karma, lkCompare.operator, lkCompare.value);
-                    }
-                    if ((include && !lkMatch) || (!include && lkMatch)) {
-                        return false;
-                    }
-                    break;
-                case 'commentKarma':
-                    const ckCompare = parseGenericValueOrPercentComparison(await authorOpts.commentKarma as string);
-                    let ckMatch;
-                    if (ckCompare.isPercent) {
-                        // @ts-ignore
-                        const ck = author.total_karma as number;
-                        ckMatch = comparisonTextOp(author.comment_karma / ck, ckCompare.operator, ckCompare.value / 100);
-                    } else {
-                        ckMatch = comparisonTextOp(author.comment_karma, ckCompare.operator, ckCompare.value);
-                    }
-                    if ((include && !ckMatch) || (!include && ckMatch)) {
-                        return false;
-                    }
-                    break;
-                case 'totalKarma':
-                    const tkCompare = parseGenericValueComparison(await authorOpts.totalKarma as string);
-                    if (tkCompare.isPercent) {
-                        throw new SimpleError(`'totalKarma' value on AuthorCriteria cannot be a percentage`);
-                    }
-                    // @ts-ignore
-                    const totalKarma = author.total_karma as number;
-                    const tkMatch = comparisonTextOp(totalKarma, tkCompare.operator, tkCompare.value);
-                    if ((include && !tkMatch) || (!include && tkMatch)) {
-                        return false;
-                    }
-                    break;
-                case 'verified':
-                    const vMatch = await author.has_verified_mail === authorOpts.verified as boolean;
-                    if ((include && !vMatch) || (!include && vMatch)) {
-                        return false;
-                    }
-                    break;
-                case 'userNotes':
-                    const notes = await userNotes.getUserNotes(item.author);
-                    const notePass = () => {
-                        for (const noteCriteria of authorOpts[k] as UserNoteCriteria[]) {
-                            const {count = '>= 1', search = 'current', type} = noteCriteria;
-                            const {
-                                value,
-                                operator,
-                                isPercent,
-                                extra = ''
-                            } = parseGenericValueOrPercentComparison(count);
-                            const order = extra.includes('asc') ? 'ascending' : 'descending';
-                            switch (search) {
-                                case 'current':
-                                    if (notes.length > 0 && notes[notes.length - 1].noteType === type) {
-                                        return true;
-                                    }
-                                    break;
-                                case 'consecutive':
-                                    let orderedNotes = notes;
-                                    if (order === 'descending') {
-                                        orderedNotes = [...notes];
-                                        orderedNotes.reverse();
-                                    }
-                                    let currCount = 0;
-                                    for (const note of orderedNotes) {
-                                        if (note.noteType === type) {
-                                            currCount++;
-                                        } else {
-                                            currCount = 0;
-                                        }
-                                        if (isPercent) {
-                                            throw new SimpleError(`When comparing UserNotes with 'consecutive' search 'count' cannot be a percentage. Given: ${count}`);
-                                        }
-                                        if (comparisonTextOp(currCount, operator, value)) {
-                                            return true;
-                                        }
-                                    }
-                                    break;
-                                case 'total':
-                                    if (isPercent) {
-                                        if (comparisonTextOp(notes.filter(x => x.noteType === type).length / notes.length, operator, value / 100)) {
-                                            return true;
-                                        }
-                                    } else if (comparisonTextOp(notes.filter(x => x.noteType === type).length, operator, value)) {
-                                        return true;
-                                    }
-                            }
-                        }
-                        return false;
-                    }
-                    const noteResult = notePass();
-                    if ((include && !noteResult) || (!include && noteResult)) {
-                        return false;
-                    }
-                    break;
+    const {shadowBanned, ...rest} = authorOpts;
+
+    if(shadowBanned !== undefined) {
+        try {
+            // @ts-ignore
+            await item.author.fetch();
+            // user is not shadowbanned
+            // if criteria specifies they SHOULD be shadowbanned then return false now
+            if(shadowBanned) {
+                return false;
+            }
+        } catch (err) {
+            if(isStatusError(err) && err.statusCode === 404) {
+                // user is shadowbanned
+                // if criteria specifies they should not be shadowbanned then return false now
+                if(!shadowBanned) {
+                    return false;
+                }
+            } else {
+                throw err;
             }
         }
     }
-    return true;
+
+    try {
+        const authorName = getActivityAuthorName(item.author);
+
+        for (const k of Object.keys(rest)) {
+            // @ts-ignore
+            if (authorOpts[k] !== undefined) {
+                switch (k) {
+                    case 'name':
+                        const authPass = () => {
+                            // @ts-ignore
+                            for (const n of authorOpts[k]) {
+                                if (n.toLowerCase() === authorName.toLowerCase()) {
+                                    return true;
+                                }
+                            }
+                            return false;
+                        }
+                        const authResult = authPass();
+                        if ((include && !authResult) || (!include && authResult)) {
+                            return false;
+                        }
+                        break;
+                    case 'flairCssClass':
+                        const css = await item.author_flair_css_class;
+                        const cssPass = () => {
+                            // @ts-ignore
+                            for (const c of authorOpts[k]) {
+                                if (c === css) {
+                                    return;
+                                }
+                            }
+                            return false;
+                        }
+                        const cssResult = cssPass();
+                        if ((include && !cssResult) || (!include && cssResult)) {
+                            return false;
+                        }
+                        break;
+                    case 'flairText':
+                        const text = await item.author_flair_text;
+                        const textPass = () => {
+                            // @ts-ignore
+                            for (const c of authorOpts[k]) {
+                                if (c === text) {
+                                    return
+                                }
+                            }
+                            return false;
+                        };
+                        const textResult = textPass();
+                        if ((include && !textResult) || (!include && textResult)) {
+                            return false;
+                        }
+                        break;
+                    case 'isMod':
+                        const mods: RedditUser[] = await item.subreddit.getModerators();
+                        const isModerator = mods.some(x => x.name === authorName);
+                        const modMatch = authorOpts.isMod === isModerator;
+                        if ((include && !modMatch) || (!include && modMatch)) {
+                            return false;
+                        }
+                        break;
+                    case 'age':
+                        const ageTest = compareDurationValue(parseDurationComparison(await authorOpts.age as string), dayjs.unix(await item.author.created));
+                        if ((include && !ageTest) || (!include && ageTest)) {
+                            return false;
+                        }
+                        break;
+                    case 'linkKarma':
+                        const lkCompare = parseGenericValueOrPercentComparison(await authorOpts.linkKarma as string);
+                        let lkMatch;
+                        if (lkCompare.isPercent) {
+                            // @ts-ignore
+                            const tk = await item.author.total_karma as number;
+                            lkMatch = comparisonTextOp(item.author.link_karma / tk, lkCompare.operator, lkCompare.value / 100);
+                        } else {
+                            lkMatch = comparisonTextOp(item.author.link_karma, lkCompare.operator, lkCompare.value);
+                        }
+                        if ((include && !lkMatch) || (!include && lkMatch)) {
+                            return false;
+                        }
+                        break;
+                    case 'commentKarma':
+                        const ckCompare = parseGenericValueOrPercentComparison(await authorOpts.commentKarma as string);
+                        let ckMatch;
+                        if (ckCompare.isPercent) {
+                            // @ts-ignore
+                            const ck = await item.author.total_karma as number;
+                            ckMatch = comparisonTextOp(item.author.comment_karma / ck, ckCompare.operator, ckCompare.value / 100);
+                        } else {
+                            ckMatch = comparisonTextOp(item.author.comment_karma, ckCompare.operator, ckCompare.value);
+                        }
+                        if ((include && !ckMatch) || (!include && ckMatch)) {
+                            return false;
+                        }
+                        break;
+                    case 'totalKarma':
+                        const tkCompare = parseGenericValueComparison(await authorOpts.totalKarma as string);
+                        if (tkCompare.isPercent) {
+                            throw new SimpleError(`'totalKarma' value on AuthorCriteria cannot be a percentage`);
+                        }
+                        // @ts-ignore
+                        const totalKarma = await item.author.total_karma as number;
+                        const tkMatch = comparisonTextOp(totalKarma, tkCompare.operator, tkCompare.value);
+                        if ((include && !tkMatch) || (!include && tkMatch)) {
+                            return false;
+                        }
+                        break;
+                    case 'verified':
+                        const vMatch = await item.author.has_verified_mail === authorOpts.verified as boolean;
+                        if ((include && !vMatch) || (!include && vMatch)) {
+                            return false;
+                        }
+                        break;
+                    case 'userNotes':
+                        const notes = await userNotes.getUserNotes(item.author);
+                        const notePass = () => {
+                            for (const noteCriteria of authorOpts[k] as UserNoteCriteria[]) {
+                                const {count = '>= 1', search = 'current', type} = noteCriteria;
+                                const {
+                                    value,
+                                    operator,
+                                    isPercent,
+                                    extra = ''
+                                } = parseGenericValueOrPercentComparison(count);
+                                const order = extra.includes('asc') ? 'ascending' : 'descending';
+                                switch (search) {
+                                    case 'current':
+                                        if (notes.length > 0 && notes[notes.length - 1].noteType === type) {
+                                            return true;
+                                        }
+                                        break;
+                                    case 'consecutive':
+                                        let orderedNotes = notes;
+                                        if (order === 'descending') {
+                                            orderedNotes = [...notes];
+                                            orderedNotes.reverse();
+                                        }
+                                        let currCount = 0;
+                                        for (const note of orderedNotes) {
+                                            if (note.noteType === type) {
+                                                currCount++;
+                                            } else {
+                                                currCount = 0;
+                                            }
+                                            if (isPercent) {
+                                                throw new SimpleError(`When comparing UserNotes with 'consecutive' search 'count' cannot be a percentage. Given: ${count}`);
+                                            }
+                                            if (comparisonTextOp(currCount, operator, value)) {
+                                                return true;
+                                            }
+                                        }
+                                        break;
+                                    case 'total':
+                                        if (isPercent) {
+                                            if (comparisonTextOp(notes.filter(x => x.noteType === type).length / notes.length, operator, value / 100)) {
+                                                return true;
+                                            }
+                                        } else if (comparisonTextOp(notes.filter(x => x.noteType === type).length, operator, value)) {
+                                            return true;
+                                        }
+                                }
+                            }
+                            return false;
+                        }
+                        const noteResult = notePass();
+                        if ((include && !noteResult) || (!include && noteResult)) {
+                            return false;
+                        }
+                        break;
+                }
+            }
+        }
+        return true;
+    } catch (err) {
+        if(isStatusError(err) && err.statusCode === 404) {
+            throw new SimpleError('Reddit returned a 404 while trying to retrieve User profile. It is likely this user is shadowbanned.');
+        } else {
+            throw err;
+        }
+    }
 }
 
 export interface ItemContent {
@@ -607,6 +653,9 @@ export const getAttributionIdentifier = (sub: Submission, useParentMediaDomain =
     if (displayDomain === '') {
         displayDomain = domain;
     }
+    if(domainIdents.length === 0 && domain !== '') {
+        domainIdents.push(domain);
+    }
 
     return {display: displayDomain, domain, aliases: domainIdents, provider, mediaType};
 }

src/Web/Common/defaults.ts

@@ -1,13 +1,10 @@
 import {BotStats, BotStatusResponse, SubredditDataResponse} from "./interfaces";
-import {ManagerStats, RunningState} from "../../Subreddit/Manager";
-import {Invokee, RunState} from "../../Common/interfaces";
-import {cacheStats} from "../../util";
+import {RunningState} from "../../Subreddit/Manager";
+import {Invokee, ManagerStats, RunState} from "../../Common/interfaces";
+import {cacheStats, createHistoricalStatsDisplay} from "../../util";
+import {createHistoricalDefaults, historicalDefaults} from "../../Common/defaults";
 
 const managerStats: ManagerStats = {
-    actionsRun: 0,
-    actionsRunSinceStart: 0,
-    actionsRunSinceStartTotal: 0,
-    actionsRunTotal: 0,
     cache: {
         currentKeyCount: 0,
         isShared: false,
@@ -18,22 +15,12 @@ const managerStats: ManagerStats = {
         totalRequests: 0,
         types: cacheStats()
     },
-    checksRunSinceStartTotal: 0,
-    checksRunTotal: 0,
-    checksTriggered: 0,
-    checksTriggeredSinceStart: 0,
-    checksTriggeredSinceStartTotal: 0,
-    checksTriggeredTotal: 0,
+    historical: {
+        lastReload: createHistoricalStatsDisplay(createHistoricalDefaults()),
+        allTime: createHistoricalStatsDisplay(createHistoricalDefaults()),
+    },
     eventsAvg: 0,
-    eventsCheckedSinceStartTotal: 0,
-    eventsCheckedTotal: 0,
     rulesAvg: 0,
-    rulesCachedSinceStartTotal: 0,
-    rulesCachedTotal: 0,
-    rulesRunSinceStartTotal: 0,
-    rulesRunTotal: 0,
-    rulesTriggeredSinceStartTotal: 0,
-    rulesTriggeredTotal: 0,
 };
 const botStats: BotStats = {
     apiAvg: '-',

src/Web/Common/interfaces.ts

@@ -1,4 +1,5 @@
-import {ManagerStats, RunningState} from "../../Subreddit/Manager";
+import {RunningState} from "../../Subreddit/Manager";
+import {ManagerStats} from "../../Common/interfaces";
 
 export interface BotStats {
     startedAtHuman: string,

src/Web/Server/routes/authenticated/user/action.ts

@@ -67,7 +67,7 @@ const action = async (req: express.Request, res: express.Response) => {
                     if (type === 'unmoderated') {
                         const activities = await manager.subreddit.getUnmoderated({limit: 100});
                         for (const a of activities.reverse()) {
-                            await manager.queue.push({
+                            await manager.firehose.push({
                                 checkType: a instanceof Submission ? 'Submission' : 'Comment',
                                 activity: a,
                             });
@@ -75,7 +75,7 @@ const action = async (req: express.Request, res: express.Response) => {
                     } else {
                         const activities = await manager.subreddit.getModqueue({limit: 100});
                         for (const a of activities.reverse()) {
-                            await manager.queue.push({
+                            await manager.firehose.push({
                                 checkType: a instanceof Submission ? 'Submission' : 'Comment',
                                 activity: a,
                             });

src/Web/Server/routes/authenticated/user/status.ts

@@ -160,13 +160,19 @@ const status = () => {
                     submissions: acc.checks.submissions + curr.checks.submissions,
                     comments: acc.checks.comments + curr.checks.comments,
                 },
-                eventsCheckedTotal: acc.eventsCheckedTotal + curr.stats.eventsCheckedTotal,
-                checksRunTotal: acc.checksRunTotal + curr.stats.checksRunTotal,
-                checksTriggeredTotal: acc.checksTriggeredTotal + curr.stats.checksTriggeredTotal,
-                rulesRunTotal: acc.rulesRunTotal + curr.stats.rulesRunTotal,
-                rulesCachedTotal: acc.rulesCachedTotal + curr.stats.rulesCachedTotal,
-                rulesTriggeredTotal: acc.rulesTriggeredTotal + curr.stats.rulesTriggeredTotal,
-                actionsRunTotal: acc.actionsRunTotal + curr.stats.actionsRunTotal,
+                historical: {
+                    allTime: {
+                        eventsCheckedTotal: acc.historical.allTime.eventsCheckedTotal + curr.stats.historical.allTime.eventsCheckedTotal,
+                        eventsActionedTotal: acc.historical.allTime.eventsActionedTotal + curr.stats.historical.allTime.eventsActionedTotal,
+                        checksRunTotal: acc.historical.allTime.checksRunTotal + curr.stats.historical.allTime.checksRunTotal,
+                        checksFromCacheTotal: acc.historical.allTime.checksFromCacheTotal + curr.stats.historical.allTime.checksFromCacheTotal,
+                        checksTriggeredTotal: acc.historical.allTime.checksTriggeredTotal + curr.stats.historical.allTime.checksTriggeredTotal,
+                        rulesRunTotal: acc.historical.allTime.rulesRunTotal + curr.stats.historical.allTime.rulesRunTotal,
+                        rulesCachedTotal: acc.historical.allTime.rulesCachedTotal + curr.stats.historical.allTime.rulesCachedTotal,
+                        rulesTriggeredTotal: acc.historical.allTime.rulesTriggeredTotal + curr.stats.historical.allTime.rulesTriggeredTotal,
+                        actionsRunTotal: acc.historical.allTime.actionsRunTotal + curr.stats.historical.allTime.actionsRunTotal,
+                    }
+                },
                 maxWorkers: acc.maxWorkers + curr.maxWorkers,
                 subMaxWorkers: acc.subMaxWorkers + curr.subMaxWorkers,
                 globalMaxWorkers: acc.globalMaxWorkers + curr.globalMaxWorkers,
@@ -178,13 +184,19 @@ const status = () => {
                 submissions: 0,
                 comments: 0,
             },
-            eventsCheckedTotal: 0,
-            checksRunTotal: 0,
-            checksTriggeredTotal: 0,
-            rulesRunTotal: 0,
-            rulesCachedTotal: 0,
-            rulesTriggeredTotal: 0,
-            actionsRunTotal: 0,
+            historical: {
+                allTime: {
+                    eventsCheckedTotal: 0,
+                    eventsActionedTotal: 0,
+                    checksRunTotal: 0,
+                    checksFromCacheTotal: 0,
+                    checksTriggeredTotal: 0,
+                    rulesRunTotal: 0,
+                    rulesCachedTotal: 0,
+                    rulesTriggeredTotal: 0,
+                    actionsRunTotal: 0,
+                }
+            },
             maxWorkers: 0,
             subMaxWorkers: 0,
             globalMaxWorkers: 0,

src/Web/assets/views/status.ejs

@@ -333,64 +333,63 @@
                                                 <% } %>
                                                 <% if (data.name !== 'All') { %>
                                                     <div data-subreddit="<%= data.name %>"
-                                                         class="stats botStats reloadStats">
+                                                         class="stats botStats reloadStats mb-2">
                                                         <label>Events</label>
                                                         <span>
-                                                            <%= data.stats.eventsCheckedSinceStartTotal === undefined ? '-' : data.stats.eventsCheckedSinceStartTotal %>
+                                                            <%= data.stats.historical.lastReload.eventsCheckedTotal === undefined ? '-' : data.stats.historical.lastReload.eventsCheckedTotal %>
                                                         </span>
                                                         <label>Checks</label>
                                                         <span class="has-tooltip">
                                                         <span class='tooltip rounded shadow-lg p-1 bg-gray-100 text-black -mt-2'>
-                                                            <span><%= data.stats.checksTriggeredSinceStartTotal %></span> Triggered / <span><%= data.stats.checksRunSinceStartTotal %></span> Run
+                                                            <span><%= data.stats.historical.lastReload.checksTriggeredTotal %></span> Triggered / <span><%= data.stats.historical.lastReload.checksRunTotal %></span> Run / <span><%= data.stats.historical.lastReload.checksFromCacheTotal %></span> Cached
                                                         </span>
-                                                        <% if (data.name !== 'All') { %>
-                                                            <a target="_blank" href="/events?instance=<%= instanceId %>&bot=<%= bot.system.name %>&subreddit=<%= data.name %>" class="underline" style="text-decoration-style: dotted"><%= data.stats.checksTriggeredSinceStartTotal %> T</a>
-                                                        <% } else { %>
-                                                                <%= data.stats.checksTriggeredSinceStartTotal %> T
-                                                        <% } %>/ <span><%= data.stats.checksRunSinceStartTotal %></span> R
+                                                        <span class="cursor-help underline" style="text-decoration-style: dotted"><%= data.stats.historical.lastReload.checksTriggeredTotal %> Triggered</span>
                                                     </span>
 
                                                         <label>Rules</label>
                                                         <span class="has-tooltip">
                                                     <span class='tooltip rounded shadow-lg p-1 bg-gray-100 text-black -mt-2'>
-                                                        <span><%= data.stats.rulesTriggeredSinceStartTotal %></span> Triggered / <span><%= data.stats.rulesCachedSinceStartTotal %></span> Cached / <span><%= data.stats.rulesRunSinceStartTotal %></span> Run
+                                                        <span><%= data.stats.historical.lastReload.rulesTriggeredTotal %></span> Triggered / <span><%= data.stats.historical.lastReload.rulesCachedTotal %></span> Cached / <span><%= data.stats.historical.lastReload.rulesRunTotal %></span> Run
                                                     </span>
-                                                    <span class="cursor-help">
-                                                        <span><%= data.stats.rulesTriggeredSinceStartTotal %></span> T / <span><%= data.stats.rulesCachedSinceStartTotal %></span> C / <span><%= data.stats.rulesRunSinceStartTotal %></span> R</span>
+                                                    <span class="cursor-help cursor-help underline" style="text-decoration-style: dotted">
+                                                        <span><%= data.stats.historical.lastReload.rulesTriggeredTotal %></span> Triggered</span>
                                                 </span>
 
                                                         <label>Actions</label>
-                                                        <span class="cursor-help"><%= data.stats.actionsRunSinceStartTotal === undefined ? '-' : data.stats.actionsRunSinceStartTotal %></span>
+                                                        <span><%= data.stats.historical.lastReload.actionsRunTotal === undefined ? '0' : data.stats.historical.lastReload.actionsRunTotal %> Run</span>
                                                     </div>
                                                 <% } %>
-                                                <div data-subreddit="<%= data.name %>" class="stats botStats allStats">
+                                                <div data-subreddit="<%= data.name %>" class="stats botStats allStats mb-2">
                                                     <label>Events</label>
                                                     <span>
-                                                        <%= data.stats.eventsCheckedTotal %>
+                                                        <%= data.stats.historical.allTime.eventsCheckedTotal %>
                                                     </span>
                                                     <label>Checks</label>
                                                     <span class="has-tooltip">
                                                         <span class='tooltip rounded shadow-lg p-1 bg-gray-100 text-black -mt-2'>
-                                                            <span><%= data.stats.checksTriggeredTotal %></span> Triggered / <span><%= data.stats.checksRunTotal %></span> Run
+                                                            <span><%= data.stats.historical.allTime.checksTriggeredTotal %></span> Triggered / <span><%= data.stats.historical.allTime.checksRunTotal %></span> Run / <span><%= data.stats.historical.allTime.checksFromCacheTotal %></span> Cached
                                                         </span>
-                                                        <span><%= data.stats.checksTriggeredTotal %> T / <span><%= data.stats.checksRunTotal %></span> R</span>
+                                                        <span class="cursor-help underline" style="text-decoration-style: dotted"><%= data.stats.historical.allTime.checksTriggeredTotal %> Triggered</span>
                                                     </span>
 
                                                     <label>Rules</label>
                                                     <span class="has-tooltip">
                                                         <span class='tooltip rounded shadow-lg p-1 bg-gray-100 text-black -mt-2'>
-                                                            <span><%= data.stats.rulesTriggeredTotal %></span> Triggered / <span><%= data.stats.rulesCachedTotal %></span> Cached / <span><%= data.stats.rulesRunTotal %></span> Run
+                                                            <span><%= data.stats.historical.allTime.rulesTriggeredTotal %></span> Triggered / <span><%= data.stats.historical.allTime.rulesCachedTotal %></span> Cached / <span><%= data.stats.historical.allTime.rulesRunTotal %></span> Run
                                                         </span>
-                                                        <span class="cursor-help"><span><%= data.stats.rulesTriggeredTotal %></span> T / <span><%= data.stats.rulesCachedTotal %></span> C / <span><%= data.stats.rulesRunTotal %></span> R</span>
+                                                        <span class="cursor-help underline" style="text-decoration-style: dotted"><span><%= data.stats.historical.allTime.rulesTriggeredTotal %></span> Triggered</span>
                                                     </span>
 
                                                     <label>Actions</label>
-                                                    <% if (data.name !== 'All') { %>
-                                                        <a target="_blank" href="/events?instance=<%= instanceId %>&bot=<%= bot.system.name %>&subreddit=<%= data.name %>" class="underline" style="text-decoration-style: dotted"><%= data.stats.actionsRunTotal %> Run</a>
-                                                    <% } else { %>
-                                                        <a target="_blank" href="/events?instance=<%= instanceId %>&bot=<%= bot.system.name %>" class="underline" style="text-decoration-style: dotted"><%= data.stats.actionsRunTotal %> Run</a>
-                                                    <% } %>
+                                                    <span>
+                                                        <span><%= data.stats.historical.allTime.actionsRunTotal === undefined ? '0' : data.stats.historical.allTime.actionsRunTotal %> Run</span>
+                                                    </span>
                                                 </div>
+                                                <% if (data.name !== 'All') { %>
+                                                    <a target="_blank" href="/events?instance=<%= instanceId %>&bot=<%= bot.system.name %>&subreddit=<%= data.name %>" style="text-decoration-style: dotted">Actioned Events</a>
+                                                <% } else { %>
+                                                    <a target="_blank" href="/events?instance=<%= instanceId %>&bot=<%= bot.system.name %>">Actioned Events</a>
+                                                <% } %>
                                             </div>
                                             <div>
                                                 <div class="text-left pb-2">

src/index.ts

@@ -186,7 +186,7 @@ const program = new Command();
                     for(const manager of b.subManagers) {
                         const activities = await manager.subreddit.getUnmoderated();
                         for (const a of activities.reverse()) {
-                            manager.queue.push({
+                            manager.firehose.push({
                                 checkType: a instanceof Submission ? 'Submission' : 'Comment',
                                 activity: a,
                                 options: {checkNames: checks}

src/util.ts

@@ -1,7 +1,7 @@
 import winston, {Logger} from "winston";
 import jsonStringify from 'safe-stable-stringify';
 import dayjs, {Dayjs, OpUnitType} from 'dayjs';
-import {isRuleSetResult, RulePremise, RuleResult, RuleSetResult} from "./Rule";
+import {FormattedRuleResult, isRuleSetResult, RulePremise, RuleResult, RuleSetResult} from "./Rule";
 import deepEqual from "fast-deep-equal";
 import {Duration} from 'dayjs/plugin/duration.js';
 import Ajv from "ajv";
@@ -9,12 +9,13 @@ import {InvalidOptionArgumentError} from "commander";
 import Submission from "snoowrap/dist/objects/Submission";
 import {Comment} from "snoowrap";
 import {inflateSync, deflateSync} from "zlib";
+import sizeOf from 'image-size';
 import {
     ActivityWindowCriteria, CacheOptions, CacheProvider,
     DurationComparison,
-    GenericComparison, LogInfo, NamedGroup,
-    PollingOptionsStrong, RegExResult, ResourceStats,
-    StringOperator
+    GenericComparison, HistoricalStats, HistoricalStatsDisplay, ImageData, ImageDetection, LogInfo, NamedGroup,
+    PollingOptionsStrong, RedditEntity, RedditEntityType, RegExResult, ResembleResult, ResourceStats, StatusCodeError,
+    StringOperator, StrongSubredditState, SubredditState
 } from "./Common/interfaces";
 import JSON5 from "json5";
 import yaml, {JSON_SCHEMA} from "js-yaml";
@@ -29,6 +30,10 @@ import Autolinker from 'autolinker';
 import {create as createMemoryStore} from './Utils/memoryStore';
 import {MESSAGE} from "triple-beam";
 import {RedditUser} from "snoowrap/dist/objects";
+import reRegExp from '@stdlib/regexp-regexp';
+import fetch, {Response} from "node-fetch";
+
+const ReReg = reRegExp();
 
 const {format} = winston;
 const {combine, printf, timestamp, label, splat, errors} = format;
@@ -570,6 +575,24 @@ export const parseSubredditName = (val:string): string => {
     return matches[1] as string;
 }
 
+export const REDDIT_ENTITY_REGEX: RegExp = /^\s*(?<entityType>\/[ru]\/|[ru]\/)*(?<name>\w+)*\s*$/;
+export const REDDIT_ENTITY_REGEX_URL = 'https://regexr.com/65r9b';
+export const parseRedditEntity = (val:string): RedditEntity => {
+    const matches = val.match(REDDIT_ENTITY_REGEX);
+    if (matches === null) {
+        throw new InvalidRegexError(REDDIT_ENTITY_REGEX, val, REDDIT_ENTITY_REGEX_URL)
+    }
+    const groups = matches.groups as any;
+    let eType: RedditEntityType = 'user';
+    if(groups.entityType !== undefined && typeof groups.entityType === 'string' && groups.entityType.includes('r')) {
+        eType = 'subreddit';
+    }
+    return {
+        name: groups.name,
+        type: eType,
+    }
+}
+
 const WIKI_REGEX: RegExp = /^\s*wiki:(?<url>[^|]+)\|*(?<subreddit>[^\s]*)\s*$/;
 const WIKI_REGEX_URL = 'https://regexr.com/61bq1';
 const URL_REGEX: RegExp = /^\s*url:(?<url>[^\s]+)\s*$/;
@@ -820,17 +843,27 @@ export const boolToString = (val: boolean): string => {
     return val ? 'Yes' : 'No';
 }
 
-export const isRedditMedia = (act: Submission): boolean => {
-    return act.is_reddit_media_domain || act.is_video || ['v.redd.it','i.redd.it'].includes(act.domain);
+export const isRedditMedia = (act: Comment | Submission): boolean => {
+    return asSubmission(act) && (act.is_reddit_media_domain || act.is_video || ['v.redd.it','i.redd.it'].includes(act.domain));
 }
 
 export const isExternalUrlSubmission = (act: Comment | Submission): boolean => {
     return asSubmission(act) && !act.is_self && !isRedditMedia(act);
 }
 
-export const parseRegex = (r: string | RegExp, val: string, flags?: string): RegExResult => {
+export const parseStringToRegex = (val: string, defaultFlags?: string): RegExp | undefined => {
+    const result = ReReg.exec(val);
+    if (result === null) {
+        return undefined;
+    }
+    // index 0 => full string
+    // index 1 => regex without flags and forward slashes
+    // index 2 => flags
+    const flags = result[2] === '' ? (defaultFlags || '') : result[2];
+    return new RegExp(result[1], flags);
+}
 
-    const reg = r instanceof RegExp ? r : new RegExp(r, flags);
+export const parseRegex = (reg: RegExp, val: string): RegExResult => {
 
     if(reg.global) {
         const g = Array.from(val.matchAll(reg));
@@ -856,6 +889,46 @@ export const parseRegex = (r: string | RegExp, val: string, flags?: string): Reg
     }
 }
 
+export const isStrongSubredditState = (value: SubredditState | StrongSubredditState) => {
+    return value.name === undefined || value.name instanceof RegExp;
+}
+
+export const asStrongSubredditState = (value: any): value is StrongSubredditState => {
+    return isStrongSubredditState(value);
+}
+
+export interface StrongSubredditStateOptions {
+    defaultFlags?: string
+    generateDescription?: boolean
+}
+
+export const toStrongSubredditState = (s: SubredditState, opts?: StrongSubredditStateOptions): StrongSubredditState => {
+    const {defaultFlags, generateDescription = false} = opts || {};
+    const {name: nameVal, stateDescription} = s;
+
+    let nameReg: RegExp | undefined;
+    if (nameVal !== undefined) {
+        if (!(nameVal instanceof RegExp)) {
+            nameReg = parseStringToRegex(nameVal, defaultFlags);
+            if (nameReg === undefined) {
+                nameReg = new RegExp(parseSubredditName(nameVal), defaultFlags);
+            }
+        } else {
+            nameReg = nameVal;
+        }
+    }
+    const strongState = {
+        ...s,
+        name: nameReg
+    };
+
+    if (generateDescription && stateDescription === undefined) {
+        strongState.stateDescription = objectToStringSummary(strongState);
+    }
+
+    return strongState;
+}
+
 export async function readConfigFile(path: string, opts: any) {
     const {log, throwOnNotFound = true} = opts;
     try {
@@ -932,10 +1005,12 @@ export const cacheStats = (): ResourceStats => {
         author: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
         authorCrit: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
         itemCrit: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
+        subredditCrit: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
         content: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
         userNotes: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
         submission: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
         comment: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
+        subreddit: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
         commentCheck: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0}
     };
 }
@@ -1002,6 +1077,10 @@ export const isScopeError = (err: any): boolean => {
     return false;
 }
 
+export const isStatusError = (err: any): err is StatusCodeError => {
+    return typeof err === 'object' && err.name === 'StatusCodeError' && err.response !== undefined;
+}
+
 /**
  * Cached activities lose type information when deserialized so need to check properties as well to see if the object is the shape of a Submission
  * */
@@ -1040,3 +1119,119 @@ export const buildCachePrefix = (parts: any[]): string => {
     }
     return prefix;
 }
+
+export const objectToStringSummary = (obj: object): string => {
+    const parts = [];
+    for(const [key, val] of Object.entries(obj)) {
+        parts.push(`${key}: ${val}`);
+    }
+    return parts.join(' | ');
+}
+
+/**
+ * Returns the index of the last element in the array where predicate is true, and -1
+ * otherwise.
+ * @param array The source array to search in
+ * @param predicate find calls predicate once for each element of the array, in descending
+ * order, until it finds one where predicate returns true. If such an element is found,
+ * findLastIndex immediately returns that element index. Otherwise, findLastIndex returns -1.
+ *
+ * @see https://stackoverflow.com/a/53187807/1469797
+ */
+export function findLastIndex<T>(array: Array<T>, predicate: (value: T, index: number, obj: T[]) => boolean): number {
+    let l = array.length;
+    while (l--) {
+        if (predicate(array[l], l, array))
+            return l;
+    }
+    return -1;
+}
+
+export const parseRuleResultsToMarkdownSummary = (ruleResults: RuleResult[]): string => {
+    const results = ruleResults.map((y: any) => {
+        const {triggered, result, name, ...restY} = y;
+        let t = triggeredIndicator(false);
+        if(triggered === null) {
+            t = 'Skipped';
+        } else if(triggered === true) {
+            t = triggeredIndicator(true);
+        }
+        return `* ${name} - ${t} - ${result || '-'}`;
+    });
+    return results.join('\r\n');
+}
+
+export const isValidImageURL = (str: string): boolean => {
+    return !!str.match(/\w+\.(jpg|jpeg|gif|png|tiff|bmp)$/gi);
+}
+
+export const getImageDataFromUrl = async (url: string, aggressive = false): Promise<[Response?, ImageData?, string?]> => {
+    if (!aggressive && !isValidImageURL(url)) {
+        return [undefined, undefined, 'URL did not end with a valid image extension'];
+    }
+    try {
+        const response = await fetch(url);
+        if (response.ok) {
+            const ct = response.headers.get('Content-Type');
+            if (ct !== null && ct.includes('image')) {
+                let buffer = await response.buffer();
+                const dimensions = sizeOf(buffer);
+                return [response, {
+                    data: buffer,
+                    width: dimensions.width as number - 5,
+                    height: dimensions.height as number - 5,
+                    pixels: (dimensions.height as number - 5) * (dimensions.width as number - 5)
+                }];
+            }
+            return [response, undefined, 'Content-Type for fetched URL did not contain "image"'];
+        }
+        return [response, undefined, `URL response was not OK: (${response.status})${response.statusText}`];
+    } catch (err) {
+        return [undefined, undefined, `Error occurred while fetching response from URL: ${err.message}`];
+    }
+}
+
+let resembleCIFunc: Function;
+
+const getCIFunc = async () => {
+    if (resembleCIFunc === undefined) {
+        // @ts-ignore
+        const resembleModule = await import('resemblejs/compareImages');
+        if (resembleModule === undefined) {
+            throw new Error('Could not import resemblejs');
+        }
+        resembleCIFunc = resembleModule.default;
+    }
+    return resembleCIFunc;
+}
+
+export const compareImages = async (data1: ImageData, data2: ImageData, threshold?: number): Promise<[ResembleResult, boolean?]> => {
+    let ci: Function;
+
+    try {
+        ci = await getCIFunc();
+    } catch (err) {
+        err.message = `Unable to do image comparison due to an issue importing the comparison library. It is likely 'node-canvas' is not installed (see ContextMod docs). Error Message: ${err.message}`;
+        throw err;
+    }
+    const results = await ci(data1.data, data2.data, {
+        returnEarlyThreshold: threshold !== undefined ? Math.min(threshold + 5, 100) : undefined,
+    }) as ResembleResult;
+
+    const sameImage = threshold === undefined ? undefined : results.rawMisMatchPercentage < threshold;
+    return [results, sameImage];
+}
+
+export const createHistoricalStatsDisplay = (data: HistoricalStats): HistoricalStatsDisplay => {
+    const display: any = {};
+    for(const [k, v] of Object.entries(data)) {
+        if(v instanceof Map) {
+            display[k] = v;
+            display[`${k}Total`] = Array.from(v.values()).reduce((acc, curr) => acc + curr, 0);
+        } else {
+            display[k] = v;
+        }
+    }
+
+    return display as HistoricalStatsDisplay;
+}