Use fixed array instead of Set to maybe reduce memory usage

* Specify docker-compose minimum version
* Change commands to use new syntax

docker-compose.yml

@@ -24,6 +24,9 @@ services:
 
   cache:
     image: 'redis:7-alpine'
+    volumes:
+      # on linux will need to make sure this directory has correct permissions for container to access
+      - './data/cache:/data'
 
   database:
     image: 'mariadb:10.9.3'

docs/operator/installation.md

@@ -55,6 +55,8 @@ The included [`docker-compose.yml`](/docker-compose.yml) provides production-rea
 
 #### Setup
 
+The included `docker-compose.yml` file is written for **Docker Compose v2.**
+
 For new installations copy [`config.yaml`](/docker/config/docker-compose/config.yaml) into a folder named `data` in the same folder `docker-compose.yml` will be run from. For users migrating their existing CM instances to docker-compose, copy your existing `config.yaml` into the same `data` folder.
 
 Read through the comments in both `docker-compose.yml` and `config.yaml` and makes changes to any relevant settings (passwords, usernames, etc...). Ensure that any settings used in both files (EX mariaDB passwords) match.
@@ -62,13 +64,13 @@ Read through the comments in both `docker-compose.yml` and `config.yaml` and mak
 To build and start CM:
 
 ```bash
-docker-compose up -d
+docker compose up -d
 ```
 
 To include Grafana/Influx dependencies run:
 
 ```bash
-docker-compose --profile full up -d
+docker compose --profile full up -d
 ```
 
 ## Locally

docs/subreddit-configuration/actionTemplating.md

@@ -57,6 +57,29 @@ All Actions with `content` have access to this data:
 | `title`      | As comments => the body of the comment. As Submission => title                                      | Test post please ignore                                  |
 | `shortTitle` | The same as `title` but truncated to 15 characters                                                  | test post pleas...                                       |
 
+#### Common Author
+
+Additionally, `author` has these properties accessible:
+
+|      Name      |             Description             | Example  |
+|----------------|-------------------------------------|----------|
+| `age`          | (Approximate) Age of account        | 3 months |
+| `linkKarma`    | Amount of link karma                | 10       |
+| `commentKarma` | Amount of comment karma             | 3        |
+| `totalKarma`   | Combined link+comment karma         | 13       |
+| `verified`     | Does account have a verified email? | true     |
+
+NOTE: Accessing these properties may require an additional API call so use sparingly on high-volume comments
+
+##### Example Usage
+
+```
+The user {{item.author}} has been a redditor for {{item.author.age}}
+```
+Produces:
+
+> The user FoxxMD has been a redditor for 3 months
+
 ### Submissions
 
 If the **Activity** is a Submission these additional properties are accessible:

package-lock.json

@@ -46,6 +46,7 @@
         "express-session-cache-manager": "^1.0.2",
         "express-socket.io-session": "^1.3.5",
         "fast-deep-equal": "^3.1.3",
+        "fixed-size-list": "^0.3.0",
         "globrex": "^0.1.2",
         "got": "^11.8.2",
         "he": "^1.2.0",
@@ -3979,6 +3980,14 @@
         "micromatch": "^4.0.2"
       }
     },
+    "node_modules/fixed-size-list": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/fixed-size-list/-/fixed-size-list-0.3.0.tgz",
+      "integrity": "sha512-c6I8wEE4ZtjKz35BaodH7yWuWmcaUVQwgBeNcI3LxJu79YH+ezHvf1oS9VkgJmyVy5eQ8Wh6jNVcj2rB4rgVgA==",
+      "dependencies": {
+        "mitt": "^1.2.0"
+      }
+    },
     "node_modules/flat": {
       "version": "5.0.2",
       "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz",
@@ -6113,6 +6122,11 @@
       "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
       "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q=="
     },
+    "node_modules/mitt": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/mitt/-/mitt-1.2.0.tgz",
+      "integrity": "sha512-r6lj77KlwqLhIUku9UWYes7KJtsczvolZkzp8hbaDPPaE24OmWl5s539Mytlj22siEQKosZ26qCBgda2PKwoJw=="
+    },
     "node_modules/mkdirp": {
       "version": "0.5.6",
       "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
@@ -13588,6 +13602,14 @@
         "micromatch": "^4.0.2"
       }
     },
+    "fixed-size-list": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/fixed-size-list/-/fixed-size-list-0.3.0.tgz",
+      "integrity": "sha512-c6I8wEE4ZtjKz35BaodH7yWuWmcaUVQwgBeNcI3LxJu79YH+ezHvf1oS9VkgJmyVy5eQ8Wh6jNVcj2rB4rgVgA==",
+      "requires": {
+        "mitt": "^1.2.0"
+      }
+    },
     "flat": {
       "version": "5.0.2",
       "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz",
@@ -15191,6 +15213,11 @@
       "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
       "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q=="
     },
+    "mitt": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/mitt/-/mitt-1.2.0.tgz",
+      "integrity": "sha512-r6lj77KlwqLhIUku9UWYes7KJtsczvolZkzp8hbaDPPaE24OmWl5s539Mytlj22siEQKosZ26qCBgda2PKwoJw=="
+    },
     "mkdirp": {
       "version": "0.5.6",
       "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",

package.json

@@ -68,6 +68,7 @@
     "express-session-cache-manager": "^1.0.2",
     "express-socket.io-session": "^1.3.5",
     "fast-deep-equal": "^3.1.3",
+    "fixed-size-list": "^0.3.0",
     "globrex": "^0.1.2",
     "got": "^11.8.2",
     "he": "^1.2.0",

src/Action/SubmissionAction/FlairAction.ts

@@ -30,15 +30,14 @@ export class FlairAction extends Action {
     async process(item: Comment | Submission, ruleResults: RuleResultEntity[], actionResults: ActionResultEntity[], options: runCheckOptions): Promise<ActionProcessResult> {
         const dryRun = this.getRuntimeAwareDryrun(options);
         let flairParts = [];
-        if(this.text !== '') {
-            flairParts.push(`Text: ${this.text}`);
-        }
-        if(this.css !== '') {
-            flairParts.push(`CSS: ${this.css}`);
-        }
-        if(this.flair_template_id !== '') {
-            flairParts.push(`Template: ${this.flair_template_id}`);
-        }
+        const renderedText = this.text === '' ? '' : await this.renderContent(this.text, item, ruleResults, actionResults) as string;
+        flairParts.push(`Text: ${renderedText === '' ? '(None)' : renderedText}`);
+
+        const renderedCss = this.css === '' ? '' : await this.renderContent(this.css, item, ruleResults, actionResults) as string;
+        flairParts.push(`CSS: ${renderedCss === '' ? '(None)' : renderedCss}`);
+
+        flairParts.push(`Template: ${this.flair_template_id === '' ? '(None)' : this.flair_template_id}`);
+
         const flairSummary = flairParts.length === 0 ? 'No flair (unflaired)' : flairParts.join(' | ');
         this.logger.verbose(flairSummary);
         if (item instanceof Submission) {
@@ -51,9 +50,9 @@ export class FlairAction extends Action {
                     await item.assignFlair({flair_template_id: this.flair_template_id}).then(() => {});
                     item.link_flair_template_id = this.flair_template_id;
                 } else {
-                    await item.assignFlair({text: this.text, cssClass: this.css}).then(() => {});
-                    item.link_flair_css_class = this.css;
-                    item.link_flair_text = this.text;
+                    await item.assignFlair({text: renderedText, cssClass: renderedCss}).then(() => {});
+                    item.link_flair_css_class = renderedCss;
+                    item.link_flair_text = renderedText;
                 }
                 await this.resources.resetCacheForItem(item);
             }

src/Action/UserFlairAction.ts

@@ -26,6 +26,8 @@ export class UserFlairAction extends Action {
   async process(item: Comment | Submission, ruleResults: RuleResultEntity[], actionResults: ActionResultEntity[], options: runCheckOptions): Promise<ActionProcessResult> {
     const dryRun = this.getRuntimeAwareDryrun(options);
     let flairParts = [];
+    let renderedText: string | undefined = undefined;
+    let renderedCss: string | undefined = undefined;
 
     if (this.flair_template_id !== undefined) {
       flairParts.push(`Flair template ID: ${this.flair_template_id}`)
@@ -34,10 +36,12 @@ export class UserFlairAction extends Action {
       }
     } else {
       if (this.text !== undefined) {
-        flairParts.push(`Text: ${this.text}`);
+        renderedText = await this.renderContent(this.text, item, ruleResults, actionResults) as string;
+        flairParts.push(`Text: ${renderedText}`);
       }
       if (this.css !== undefined) {
-        flairParts.push(`CSS: ${this.css}`);
+        renderedCss = await this.renderContent(this.css, item, ruleResults, actionResults) as string;
+        flairParts.push(`CSS: ${renderedCss}`);
       }
     }
 
@@ -58,7 +62,7 @@ export class UserFlairAction extends Action {
           this.logger.error('Either the flair template ID is incorrect or you do not have permission to access it.');
           throw err;
         }
-      } else if (this.text === undefined && this.css === undefined) {
+      } else if (renderedText === undefined && renderedCss === undefined) {
         // @ts-ignore
         await item.subreddit.deleteUserFlair(item.author.name);
         item.author_flair_css_class = null;
@@ -68,11 +72,11 @@ export class UserFlairAction extends Action {
         // @ts-ignore
         await item.author.assignFlair({
           subredditName: item.subreddit.display_name,
-          cssClass: this.css,
-          text: this.text,
+          cssClass: renderedCss,
+          text: renderedText,
         });
-        item.author_flair_text = this.text ?? null;
-        item.author_flair_css_class = this.css ?? null;
+        item.author_flair_text = renderedText ?? null;
+        item.author_flair_css_class = renderedCss ?? null;
       }
       await this.resources.resetCacheForItem(item);
       if(typeof item.author !== 'string') {

src/Bot/index.ts

@@ -46,7 +46,13 @@ import {RunStateType} from "../Common/Entities/RunStateType";
 import {QueueRunState} from "../Common/Entities/EntityRunState/QueueRunState";
 import {EventsRunState} from "../Common/Entities/EntityRunState/EventsRunState";
 import {ManagerRunState} from "../Common/Entities/EntityRunState/ManagerRunState";
-import {Invokee, PollOn} from "../Common/Infrastructure/Atomic";
+import {
+    Invokee,
+    POLLING_COMMENTS, POLLING_MODQUEUE,
+    POLLING_SUBMISSIONS,
+    POLLING_UNMODERATED,
+    PollOn
+} from "../Common/Infrastructure/Atomic";
 import {FilterCriteriaDefaults} from "../Common/Infrastructure/Filters/FilterShapes";
 import {snooLogWrapper} from "../Utils/loggerFactory";
 import {InfluxClient} from "../Common/Influx/InfluxClient";
@@ -558,15 +564,15 @@ class Bot implements BotInstanceFunctions {
 
     parseSharedStreams() {
 
-        const sharedCommentsSubreddits = !this.sharedStreams.includes('newComm') ? [] : this.subManagers.filter(x => x.isPollingShared('newComm')).map(x => x.subreddit.display_name);
+        const sharedCommentsSubreddits = !this.sharedStreams.includes(POLLING_COMMENTS) ? [] : this.subManagers.filter(x => x.isPollingShared(POLLING_COMMENTS)).map(x => x.subreddit.display_name);
         if (sharedCommentsSubreddits.length > 0) {
-            const stream = this.cacheManager.modStreams.get('newComm');
+            const stream = this.cacheManager.modStreams.get(POLLING_COMMENTS);
             if (stream === undefined || stream.subreddit !== sharedCommentsSubreddits.join('+')) {
                 let processed;
                 if (stream !== undefined) {
                     this.logger.info('Restarting SHARED COMMENT STREAM due to a subreddit config change');
                     stream.end('Replacing with a new stream with updated subreddits');
-                    processed = stream.processed;
+                    processed = stream.processedBuffer;
                 }
                 if (sharedCommentsSubreddits.length > 100) {
                     this.logger.warn(`SHARED COMMENT STREAM => Reddit can only combine 100 subreddits for getting new Comments but this bot has ${sharedCommentsSubreddits.length}`);
@@ -580,26 +586,26 @@ class Bot implements BotInstanceFunctions {
                     label: 'Shared Polling'
                 });
                 // @ts-ignore
-                defaultCommentStream.on('error', this.createSharedStreamErrorListener('newComm'));
-                defaultCommentStream.on('listing', this.createSharedStreamListingListener('newComm'));
-                this.cacheManager.modStreams.set('newComm', defaultCommentStream);
+                defaultCommentStream.on('error', this.createSharedStreamErrorListener(POLLING_COMMENTS));
+                defaultCommentStream.on('listing', this.createSharedStreamListingListener(POLLING_COMMENTS));
+                this.cacheManager.modStreams.set(POLLING_COMMENTS, defaultCommentStream);
             }
         } else {
-            const stream = this.cacheManager.modStreams.get('newComm');
+            const stream = this.cacheManager.modStreams.get(POLLING_COMMENTS);
             if (stream !== undefined) {
                 stream.end('Determined no managers are listening on shared stream parsing');
             }
         }
 
-        const sharedSubmissionsSubreddits = !this.sharedStreams.includes('newSub') ? [] : this.subManagers.filter(x => x.isPollingShared('newSub')).map(x => x.subreddit.display_name);
+        const sharedSubmissionsSubreddits = !this.sharedStreams.includes(POLLING_SUBMISSIONS) ? [] : this.subManagers.filter(x => x.isPollingShared(POLLING_SUBMISSIONS)).map(x => x.subreddit.display_name);
         if (sharedSubmissionsSubreddits.length > 0) {
-            const stream = this.cacheManager.modStreams.get('newSub');
+            const stream = this.cacheManager.modStreams.get(POLLING_SUBMISSIONS);
             if (stream === undefined || stream.subreddit !== sharedSubmissionsSubreddits.join('+')) {
                 let processed;
                 if (stream !== undefined) {
                     this.logger.info('Restarting SHARED SUBMISSION STREAM due to a subreddit config change');
                     stream.end('Replacing with a new stream with updated subreddits');
-                    processed = stream.processed;
+                    processed = stream.processedBuffer;
                 }
                 if (sharedSubmissionsSubreddits.length > 100) {
                     this.logger.warn(`SHARED SUBMISSION STREAM => Reddit can only combine 100 subreddits for getting new Submissions but this bot has ${sharedSubmissionsSubreddits.length}`);
@@ -613,19 +619,19 @@ class Bot implements BotInstanceFunctions {
                     label: 'Shared Polling'
                 });
                 // @ts-ignore
-                defaultSubStream.on('error', this.createSharedStreamErrorListener('newSub'));
-                defaultSubStream.on('listing', this.createSharedStreamListingListener('newSub'));
-                this.cacheManager.modStreams.set('newSub', defaultSubStream);
+                defaultSubStream.on('error', this.createSharedStreamErrorListener(POLLING_SUBMISSIONS));
+                defaultSubStream.on('listing', this.createSharedStreamListingListener(POLLING_SUBMISSIONS));
+                this.cacheManager.modStreams.set(POLLING_SUBMISSIONS, defaultSubStream);
             }
         } else {
-            const stream = this.cacheManager.modStreams.get('newSub');
+            const stream = this.cacheManager.modStreams.get(POLLING_SUBMISSIONS);
             if (stream !== undefined) {
                 stream.end('Determined no managers are listening on shared stream parsing');
             }
         }
 
-        const isUnmoderatedShared = !this.sharedStreams.includes('unmoderated') ? false : this.subManagers.some(x => x.isPollingShared('unmoderated'));
-        const unmoderatedstream = this.cacheManager.modStreams.get('unmoderated');
+        const isUnmoderatedShared = !this.sharedStreams.includes(POLLING_UNMODERATED) ? false : this.subManagers.some(x => x.isPollingShared(POLLING_UNMODERATED));
+        const unmoderatedstream = this.cacheManager.modStreams.get(POLLING_UNMODERATED);
         if (isUnmoderatedShared && unmoderatedstream === undefined) {
             const defaultUnmoderatedStream = new UnmoderatedStream(this.client, {
                 subreddit: 'mod',
@@ -634,15 +640,15 @@ class Bot implements BotInstanceFunctions {
                 label: 'Shared Polling'
             });
             // @ts-ignore
-            defaultUnmoderatedStream.on('error', this.createSharedStreamErrorListener('unmoderated'));
-            defaultUnmoderatedStream.on('listing', this.createSharedStreamListingListener('unmoderated'));
-            this.cacheManager.modStreams.set('unmoderated', defaultUnmoderatedStream);
+            defaultUnmoderatedStream.on('error', this.createSharedStreamErrorListener(POLLING_UNMODERATED));
+            defaultUnmoderatedStream.on('listing', this.createSharedStreamListingListener(POLLING_UNMODERATED));
+            this.cacheManager.modStreams.set(POLLING_UNMODERATED, defaultUnmoderatedStream);
         } else if (!isUnmoderatedShared && unmoderatedstream !== undefined) {
             unmoderatedstream.end('Determined no managers are listening on shared stream parsing');
         }
 
-        const isModqueueShared = !this.sharedStreams.includes('modqueue') ? false : this.subManagers.some(x => x.isPollingShared('modqueue'));
-        const modqueuestream = this.cacheManager.modStreams.get('modqueue');
+        const isModqueueShared = !this.sharedStreams.includes(POLLING_MODQUEUE) ? false : this.subManagers.some(x => x.isPollingShared(POLLING_MODQUEUE));
+        const modqueuestream = this.cacheManager.modStreams.get(POLLING_MODQUEUE);
         if (isModqueueShared && modqueuestream === undefined) {
             const defaultModqueueStream = new ModQueueStream(this.client, {
                 subreddit: 'mod',
@@ -651,9 +657,9 @@ class Bot implements BotInstanceFunctions {
                 label: 'Shared Polling'
             });
             // @ts-ignore
-            defaultModqueueStream.on('error', this.createSharedStreamErrorListener('modqueue'));
-            defaultModqueueStream.on('listing', this.createSharedStreamListingListener('modqueue'));
-            this.cacheManager.modStreams.set('modqueue', defaultModqueueStream);
+            defaultModqueueStream.on('error', this.createSharedStreamErrorListener(POLLING_MODQUEUE));
+            defaultModqueueStream.on('listing', this.createSharedStreamListingListener(POLLING_MODQUEUE));
+            this.cacheManager.modStreams.set(POLLING_MODQUEUE, defaultModqueueStream);
         } else if (isModqueueShared && modqueuestream !== undefined) {
             modqueuestream.end('Determined no managers are listening on shared stream parsing');
         }

src/Common/Infrastructure/Atomic.ts

@@ -111,6 +111,19 @@ export interface DurationObject {
 
 export type JoinOperands = 'OR' | 'AND';
 export type PollOn = 'unmoderated' | 'modqueue' | 'newSub' | 'newComm';
+export const POLLING_UNMODERATED: PollOn = 'unmoderated';
+export const POLLING_MODQUEUE: PollOn = 'modqueue';
+export const POLLING_SUBMISSIONS: PollOn = 'newSub';
+export const POLLING_COMMENTS: PollOn = 'newComm';
+export const pollOnTypes: PollOn[] = [POLLING_UNMODERATED, POLLING_MODQUEUE, POLLING_SUBMISSIONS, POLLING_COMMENTS];
+export const pollOnTypeMapping: Map<string, PollOn> = new Map([
+    ['unmoderated', POLLING_UNMODERATED],
+    ['modqueue', POLLING_MODQUEUE],
+    ['newsub', POLLING_SUBMISSIONS],
+    ['newcomm', POLLING_COMMENTS],
+    // be nice if user mispelled
+    ['newcom', POLLING_COMMENTS]
+]);
 export type ModeratorNames = 'self' | 'automod' | 'automoderator' | string;
 export type Invokee = 'system' | 'user';
 export type RunState = 'running' | 'paused' | 'stopped';

src/Common/interfaces.ts

@@ -372,7 +372,7 @@ export interface PollingOptions extends PollingDefaults {
      * * after they have been manually approved from modqueue
      *
      * */
-    pollOn: 'unmoderated' | 'modqueue' | 'newSub' | 'newComm'
+    pollOn: PollOn
 }
 
 export interface TTLConfig {

src/ConfigBuilder.ts

@@ -8,7 +8,7 @@ import {
     overwriteMerge,
     parseBool, parseExternalUrl, parseUrlContext, parseWikiContext, randomId,
     readConfigFile,
-    removeUndefinedKeys, resolvePathFromEnvWithRelative, toStrongSharingACLConfig
+    removeUndefinedKeys, resolvePathFromEnvWithRelative, toPollOn, toStrongSharingACLConfig
 } from "./util";
 
 import Ajv, {Schema} from 'ajv';
@@ -74,8 +74,8 @@ import {ErrorWithCause} from "pony-cause";
 import {RunConfigHydratedData, RunConfigData, RunConfigObject} from "./Run";
 import {AuthorRuleConfig} from "./Rule/AuthorRule";
 import {
-    CacheProvider, ConfigFormat, ConfigFragmentParseFunc,
-    PollOn
+    CacheProvider, ConfigFormat, ConfigFragmentParseFunc, POLLING_MODQUEUE, POLLING_UNMODERATED,
+    PollOn, pollOnTypes
 } from "./Common/Infrastructure/Atomic";
 import {
     asFilterOptionsJson,
@@ -452,27 +452,31 @@ export class ConfigBuilder {
 
 export const buildPollingOptions = (values: (string | PollingOptions)[]): PollingOptionsStrong[] => {
     let opts: PollingOptionsStrong[] = [];
+    let rawOpts: PollingOptions;
     for (const v of values) {
         if (typeof v === 'string') {
-            opts.push({
-                pollOn: v as PollOn,
-                interval: DEFAULT_POLLING_INTERVAL,
-                limit: DEFAULT_POLLING_LIMIT,
-            });
+            rawOpts = {pollOn: v as PollOn}; // maybeee
         } else {
-            const {
-                pollOn: p,
-                interval = DEFAULT_POLLING_INTERVAL,
-                limit = DEFAULT_POLLING_LIMIT,
-                delayUntil,
-            } = v;
-            opts.push({
-                pollOn: p as PollOn,
-                interval,
-                limit,
-                delayUntil,
-            });
+            rawOpts = v;
         }
+
+        const {
+            pollOn: p,
+            interval = DEFAULT_POLLING_INTERVAL,
+            limit = DEFAULT_POLLING_LIMIT,
+            delayUntil,
+        } = rawOpts;
+
+        const pVal = toPollOn(p);
+        if (opts.some(x => x.pollOn === pVal)) {
+            throw new SimpleError(`Polling source ${pVal} cannot appear more than once in polling options`);
+        }
+        opts.push({
+            pollOn: pVal,
+            interval,
+            limit,
+            delayUntil,
+        });
     }
     return opts;
 }
@@ -796,7 +800,7 @@ export const parseDefaultBotInstanceFromArgs = (args: any): BotInstanceJsonConfi
             heartbeatInterval: heartbeat,
         },
         polling: {
-            shared: sharedMod ? ['unmoderated', 'modqueue'] : undefined,
+            shared: sharedMod ? [POLLING_UNMODERATED, POLLING_MODQUEUE] : undefined,
         },
         nanny: {
             softLimit,
@@ -908,7 +912,7 @@ export const parseDefaultBotInstanceFromEnv = (): BotInstanceJsonConfig => {
             heartbeatInterval: process.env.HEARTBEAT !== undefined ? parseInt(process.env.HEARTBEAT) : undefined,
         },
         polling: {
-            shared: parseBool(process.env.SHARE_MOD) ? ['unmoderated', 'modqueue'] : undefined,
+            shared: parseBool(process.env.SHARE_MOD) ? [POLLING_UNMODERATED, POLLING_MODQUEUE] : undefined,
         },
         nanny: {
             softLimit: process.env.SOFT_LIMIT !== undefined ? parseInt(process.env.SOFT_LIMIT) : undefined,
@@ -1525,10 +1529,10 @@ export const buildBotConfig = (data: BotInstanceJsonConfig, opConfig: OperatorCo
         botCache.provider.prefix = buildCachePrefix([botCache.provider.prefix, 'bot', (botName || objectHash.sha1(botCreds))]);
     }
 
-    let realShared = shared === true ? ['unmoderated', 'modqueue', 'newComm', 'newSub'] : shared;
+    let realShared: PollOn[] = shared === true ? pollOnTypes : shared.map(toPollOn);
     if (sharedMod === true) {
-        realShared.push('unmoderated');
-        realShared.push('modqueue');
+        realShared.push(POLLING_UNMODERATED);
+        realShared.push(POLLING_MODQUEUE);
     }
 
     const botLevelStatDefaults = {...statDefaultsFromOp, ...databaseStatisticsDefaults};
@@ -1566,7 +1570,7 @@ export const buildBotConfig = (data: BotInstanceJsonConfig, opConfig: OperatorCo
         caching: botCache,
         userAgent,
         polling: {
-            shared: [...new Set(realShared)] as PollOn[],
+            shared: Array.from(new Set(realShared)),
             stagger,
             limit,
             interval,

src/Rule/RecentActivityRule.ts

@@ -126,28 +126,7 @@ export class RecentActivityRule extends Rule {
     async process(item: Submission | Comment): Promise<[boolean, RuleResult]> {
         let activities;
 
-        // ACID is a bitch
-        // reddit may not return the activity being checked in the author's recent history due to availability/consistency issues or *something*
-        // so make sure we add it in if config is checking the same type and it isn't included
-        // TODO refactor this for SubredditState everywhere branch
-        let shouldIncludeSelf = true;
         const strongWindow = windowConfigToWindowCriteria(this.window);
-        const {
-            filterOn: {
-                post: {
-                    subreddits: {
-                        include = [],
-                        exclude = []
-                    } = {},
-                } = {},
-            } = {}
-        } = strongWindow;
-        // typeof x === string -- a patch for now...technically this is all it supports but eventually will need to be able to do any SubredditState
-        if (include.length > 0 && !include.some(x => x.name !== undefined && x.name.toLocaleLowerCase() === item.subreddit.display_name.toLocaleLowerCase())) {
-            shouldIncludeSelf = false;
-        } else if (exclude.length > 0 && exclude.some(x => x.name !== undefined && x.name.toLocaleLowerCase() === item.subreddit.display_name.toLocaleLowerCase())) {
-            shouldIncludeSelf = false;
-        }
 
         if(strongWindow.fetch === undefined && this.lookAt !== undefined) {
             switch(this.lookAt) {
@@ -159,25 +138,10 @@ export class RecentActivityRule extends Rule {
             }
         }
 
-        activities = await this.resources.getAuthorActivities(item.author, strongWindow);
-
-        switch (strongWindow.fetch) {
-            case 'comment':
-                if (shouldIncludeSelf && item instanceof Comment && !activities.some(x => x.name === item.name)) {
-                    activities.unshift(item);
-                }
-                break;
-            case 'submission':
-                if (shouldIncludeSelf && item instanceof Submission && !activities.some(x => x.name === item.name)) {
-                    activities.unshift(item);
-                }
-                break;
-            default:
-                if (shouldIncludeSelf && !activities.some(x => x.name === item.name)) {
-                    activities.unshift(item);
-                }
-                break;
-        }
+        // ACID is a bitch
+        // reddit may not return the activity being checked in the author's recent history due to availability/consistency issues or *something*
+        // so add current activity as a prefetched activity and add it to the returned activities (after it goes through filtering)
+        activities = await this.resources.getAuthorActivities(item.author, strongWindow, undefined, [item]);
 
         let viableActivity = activities;
         // if config does not specify reference then we set the default based on whether the item is a submission or not

src/Subreddit/Manager.ts

@@ -93,8 +93,8 @@ import {EntityRunState} from "../Common/Entities/EntityRunState/EntityRunState";
 import {
     ActivitySourceValue,
     EventRetentionPolicyRange,
-    Invokee,
-    PollOn,
+    Invokee, POLLING_COMMENTS, POLLING_MODQUEUE, POLLING_SUBMISSIONS, POLLING_UNMODERATED,
+    PollOn, pollOnTypes,
     recordOutputTypes,
     RunState
 } from "../Common/Infrastructure/Atomic";
@@ -635,7 +635,7 @@ export class Manager extends EventEmitter implements RunningStates {
             const configBuilder = new ConfigBuilder({logger: this.logger});
             const validJson = configBuilder.validateJson(configObj);
             const {
-                polling = [{pollOn: 'unmoderated', limit: DEFAULT_POLLING_LIMIT, interval: DEFAULT_POLLING_INTERVAL}],
+                polling = [{pollOn: POLLING_SUBMISSIONS, limit: DEFAULT_POLLING_LIMIT, interval: DEFAULT_POLLING_INTERVAL}],
                 caching,
                 credentials,
                 dryRun,
@@ -957,7 +957,7 @@ export class Manager extends EventEmitter implements RunningStates {
         await this.resources.setActivityLastSeenDate(item.name);
 
         // if modqueue is running then we know we are checking for new reports every X seconds
-        if(options.activitySource.identifier === 'modqueue') {
+        if(options.activitySource.identifier === POLLING_MODQUEUE) {
             // if the activity is from modqueue and only has one report then we know that report was just created
             if(item.num_reports === 1
                 // otherwise if it has more than one report AND we have seen it (its only seen if it has already been stored (in below block))
@@ -1325,25 +1325,20 @@ export class Manager extends EventEmitter implements RunningStates {
         }
     }
 
-    isPollingShared(streamName: string): boolean {
+    isPollingShared(streamName: PollOn): boolean {
         const pollOption = this.pollOptions.find(x => x.pollOn === streamName);
-        return pollOption !== undefined && pollOption.limit === DEFAULT_POLLING_LIMIT && pollOption.interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(streamName as PollOn);
+        return pollOption !== undefined && pollOption.limit === DEFAULT_POLLING_LIMIT && pollOption.interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(streamName);
     }
 
     async buildPolling() {
 
-        const sources: PollOn[] = ['unmoderated', 'modqueue', 'newComm', 'newSub'];
+        const sources = [...pollOnTypes];
 
         const subName = this.subreddit.display_name;
 
         for (const source of sources) {
 
-            if (!sources.includes(source)) {
-                this.logger.error(`'${source}' is not a valid polling source. Valid sources: unmoderated | modqueue | newComm | newSub`);
-                continue;
-            }
-
-            const pollOpt = this.pollOptions.find(x => x.pollOn.toLowerCase() === source.toLowerCase());
+            const pollOpt = this.pollOptions.find(x => x.pollOn === source);
             if (pollOpt === undefined) {
                 if(this.sharedStreamCallbacks.has(source)) {
                     this.logger.debug(`Removing listener for shared polling on ${source.toUpperCase()} because it no longer exists in config`);
@@ -1366,11 +1361,11 @@ export class Manager extends EventEmitter implements RunningStates {
                 let modStreamType: string | undefined;
 
                 switch (source) {
-                    case 'unmoderated':
+                    case POLLING_UNMODERATED:
                         if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
-                            modStreamType = 'unmoderated';
+                            modStreamType = POLLING_UNMODERATED;
                             // use default mod stream from resources
-                            stream = this.cacheManager.modStreams.get('unmoderated') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                            stream = this.cacheManager.modStreams.get(POLLING_UNMODERATED) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
                         } else {
                             stream = new UnmoderatedStream(this.client, {
                                 subreddit: this.subreddit.display_name,
@@ -1380,11 +1375,11 @@ export class Manager extends EventEmitter implements RunningStates {
                             });
                         }
                         break;
-                    case 'modqueue':
+                    case POLLING_MODQUEUE:
                         if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
-                            modStreamType = 'modqueue';
+                            modStreamType = POLLING_MODQUEUE;
                             // use default mod stream from resources
-                            stream = this.cacheManager.modStreams.get('modqueue') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                            stream = this.cacheManager.modStreams.get(POLLING_MODQUEUE) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
                         } else {
                             stream = new ModQueueStream(this.client, {
                                 subreddit: this.subreddit.display_name,
@@ -1394,11 +1389,11 @@ export class Manager extends EventEmitter implements RunningStates {
                             });
                         }
                         break;
-                    case 'newSub':
+                    case POLLING_SUBMISSIONS:
                         if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
-                            modStreamType = 'newSub';
+                            modStreamType = POLLING_SUBMISSIONS;
                             // use default mod stream from resources
-                            stream = this.cacheManager.modStreams.get('newSub') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                            stream = this.cacheManager.modStreams.get(POLLING_SUBMISSIONS) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
                         } else {
                             stream = new SubmissionStream(this.client, {
                                 subreddit: this.subreddit.display_name,
@@ -1408,11 +1403,11 @@ export class Manager extends EventEmitter implements RunningStates {
                             });
                         }
                         break;
-                    case 'newComm':
+                    case POLLING_COMMENTS:
                         if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
-                            modStreamType = 'newComm';
+                            modStreamType = POLLING_COMMENTS;
                             // use default mod stream from resources
-                            stream = this.cacheManager.modStreams.get('newComm') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                            stream = this.cacheManager.modStreams.get(POLLING_COMMENTS) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
                         } else {
                             stream = new CommentStream(this.client, {
                                 subreddit: this.subreddit.display_name,
@@ -1422,6 +1417,8 @@ export class Manager extends EventEmitter implements RunningStates {
                             });
                         }
                         break;
+                    default:
+                        throw new CMError(`This shouldn't happen! All polling sources are enumerated in switch. Source value: ${source}`)
                 }
 
                 if (stream === undefined) {
@@ -1514,10 +1511,10 @@ export class Manager extends EventEmitter implements RunningStates {
     }
 
     noChecksWarning = (source: PollOn) => (listing: any) => {
-        if (this.commentChecks.length === 0 && ['modqueue', 'newComm'].some(x => x === source)) {
+        if (this.commentChecks.length === 0 && [POLLING_MODQUEUE, POLLING_COMMENTS].some(x => x === source)) {
             this.logger.warn(`Polling '${source.toUpperCase()}' may return Comments but no comments checks were configured.`);
         }
-        if (this.submissionChecks.length === 0 && ['unmoderated', 'modqueue', 'newSub'].some(x => x === source)) {
+        if (this.submissionChecks.length === 0 && [POLLING_UNMODERATED, POLLING_MODQUEUE, POLLING_SUBMISSIONS].some(x => x === source)) {
             this.logger.warn(`Polling '${source.toUpperCase()}' may return Submissions but no submission checks were configured.`);
         }
     }
@@ -1670,7 +1667,7 @@ export class Manager extends EventEmitter implements RunningStates {
             }
             this.startedAt = dayjs();
 
-            const modQueuePollOpts = this.pollOptions.find(x => x.pollOn === 'modqueue');
+            const modQueuePollOpts = this.pollOptions.find(x => x.pollOn === POLLING_MODQUEUE);
             if(modQueuePollOpts !== undefined) {
                 this.modqueueInterval = modQueuePollOpts.interval;
             }

src/Subreddit/Streams.ts

@@ -7,6 +7,7 @@ import {mergeArr, parseDuration, random} from "../util";
 import { Logger } from "winston";
 import {ErrorWithCause} from "pony-cause";
 import dayjs, {Dayjs as DayjsObj} from "dayjs";
+import { FixedSizeList } from 'fixed-size-list'
 
 type Awaitable<T> = Promise<T> | T;
 
@@ -14,10 +15,12 @@ interface RCBPollingOptions<T> extends SnooStormOptions {
     subreddit: string,
     enforceContinuity?: boolean
     logger: Logger
+    sort?: string
     name?: string,
-    processed?: Set<T[keyof T]>
+    processed?: FixedSizeList<T[keyof T]>
     label?: string
     dateCutoff?: boolean
+    maxHistory?: number
 }
 
 interface RCBPollConfiguration<T> extends PollConfiguration<T>,RCBPollingOptions<T> {
@@ -40,6 +43,9 @@ export class SPoll<T extends RedditContent<object>> extends Poll<T> {
     name: string = 'Reddit Stream';
     logger: Logger;
     subreddit: string;
+    // using a fixed sized "regular" array means slightly more memory usage vs. a Set when holding N items
+    // BUT now we can limit N items to something reasonable instead of having a crazy big Set with all items seen since stream was started
+    processedBuffer: FixedSizeList<T[keyof T]>;
 
     constructor(options: RCBPollConfiguration<T>) {
         super(options);
@@ -54,6 +60,7 @@ export class SPoll<T extends RedditContent<object>> extends Poll<T> {
             label = 'Polling',
             processed,
             dateCutoff,
+            maxHistory = 300,
         } = options;
         this.subreddit = subreddit;
         this.name = name !== undefined ? name : this.name;
@@ -67,8 +74,10 @@ export class SPoll<T extends RedditContent<object>> extends Poll<T> {
         // if we pass in processed on init the intention is to "continue" from where the previous stream left off
         // WITHOUT new start behavior
         if (processed !== undefined) {
-            this.processed = processed;
+            this.processedBuffer = processed;
             this.newStart = false;
+        } else {
+            this.processedBuffer = new FixedSizeList<T[keyof T]>(maxHistory);
         }
 
         clearInterval(this.interval);
@@ -97,14 +106,14 @@ export class SPoll<T extends RedditContent<object>> extends Poll<T> {
                         }
                         for (const item of batch) {
                             const id = item[self.identifier];
-                            if (self.processed.has(id)) {
+                            if (self.processedBuffer.data.some(x => x === id)) {
                                 anyAlreadySeen = true;
                                 continue;
                             }
 
                             // add new item to list and set as processed
                             newItems.push(item);
-                            self.processed.add(id);
+                            self.processedBuffer.add(id);
                         }
                         page++;
                     }

src/Subreddit/SubredditResources.ts

@@ -1030,13 +1030,13 @@ export class SubredditResources {
         }
     }
 
-    async getAuthorActivities(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing): Promise<SnoowrapActivity[]> {
+    async getAuthorActivities(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing, prefetchedActivities?: SnoowrapActivity[]): Promise<SnoowrapActivity[]> {
 
-        const {post} = await this.getAuthorActivitiesWithFilter(user, options, customListing);
+        const {post} = await this.getAuthorActivitiesWithFilter(user, options, customListing, prefetchedActivities);
         return post;
     }
 
-    async getAuthorActivitiesWithFilter(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing): Promise<FetchedActivitiesResult> {
+    async getAuthorActivitiesWithFilter(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing, prefetchedActivities?: SnoowrapActivity[]): Promise<FetchedActivitiesResult> {
         let listFuncName: string;
         let listFunc: ListingFunc;
 
@@ -1064,21 +1064,21 @@ export class SubredditResources {
             ...(cloneDeep(options)),
         }
 
-        return await this.getActivities(user, criteriaWithDefaults, {func: listFunc, name: listFuncName});
+        return await this.getActivities(user, criteriaWithDefaults, {func: listFunc, name: listFuncName}, prefetchedActivities);
     }
 
-    async getAuthorComments(user: RedditUser, options: ActivityWindowCriteria): Promise<Comment[]> {
-        return await this.getAuthorActivities(user, {...options, fetch: 'comment'}) as unknown as Promise<Comment[]>;
+    async getAuthorComments(user: RedditUser, options: ActivityWindowCriteria, prefetchedActivities?: SnoowrapActivity[]): Promise<Comment[]> {
+        return await this.getAuthorActivities(user, {...options, fetch: 'comment'}, undefined, prefetchedActivities) as unknown as Promise<Comment[]>;
     }
 
-    async getAuthorSubmissions(user: RedditUser, options: ActivityWindowCriteria): Promise<Submission[]> {
+    async getAuthorSubmissions(user: RedditUser, options: ActivityWindowCriteria, prefetchedActivities?: SnoowrapActivity[]): Promise<Submission[]> {
         return await this.getAuthorActivities(user, {
             ...options,
             fetch: 'submission'
-        }) as unknown as Promise<Submission[]>;
+        }, undefined,prefetchedActivities) as unknown as Promise<Submission[]>;
     }
 
-    async getActivities(user: RedditUser, options: ActivityWindowCriteria, listingData: NamedListing): Promise<FetchedActivitiesResult> {
+    async getActivities(user: RedditUser, options: ActivityWindowCriteria, listingData: NamedListing, prefetchedActivities: SnoowrapActivity[] = []): Promise<FetchedActivitiesResult> {
 
         try {
 
@@ -1213,12 +1213,24 @@ export class SubredditResources {
                     }
                 }
 
-                let unFilteredItems: SnoowrapActivity[] | undefined;
-
+                let preFilteredPrefetchedActivities = [...prefetchedActivities];
+                if(preFilteredPrefetchedActivities.length > 0) {
+                    switch(options.fetch) {
+                        // TODO this may not work if using a custom listingFunc that does not include fetch type
+                        case 'comment':
+                            preFilteredPrefetchedActivities = preFilteredPrefetchedActivities.filter(x => asComment(x));
+                            break;
+                        case 'submission':
+                            preFilteredPrefetchedActivities = preFilteredPrefetchedActivities.filter(x => asSubmission(x));
+                            break;
+                    }
+                    preFilteredPrefetchedActivities = await this.filterListingWithHistoryOptions(preFilteredPrefetchedActivities, user, options.filterOn?.pre);
+                }
+                let unFilteredItems: SnoowrapActivity[] | undefined = [...preFilteredPrefetchedActivities];
+                pre = pre.concat(preFilteredPrefetchedActivities);
 
                 const { func: listingFunc } = listingData;
 
-
                 let listing = await listingFunc(getAuthorHistoryAPIOptions(options));
                 let hitEnd = false;
                 let offset = chunkSize;
@@ -1228,6 +1240,9 @@ export class SubredditResources {
                         timeOk = false;
 
                     let listSlice = listing.slice(offset - chunkSize);
+                    // filter out any from slice that were already included from prefetched list so that prefetched aren't included twice
+                    listSlice = preFilteredPrefetchedActivities.length === 0 ? listSlice : listSlice.filter(x => !preFilteredPrefetchedActivities.some(y => y.name === x.name));
+
                     let preListSlice = await this.filterListingWithHistoryOptions(listSlice, user, options.filterOn?.pre);
 
                     // its more likely the time criteria is going to be hit before the count criteria
@@ -1502,6 +1517,7 @@ export class SubredditResources {
             usernotes,
             ruleResults,
             actionResults,
+            author: (val) => this.getAuthor(val)
         });
     }
 

src/Utils/SnoowrapUtils.ts

@@ -133,6 +133,7 @@ export interface TemplateContext {
     ruleResults?: RuleResultEntity[]
     actionResults?: ActionResultEntity[]
     activity?: SnoowrapActivity
+    author?: (val: string | RedditUser) => Promise<RedditUser>
     [key: string]: any
 }
 
@@ -140,11 +141,25 @@ export const renderContent = async (template: string, data: TemplateContext = {}
     const {
         usernotes,
         ruleResults,
+        author,
         actionResults,
         activity,
         ...restContext
     } = data;
 
+    let fetchedUser: RedditUser | undefined;
+    // @ts-ignore
+    const user = async (): Promise<RedditUser> => {
+        if(fetchedUser === undefined) {
+            if(author !== undefined) {
+                // @ts-ignore
+                fetchedUser = await author(activity.author);
+            }
+        }
+        // @ts-ignore
+        return fetchedUser;
+    }
+
     let view: GenericContentTemplateData = {
         botLink: BOT_LINK,
         ...restContext
@@ -171,10 +186,24 @@ export const renderContent = async (template: string, data: TemplateContext = {}
 
         view.modmailLink = `https://www.reddit.com/message/compose?to=%2Fr%2F${subreddit}&message=${encodeURIComponent(permalink)}`;
 
+        const author: any = {
+            toString: () => getActivityAuthorName(activity.author)
+        };
+
+        if(template.includes('{{item.author.')) {
+            // @ts-ignore
+            const auth = await user();
+
+            author.age = dayjs.unix(auth.created).fromNow(true);
+            author.linkKarma = auth.link_karma;
+            author.commentKarma = auth.comment_karma;
+            author.totalKarma = auth.comment_karma + auth.link_karma;
+            author.verified = auth.has_verified_email;
+        }
+
         const templateData: any = {
             kind: activity instanceof Submission ? 'submission' : 'comment',
-            // @ts-ignore
-            author: getActivityAuthorName(await activity.author),
+            author,
             votes: activity.score,
             age: dayjs.duration(dayjs().diff(dayjs.unix(activity.created))).humanize(),
             permalink,

src/util.ts

@@ -77,6 +77,7 @@ import {
     ImageHashCacheData,
     ModUserNoteLabel,
     modUserNoteLabels,
+    PollOn, pollOnTypeMapping, pollOnTypes,
     RedditEntity,
     RedditEntityType,
     RelativeDateTimeMatch,
@@ -3088,3 +3089,12 @@ export const toStrongSharingACLConfig = (data: SharingACLConfig | string[]): Str
         exclude: (data.exclude ?? []).map(x => parseStringToRegexOrLiteralSearch(x))
     }
 }
+
+export const toPollOn = (val: string | PollOn): PollOn => {
+    const clean = val.toLowerCase().trim();
+    const pVal = pollOnTypeMapping.get(clean);
+    if(pVal !== undefined) {
+        return pVal;
+    }
+    throw new SimpleError(`'${val}' is not a valid polling source. Valid sources: ${pollOnTypes.join(' | ')}`);
+}