Use fixed array instead of Set to maybe reduce memory usage

feat: Implement templating for flair actions
docs: Update docker compose instructions
2026-01-14 16:08:02 -05:00 · 2022-11-21 15:56:56 -05:00 · 2022-11-21 11:43:34 -05:00 · 2022-11-21 11:29:14 -05:00 · 2022-11-17 13:16:10 -05:00 · 2022-11-17 12:03:55 -05:00
23 changed files with 318 additions and 376 deletions
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -24,6 +24,9 @@ services:

  cache:
    image: 'redis:7-alpine'
+    volumes:
+      # on linux will need to make sure this directory has correct permissions for container to access
+      - './data/cache:/data'

  database:
    image: 'mariadb:10.9.3'
--- a/docs/operator/installation.md
+++ b/docs/operator/installation.md
@@ -55,6 +55,8 @@ The included [`docker-compose.yml`](/docker-compose.yml) provides production-rea

 #### Setup

+The included `docker-compose.yml` file is written for **Docker Compose v2.**
+
 For new installations copy [`config.yaml`](/docker/config/docker-compose/config.yaml) into a folder named `data` in the same folder `docker-compose.yml` will be run from. For users migrating their existing CM instances to docker-compose, copy your existing `config.yaml` into the same `data` folder.

 Read through the comments in both `docker-compose.yml` and `config.yaml` and makes changes to any relevant settings (passwords, usernames, etc...). Ensure that any settings used in both files (EX mariaDB passwords) match.
@@ -62,13 +64,13 @@ Read through the comments in both `docker-compose.yml` and `config.yaml` and mak
 To build and start CM:

 ```bash
-docker-compose up -d
+docker compose up -d
 ```

 To include Grafana/Influx dependencies run:

 ```bash
-docker-compose --profile full up -d
+docker compose --profile full up -d
 ```

 ## Locally
--- a/docs/subreddit-configuration/actionTemplating.md
+++ b/docs/subreddit-configuration/actionTemplating.md
@@ -57,6 +57,29 @@ All Actions with `content` have access to this data:
 | `title`      | As comments => the body of the comment. As Submission => title                                      | Test post please ignore                                  |
 | `shortTitle` | The same as `title` but truncated to 15 characters                                                  | test post pleas...                                       |

+#### Common Author
+
+Additionally, `author` has these properties accessible:
+
+|      Name      |             Description             | Example  |
+|----------------|-------------------------------------|----------|
+| `age`          | (Approximate) Age of account        | 3 months |
+| `linkKarma`    | Amount of link karma                | 10       |
+| `commentKarma` | Amount of comment karma             | 3        |
+| `totalKarma`   | Combined link+comment karma         | 13       |
+| `verified`     | Does account have a verified email? | true     |
+
+NOTE: Accessing these properties may require an additional API call so use sparingly on high-volume comments
+
+##### Example Usage
+
+```
+The user {{item.author}} has been a redditor for {{item.author.age}}
+```
+Produces:
+
+> The user FoxxMD has been a redditor for 3 months
+
 ### Submissions

 If the **Activity** is a Submission these additional properties are accessible:
--- a/package-lock.json
+++ b/package-lock.json
@@ -46,6 +46,7 @@
        "express-session-cache-manager": "^1.0.2",
        "express-socket.io-session": "^1.3.5",
        "fast-deep-equal": "^3.1.3",
+        "fixed-size-list": "^0.3.0",
        "globrex": "^0.1.2",
        "got": "^11.8.2",
        "he": "^1.2.0",
@@ -3979,6 +3980,14 @@
        "micromatch": "^4.0.2"
      }
    },
+    "node_modules/fixed-size-list": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/fixed-size-list/-/fixed-size-list-0.3.0.tgz",
+      "integrity": "sha512-c6I8wEE4ZtjKz35BaodH7yWuWmcaUVQwgBeNcI3LxJu79YH+ezHvf1oS9VkgJmyVy5eQ8Wh6jNVcj2rB4rgVgA==",
+      "dependencies": {
+        "mitt": "^1.2.0"
+      }
+    },
    "node_modules/flat": {
      "version": "5.0.2",
      "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz",
@@ -6113,6 +6122,11 @@
      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
      "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q=="
    },
+    "node_modules/mitt": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/mitt/-/mitt-1.2.0.tgz",
+      "integrity": "sha512-r6lj77KlwqLhIUku9UWYes7KJtsczvolZkzp8hbaDPPaE24OmWl5s539Mytlj22siEQKosZ26qCBgda2PKwoJw=="
+    },
    "node_modules/mkdirp": {
      "version": "0.5.6",
      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
@@ -13588,6 +13602,14 @@
        "micromatch": "^4.0.2"
      }
    },
+    "fixed-size-list": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/fixed-size-list/-/fixed-size-list-0.3.0.tgz",
+      "integrity": "sha512-c6I8wEE4ZtjKz35BaodH7yWuWmcaUVQwgBeNcI3LxJu79YH+ezHvf1oS9VkgJmyVy5eQ8Wh6jNVcj2rB4rgVgA==",
+      "requires": {
+        "mitt": "^1.2.0"
+      }
+    },
    "flat": {
      "version": "5.0.2",
      "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz",
@@ -15191,6 +15213,11 @@
      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
      "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q=="
    },
+    "mitt": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/mitt/-/mitt-1.2.0.tgz",
+      "integrity": "sha512-r6lj77KlwqLhIUku9UWYes7KJtsczvolZkzp8hbaDPPaE24OmWl5s539Mytlj22siEQKosZ26qCBgda2PKwoJw=="
+    },
    "mkdirp": {
      "version": "0.5.6",
      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
--- a/package.json
+++ b/package.json
@@ -68,6 +68,7 @@
    "express-session-cache-manager": "^1.0.2",
    "express-socket.io-session": "^1.3.5",
    "fast-deep-equal": "^3.1.3",
+    "fixed-size-list": "^0.3.0",
    "globrex": "^0.1.2",
    "got": "^11.8.2",
    "he": "^1.2.0",
--- a/src/Action/SubmissionAction/FlairAction.ts
+++ b/src/Action/SubmissionAction/FlairAction.ts
@@ -30,15 +30,14 @@ export class FlairAction extends Action {
    async process(item: Comment | Submission, ruleResults: RuleResultEntity[], actionResults: ActionResultEntity[], options: runCheckOptions): Promise<ActionProcessResult> {
        const dryRun = this.getRuntimeAwareDryrun(options);
        let flairParts = [];
-        if(this.text !== '') {
-            flairParts.push(`Text: ${this.text}`);
-        }
-        if(this.css !== '') {
-            flairParts.push(`CSS: ${this.css}`);
-        }
-        if(this.flair_template_id !== '') {
-            flairParts.push(`Template: ${this.flair_template_id}`);
-        }
+        const renderedText = this.text === '' ? '' : await this.renderContent(this.text, item, ruleResults, actionResults) as string;
+        flairParts.push(`Text: ${renderedText === '' ? '(None)' : renderedText}`);
+
+        const renderedCss = this.css === '' ? '' : await this.renderContent(this.css, item, ruleResults, actionResults) as string;
+        flairParts.push(`CSS: ${renderedCss === '' ? '(None)' : renderedCss}`);
+
+        flairParts.push(`Template: ${this.flair_template_id === '' ? '(None)' : this.flair_template_id}`);
+
        const flairSummary = flairParts.length === 0 ? 'No flair (unflaired)' : flairParts.join(' | ');
        this.logger.verbose(flairSummary);
        if (item instanceof Submission) {
@@ -51,9 +50,9 @@ export class FlairAction extends Action {
                    await item.assignFlair({flair_template_id: this.flair_template_id}).then(() => {});
                    item.link_flair_template_id = this.flair_template_id;
                } else {
-                    await item.assignFlair({text: this.text, cssClass: this.css}).then(() => {});
-                    item.link_flair_css_class = this.css;
-                    item.link_flair_text = this.text;
+                    await item.assignFlair({text: renderedText, cssClass: renderedCss}).then(() => {});
+                    item.link_flair_css_class = renderedCss;
+                    item.link_flair_text = renderedText;
                }
                await this.resources.resetCacheForItem(item);
            }
--- a/src/Action/UserFlairAction.ts
+++ b/src/Action/UserFlairAction.ts
@@ -26,6 +26,8 @@ export class UserFlairAction extends Action {
  async process(item: Comment | Submission, ruleResults: RuleResultEntity[], actionResults: ActionResultEntity[], options: runCheckOptions): Promise<ActionProcessResult> {
    const dryRun = this.getRuntimeAwareDryrun(options);
    let flairParts = [];
+    let renderedText: string | undefined = undefined;
+    let renderedCss: string | undefined = undefined;

    if (this.flair_template_id !== undefined) {
      flairParts.push(`Flair template ID: ${this.flair_template_id}`)
@@ -34,10 +36,12 @@ export class UserFlairAction extends Action {
      }
    } else {
      if (this.text !== undefined) {
-        flairParts.push(`Text: ${this.text}`);
+        renderedText = await this.renderContent(this.text, item, ruleResults, actionResults) as string;
+        flairParts.push(`Text: ${renderedText}`);
      }
      if (this.css !== undefined) {
-        flairParts.push(`CSS: ${this.css}`);
+        renderedCss = await this.renderContent(this.css, item, ruleResults, actionResults) as string;
+        flairParts.push(`CSS: ${renderedCss}`);
      }
    }

@@ -58,7 +62,7 @@ export class UserFlairAction extends Action {
          this.logger.error('Either the flair template ID is incorrect or you do not have permission to access it.');
          throw err;
        }
-      } else if (this.text === undefined && this.css === undefined) {
+      } else if (renderedText === undefined && renderedCss === undefined) {
        // @ts-ignore
        await item.subreddit.deleteUserFlair(item.author.name);
        item.author_flair_css_class = null;
@@ -68,11 +72,11 @@ export class UserFlairAction extends Action {
        // @ts-ignore
        await item.author.assignFlair({
          subredditName: item.subreddit.display_name,
-          cssClass: this.css,
-          text: this.text,
+          cssClass: renderedCss,
+          text: renderedText,
        });
-        item.author_flair_text = this.text ?? null;
-        item.author_flair_css_class = this.css ?? null;
+        item.author_flair_text = renderedText ?? null;
+        item.author_flair_css_class = renderedCss ?? null;
      }
      await this.resources.resetCacheForItem(item);
      if(typeof item.author !== 'string') {
--- a/src/Bot/index.ts
+++ b/src/Bot/index.ts
@@ -46,7 +46,13 @@ import {RunStateType} from "../Common/Entities/RunStateType";
 import {QueueRunState} from "../Common/Entities/EntityRunState/QueueRunState";
 import {EventsRunState} from "../Common/Entities/EntityRunState/EventsRunState";
 import {ManagerRunState} from "../Common/Entities/EntityRunState/ManagerRunState";
-import {Invokee, PollOn} from "../Common/Infrastructure/Atomic";
+import {
+    Invokee,
+    POLLING_COMMENTS, POLLING_MODQUEUE,
+    POLLING_SUBMISSIONS,
+    POLLING_UNMODERATED,
+    PollOn
+} from "../Common/Infrastructure/Atomic";
 import {FilterCriteriaDefaults} from "../Common/Infrastructure/Filters/FilterShapes";
 import {snooLogWrapper} from "../Utils/loggerFactory";
 import {InfluxClient} from "../Common/Influx/InfluxClient";
@@ -558,15 +564,15 @@ class Bot implements BotInstanceFunctions {

    parseSharedStreams() {

-        const sharedCommentsSubreddits = !this.sharedStreams.includes('newComm') ? [] : this.subManagers.filter(x => x.isPollingShared('newComm')).map(x => x.subreddit.display_name);
+        const sharedCommentsSubreddits = !this.sharedStreams.includes(POLLING_COMMENTS) ? [] : this.subManagers.filter(x => x.isPollingShared(POLLING_COMMENTS)).map(x => x.subreddit.display_name);
        if (sharedCommentsSubreddits.length > 0) {
-            const stream = this.cacheManager.modStreams.get('newComm');
+            const stream = this.cacheManager.modStreams.get(POLLING_COMMENTS);
            if (stream === undefined || stream.subreddit !== sharedCommentsSubreddits.join('+')) {
                let processed;
                if (stream !== undefined) {
                    this.logger.info('Restarting SHARED COMMENT STREAM due to a subreddit config change');
                    stream.end('Replacing with a new stream with updated subreddits');
-                    processed = stream.processed;
+                    processed = stream.processedBuffer;
                }
                if (sharedCommentsSubreddits.length > 100) {
                    this.logger.warn(`SHARED COMMENT STREAM => Reddit can only combine 100 subreddits for getting new Comments but this bot has ${sharedCommentsSubreddits.length}`);
@@ -580,26 +586,26 @@ class Bot implements BotInstanceFunctions {
                    label: 'Shared Polling'
                });
                // @ts-ignore
-                defaultCommentStream.on('error', this.createSharedStreamErrorListener('newComm'));
-                defaultCommentStream.on('listing', this.createSharedStreamListingListener('newComm'));
-                this.cacheManager.modStreams.set('newComm', defaultCommentStream);
+                defaultCommentStream.on('error', this.createSharedStreamErrorListener(POLLING_COMMENTS));
+                defaultCommentStream.on('listing', this.createSharedStreamListingListener(POLLING_COMMENTS));
+                this.cacheManager.modStreams.set(POLLING_COMMENTS, defaultCommentStream);
            }
        } else {
-            const stream = this.cacheManager.modStreams.get('newComm');
+            const stream = this.cacheManager.modStreams.get(POLLING_COMMENTS);
            if (stream !== undefined) {
                stream.end('Determined no managers are listening on shared stream parsing');
            }
        }

-        const sharedSubmissionsSubreddits = !this.sharedStreams.includes('newSub') ? [] : this.subManagers.filter(x => x.isPollingShared('newSub')).map(x => x.subreddit.display_name);
+        const sharedSubmissionsSubreddits = !this.sharedStreams.includes(POLLING_SUBMISSIONS) ? [] : this.subManagers.filter(x => x.isPollingShared(POLLING_SUBMISSIONS)).map(x => x.subreddit.display_name);
        if (sharedSubmissionsSubreddits.length > 0) {
-            const stream = this.cacheManager.modStreams.get('newSub');
+            const stream = this.cacheManager.modStreams.get(POLLING_SUBMISSIONS);
            if (stream === undefined || stream.subreddit !== sharedSubmissionsSubreddits.join('+')) {
                let processed;
                if (stream !== undefined) {
                    this.logger.info('Restarting SHARED SUBMISSION STREAM due to a subreddit config change');
                    stream.end('Replacing with a new stream with updated subreddits');
-                    processed = stream.processed;
+                    processed = stream.processedBuffer;
                }
                if (sharedSubmissionsSubreddits.length > 100) {
                    this.logger.warn(`SHARED SUBMISSION STREAM => Reddit can only combine 100 subreddits for getting new Submissions but this bot has ${sharedSubmissionsSubreddits.length}`);
@@ -613,19 +619,19 @@ class Bot implements BotInstanceFunctions {
                    label: 'Shared Polling'
                });
                // @ts-ignore
-                defaultSubStream.on('error', this.createSharedStreamErrorListener('newSub'));
-                defaultSubStream.on('listing', this.createSharedStreamListingListener('newSub'));
-                this.cacheManager.modStreams.set('newSub', defaultSubStream);
+                defaultSubStream.on('error', this.createSharedStreamErrorListener(POLLING_SUBMISSIONS));
+                defaultSubStream.on('listing', this.createSharedStreamListingListener(POLLING_SUBMISSIONS));
+                this.cacheManager.modStreams.set(POLLING_SUBMISSIONS, defaultSubStream);
            }
        } else {
-            const stream = this.cacheManager.modStreams.get('newSub');
+            const stream = this.cacheManager.modStreams.get(POLLING_SUBMISSIONS);
            if (stream !== undefined) {
                stream.end('Determined no managers are listening on shared stream parsing');
            }
        }

-        const isUnmoderatedShared = !this.sharedStreams.includes('unmoderated') ? false : this.subManagers.some(x => x.isPollingShared('unmoderated'));
-        const unmoderatedstream = this.cacheManager.modStreams.get('unmoderated');
+        const isUnmoderatedShared = !this.sharedStreams.includes(POLLING_UNMODERATED) ? false : this.subManagers.some(x => x.isPollingShared(POLLING_UNMODERATED));
+        const unmoderatedstream = this.cacheManager.modStreams.get(POLLING_UNMODERATED);
        if (isUnmoderatedShared && unmoderatedstream === undefined) {
            const defaultUnmoderatedStream = new UnmoderatedStream(this.client, {
                subreddit: 'mod',
@@ -634,15 +640,15 @@ class Bot implements BotInstanceFunctions {
                label: 'Shared Polling'
            });
            // @ts-ignore
-            defaultUnmoderatedStream.on('error', this.createSharedStreamErrorListener('unmoderated'));
-            defaultUnmoderatedStream.on('listing', this.createSharedStreamListingListener('unmoderated'));
-            this.cacheManager.modStreams.set('unmoderated', defaultUnmoderatedStream);
+            defaultUnmoderatedStream.on('error', this.createSharedStreamErrorListener(POLLING_UNMODERATED));
+            defaultUnmoderatedStream.on('listing', this.createSharedStreamListingListener(POLLING_UNMODERATED));
+            this.cacheManager.modStreams.set(POLLING_UNMODERATED, defaultUnmoderatedStream);
        } else if (!isUnmoderatedShared && unmoderatedstream !== undefined) {
            unmoderatedstream.end('Determined no managers are listening on shared stream parsing');
        }

-        const isModqueueShared = !this.sharedStreams.includes('modqueue') ? false : this.subManagers.some(x => x.isPollingShared('modqueue'));
-        const modqueuestream = this.cacheManager.modStreams.get('modqueue');
+        const isModqueueShared = !this.sharedStreams.includes(POLLING_MODQUEUE) ? false : this.subManagers.some(x => x.isPollingShared(POLLING_MODQUEUE));
+        const modqueuestream = this.cacheManager.modStreams.get(POLLING_MODQUEUE);
        if (isModqueueShared && modqueuestream === undefined) {
            const defaultModqueueStream = new ModQueueStream(this.client, {
                subreddit: 'mod',
@@ -651,9 +657,9 @@ class Bot implements BotInstanceFunctions {
                label: 'Shared Polling'
            });
            // @ts-ignore
-            defaultModqueueStream.on('error', this.createSharedStreamErrorListener('modqueue'));
-            defaultModqueueStream.on('listing', this.createSharedStreamListingListener('modqueue'));
-            this.cacheManager.modStreams.set('modqueue', defaultModqueueStream);
+            defaultModqueueStream.on('error', this.createSharedStreamErrorListener(POLLING_MODQUEUE));
+            defaultModqueueStream.on('listing', this.createSharedStreamListingListener(POLLING_MODQUEUE));
+            this.cacheManager.modStreams.set(POLLING_MODQUEUE, defaultModqueueStream);
        } else if (isModqueueShared && modqueuestream !== undefined) {
            modqueuestream.end('Determined no managers are listening on shared stream parsing');
        }
--- a/src/Common/Entities/Activity.ts
+++ b/src/Common/Entities/Activity.ts
@@ -1,4 +1,4 @@
-import {Entity, Column, ManyToOne, PrimaryColumn, OneToMany, Index, DataSource, JoinColumn} from "typeorm";
+import {Entity, Column, ManyToOne, PrimaryColumn, OneToMany, Index} from "typeorm";
 import {AuthorEntity} from "./AuthorEntity";
 import {Subreddit} from "./Subreddit";
 import {CMEvent} from "./CMEvent";
@@ -6,8 +6,6 @@ import {asComment, getActivityAuthorName, parseRedditFullname, redditThingTypeTo
 import {activityReports, ActivityType, Report, SnoowrapActivity} from "../Infrastructure/Reddit";
 import {ActivityReport} from "./ActivityReport";
 import dayjs, {Dayjs} from "dayjs";
-import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients";
-import {Comment, Submission} from 'snoowrap/dist/objects';

 export interface ActivityEntityOptions {
    id: string
@@ -47,7 +45,7 @@ export class Activity {
    @Column({name: 'name'})
    name!: string;

-    @ManyToOne(type => Subreddit, sub => sub.activities, {cascade: ['insert'], eager: true})
+    @ManyToOne(type => Subreddit, sub => sub.activities, {cascade: ['insert']})
    subreddit!: Subreddit;

    @Column("varchar", {length: 20})
@@ -60,18 +58,17 @@ export class Activity {
    @Column("text")
    permalink!: string;

-    @ManyToOne(type => AuthorEntity, author => author.activities, {cascade: ['insert'], eager: true})
+    @ManyToOne(type => AuthorEntity, author => author.activities, {cascade: ['insert']})
    author!: AuthorEntity;

-    @OneToMany(type => CMEvent, act => act.activity)
+    @OneToMany(type => CMEvent, act => act.activity) // note: we will create author property in the Photo class below
    actionedEvents!: CMEvent[]

-    @ManyToOne('Activity', 'comments', {nullable: true, cascade: ['insert']})
-    @JoinColumn({name: 'submission_id'})
+    @ManyToOne(type => Activity, obj => obj.comments, {nullable: true})
    submission?: Activity;

-    @OneToMany('Activity', 'submission', {nullable: true})
-    comments?: Activity[];
+    @OneToMany(type => Activity, obj => obj.submission, {nullable: true})
+    comments!: Activity[];

    @OneToMany(type => ActivityReport, act => act.activity, {cascade: ['insert'], eager: true})
    reports: ActivityReport[] | undefined
@@ -154,12 +151,10 @@ export class Activity {
        return false;
    }

-    static async fromSnoowrapActivity(activity: SnoowrapActivity, options: fromSnoowrapOptions | undefined = {}) {
-
+    static fromSnoowrapActivity(subreddit: Subreddit, activity: SnoowrapActivity, lastKnownStateTimestamp?: dayjs.Dayjs | undefined) {
        let submission: Activity | undefined;
        let type: ActivityType = 'submission';
        let content: string;
-        const subreddit = await Subreddit.fromSnoowrap(activity.subreddit, options?.db);
        if(asComment(activity)) {
            type = 'comment';
            content = activity.body;
@@ -184,30 +179,8 @@ export class Activity {
            submission
        });

-        entity.syncReports(activity, options.lastKnownStateTimestamp);
+        entity.syncReports(activity, lastKnownStateTimestamp);

        return entity;
    }
-
-    toSnoowrap(client: ExtendedSnoowrap): SnoowrapActivity {
-        let act: SnoowrapActivity;
-        if(this.type === 'submission') {
-            act = new Submission({name: this.id, id: this.name}, client, false);
-            act.title = this.content;
-        } else {
-            act = new Comment({name: this.id, id: this.name}, client, false);
-            act.link_id = this.submission?.id as string;
-            act.body = this.content;
-        }
-        act.permalink = this.permalink;
-        act.subreddit = this.subreddit.toSnoowrap(client);
-        act.author = this.author.toSnoowrap(client);
-
-        return act;
-    }
-}
-
-export interface fromSnoowrapOptions {
-    lastKnownStateTimestamp?: dayjs.Dayjs | undefined
-    db?: DataSource
 }
--- a/src/Common/Entities/AuthorEntity.ts
+++ b/src/Common/Entities/AuthorEntity.ts
@@ -1,8 +1,5 @@
 import {Entity, Column, PrimaryColumn, OneToMany} from "typeorm";
 import {Activity} from "./Activity";
-import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients";
-import {SnoowrapActivity} from "../Infrastructure/Reddit";
-import {RedditUser} from "snoowrap/dist/objects";

@Entity({name: 'Author'})
 export class AuthorEntity {
@@ -14,15 +11,11 @@ export class AuthorEntity {
    name!: string;

    @OneToMany(type => Activity, act => act.author)
-    activities!: Promise<Activity[]>
+    activities!: Activity[]

    constructor(data?: any) {
        if(data !== undefined) {
            this.name = data.name;
        }
    }
-
-    toSnoowrap(client: ExtendedSnoowrap): RedditUser {
-        return new RedditUser({name: this.name, id: this.id}, client, false);
-    }
 }
--- a/src/Common/Entities/DispatchedEntity.ts
+++ b/src/Common/Entities/DispatchedEntity.ts
@@ -6,7 +6,7 @@ import {
    ManyToOne,
    PrimaryColumn,
    BeforeInsert,
-    AfterLoad, JoinColumn
+    AfterLoad
 } from "typeorm";
 import {
    ActivityDispatch
@@ -22,15 +22,15 @@ import Comment from "snoowrap/dist/objects/Comment";
 import {ColumnDurationTransformer} from "./Transformers";
 import { RedditUser } from "snoowrap/dist/objects";
 import {ActivitySourceTypes, DurationVal, NonDispatchActivitySourceValue, onExistingFoundBehavior} from "../Infrastructure/Atomic";
-import {Activity} from "./Activity";

@Entity({name: 'DispatchedAction'})
 export class DispatchedEntity extends TimeAwareRandomBaseEntity {

-    //@ManyToOne(type => Activity, obj => obj.dispatched, {cascade: ['insert'], eager: true, nullable: false})
-    @ManyToOne(type => Activity, undefined, {cascade: ['insert'], eager: true, nullable: false})
-    @JoinColumn({name: 'activityId'})
-    activity!: Activity
+    @Column()
+    activityId!: string
+
+    @Column()
+    author!: string

    @Column({
        type: 'int',
@@ -82,10 +82,11 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity {
        }})
    tardyTolerant!: boolean | Duration

-    constructor(data?: HydratedActivityDispatch) {
+    constructor(data?: ActivityDispatch & { manager: ManagerEntity }) {
        super();
        if (data !== undefined) {
-            this.activity = data.activity;
+            this.activityId = data.activity.name;
+            this.author = getActivityAuthorName(data.activity.author);
            this.delay = data.delay;
            this.createdAt = data.queuedAt;
            this.type = data.type;
@@ -150,7 +151,20 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity {
    }

    async toActivityDispatch(client: ExtendedSnoowrap): Promise<ActivityDispatch> {
-        let activity = this.activity.toSnoowrap(client);
+        const redditThing = parseRedditFullname(this.activityId);
+        if(redditThing === undefined) {
+            throw new Error(`Could not parse reddit ID from value '${this.activityId}'`);
+        }
+        let activity: Comment | Submission;
+        if (redditThing?.type === 'comment') {
+            // @ts-ignore
+            activity = await client.getComment(redditThing.id);
+        } else {
+            // @ts-ignore
+            activity = await client.getSubmission(redditThing.id);
+        }
+        activity.author = new RedditUser({name: this.author}, client, false);
+        activity.id  = redditThing.id;
        return {
            id: this.id,
            queuedAt: this.createdAt,
@@ -162,13 +176,8 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity {
            cancelIfQueued: this.cancelIfQueued,
            identifier: this.identifier,
            type: this.type,
-            author: activity.author.name,
+            author: this.author,
            dryRun: this.dryRun
        }
    }
 }
-
-export interface HydratedActivityDispatch extends Omit<ActivityDispatch, 'activity'> {
-    activity: Activity
-    manager: ManagerEntity
-}
--- a/src/Common/Entities/Subreddit.ts
+++ b/src/Common/Entities/Subreddit.ts
@@ -1,7 +1,5 @@
-import {Entity, Column, PrimaryColumn, OneToMany, Index, DataSource} from "typeorm";
+import {Entity, Column, PrimaryColumn, OneToMany, Index} from "typeorm";
 import {Activity} from "./Activity";
-import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients";
-import {Subreddit as SnoowrapSubreddit} from "snoowrap/dist/objects";

 export interface SubredditEntityOptions {
    id: string
@@ -27,18 +25,4 @@ export class Subreddit {
            this.name = data.name;
        }
    }
-
-    toSnoowrap(client: ExtendedSnoowrap): SnoowrapSubreddit {
-        return new SnoowrapSubreddit({display_name: this.name, name: this.id}, client, false);
-    }
-
-    static async fromSnoowrap(subreddit: SnoowrapSubreddit, db?: DataSource) {
-        if(db !== undefined) {
-           const existing = await db.getRepository(Subreddit).findOneBy({name: subreddit.display_name});
-           if(existing) {
-               return existing;
-           }
-        }
-        return new Subreddit({id: await subreddit.name, name: await subreddit.display_name});
-    }
 }
--- a/src/Common/Infrastructure/Atomic.ts
+++ b/src/Common/Infrastructure/Atomic.ts
@@ -111,6 +111,19 @@ export interface DurationObject {

 export type JoinOperands = 'OR' | 'AND';
 export type PollOn = 'unmoderated' | 'modqueue' | 'newSub' | 'newComm';
+export const POLLING_UNMODERATED: PollOn = 'unmoderated';
+export const POLLING_MODQUEUE: PollOn = 'modqueue';
+export const POLLING_SUBMISSIONS: PollOn = 'newSub';
+export const POLLING_COMMENTS: PollOn = 'newComm';
+export const pollOnTypes: PollOn[] = [POLLING_UNMODERATED, POLLING_MODQUEUE, POLLING_SUBMISSIONS, POLLING_COMMENTS];
+export const pollOnTypeMapping: Map<string, PollOn> = new Map([
+    ['unmoderated', POLLING_UNMODERATED],
+    ['modqueue', POLLING_MODQUEUE],
+    ['newsub', POLLING_SUBMISSIONS],
+    ['newcomm', POLLING_COMMENTS],
+    // be nice if user mispelled
+    ['newcom', POLLING_COMMENTS]
+]);
 export type ModeratorNames = 'self' | 'automod' | 'automoderator' | string;
 export type Invokee = 'system' | 'user';
 export type RunState = 'running' | 'paused' | 'stopped';
--- a/src/Common/Migrations/Database/Server/1667415256831-delayedReset.ts
+++ b/src/Common/Migrations/Database/Server/1667415256831-delayedReset.ts
@@ -1,19 +0,0 @@
-import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"
-
-export class delayedReset1667415256831 implements MigrationInterface {
-
-    public async up(queryRunner: QueryRunner): Promise<void> {
-        queryRunner.connection.logger.logSchemaBuild('Truncating (removing) existing Dispatched Actions due to internal structural changes');
-        await queryRunner.clearTable('DispatchedAction');
-        await queryRunner.changeColumn('DispatchedAction', 'author', new TableColumn({
-            name: 'author',
-            type: 'varchar',
-            length: '150',
-            isNullable: true
-        }));
-    }
-
-    public async down(queryRunner: QueryRunner): Promise<void> {
-    }
-
-}
--- a/src/Common/interfaces.ts
+++ b/src/Common/interfaces.ts
@@ -372,7 +372,7 @@ export interface PollingOptions extends PollingDefaults {
     * * after they have been manually approved from modqueue
     *
     * */
-    pollOn: 'unmoderated' | 'modqueue' | 'newSub' | 'newComm'
+    pollOn: PollOn
 }

 export interface TTLConfig {
--- a/src/ConfigBuilder.ts
+++ b/src/ConfigBuilder.ts
@@ -8,7 +8,7 @@ import {
    overwriteMerge,
    parseBool, parseExternalUrl, parseUrlContext, parseWikiContext, randomId,
    readConfigFile,
-    removeUndefinedKeys, resolvePathFromEnvWithRelative, toStrongSharingACLConfig
+    removeUndefinedKeys, resolvePathFromEnvWithRelative, toPollOn, toStrongSharingACLConfig
 } from "./util";

 import Ajv, {Schema} from 'ajv';
@@ -74,8 +74,8 @@ import {ErrorWithCause} from "pony-cause";
 import {RunConfigHydratedData, RunConfigData, RunConfigObject} from "./Run";
 import {AuthorRuleConfig} from "./Rule/AuthorRule";
 import {
-    CacheProvider, ConfigFormat, ConfigFragmentParseFunc,
-    PollOn
+    CacheProvider, ConfigFormat, ConfigFragmentParseFunc, POLLING_MODQUEUE, POLLING_UNMODERATED,
+    PollOn, pollOnTypes
 } from "./Common/Infrastructure/Atomic";
 import {
    asFilterOptionsJson,
@@ -452,27 +452,31 @@ export class ConfigBuilder {

 export const buildPollingOptions = (values: (string | PollingOptions)[]): PollingOptionsStrong[] => {
    let opts: PollingOptionsStrong[] = [];
+    let rawOpts: PollingOptions;
    for (const v of values) {
        if (typeof v === 'string') {
-            opts.push({
-                pollOn: v as PollOn,
-                interval: DEFAULT_POLLING_INTERVAL,
-                limit: DEFAULT_POLLING_LIMIT,
-            });
+            rawOpts = {pollOn: v as PollOn}; // maybeee
        } else {
-            const {
-                pollOn: p,
-                interval = DEFAULT_POLLING_INTERVAL,
-                limit = DEFAULT_POLLING_LIMIT,
-                delayUntil,
-            } = v;
-            opts.push({
-                pollOn: p as PollOn,
-                interval,
-                limit,
-                delayUntil,
-            });
+            rawOpts = v;
        }
+
+        const {
+            pollOn: p,
+            interval = DEFAULT_POLLING_INTERVAL,
+            limit = DEFAULT_POLLING_LIMIT,
+            delayUntil,
+        } = rawOpts;
+
+        const pVal = toPollOn(p);
+        if (opts.some(x => x.pollOn === pVal)) {
+            throw new SimpleError(`Polling source ${pVal} cannot appear more than once in polling options`);
+        }
+        opts.push({
+            pollOn: pVal,
+            interval,
+            limit,
+            delayUntil,
+        });
    }
    return opts;
 }
@@ -796,7 +800,7 @@ export const parseDefaultBotInstanceFromArgs = (args: any): BotInstanceJsonConfi
            heartbeatInterval: heartbeat,
        },
        polling: {
-            shared: sharedMod ? ['unmoderated', 'modqueue'] : undefined,
+            shared: sharedMod ? [POLLING_UNMODERATED, POLLING_MODQUEUE] : undefined,
        },
        nanny: {
            softLimit,
@@ -908,7 +912,7 @@ export const parseDefaultBotInstanceFromEnv = (): BotInstanceJsonConfig => {
            heartbeatInterval: process.env.HEARTBEAT !== undefined ? parseInt(process.env.HEARTBEAT) : undefined,
        },
        polling: {
-            shared: parseBool(process.env.SHARE_MOD) ? ['unmoderated', 'modqueue'] : undefined,
+            shared: parseBool(process.env.SHARE_MOD) ? [POLLING_UNMODERATED, POLLING_MODQUEUE] : undefined,
        },
        nanny: {
            softLimit: process.env.SOFT_LIMIT !== undefined ? parseInt(process.env.SOFT_LIMIT) : undefined,
@@ -1525,10 +1529,10 @@ export const buildBotConfig = (data: BotInstanceJsonConfig, opConfig: OperatorCo
        botCache.provider.prefix = buildCachePrefix([botCache.provider.prefix, 'bot', (botName || objectHash.sha1(botCreds))]);
    }

-    let realShared = shared === true ? ['unmoderated', 'modqueue', 'newComm', 'newSub'] : shared;
+    let realShared: PollOn[] = shared === true ? pollOnTypes : shared.map(toPollOn);
    if (sharedMod === true) {
-        realShared.push('unmoderated');
-        realShared.push('modqueue');
+        realShared.push(POLLING_UNMODERATED);
+        realShared.push(POLLING_MODQUEUE);
    }

    const botLevelStatDefaults = {...statDefaultsFromOp, ...databaseStatisticsDefaults};
@@ -1566,7 +1570,7 @@ export const buildBotConfig = (data: BotInstanceJsonConfig, opConfig: OperatorCo
        caching: botCache,
        userAgent,
        polling: {
-            shared: [...new Set(realShared)] as PollOn[],
+            shared: Array.from(new Set(realShared)),
            stagger,
            limit,
            interval,
--- a/src/Rule/RecentActivityRule.ts
+++ b/src/Rule/RecentActivityRule.ts
@@ -126,28 +126,7 @@ export class RecentActivityRule extends Rule {
    async process(item: Submission | Comment): Promise<[boolean, RuleResult]> {
        let activities;

-        // ACID is a bitch
-        // reddit may not return the activity being checked in the author's recent history due to availability/consistency issues or *something*
-        // so make sure we add it in if config is checking the same type and it isn't included
-        // TODO refactor this for SubredditState everywhere branch
-        let shouldIncludeSelf = true;
        const strongWindow = windowConfigToWindowCriteria(this.window);
-        const {
-            filterOn: {
-                post: {
-                    subreddits: {
-                        include = [],
-                        exclude = []
-                    } = {},
-                } = {},
-            } = {}
-        } = strongWindow;
-        // typeof x === string -- a patch for now...technically this is all it supports but eventually will need to be able to do any SubredditState
-        if (include.length > 0 && !include.some(x => x.name !== undefined && x.name.toLocaleLowerCase() === item.subreddit.display_name.toLocaleLowerCase())) {
-            shouldIncludeSelf = false;
-        } else if (exclude.length > 0 && exclude.some(x => x.name !== undefined && x.name.toLocaleLowerCase() === item.subreddit.display_name.toLocaleLowerCase())) {
-            shouldIncludeSelf = false;
-        }

        if(strongWindow.fetch === undefined && this.lookAt !== undefined) {
            switch(this.lookAt) {
@@ -159,25 +138,10 @@ export class RecentActivityRule extends Rule {
            }
        }

-        activities = await this.resources.getAuthorActivities(item.author, strongWindow);
-
-        switch (strongWindow.fetch) {
-            case 'comment':
-                if (shouldIncludeSelf && item instanceof Comment && !activities.some(x => x.name === item.name)) {
-                    activities.unshift(item);
-                }
-                break;
-            case 'submission':
-                if (shouldIncludeSelf && item instanceof Submission && !activities.some(x => x.name === item.name)) {
-                    activities.unshift(item);
-                }
-                break;
-            default:
-                if (shouldIncludeSelf && !activities.some(x => x.name === item.name)) {
-                    activities.unshift(item);
-                }
-                break;
-        }
+        // ACID is a bitch
+        // reddit may not return the activity being checked in the author's recent history due to availability/consistency issues or *something*
+        // so add current activity as a prefetched activity and add it to the returned activities (after it goes through filtering)
+        activities = await this.resources.getAuthorActivities(item.author, strongWindow, undefined, [item]);

        let viableActivity = activities;
        // if config does not specify reference then we set the default based on whether the item is a submission or not
--- a/src/Subreddit/Manager.ts
+++ b/src/Subreddit/Manager.ts
@@ -93,8 +93,8 @@ import {EntityRunState} from "../Common/Entities/EntityRunState/EntityRunState";
 import {
    ActivitySourceValue,
    EventRetentionPolicyRange,
-    Invokee,
-    PollOn,
+    Invokee, POLLING_COMMENTS, POLLING_MODQUEUE, POLLING_SUBMISSIONS, POLLING_UNMODERATED,
+    PollOn, pollOnTypes,
    recordOutputTypes,
    RunState
 } from "../Common/Infrastructure/Atomic";
@@ -635,7 +635,7 @@ export class Manager extends EventEmitter implements RunningStates {
            const configBuilder = new ConfigBuilder({logger: this.logger});
            const validJson = configBuilder.validateJson(configObj);
            const {
-                polling = [{pollOn: 'unmoderated', limit: DEFAULT_POLLING_LIMIT, interval: DEFAULT_POLLING_INTERVAL}],
+                polling = [{pollOn: POLLING_SUBMISSIONS, limit: DEFAULT_POLLING_LIMIT, interval: DEFAULT_POLLING_INTERVAL}],
                caching,
                credentials,
                dryRun,
@@ -957,7 +957,7 @@ export class Manager extends EventEmitter implements RunningStates {
        await this.resources.setActivityLastSeenDate(item.name);

        // if modqueue is running then we know we are checking for new reports every X seconds
-        if(options.activitySource.identifier === 'modqueue') {
+        if(options.activitySource.identifier === POLLING_MODQUEUE) {
            // if the activity is from modqueue and only has one report then we know that report was just created
            if(item.num_reports === 1
                // otherwise if it has more than one report AND we have seen it (its only seen if it has already been stored (in below block))
@@ -975,7 +975,7 @@ export class Manager extends EventEmitter implements RunningStates {
        let shouldPersistReports = false;

        if (existingEntity === null) {
-            activityEntity = await Activity.fromSnoowrapActivity(activity, {lastKnownStateTimestamp, db: this.resources.database});
+            activityEntity = Activity.fromSnoowrapActivity(this.managerEntity.subreddit, activity, lastKnownStateTimestamp);
            // always persist if activity is not already persisted and any reports exist
            if (item.num_reports > 0) {
                shouldPersistReports = true;
@@ -1189,7 +1189,7 @@ export class Manager extends EventEmitter implements RunningStates {
                                    // @ts-ignore
                                    const subProxy = await this.client.getSubmission((item as Comment).link_id);
                                    const sub = await this.resources.getActivity(subProxy);
-                                    subActivity = await this.activityRepo.save(await Activity.fromSnoowrapActivity(sub, {db: this.resources.database}));
+                                    subActivity = await this.activityRepo.save(Activity.fromSnoowrapActivity(this.managerEntity.subreddit, sub));
                                }
                                event.activity.submission = subActivity;

@@ -1325,25 +1325,20 @@ export class Manager extends EventEmitter implements RunningStates {
        }
    }

-    isPollingShared(streamName: string): boolean {
+    isPollingShared(streamName: PollOn): boolean {
        const pollOption = this.pollOptions.find(x => x.pollOn === streamName);
-        return pollOption !== undefined && pollOption.limit === DEFAULT_POLLING_LIMIT && pollOption.interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(streamName as PollOn);
+        return pollOption !== undefined && pollOption.limit === DEFAULT_POLLING_LIMIT && pollOption.interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(streamName);
    }

    async buildPolling() {

-        const sources: PollOn[] = ['unmoderated', 'modqueue', 'newComm', 'newSub'];
+        const sources = [...pollOnTypes];

        const subName = this.subreddit.display_name;

        for (const source of sources) {

-            if (!sources.includes(source)) {
-                this.logger.error(`'${source}' is not a valid polling source. Valid sources: unmoderated | modqueue | newComm | newSub`);
-                continue;
-            }
-
-            const pollOpt = this.pollOptions.find(x => x.pollOn.toLowerCase() === source.toLowerCase());
+            const pollOpt = this.pollOptions.find(x => x.pollOn === source);
            if (pollOpt === undefined) {
                if(this.sharedStreamCallbacks.has(source)) {
                    this.logger.debug(`Removing listener for shared polling on ${source.toUpperCase()} because it no longer exists in config`);
@@ -1366,11 +1361,11 @@ export class Manager extends EventEmitter implements RunningStates {
                let modStreamType: string | undefined;

                switch (source) {
-                    case 'unmoderated':
+                    case POLLING_UNMODERATED:
                        if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
-                            modStreamType = 'unmoderated';
+                            modStreamType = POLLING_UNMODERATED;
                            // use default mod stream from resources
-                            stream = this.cacheManager.modStreams.get('unmoderated') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                            stream = this.cacheManager.modStreams.get(POLLING_UNMODERATED) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
                        } else {
                            stream = new UnmoderatedStream(this.client, {
                                subreddit: this.subreddit.display_name,
@@ -1380,11 +1375,11 @@ export class Manager extends EventEmitter implements RunningStates {
                            });
                        }
                        break;
-                    case 'modqueue':
+                    case POLLING_MODQUEUE:
                        if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
-                            modStreamType = 'modqueue';
+                            modStreamType = POLLING_MODQUEUE;
                            // use default mod stream from resources
-                            stream = this.cacheManager.modStreams.get('modqueue') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                            stream = this.cacheManager.modStreams.get(POLLING_MODQUEUE) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
                        } else {
                            stream = new ModQueueStream(this.client, {
                                subreddit: this.subreddit.display_name,
@@ -1394,11 +1389,11 @@ export class Manager extends EventEmitter implements RunningStates {
                            });
                        }
                        break;
-                    case 'newSub':
+                    case POLLING_SUBMISSIONS:
                        if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
-                            modStreamType = 'newSub';
+                            modStreamType = POLLING_SUBMISSIONS;
                            // use default mod stream from resources
-                            stream = this.cacheManager.modStreams.get('newSub') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                            stream = this.cacheManager.modStreams.get(POLLING_SUBMISSIONS) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
                        } else {
                            stream = new SubmissionStream(this.client, {
                                subreddit: this.subreddit.display_name,
@@ -1408,11 +1403,11 @@ export class Manager extends EventEmitter implements RunningStates {
                            });
                        }
                        break;
-                    case 'newComm':
+                    case POLLING_COMMENTS:
                        if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
-                            modStreamType = 'newComm';
+                            modStreamType = POLLING_COMMENTS;
                            // use default mod stream from resources
-                            stream = this.cacheManager.modStreams.get('newComm') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                            stream = this.cacheManager.modStreams.get(POLLING_COMMENTS) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
                        } else {
                            stream = new CommentStream(this.client, {
                                subreddit: this.subreddit.display_name,
@@ -1422,6 +1417,8 @@ export class Manager extends EventEmitter implements RunningStates {
                            });
                        }
                        break;
+                    default:
+                        throw new CMError(`This shouldn't happen! All polling sources are enumerated in switch. Source value: ${source}`)
                }

                if (stream === undefined) {
@@ -1514,10 +1511,10 @@ export class Manager extends EventEmitter implements RunningStates {
    }

    noChecksWarning = (source: PollOn) => (listing: any) => {
-        if (this.commentChecks.length === 0 && ['modqueue', 'newComm'].some(x => x === source)) {
+        if (this.commentChecks.length === 0 && [POLLING_MODQUEUE, POLLING_COMMENTS].some(x => x === source)) {
            this.logger.warn(`Polling '${source.toUpperCase()}' may return Comments but no comments checks were configured.`);
        }
-        if (this.submissionChecks.length === 0 && ['unmoderated', 'modqueue', 'newSub'].some(x => x === source)) {
+        if (this.submissionChecks.length === 0 && [POLLING_UNMODERATED, POLLING_MODQUEUE, POLLING_SUBMISSIONS].some(x => x === source)) {
            this.logger.warn(`Polling '${source.toUpperCase()}' may return Submissions but no submission checks were configured.`);
        }
    }
@@ -1670,7 +1667,7 @@ export class Manager extends EventEmitter implements RunningStates {
            }
            this.startedAt = dayjs();

-            const modQueuePollOpts = this.pollOptions.find(x => x.pollOn === 'modqueue');
+            const modQueuePollOpts = this.pollOptions.find(x => x.pollOn === POLLING_MODQUEUE);
            if(modQueuePollOpts !== undefined) {
                this.modqueueInterval = modQueuePollOpts.interval;
            }
--- a/src/Subreddit/Streams.ts
+++ b/src/Subreddit/Streams.ts
@@ -7,6 +7,7 @@ import {mergeArr, parseDuration, random} from "../util";
 import { Logger } from "winston";
 import {ErrorWithCause} from "pony-cause";
 import dayjs, {Dayjs as DayjsObj} from "dayjs";
+import { FixedSizeList } from 'fixed-size-list'

 type Awaitable<T> = Promise<T> | T;

@@ -14,10 +15,12 @@ interface RCBPollingOptions<T> extends SnooStormOptions {
    subreddit: string,
    enforceContinuity?: boolean
    logger: Logger
+    sort?: string
    name?: string,
-    processed?: Set<T[keyof T]>
+    processed?: FixedSizeList<T[keyof T]>
    label?: string
    dateCutoff?: boolean
+    maxHistory?: number
 }

 interface RCBPollConfiguration<T> extends PollConfiguration<T>,RCBPollingOptions<T> {
@@ -40,6 +43,9 @@ export class SPoll<T extends RedditContent<object>> extends Poll<T> {
    name: string = 'Reddit Stream';
    logger: Logger;
    subreddit: string;
+    // using a fixed sized "regular" array means slightly more memory usage vs. a Set when holding N items
+    // BUT now we can limit N items to something reasonable instead of having a crazy big Set with all items seen since stream was started
+    processedBuffer: FixedSizeList<T[keyof T]>;

    constructor(options: RCBPollConfiguration<T>) {
        super(options);
@@ -54,6 +60,7 @@ export class SPoll<T extends RedditContent<object>> extends Poll<T> {
            label = 'Polling',
            processed,
            dateCutoff,
+            maxHistory = 300,
        } = options;
        this.subreddit = subreddit;
        this.name = name !== undefined ? name : this.name;
@@ -67,8 +74,10 @@ export class SPoll<T extends RedditContent<object>> extends Poll<T> {
        // if we pass in processed on init the intention is to "continue" from where the previous stream left off
        // WITHOUT new start behavior
        if (processed !== undefined) {
-            this.processed = processed;
+            this.processedBuffer = processed;
            this.newStart = false;
+        } else {
+            this.processedBuffer = new FixedSizeList<T[keyof T]>(maxHistory);
        }

        clearInterval(this.interval);
@@ -97,14 +106,14 @@ export class SPoll<T extends RedditContent<object>> extends Poll<T> {
                        }
                        for (const item of batch) {
                            const id = item[self.identifier];
-                            if (self.processed.has(id)) {
+                            if (self.processedBuffer.data.some(x => x === id)) {
                                anyAlreadySeen = true;
                                continue;
                            }

                            // add new item to list and set as processed
                            newItems.push(item);
-                            self.processed.add(id);
+                            self.processedBuffer.add(id);
                        }
                        page++;
                    }
--- a/src/Subreddit/SubredditResources.ts
+++ b/src/Subreddit/SubredditResources.ts
@@ -69,16 +69,7 @@ import {cacheTTLDefaults, createHistoricalDisplayDefaults,} from "../Common/defa
 import {ExtendedSnoowrap} from "../Utils/SnoowrapClients";
 import dayjs, {Dayjs} from "dayjs";
 import ImageData from "../Common/ImageData";
-import {
-    Between, Brackets,
-    DataSource,
-    DeleteQueryBuilder,
-    In,
-    LessThan,
-    NotBrackets,
-    Repository,
-    SelectQueryBuilder
-} from "typeorm";
+import {Between, DataSource, DeleteQueryBuilder, LessThan, Repository, SelectQueryBuilder} from "typeorm";
 import {CMEvent as ActionedEventEntity, CMEvent} from "../Common/Entities/CMEvent";
 import {RuleResultEntity} from "../Common/Entities/RuleResultEntity";
 import globrex from 'globrex';
@@ -171,8 +162,6 @@ import {ActivitySource} from "../Common/ActivitySource";
 import {SubredditResourceOptions} from "../Common/Subreddit/SubredditResourceInterfaces";
 import {SubredditStats} from "./Stats";
 import {CMCache} from "../Common/Cache";
-import { Activity } from '../Common/Entities/Activity';
-import {FindOptionsWhere} from "typeorm/find-options/FindOptionsWhere";

 export const DEFAULT_FOOTER = '\r\n*****\r\nThis action was performed by [a bot.]({{botLink}}) Mention a moderator or [send a modmail]({{modmailLink}}) if you have any ideas, questions, or concerns about this action.';

@@ -204,7 +193,6 @@ export class SubredditResources {
    botAccount?: string;
    dispatchedActivityRepo: Repository<DispatchedEntity>
    activitySourceRepo: Repository<ActivitySourceEntity>
-    activityRepo: Repository<Activity>
    retention?: EventRetentionPolicyRange
    managerEntity: ManagerEntity
    botEntity: Bot
@@ -241,7 +229,6 @@ export class SubredditResources {
        this.database = database;
        this.dispatchedActivityRepo = this.database.getRepository(DispatchedEntity);
        this.activitySourceRepo = this.database.getRepository(ActivitySourceEntity);
-        this.activityRepo = this.database.getRepository(Activity);
        this.retention = retention;
        //this.prefix = prefix;
        this.client = client;
@@ -417,25 +404,21 @@ export class SubredditResources {
                    }
                },
                relations: {
-                    manager: true,
-                    activity: {
-                        submission: true
-                    }
+                    manager: true
                }
            });
            const now = dayjs();
-            const toRemove = [];
            for(const dAct of dispatchedActivities) {
                const shouldDispatchAt = dAct.createdAt.add(dAct.delay.asSeconds(), 'seconds');
                let tardyHint = '';
                if(shouldDispatchAt.isBefore(now)) {
-                    let tardyHint = `Activity ${dAct.activity.id} queued at ${dAct.createdAt.format('YYYY-MM-DD HH:mm:ssZ')} for ${dAct.delay.humanize()} is now LATE`;
+                    let tardyHint = `Activity ${dAct.activityId} queued at ${dAct.createdAt.format('YYYY-MM-DD HH:mm:ssZ')} for ${dAct.delay.humanize()} is now LATE`;
                    if(dAct.tardyTolerant === true) {
                        tardyHint += ` but was configured as ALWAYS 'tardy tolerant' so will be dispatched immediately`;
                    } else if(dAct.tardyTolerant === false) {
                        tardyHint += ` and was not configured as 'tardy tolerant' so will be dropped`;
                        this.logger.warn(tardyHint);
-                        toRemove.push(dAct.id);
+                        await this.removeDelayedActivity(dAct.id);
                        continue;
                    } else {
                        // see if its within tolerance
@@ -443,7 +426,7 @@ export class SubredditResources {
                        if(latest.isBefore(now)) {
                            tardyHint += ` and IS NOT within tardy tolerance of ${dAct.tardyTolerant.humanize()} of planned dispatch time so will be dropped`;
                            this.logger.warn(tardyHint);
-                            toRemove.push(dAct.id);
+                            await this.removeDelayedActivity(dAct.id);
                            continue;
                        } else {
                            tardyHint += `but is within tardy tolerance of ${dAct.tardyTolerant.humanize()} of planned dispatch time so will be dispatched immediately`;
@@ -456,115 +439,27 @@ export class SubredditResources {
                try {
                    this.delayedItems.push(await dAct.toActivityDispatch(this.client))
                } catch (e) {
-                    this.logger.warn(new ErrorWithCause(`Unable to add Activity ${dAct.activity.id} from database delayed activities to in-app delayed activities queue`, {cause: e}));
+                    this.logger.warn(new ErrorWithCause(`Unable to add Activity ${dAct.activityId} from database delayed activities to in-app delayed activities queue`, {cause: e}));
                }
            }
-            if(toRemove.length > 0) {
-                await this.removeDelayedActivity(toRemove);
-            }
        }
    }

    async addDelayedActivity(data: ActivityDispatch) {
-        // TODO merge this with getActivity or something...
-        if(asComment(data.activity)) {
-            const existingSub =  await this.activityRepo.findOneBy({_id: data.activity.link_id});
-            if(existingSub === null) {
-                const sub = await this.getActivity(new Submission({name: data.activity.link_id}, this.client, false));
-                await this.activityRepo.save(await Activity.fromSnoowrapActivity(sub, {db: this.database}));
-            }
-        }
-        const dEntity = await this.dispatchedActivityRepo.save(new DispatchedEntity({...data, manager: this.managerEntity, activity: await Activity.fromSnoowrapActivity(data.activity, {db: this.database})}));
+        const dEntity = await this.dispatchedActivityRepo.save(new DispatchedEntity({...data, manager: this.managerEntity}));
        data.id = dEntity.id;
        this.delayedItems.push(data);
    }

    async removeDelayedActivity(val?: string | string[]) {
-        let dispatched: DispatchedEntity[] = [];
-        const where: FindOptionsWhere<DispatchedEntity> = {
-            manager: {
-                id: this.managerEntity.id
-            }
-        };
-
-        if(val !== undefined) {
-            const ids = typeof val === 'string' ? [val] : val;
-            where.id = In(ids);
-        }
-
-        dispatched = await this.dispatchedActivityRepo.find({
-            where,
-            relations: {
-                manager: true,
-                activity: {
-                    actionedEvents: true,
-                    submission: {
-                        actionedEvents: true
-                    }
-                }
-            }
-        });
-
-        const actualDispatchedIds = dispatched.map(x => x.id);
-        this.logger.debug(`${actualDispatchedIds.length} marked for deletion`, {leaf: 'Delayed Activities'});
-
-        // get potential activities to delete
-        // but only include activities that don't have any actionedEvents
-        let activityIdsToDelete = Array.from(dispatched.reduce((acc, curr) => {
-            if(curr.activity.actionedEvents === null || curr.activity.actionedEvents.length === 0) {
-                acc.add(curr.activity.id);
-            }
-            if(curr.activity.submission !== undefined && curr.activity.submission !== null) {
-                if(curr.activity.submission.actionedEvents === null || curr.activity.submission.actionedEvents.length === 0) {
-                    acc.add(curr.activity.submission.id);
-                }
-            }
-            return acc;
-        }, new Set<string>()));
-        const rawActCount = activityIdsToDelete.length;
-        let activeActCount = 0;
-
-        // if we have any potential activities to delete we now need to get any dispatched actions that reference these activities
-        // that are NOT the ones we are going to delete
-        if(activityIdsToDelete.length > 0) {
-            const activeDispatchedQuery = this.dispatchedActivityRepo.createQueryBuilder('dis')
-                .leftJoinAndSelect('dis.activity', 'activity')
-                .leftJoinAndSelect('activity.submission', 'submission')
-                .where(new NotBrackets((qb) => {
-                    qb.where('dis.id IN (:...currIds)', {currIds: actualDispatchedIds});
-                }))
-                .andWhere(new Brackets((qb) => {
-                    qb.where('activity._id IN (:...actMainIds)', {actMainIds: activityIdsToDelete})
-                    qb.orWhere('submission._id IN (:...actSubIds)', {actSubIds: activityIdsToDelete})
-                }));
-            //const sql = activeDispatchedQuery.getSql();
-            const activeDispatched = await activeDispatchedQuery.getMany();
-
-            // all activity ids, from the actions to delete, that are being used by dispatched actions that are NOT the ones we are going to delete
-            const activeDispatchedIds = Array.from(activeDispatched.reduce((acc, curr) => {
-                acc.add(curr.activity.id);
-                if(curr.activity.submission !== undefined && curr.activity.submission !== null) {
-                    acc.add(curr.activity.submission.id);
-                }
-                return acc;
-            }, new Set<string>()));
-            activeActCount = activeDispatchedIds.length;
-
-            // filter out any that are still in use
-            activityIdsToDelete = activityIdsToDelete.filter(x => !activeDispatchedIds.includes(x));
-        }
-
-        this.logger.debug(`Marked ${activityIdsToDelete.length} Activities created, by Delayed, for deletion (${rawActCount} w/o Events | ${activeActCount} used by other Delayed Activities)`, {leaf: 'Delayed Activities'});
-
-        if(actualDispatchedIds.length > 0) {
-            await this.dispatchedActivityRepo.delete(actualDispatchedIds);
+        if(val === undefined) {
+            await this.dispatchedActivityRepo.delete({manager: {id: this.managerEntity.id}});
+            this.delayedItems = [];
        } else {
-            this.logger.warn('No dispatched ids found to delete');
+            const ids = typeof val === 'string' ? [val] : val;
+            await this.dispatchedActivityRepo.delete(ids);
+            this.delayedItems = this.delayedItems.filter(x => !ids.includes(x.id));
        }
-        if(activityIdsToDelete.length > 0) {
-            await this.activityRepo.delete(activityIdsToDelete);
-        }
-        this.delayedItems = this.delayedItems.filter(x => !actualDispatchedIds.includes(x.id));
    }

    async initStats() {
@@ -1135,13 +1030,13 @@ export class SubredditResources {
        }
    }

-    async getAuthorActivities(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing): Promise<SnoowrapActivity[]> {
+    async getAuthorActivities(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing, prefetchedActivities?: SnoowrapActivity[]): Promise<SnoowrapActivity[]> {

-        const {post} = await this.getAuthorActivitiesWithFilter(user, options, customListing);
+        const {post} = await this.getAuthorActivitiesWithFilter(user, options, customListing, prefetchedActivities);
        return post;
    }

-    async getAuthorActivitiesWithFilter(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing): Promise<FetchedActivitiesResult> {
+    async getAuthorActivitiesWithFilter(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing, prefetchedActivities?: SnoowrapActivity[]): Promise<FetchedActivitiesResult> {
        let listFuncName: string;
        let listFunc: ListingFunc;

@@ -1169,21 +1064,21 @@ export class SubredditResources {
            ...(cloneDeep(options)),
        }

-        return await this.getActivities(user, criteriaWithDefaults, {func: listFunc, name: listFuncName});
+        return await this.getActivities(user, criteriaWithDefaults, {func: listFunc, name: listFuncName}, prefetchedActivities);
    }

-    async getAuthorComments(user: RedditUser, options: ActivityWindowCriteria): Promise<Comment[]> {
-        return await this.getAuthorActivities(user, {...options, fetch: 'comment'}) as unknown as Promise<Comment[]>;
+    async getAuthorComments(user: RedditUser, options: ActivityWindowCriteria, prefetchedActivities?: SnoowrapActivity[]): Promise<Comment[]> {
+        return await this.getAuthorActivities(user, {...options, fetch: 'comment'}, undefined, prefetchedActivities) as unknown as Promise<Comment[]>;
    }

-    async getAuthorSubmissions(user: RedditUser, options: ActivityWindowCriteria): Promise<Submission[]> {
+    async getAuthorSubmissions(user: RedditUser, options: ActivityWindowCriteria, prefetchedActivities?: SnoowrapActivity[]): Promise<Submission[]> {
        return await this.getAuthorActivities(user, {
            ...options,
            fetch: 'submission'
-        }) as unknown as Promise<Submission[]>;
+        }, undefined,prefetchedActivities) as unknown as Promise<Submission[]>;
    }

-    async getActivities(user: RedditUser, options: ActivityWindowCriteria, listingData: NamedListing): Promise<FetchedActivitiesResult> {
+    async getActivities(user: RedditUser, options: ActivityWindowCriteria, listingData: NamedListing, prefetchedActivities: SnoowrapActivity[] = []): Promise<FetchedActivitiesResult> {

        try {

@@ -1318,12 +1213,24 @@ export class SubredditResources {
                    }
                }

-                let unFilteredItems: SnoowrapActivity[] | undefined;
-
+                let preFilteredPrefetchedActivities = [...prefetchedActivities];
+                if(preFilteredPrefetchedActivities.length > 0) {
+                    switch(options.fetch) {
+                        // TODO this may not work if using a custom listingFunc that does not include fetch type
+                        case 'comment':
+                            preFilteredPrefetchedActivities = preFilteredPrefetchedActivities.filter(x => asComment(x));
+                            break;
+                        case 'submission':
+                            preFilteredPrefetchedActivities = preFilteredPrefetchedActivities.filter(x => asSubmission(x));
+                            break;
+                    }
+                    preFilteredPrefetchedActivities = await this.filterListingWithHistoryOptions(preFilteredPrefetchedActivities, user, options.filterOn?.pre);
+                }
+                let unFilteredItems: SnoowrapActivity[] | undefined = [...preFilteredPrefetchedActivities];
+                pre = pre.concat(preFilteredPrefetchedActivities);

                const { func: listingFunc } = listingData;

-
                let listing = await listingFunc(getAuthorHistoryAPIOptions(options));
                let hitEnd = false;
                let offset = chunkSize;
@@ -1333,6 +1240,9 @@ export class SubredditResources {
                        timeOk = false;

                    let listSlice = listing.slice(offset - chunkSize);
+                    // filter out any from slice that were already included from prefetched list so that prefetched aren't included twice
+                    listSlice = preFilteredPrefetchedActivities.length === 0 ? listSlice : listSlice.filter(x => !preFilteredPrefetchedActivities.some(y => y.name === x.name));
+
                    let preListSlice = await this.filterListingWithHistoryOptions(listSlice, user, options.filterOn?.pre);

                    // its more likely the time criteria is going to be hit before the count criteria
@@ -1607,6 +1517,7 @@ export class SubredditResources {
            usernotes,
            ruleResults,
            actionResults,
+            author: (val) => this.getAuthor(val)
        });
    }

--- a/src/Utils/SnoowrapUtils.ts
+++ b/src/Utils/SnoowrapUtils.ts
@@ -133,6 +133,7 @@ export interface TemplateContext {
    ruleResults?: RuleResultEntity[]
    actionResults?: ActionResultEntity[]
    activity?: SnoowrapActivity
+    author?: (val: string | RedditUser) => Promise<RedditUser>
    [key: string]: any
 }

@@ -140,11 +141,25 @@ export const renderContent = async (template: string, data: TemplateContext = {}
    const {
        usernotes,
        ruleResults,
+        author,
        actionResults,
        activity,
        ...restContext
    } = data;

+    let fetchedUser: RedditUser | undefined;
+    // @ts-ignore
+    const user = async (): Promise<RedditUser> => {
+        if(fetchedUser === undefined) {
+            if(author !== undefined) {
+                // @ts-ignore
+                fetchedUser = await author(activity.author);
+            }
+        }
+        // @ts-ignore
+        return fetchedUser;
+    }
+
    let view: GenericContentTemplateData = {
        botLink: BOT_LINK,
        ...restContext
@@ -171,10 +186,24 @@ export const renderContent = async (template: string, data: TemplateContext = {}

        view.modmailLink = `https://www.reddit.com/message/compose?to=%2Fr%2F${subreddit}&message=${encodeURIComponent(permalink)}`;

+        const author: any = {
+            toString: () => getActivityAuthorName(activity.author)
+        };
+
+        if(template.includes('{{item.author.')) {
+            // @ts-ignore
+            const auth = await user();
+
+            author.age = dayjs.unix(auth.created).fromNow(true);
+            author.linkKarma = auth.link_karma;
+            author.commentKarma = auth.comment_karma;
+            author.totalKarma = auth.comment_karma + auth.link_karma;
+            author.verified = auth.has_verified_email;
+        }
+
        const templateData: any = {
            kind: activity instanceof Submission ? 'submission' : 'comment',
-            // @ts-ignore
-            author: getActivityAuthorName(await activity.author),
+            author,
            votes: activity.score,
            age: dayjs.duration(dayjs().diff(dayjs.unix(activity.created))).humanize(),
            permalink,
--- a/src/Web/assets/views/status.ejs
+++ b/src/Web/assets/views/status.ejs
@@ -1297,7 +1297,7 @@
                    const durationDayjs = dayjs.duration(x.duration, 'seconds');
                    const durationDisplay = durationDayjs.humanize();
                    const cancelLink = `<a href="#" data-id="${x.id}" data-subreddit="${x.subreddit}" class="delayCancel">CANCEL</a>`;
-                    return `<div>A <a href="https://reddit.com${x.permalink}">${x.submissionId !== undefined ? 'Comment' : 'Submission'}</a> by <a href="https://reddit.com/u/${x.author}">${x.author}</a>${isAll ? `, dispatched in <a href="https://reddit.com${x.subreddit}">${x.subreddit}</a> ,` : ''} queued by ${x.source} at ${queuedAtDisplay} for ${durationDisplay} (dispatches ${durationUntilNow.humanize(true)}) -- ${cancelLink}</div>`;
+                    return `<div>A <a href="https://reddit.com${x.permalink}">${x.submissionId !== undefined ? 'Comment' : 'Submission'}</a>${isAll ? ` in <a href="https://reddit.com${x.subreddit}">${x.subreddit}</a> ` : ''} by <a href="https://reddit.com/u/${x.author}">${x.author}</a> queued by ${x.source} at ${queuedAtDisplay} for ${durationDisplay} (dispatches ${durationUntilNow.humanize(true)}) -- ${cancelLink}</div>`;
                });
                //let sub = resp.name;
                if(sub === 'All') {
--- a/src/util.ts
+++ b/src/util.ts
@@ -77,6 +77,7 @@ import {
    ImageHashCacheData,
    ModUserNoteLabel,
    modUserNoteLabels,
+    PollOn, pollOnTypeMapping, pollOnTypes,
    RedditEntity,
    RedditEntityType,
    RelativeDateTimeMatch,
@@ -2855,7 +2856,7 @@ export const generateSnoowrapEntityFromRedditThing = (data: RedditThing, client:
        case 'user':
            return new RedditUser({id: data.val}, client, false);
        case 'subreddit':
-            return new Subreddit({name: data.val}, client, false);
+            return new Subreddit({id: data.val}, client, false);
        case 'message':
            return new PrivateMessage({id: data.val}, client, false)

@@ -3088,3 +3089,12 @@ export const toStrongSharingACLConfig = (data: SharingACLConfig | string[]): Str
        exclude: (data.exclude ?? []).map(x => parseStringToRegexOrLiteralSearch(x))
    }
 }
+
+export const toPollOn = (val: string | PollOn): PollOn => {
+    const clean = val.toLowerCase().trim();
+    const pVal = pollOnTypeMapping.get(clean);
+    if(pVal !== undefined) {
+        return pVal;
+    }
+    throw new SimpleError(`'${val}' is not a valid polling source. Valid sources: ${pollOnTypes.join(' | ')}`);
+}
Author	SHA1	Message	Date
FoxxMD	14a317ace4	Use fixed array instead of Set to maybe reduce memory usage	2022-11-21 15:56:56 -05:00
FoxxMD	e79779d980	feat: Implement templating for flair actions	2022-11-21 11:43:34 -05:00
FoxxMD	b094b72d4a	docs: Update docker compose instructions * Specify docker-compose minimum version * Change commands to use new syntax	2022-11-21 11:29:14 -05:00
FoxxMD	d90e88360d	feat: Add some author properties for templating	2022-11-17 13:16:10 -05:00
FoxxMD	9031f7fec8	refactor(polling): Improve resilience for polling source parsing * Replace hard-coded polling sources with string constants * Implement string to PollOn parsing which is case-insensitive and forgives mispelling * Check that manager specifies only one of each polling source type when build config	2022-11-17 12:03:55 -05:00
FoxxMD	0a2b13e4c4	fix: Fix including self activities in Recent Activity without filtering * Consolidate ACID check for author history results into authorActivities function so it can be used everywhere * Remove self check in Recent Activity and used consolidated functionality so that filtering still occurs on current activity	2022-11-16 10:18:37 -05:00