fix(delayed): Prevent delete call when no ids found

Make sure we delete Activities that were inserted on dispatched actions BUT ONLY if they are not used anywhere else (events or other delayed activities)

docker-compose.yml

@@ -24,9 +24,6 @@ services:
 
   cache:
     image: 'redis:7-alpine'
-    volumes:
-      # on linux will need to make sure this directory has correct permissions for container to access
-      - './data/cache:/data'
 
   database:
     image: 'mariadb:10.9.3'

docs/operator/configuration.md

@@ -36,7 +36,7 @@ configuration.
 * **FILE** -- Values specified in a YAML/JSON configuration file using the structure [in the schema](https://json-schema.app/view/%23?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Fcontext-mod%2Fmaster%2Fsrc%2FSchema%2FOperatorConfig.json)
   * When reading the **schema** if the variable is available at a level of configuration other than **FILE** it will be
     noted with the same symbol as above. The value shown is the default.
-* **ARG** -- Values specified as CLI arguments to the program (see [CLI Usage](#cli-usage) below)
+* **ARG** -- Values specified as CLI arguments to the program (see [ClI Usage](#cli-usage) below)
 
 ## File Configuration (Recommended)
 

docs/operator/installation.md

@@ -55,8 +55,6 @@ The included [`docker-compose.yml`](/docker-compose.yml) provides production-rea
 
 #### Setup
 
-The included `docker-compose.yml` file is written for **Docker Compose v2.**
-
 For new installations copy [`config.yaml`](/docker/config/docker-compose/config.yaml) into a folder named `data` in the same folder `docker-compose.yml` will be run from. For users migrating their existing CM instances to docker-compose, copy your existing `config.yaml` into the same `data` folder.
 
 Read through the comments in both `docker-compose.yml` and `config.yaml` and makes changes to any relevant settings (passwords, usernames, etc...). Ensure that any settings used in both files (EX mariaDB passwords) match.
@@ -64,13 +62,13 @@ Read through the comments in both `docker-compose.yml` and `config.yaml` and mak
 To build and start CM:
 
 ```bash
-docker compose up -d
+docker-compose up -d
 ```
 
 To include Grafana/Influx dependencies run:
 
 ```bash
-docker compose --profile full up -d
+docker-compose --profile full up -d
 ```
 
 ## Locally

docs/subreddit-configuration/actionTemplating.md

@@ -57,29 +57,6 @@ All Actions with `content` have access to this data:
 | `title`      | As comments => the body of the comment. As Submission => title                                      | Test post please ignore                                  |
 | `shortTitle` | The same as `title` but truncated to 15 characters                                                  | test post pleas...                                       |
 
-#### Common Author
-
-Additionally, `author` has these properties accessible:
-
-|      Name      |             Description             | Example  |
-|----------------|-------------------------------------|----------|
-| `age`          | (Approximate) Age of account        | 3 months |
-| `linkKarma`    | Amount of link karma                | 10       |
-| `commentKarma` | Amount of comment karma             | 3        |
-| `totalKarma`   | Combined link+comment karma         | 13       |
-| `verified`     | Does account have a verified email? | true     |
-
-NOTE: Accessing these properties may require an additional API call so use sparingly on high-volume comments
-
-##### Example Usage
-
-```
-The user {{item.author}} has been a redditor for {{item.author.age}}
-```
-Produces:
-
-> The user FoxxMD has been a redditor for 3 months
-
 ### Submissions
 
 If the **Activity** is a Submission these additional properties are accessible:

docs/subreddit-configuration/in-depth/author/authorFilter.json5

@@ -40,7 +40,7 @@
                   // for this to pass the Author of the Submission must not have the flair "Supreme Memer" and have the name "user1" or "user2"
                   {
                     "flairText": ["Supreme Memer"],
-                    "name": ["user1","user2"]
+                    "names": ["user1","user2"]
                   },
                   {
                     // for this to pass the Author of the Submission must not have the flair "Decent Memer"

docs/subreddit-configuration/in-depth/author/authorFilter.yaml

@@ -30,7 +30,7 @@ runs:
                 # for this to pass the Author of the Submission must not have the flair "Supreme Memer" and have the name "user1" or "user2"
                 - flairText:
                     - Supreme Memer
-                  name:
+                  names:
                     - user1
                     - user2
                   # for this to pass the Author of the Submission must not have the flair "Decent Memer"

src/Action/SubmissionAction/FlairAction.ts

@@ -30,25 +30,30 @@ export class FlairAction extends Action {
     async process(item: Comment | Submission, ruleResults: RuleResultEntity[], actionResults: ActionResultEntity[], options: runCheckOptions): Promise<ActionProcessResult> {
         const dryRun = this.getRuntimeAwareDryrun(options);
         let flairParts = [];
-        const renderedText = this.text === '' ? '' : await this.renderContent(this.text, item, ruleResults, actionResults) as string;
-        flairParts.push(`Text: ${renderedText === '' ? '(None)' : renderedText}`);
-
-        const renderedCss = this.css === '' ? '' : await this.renderContent(this.css, item, ruleResults, actionResults) as string;
-        flairParts.push(`CSS: ${renderedCss === '' ? '(None)' : renderedCss}`);
-
-        flairParts.push(`Template: ${this.flair_template_id === '' ? '(None)' : this.flair_template_id}`);
-
+        if(this.text !== '') {
+            flairParts.push(`Text: ${this.text}`);
+        }
+        if(this.css !== '') {
+            flairParts.push(`CSS: ${this.css}`);
+        }
+        if(this.flair_template_id !== '') {
+            flairParts.push(`Template: ${this.flair_template_id}`);
+        }
         const flairSummary = flairParts.length === 0 ? 'No flair (unflaired)' : flairParts.join(' | ');
         this.logger.verbose(flairSummary);
         if (item instanceof Submission) {
             if(!this.dryRun) {
                 if (this.flair_template_id) {
-                    await item.selectFlair({flair_template_id: this.flair_template_id}).then(() => {});
+                    // typings are wrong for this function, flair_template_id should be accepted
+                    // assignFlair uses /api/flair (mod endpoint)
+                    // selectFlair uses /api/selectflair (self endpoint for user to choose their own flair for submission)
+                    // @ts-ignore
+                    await item.assignFlair({flair_template_id: this.flair_template_id}).then(() => {});
                     item.link_flair_template_id = this.flair_template_id;
                 } else {
-                    await item.assignFlair({text: renderedText, cssClass: renderedCss}).then(() => {});
-                    item.link_flair_css_class = renderedCss;
-                    item.link_flair_text = renderedText;
+                    await item.assignFlair({text: this.text, cssClass: this.css}).then(() => {});
+                    item.link_flair_css_class = this.css;
+                    item.link_flair_text = this.text;
                 }
                 await this.resources.resetCacheForItem(item);
             }

src/Action/UserFlairAction.ts

@@ -26,8 +26,6 @@ export class UserFlairAction extends Action {
   async process(item: Comment | Submission, ruleResults: RuleResultEntity[], actionResults: ActionResultEntity[], options: runCheckOptions): Promise<ActionProcessResult> {
     const dryRun = this.getRuntimeAwareDryrun(options);
     let flairParts = [];
-    let renderedText: string | undefined = undefined;
-    let renderedCss: string | undefined = undefined;
 
     if (this.flair_template_id !== undefined) {
       flairParts.push(`Flair template ID: ${this.flair_template_id}`)
@@ -36,12 +34,10 @@ export class UserFlairAction extends Action {
       }
     } else {
       if (this.text !== undefined) {
-        renderedText = await this.renderContent(this.text, item, ruleResults, actionResults) as string;
-        flairParts.push(`Text: ${renderedText}`);
+        flairParts.push(`Text: ${this.text}`);
       }
       if (this.css !== undefined) {
-        renderedCss = await this.renderContent(this.css, item, ruleResults, actionResults) as string;
-        flairParts.push(`CSS: ${renderedCss}`);
+        flairParts.push(`CSS: ${this.css}`);
       }
     }
 
@@ -62,7 +58,7 @@ export class UserFlairAction extends Action {
           this.logger.error('Either the flair template ID is incorrect or you do not have permission to access it.');
           throw err;
         }
-      } else if (renderedText === undefined && renderedCss === undefined) {
+      } else if (this.text === undefined && this.css === undefined) {
         // @ts-ignore
         await item.subreddit.deleteUserFlair(item.author.name);
         item.author_flair_css_class = null;
@@ -72,11 +68,11 @@ export class UserFlairAction extends Action {
         // @ts-ignore
         await item.author.assignFlair({
           subredditName: item.subreddit.display_name,
-          cssClass: renderedCss,
-          text: renderedText,
+          cssClass: this.css,
+          text: this.text,
         });
-        item.author_flair_text = renderedText ?? null;
-        item.author_flair_css_class = renderedCss ?? null;
+        item.author_flair_text = this.text ?? null;
+        item.author_flair_css_class = this.css ?? null;
       }
       await this.resources.resetCacheForItem(item);
       if(typeof item.author !== 'string') {

src/Bot/index.ts

@@ -46,13 +46,7 @@ import {RunStateType} from "../Common/Entities/RunStateType";
 import {QueueRunState} from "../Common/Entities/EntityRunState/QueueRunState";
 import {EventsRunState} from "../Common/Entities/EntityRunState/EventsRunState";
 import {ManagerRunState} from "../Common/Entities/EntityRunState/ManagerRunState";
-import {
-    Invokee,
-    POLLING_COMMENTS, POLLING_MODQUEUE,
-    POLLING_SUBMISSIONS,
-    POLLING_UNMODERATED,
-    PollOn
-} from "../Common/Infrastructure/Atomic";
+import {Invokee, PollOn} from "../Common/Infrastructure/Atomic";
 import {FilterCriteriaDefaults} from "../Common/Infrastructure/Filters/FilterShapes";
 import {snooLogWrapper} from "../Utils/loggerFactory";
 import {InfluxClient} from "../Common/Influx/InfluxClient";
@@ -564,9 +558,9 @@ class Bot implements BotInstanceFunctions {
 
     parseSharedStreams() {
 
-        const sharedCommentsSubreddits = !this.sharedStreams.includes(POLLING_COMMENTS) ? [] : this.subManagers.filter(x => x.isPollingShared(POLLING_COMMENTS)).map(x => x.subreddit.display_name);
+        const sharedCommentsSubreddits = !this.sharedStreams.includes('newComm') ? [] : this.subManagers.filter(x => x.isPollingShared('newComm')).map(x => x.subreddit.display_name);
         if (sharedCommentsSubreddits.length > 0) {
-            const stream = this.cacheManager.modStreams.get(POLLING_COMMENTS);
+            const stream = this.cacheManager.modStreams.get('newComm');
             if (stream === undefined || stream.subreddit !== sharedCommentsSubreddits.join('+')) {
                 let processed;
                 if (stream !== undefined) {
@@ -586,20 +580,20 @@ class Bot implements BotInstanceFunctions {
                     label: 'Shared Polling'
                 });
                 // @ts-ignore
-                defaultCommentStream.on('error', this.createSharedStreamErrorListener(POLLING_COMMENTS));
-                defaultCommentStream.on('listing', this.createSharedStreamListingListener(POLLING_COMMENTS));
-                this.cacheManager.modStreams.set(POLLING_COMMENTS, defaultCommentStream);
+                defaultCommentStream.on('error', this.createSharedStreamErrorListener('newComm'));
+                defaultCommentStream.on('listing', this.createSharedStreamListingListener('newComm'));
+                this.cacheManager.modStreams.set('newComm', defaultCommentStream);
             }
         } else {
-            const stream = this.cacheManager.modStreams.get(POLLING_COMMENTS);
+            const stream = this.cacheManager.modStreams.get('newComm');
             if (stream !== undefined) {
                 stream.end('Determined no managers are listening on shared stream parsing');
             }
         }
 
-        const sharedSubmissionsSubreddits = !this.sharedStreams.includes(POLLING_SUBMISSIONS) ? [] : this.subManagers.filter(x => x.isPollingShared(POLLING_SUBMISSIONS)).map(x => x.subreddit.display_name);
+        const sharedSubmissionsSubreddits = !this.sharedStreams.includes('newSub') ? [] : this.subManagers.filter(x => x.isPollingShared('newSub')).map(x => x.subreddit.display_name);
         if (sharedSubmissionsSubreddits.length > 0) {
-            const stream = this.cacheManager.modStreams.get(POLLING_SUBMISSIONS);
+            const stream = this.cacheManager.modStreams.get('newSub');
             if (stream === undefined || stream.subreddit !== sharedSubmissionsSubreddits.join('+')) {
                 let processed;
                 if (stream !== undefined) {
@@ -619,19 +613,19 @@ class Bot implements BotInstanceFunctions {
                     label: 'Shared Polling'
                 });
                 // @ts-ignore
-                defaultSubStream.on('error', this.createSharedStreamErrorListener(POLLING_SUBMISSIONS));
-                defaultSubStream.on('listing', this.createSharedStreamListingListener(POLLING_SUBMISSIONS));
-                this.cacheManager.modStreams.set(POLLING_SUBMISSIONS, defaultSubStream);
+                defaultSubStream.on('error', this.createSharedStreamErrorListener('newSub'));
+                defaultSubStream.on('listing', this.createSharedStreamListingListener('newSub'));
+                this.cacheManager.modStreams.set('newSub', defaultSubStream);
             }
         } else {
-            const stream = this.cacheManager.modStreams.get(POLLING_SUBMISSIONS);
+            const stream = this.cacheManager.modStreams.get('newSub');
             if (stream !== undefined) {
                 stream.end('Determined no managers are listening on shared stream parsing');
             }
         }
 
-        const isUnmoderatedShared = !this.sharedStreams.includes(POLLING_UNMODERATED) ? false : this.subManagers.some(x => x.isPollingShared(POLLING_UNMODERATED));
-        const unmoderatedstream = this.cacheManager.modStreams.get(POLLING_UNMODERATED);
+        const isUnmoderatedShared = !this.sharedStreams.includes('unmoderated') ? false : this.subManagers.some(x => x.isPollingShared('unmoderated'));
+        const unmoderatedstream = this.cacheManager.modStreams.get('unmoderated');
         if (isUnmoderatedShared && unmoderatedstream === undefined) {
             const defaultUnmoderatedStream = new UnmoderatedStream(this.client, {
                 subreddit: 'mod',
@@ -640,15 +634,15 @@ class Bot implements BotInstanceFunctions {
                 label: 'Shared Polling'
             });
             // @ts-ignore
-            defaultUnmoderatedStream.on('error', this.createSharedStreamErrorListener(POLLING_UNMODERATED));
-            defaultUnmoderatedStream.on('listing', this.createSharedStreamListingListener(POLLING_UNMODERATED));
-            this.cacheManager.modStreams.set(POLLING_UNMODERATED, defaultUnmoderatedStream);
+            defaultUnmoderatedStream.on('error', this.createSharedStreamErrorListener('unmoderated'));
+            defaultUnmoderatedStream.on('listing', this.createSharedStreamListingListener('unmoderated'));
+            this.cacheManager.modStreams.set('unmoderated', defaultUnmoderatedStream);
         } else if (!isUnmoderatedShared && unmoderatedstream !== undefined) {
             unmoderatedstream.end('Determined no managers are listening on shared stream parsing');
         }
 
-        const isModqueueShared = !this.sharedStreams.includes(POLLING_MODQUEUE) ? false : this.subManagers.some(x => x.isPollingShared(POLLING_MODQUEUE));
-        const modqueuestream = this.cacheManager.modStreams.get(POLLING_MODQUEUE);
+        const isModqueueShared = !this.sharedStreams.includes('modqueue') ? false : this.subManagers.some(x => x.isPollingShared('modqueue'));
+        const modqueuestream = this.cacheManager.modStreams.get('modqueue');
         if (isModqueueShared && modqueuestream === undefined) {
             const defaultModqueueStream = new ModQueueStream(this.client, {
                 subreddit: 'mod',
@@ -657,9 +651,9 @@ class Bot implements BotInstanceFunctions {
                 label: 'Shared Polling'
             });
             // @ts-ignore
-            defaultModqueueStream.on('error', this.createSharedStreamErrorListener(POLLING_MODQUEUE));
-            defaultModqueueStream.on('listing', this.createSharedStreamListingListener(POLLING_MODQUEUE));
-            this.cacheManager.modStreams.set(POLLING_MODQUEUE, defaultModqueueStream);
+            defaultModqueueStream.on('error', this.createSharedStreamErrorListener('modqueue'));
+            defaultModqueueStream.on('listing', this.createSharedStreamListingListener('modqueue'));
+            this.cacheManager.modStreams.set('modqueue', defaultModqueueStream);
         } else if (isModqueueShared && modqueuestream !== undefined) {
             modqueuestream.end('Determined no managers are listening on shared stream parsing');
         }

src/Common/Entities/Activity.ts

@@ -1,4 +1,4 @@
-import {Entity, Column, ManyToOne, PrimaryColumn, OneToMany, Index} from "typeorm";
+import {Entity, Column, ManyToOne, PrimaryColumn, OneToMany, Index, DataSource, JoinColumn} from "typeorm";
 import {AuthorEntity} from "./AuthorEntity";
 import {Subreddit} from "./Subreddit";
 import {CMEvent} from "./CMEvent";
@@ -6,6 +6,8 @@ import {asComment, getActivityAuthorName, parseRedditFullname, redditThingTypeTo
 import {activityReports, ActivityType, Report, SnoowrapActivity} from "../Infrastructure/Reddit";
 import {ActivityReport} from "./ActivityReport";
 import dayjs, {Dayjs} from "dayjs";
+import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients";
+import {Comment, Submission} from 'snoowrap/dist/objects';
 
 export interface ActivityEntityOptions {
     id: string
@@ -45,7 +47,7 @@ export class Activity {
     @Column({name: 'name'})
     name!: string;
 
-    @ManyToOne(type => Subreddit, sub => sub.activities, {cascade: ['insert']})
+    @ManyToOne(type => Subreddit, sub => sub.activities, {cascade: ['insert'], eager: true})
     subreddit!: Subreddit;
 
     @Column("varchar", {length: 20})
@@ -58,17 +60,18 @@ export class Activity {
     @Column("text")
     permalink!: string;
 
-    @ManyToOne(type => AuthorEntity, author => author.activities, {cascade: ['insert']})
+    @ManyToOne(type => AuthorEntity, author => author.activities, {cascade: ['insert'], eager: true})
     author!: AuthorEntity;
 
-    @OneToMany(type => CMEvent, act => act.activity) // note: we will create author property in the Photo class below
+    @OneToMany(type => CMEvent, act => act.activity)
     actionedEvents!: CMEvent[]
 
-    @ManyToOne(type => Activity, obj => obj.comments, {nullable: true})
+    @ManyToOne('Activity', 'comments', {nullable: true, cascade: ['insert']})
+    @JoinColumn({name: 'submission_id'})
     submission?: Activity;
 
-    @OneToMany(type => Activity, obj => obj.submission, {nullable: true})
-    comments!: Activity[];
+    @OneToMany('Activity', 'submission', {nullable: true})
+    comments?: Activity[];
 
     @OneToMany(type => ActivityReport, act => act.activity, {cascade: ['insert'], eager: true})
     reports: ActivityReport[] | undefined
@@ -151,10 +154,12 @@ export class Activity {
         return false;
     }
 
-    static fromSnoowrapActivity(subreddit: Subreddit, activity: SnoowrapActivity, lastKnownStateTimestamp?: dayjs.Dayjs | undefined) {
+    static async fromSnoowrapActivity(activity: SnoowrapActivity, options: fromSnoowrapOptions | undefined = {}) {
+
         let submission: Activity | undefined;
         let type: ActivityType = 'submission';
         let content: string;
+        const subreddit = await Subreddit.fromSnoowrap(activity.subreddit, options?.db);
         if(asComment(activity)) {
             type = 'comment';
             content = activity.body;
@@ -179,8 +184,30 @@ export class Activity {
             submission
         });
 
-        entity.syncReports(activity, lastKnownStateTimestamp);
+        entity.syncReports(activity, options.lastKnownStateTimestamp);
 
         return entity;
     }
+
+    toSnoowrap(client: ExtendedSnoowrap): SnoowrapActivity {
+        let act: SnoowrapActivity;
+        if(this.type === 'submission') {
+            act = new Submission({name: this.id, id: this.name}, client, false);
+            act.title = this.content;
+        } else {
+            act = new Comment({name: this.id, id: this.name}, client, false);
+            act.link_id = this.submission?.id as string;
+            act.body = this.content;
+        }
+        act.permalink = this.permalink;
+        act.subreddit = this.subreddit.toSnoowrap(client);
+        act.author = this.author.toSnoowrap(client);
+
+        return act;
+    }
+}
+
+export interface fromSnoowrapOptions {
+    lastKnownStateTimestamp?: dayjs.Dayjs | undefined
+    db?: DataSource
 }

src/Common/Entities/AuthorEntity.ts

@@ -1,5 +1,8 @@
 import {Entity, Column, PrimaryColumn, OneToMany} from "typeorm";
 import {Activity} from "./Activity";
+import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients";
+import {SnoowrapActivity} from "../Infrastructure/Reddit";
+import {RedditUser} from "snoowrap/dist/objects";
 
 @Entity({name: 'Author'})
 export class AuthorEntity {
@@ -11,11 +14,15 @@ export class AuthorEntity {
     name!: string;
 
     @OneToMany(type => Activity, act => act.author)
-    activities!: Activity[]
+    activities!: Promise<Activity[]>
 
     constructor(data?: any) {
         if(data !== undefined) {
             this.name = data.name;
         }
     }
+
+    toSnoowrap(client: ExtendedSnoowrap): RedditUser {
+        return new RedditUser({name: this.name, id: this.id}, client, false);
+    }
 }

src/Common/Entities/DispatchedEntity.ts

@@ -6,7 +6,7 @@ import {
     ManyToOne,
     PrimaryColumn,
     BeforeInsert,
-    AfterLoad
+    AfterLoad, JoinColumn
 } from "typeorm";
 import {
     ActivityDispatch
@@ -22,15 +22,15 @@ import Comment from "snoowrap/dist/objects/Comment";
 import {ColumnDurationTransformer} from "./Transformers";
 import { RedditUser } from "snoowrap/dist/objects";
 import {ActivitySourceTypes, DurationVal, NonDispatchActivitySourceValue, onExistingFoundBehavior} from "../Infrastructure/Atomic";
+import {Activity} from "./Activity";
 
 @Entity({name: 'DispatchedAction'})
 export class DispatchedEntity extends TimeAwareRandomBaseEntity {
 
-    @Column()
-    activityId!: string
-
-    @Column()
-    author!: string
+    //@ManyToOne(type => Activity, obj => obj.dispatched, {cascade: ['insert'], eager: true, nullable: false})
+    @ManyToOne(type => Activity, undefined, {cascade: ['insert'], eager: true, nullable: false})
+    @JoinColumn({name: 'activityId'})
+    activity!: Activity
 
     @Column({
         type: 'int',
@@ -82,11 +82,10 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity {
         }})
     tardyTolerant!: boolean | Duration
 
-    constructor(data?: ActivityDispatch & { manager: ManagerEntity }) {
+    constructor(data?: HydratedActivityDispatch) {
         super();
         if (data !== undefined) {
-            this.activityId = data.activity.name;
-            this.author = getActivityAuthorName(data.activity.author);
+            this.activity = data.activity;
             this.delay = data.delay;
             this.createdAt = data.queuedAt;
             this.type = data.type;
@@ -151,20 +150,7 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity {
     }
 
     async toActivityDispatch(client: ExtendedSnoowrap): Promise<ActivityDispatch> {
-        const redditThing = parseRedditFullname(this.activityId);
-        if(redditThing === undefined) {
-            throw new Error(`Could not parse reddit ID from value '${this.activityId}'`);
-        }
-        let activity: Comment | Submission;
-        if (redditThing?.type === 'comment') {
-            // @ts-ignore
-            activity = await client.getComment(redditThing.id);
-        } else {
-            // @ts-ignore
-            activity = await client.getSubmission(redditThing.id);
-        }
-        activity.author = new RedditUser({name: this.author}, client, false);
-        activity.id  = redditThing.id;
+        let activity = this.activity.toSnoowrap(client);
         return {
             id: this.id,
             queuedAt: this.createdAt,
@@ -176,8 +162,13 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity {
             cancelIfQueued: this.cancelIfQueued,
             identifier: this.identifier,
             type: this.type,
-            author: this.author,
+            author: activity.author.name,
             dryRun: this.dryRun
         }
     }
 }
+
+export interface HydratedActivityDispatch extends Omit<ActivityDispatch, 'activity'> {
+    activity: Activity
+    manager: ManagerEntity
+}

src/Common/Entities/Subreddit.ts

@@ -1,5 +1,7 @@
-import {Entity, Column, PrimaryColumn, OneToMany, Index} from "typeorm";
+import {Entity, Column, PrimaryColumn, OneToMany, Index, DataSource} from "typeorm";
 import {Activity} from "./Activity";
+import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients";
+import {Subreddit as SnoowrapSubreddit} from "snoowrap/dist/objects";
 
 export interface SubredditEntityOptions {
     id: string
@@ -25,4 +27,18 @@ export class Subreddit {
             this.name = data.name;
         }
     }
+
+    toSnoowrap(client: ExtendedSnoowrap): SnoowrapSubreddit {
+        return new SnoowrapSubreddit({display_name: this.name, name: this.id}, client, false);
+    }
+
+    static async fromSnoowrap(subreddit: SnoowrapSubreddit, db?: DataSource) {
+        if(db !== undefined) {
+           const existing = await db.getRepository(Subreddit).findOneBy({name: subreddit.display_name});
+           if(existing) {
+               return existing;
+           }
+        }
+        return new Subreddit({id: await subreddit.name, name: await subreddit.display_name});
+    }
 }

src/Common/Infrastructure/Atomic.ts

@@ -111,19 +111,6 @@ export interface DurationObject {
 
 export type JoinOperands = 'OR' | 'AND';
 export type PollOn = 'unmoderated' | 'modqueue' | 'newSub' | 'newComm';
-export const POLLING_UNMODERATED: PollOn = 'unmoderated';
-export const POLLING_MODQUEUE: PollOn = 'modqueue';
-export const POLLING_SUBMISSIONS: PollOn = 'newSub';
-export const POLLING_COMMENTS: PollOn = 'newComm';
-export const pollOnTypes: PollOn[] = [POLLING_UNMODERATED, POLLING_MODQUEUE, POLLING_SUBMISSIONS, POLLING_COMMENTS];
-export const pollOnTypeMapping: Map<string, PollOn> = new Map([
-    ['unmoderated', POLLING_UNMODERATED],
-    ['modqueue', POLLING_MODQUEUE],
-    ['newsub', POLLING_SUBMISSIONS],
-    ['newcomm', POLLING_COMMENTS],
-    // be nice if user mispelled
-    ['newcom', POLLING_COMMENTS]
-]);
 export type ModeratorNames = 'self' | 'automod' | 'automoderator' | string;
 export type Invokee = 'system' | 'user';
 export type RunState = 'running' | 'paused' | 'stopped';

src/Common/Migrations/Database/Server/1667415256831-delayedReset.ts

@@ -0,0 +1,19 @@
+import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"
+
+export class delayedReset1667415256831 implements MigrationInterface {
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        queryRunner.connection.logger.logSchemaBuild('Truncating (removing) existing Dispatched Actions due to internal structural changes');
+        await queryRunner.clearTable('DispatchedAction');
+        await queryRunner.changeColumn('DispatchedAction', 'author', new TableColumn({
+            name: 'author',
+            type: 'varchar',
+            length: '150',
+            isNullable: true
+        }));
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+    }
+
+}

src/Common/interfaces.ts

@@ -372,7 +372,7 @@ export interface PollingOptions extends PollingDefaults {
      * * after they have been manually approved from modqueue
      *
      * */
-    pollOn: PollOn
+    pollOn: 'unmoderated' | 'modqueue' | 'newSub' | 'newComm'
 }
 
 export interface TTLConfig {

src/ConfigBuilder.ts

@@ -8,7 +8,7 @@ import {
     overwriteMerge,
     parseBool, parseExternalUrl, parseUrlContext, parseWikiContext, randomId,
     readConfigFile,
-    removeUndefinedKeys, resolvePathFromEnvWithRelative, toPollOn, toStrongSharingACLConfig
+    removeUndefinedKeys, resolvePathFromEnvWithRelative, toStrongSharingACLConfig
 } from "./util";
 
 import Ajv, {Schema} from 'ajv';
@@ -74,8 +74,8 @@ import {ErrorWithCause} from "pony-cause";
 import {RunConfigHydratedData, RunConfigData, RunConfigObject} from "./Run";
 import {AuthorRuleConfig} from "./Rule/AuthorRule";
 import {
-    CacheProvider, ConfigFormat, ConfigFragmentParseFunc, POLLING_MODQUEUE, POLLING_UNMODERATED,
-    PollOn, pollOnTypes
+    CacheProvider, ConfigFormat, ConfigFragmentParseFunc,
+    PollOn
 } from "./Common/Infrastructure/Atomic";
 import {
     asFilterOptionsJson,
@@ -452,31 +452,27 @@ export class ConfigBuilder {
 
 export const buildPollingOptions = (values: (string | PollingOptions)[]): PollingOptionsStrong[] => {
     let opts: PollingOptionsStrong[] = [];
-    let rawOpts: PollingOptions;
     for (const v of values) {
         if (typeof v === 'string') {
-            rawOpts = {pollOn: v as PollOn}; // maybeee
+            opts.push({
+                pollOn: v as PollOn,
+                interval: DEFAULT_POLLING_INTERVAL,
+                limit: DEFAULT_POLLING_LIMIT,
+            });
         } else {
-            rawOpts = v;
+            const {
+                pollOn: p,
+                interval = DEFAULT_POLLING_INTERVAL,
+                limit = DEFAULT_POLLING_LIMIT,
+                delayUntil,
+            } = v;
+            opts.push({
+                pollOn: p as PollOn,
+                interval,
+                limit,
+                delayUntil,
+            });
         }
-
-        const {
-            pollOn: p,
-            interval = DEFAULT_POLLING_INTERVAL,
-            limit = DEFAULT_POLLING_LIMIT,
-            delayUntil,
-        } = rawOpts;
-
-        const pVal = toPollOn(p);
-        if (opts.some(x => x.pollOn === pVal)) {
-            throw new SimpleError(`Polling source ${pVal} cannot appear more than once in polling options`);
-        }
-        opts.push({
-            pollOn: pVal,
-            interval,
-            limit,
-            delayUntil,
-        });
     }
     return opts;
 }
@@ -800,7 +796,7 @@ export const parseDefaultBotInstanceFromArgs = (args: any): BotInstanceJsonConfi
             heartbeatInterval: heartbeat,
         },
         polling: {
-            shared: sharedMod ? [POLLING_UNMODERATED, POLLING_MODQUEUE] : undefined,
+            shared: sharedMod ? ['unmoderated', 'modqueue'] : undefined,
         },
         nanny: {
             softLimit,
@@ -912,7 +908,7 @@ export const parseDefaultBotInstanceFromEnv = (): BotInstanceJsonConfig => {
             heartbeatInterval: process.env.HEARTBEAT !== undefined ? parseInt(process.env.HEARTBEAT) : undefined,
         },
         polling: {
-            shared: parseBool(process.env.SHARE_MOD) ? [POLLING_UNMODERATED, POLLING_MODQUEUE] : undefined,
+            shared: parseBool(process.env.SHARE_MOD) ? ['unmoderated', 'modqueue'] : undefined,
         },
         nanny: {
             softLimit: process.env.SOFT_LIMIT !== undefined ? parseInt(process.env.SOFT_LIMIT) : undefined,
@@ -1529,10 +1525,10 @@ export const buildBotConfig = (data: BotInstanceJsonConfig, opConfig: OperatorCo
         botCache.provider.prefix = buildCachePrefix([botCache.provider.prefix, 'bot', (botName || objectHash.sha1(botCreds))]);
     }
 
-    let realShared: PollOn[] = shared === true ? pollOnTypes : shared.map(toPollOn);
+    let realShared = shared === true ? ['unmoderated', 'modqueue', 'newComm', 'newSub'] : shared;
     if (sharedMod === true) {
-        realShared.push(POLLING_UNMODERATED);
-        realShared.push(POLLING_MODQUEUE);
+        realShared.push('unmoderated');
+        realShared.push('modqueue');
     }
 
     const botLevelStatDefaults = {...statDefaultsFromOp, ...databaseStatisticsDefaults};
@@ -1570,7 +1566,7 @@ export const buildBotConfig = (data: BotInstanceJsonConfig, opConfig: OperatorCo
         caching: botCache,
         userAgent,
         polling: {
-            shared: Array.from(new Set(realShared)),
+            shared: [...new Set(realShared)] as PollOn[],
             stagger,
             limit,
             interval,

src/Rule/RecentActivityRule.ts

@@ -126,7 +126,28 @@ export class RecentActivityRule extends Rule {
     async process(item: Submission | Comment): Promise<[boolean, RuleResult]> {
         let activities;
 
+        // ACID is a bitch
+        // reddit may not return the activity being checked in the author's recent history due to availability/consistency issues or *something*
+        // so make sure we add it in if config is checking the same type and it isn't included
+        // TODO refactor this for SubredditState everywhere branch
+        let shouldIncludeSelf = true;
         const strongWindow = windowConfigToWindowCriteria(this.window);
+        const {
+            filterOn: {
+                post: {
+                    subreddits: {
+                        include = [],
+                        exclude = []
+                    } = {},
+                } = {},
+            } = {}
+        } = strongWindow;
+        // typeof x === string -- a patch for now...technically this is all it supports but eventually will need to be able to do any SubredditState
+        if (include.length > 0 && !include.some(x => x.name !== undefined && x.name.toLocaleLowerCase() === item.subreddit.display_name.toLocaleLowerCase())) {
+            shouldIncludeSelf = false;
+        } else if (exclude.length > 0 && exclude.some(x => x.name !== undefined && x.name.toLocaleLowerCase() === item.subreddit.display_name.toLocaleLowerCase())) {
+            shouldIncludeSelf = false;
+        }
 
         if(strongWindow.fetch === undefined && this.lookAt !== undefined) {
             switch(this.lookAt) {
@@ -138,10 +159,25 @@ export class RecentActivityRule extends Rule {
             }
         }
 
-        // ACID is a bitch
-        // reddit may not return the activity being checked in the author's recent history due to availability/consistency issues or *something*
-        // so add current activity as a prefetched activity and add it to the returned activities (after it goes through filtering)
-        activities = await this.resources.getAuthorActivities(item.author, strongWindow, undefined, [item]);
+        activities = await this.resources.getAuthorActivities(item.author, strongWindow);
+
+        switch (strongWindow.fetch) {
+            case 'comment':
+                if (shouldIncludeSelf && item instanceof Comment && !activities.some(x => x.name === item.name)) {
+                    activities.unshift(item);
+                }
+                break;
+            case 'submission':
+                if (shouldIncludeSelf && item instanceof Submission && !activities.some(x => x.name === item.name)) {
+                    activities.unshift(item);
+                }
+                break;
+            default:
+                if (shouldIncludeSelf && !activities.some(x => x.name === item.name)) {
+                    activities.unshift(item);
+                }
+                break;
+        }
 
         let viableActivity = activities;
         // if config does not specify reference then we set the default based on whether the item is a submission or not

src/Subreddit/Manager.ts

@@ -93,8 +93,8 @@ import {EntityRunState} from "../Common/Entities/EntityRunState/EntityRunState";
 import {
     ActivitySourceValue,
     EventRetentionPolicyRange,
-    Invokee, POLLING_COMMENTS, POLLING_MODQUEUE, POLLING_SUBMISSIONS, POLLING_UNMODERATED,
-    PollOn, pollOnTypes,
+    Invokee,
+    PollOn,
     recordOutputTypes,
     RunState
 } from "../Common/Infrastructure/Atomic";
@@ -635,7 +635,7 @@ export class Manager extends EventEmitter implements RunningStates {
             const configBuilder = new ConfigBuilder({logger: this.logger});
             const validJson = configBuilder.validateJson(configObj);
             const {
-                polling = [{pollOn: POLLING_SUBMISSIONS, limit: DEFAULT_POLLING_LIMIT, interval: DEFAULT_POLLING_INTERVAL}],
+                polling = [{pollOn: 'unmoderated', limit: DEFAULT_POLLING_LIMIT, interval: DEFAULT_POLLING_INTERVAL}],
                 caching,
                 credentials,
                 dryRun,
@@ -957,7 +957,7 @@ export class Manager extends EventEmitter implements RunningStates {
         await this.resources.setActivityLastSeenDate(item.name);
 
         // if modqueue is running then we know we are checking for new reports every X seconds
-        if(options.activitySource.identifier === POLLING_MODQUEUE) {
+        if(options.activitySource.identifier === 'modqueue') {
             // if the activity is from modqueue and only has one report then we know that report was just created
             if(item.num_reports === 1
                 // otherwise if it has more than one report AND we have seen it (its only seen if it has already been stored (in below block))
@@ -975,7 +975,7 @@ export class Manager extends EventEmitter implements RunningStates {
         let shouldPersistReports = false;
 
         if (existingEntity === null) {
-            activityEntity = Activity.fromSnoowrapActivity(this.managerEntity.subreddit, activity, lastKnownStateTimestamp);
+            activityEntity = await Activity.fromSnoowrapActivity(activity, {lastKnownStateTimestamp, db: this.resources.database});
             // always persist if activity is not already persisted and any reports exist
             if (item.num_reports > 0) {
                 shouldPersistReports = true;
@@ -1189,7 +1189,7 @@ export class Manager extends EventEmitter implements RunningStates {
                                     // @ts-ignore
                                     const subProxy = await this.client.getSubmission((item as Comment).link_id);
                                     const sub = await this.resources.getActivity(subProxy);
-                                    subActivity = await this.activityRepo.save(Activity.fromSnoowrapActivity(this.managerEntity.subreddit, sub));
+                                    subActivity = await this.activityRepo.save(await Activity.fromSnoowrapActivity(sub, {db: this.resources.database}));
                                 }
                                 event.activity.submission = subActivity;
 
@@ -1325,20 +1325,25 @@ export class Manager extends EventEmitter implements RunningStates {
         }
     }
 
-    isPollingShared(streamName: PollOn): boolean {
+    isPollingShared(streamName: string): boolean {
         const pollOption = this.pollOptions.find(x => x.pollOn === streamName);
-        return pollOption !== undefined && pollOption.limit === DEFAULT_POLLING_LIMIT && pollOption.interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(streamName);
+        return pollOption !== undefined && pollOption.limit === DEFAULT_POLLING_LIMIT && pollOption.interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(streamName as PollOn);
     }
 
     async buildPolling() {
 
-        const sources = [...pollOnTypes];
+        const sources: PollOn[] = ['unmoderated', 'modqueue', 'newComm', 'newSub'];
 
         const subName = this.subreddit.display_name;
 
         for (const source of sources) {
 
-            const pollOpt = this.pollOptions.find(x => x.pollOn === source);
+            if (!sources.includes(source)) {
+                this.logger.error(`'${source}' is not a valid polling source. Valid sources: unmoderated | modqueue | newComm | newSub`);
+                continue;
+            }
+
+            const pollOpt = this.pollOptions.find(x => x.pollOn.toLowerCase() === source.toLowerCase());
             if (pollOpt === undefined) {
                 if(this.sharedStreamCallbacks.has(source)) {
                     this.logger.debug(`Removing listener for shared polling on ${source.toUpperCase()} because it no longer exists in config`);
@@ -1361,11 +1366,11 @@ export class Manager extends EventEmitter implements RunningStates {
                 let modStreamType: string | undefined;
 
                 switch (source) {
-                    case POLLING_UNMODERATED:
+                    case 'unmoderated':
                         if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
-                            modStreamType = POLLING_UNMODERATED;
+                            modStreamType = 'unmoderated';
                             // use default mod stream from resources
-                            stream = this.cacheManager.modStreams.get(POLLING_UNMODERATED) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                            stream = this.cacheManager.modStreams.get('unmoderated') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
                         } else {
                             stream = new UnmoderatedStream(this.client, {
                                 subreddit: this.subreddit.display_name,
@@ -1375,11 +1380,11 @@ export class Manager extends EventEmitter implements RunningStates {
                             });
                         }
                         break;
-                    case POLLING_MODQUEUE:
+                    case 'modqueue':
                         if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
-                            modStreamType = POLLING_MODQUEUE;
+                            modStreamType = 'modqueue';
                             // use default mod stream from resources
-                            stream = this.cacheManager.modStreams.get(POLLING_MODQUEUE) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                            stream = this.cacheManager.modStreams.get('modqueue') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
                         } else {
                             stream = new ModQueueStream(this.client, {
                                 subreddit: this.subreddit.display_name,
@@ -1389,11 +1394,11 @@ export class Manager extends EventEmitter implements RunningStates {
                             });
                         }
                         break;
-                    case POLLING_SUBMISSIONS:
+                    case 'newSub':
                         if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
-                            modStreamType = POLLING_SUBMISSIONS;
+                            modStreamType = 'newSub';
                             // use default mod stream from resources
-                            stream = this.cacheManager.modStreams.get(POLLING_SUBMISSIONS) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                            stream = this.cacheManager.modStreams.get('newSub') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
                         } else {
                             stream = new SubmissionStream(this.client, {
                                 subreddit: this.subreddit.display_name,
@@ -1403,11 +1408,11 @@ export class Manager extends EventEmitter implements RunningStates {
                             });
                         }
                         break;
-                    case POLLING_COMMENTS:
+                    case 'newComm':
                         if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
-                            modStreamType = POLLING_COMMENTS;
+                            modStreamType = 'newComm';
                             // use default mod stream from resources
-                            stream = this.cacheManager.modStreams.get(POLLING_COMMENTS) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                            stream = this.cacheManager.modStreams.get('newComm') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
                         } else {
                             stream = new CommentStream(this.client, {
                                 subreddit: this.subreddit.display_name,
@@ -1417,8 +1422,6 @@ export class Manager extends EventEmitter implements RunningStates {
                             });
                         }
                         break;
-                    default:
-                        throw new CMError(`This shouldn't happen! All polling sources are enumerated in switch. Source value: ${source}`)
                 }
 
                 if (stream === undefined) {
@@ -1511,10 +1514,10 @@ export class Manager extends EventEmitter implements RunningStates {
     }
 
     noChecksWarning = (source: PollOn) => (listing: any) => {
-        if (this.commentChecks.length === 0 && [POLLING_MODQUEUE, POLLING_COMMENTS].some(x => x === source)) {
+        if (this.commentChecks.length === 0 && ['modqueue', 'newComm'].some(x => x === source)) {
             this.logger.warn(`Polling '${source.toUpperCase()}' may return Comments but no comments checks were configured.`);
         }
-        if (this.submissionChecks.length === 0 && [POLLING_UNMODERATED, POLLING_MODQUEUE, POLLING_SUBMISSIONS].some(x => x === source)) {
+        if (this.submissionChecks.length === 0 && ['unmoderated', 'modqueue', 'newSub'].some(x => x === source)) {
             this.logger.warn(`Polling '${source.toUpperCase()}' may return Submissions but no submission checks were configured.`);
         }
     }
@@ -1667,7 +1670,7 @@ export class Manager extends EventEmitter implements RunningStates {
             }
             this.startedAt = dayjs();
 
-            const modQueuePollOpts = this.pollOptions.find(x => x.pollOn === POLLING_MODQUEUE);
+            const modQueuePollOpts = this.pollOptions.find(x => x.pollOn === 'modqueue');
             if(modQueuePollOpts !== undefined) {
                 this.modqueueInterval = modQueuePollOpts.interval;
             }

src/Subreddit/SubredditResources.ts

@@ -69,7 +69,16 @@ import {cacheTTLDefaults, createHistoricalDisplayDefaults,} from "../Common/defa
 import {ExtendedSnoowrap} from "../Utils/SnoowrapClients";
 import dayjs, {Dayjs} from "dayjs";
 import ImageData from "../Common/ImageData";
-import {Between, DataSource, DeleteQueryBuilder, LessThan, Repository, SelectQueryBuilder} from "typeorm";
+import {
+    Between, Brackets,
+    DataSource,
+    DeleteQueryBuilder,
+    In,
+    LessThan,
+    NotBrackets,
+    Repository,
+    SelectQueryBuilder
+} from "typeorm";
 import {CMEvent as ActionedEventEntity, CMEvent} from "../Common/Entities/CMEvent";
 import {RuleResultEntity} from "../Common/Entities/RuleResultEntity";
 import globrex from 'globrex';
@@ -162,6 +171,8 @@ import {ActivitySource} from "../Common/ActivitySource";
 import {SubredditResourceOptions} from "../Common/Subreddit/SubredditResourceInterfaces";
 import {SubredditStats} from "./Stats";
 import {CMCache} from "../Common/Cache";
+import { Activity } from '../Common/Entities/Activity';
+import {FindOptionsWhere} from "typeorm/find-options/FindOptionsWhere";
 
 export const DEFAULT_FOOTER = '\r\n*****\r\nThis action was performed by [a bot.]({{botLink}}) Mention a moderator or [send a modmail]({{modmailLink}}) if you have any ideas, questions, or concerns about this action.';
 
@@ -193,6 +204,7 @@ export class SubredditResources {
     botAccount?: string;
     dispatchedActivityRepo: Repository<DispatchedEntity>
     activitySourceRepo: Repository<ActivitySourceEntity>
+    activityRepo: Repository<Activity>
     retention?: EventRetentionPolicyRange
     managerEntity: ManagerEntity
     botEntity: Bot
@@ -229,6 +241,7 @@ export class SubredditResources {
         this.database = database;
         this.dispatchedActivityRepo = this.database.getRepository(DispatchedEntity);
         this.activitySourceRepo = this.database.getRepository(ActivitySourceEntity);
+        this.activityRepo = this.database.getRepository(Activity);
         this.retention = retention;
         //this.prefix = prefix;
         this.client = client;
@@ -404,21 +417,25 @@ export class SubredditResources {
                     }
                 },
                 relations: {
-                    manager: true
+                    manager: true,
+                    activity: {
+                        submission: true
+                    }
                 }
             });
             const now = dayjs();
+            const toRemove = [];
             for(const dAct of dispatchedActivities) {
                 const shouldDispatchAt = dAct.createdAt.add(dAct.delay.asSeconds(), 'seconds');
                 let tardyHint = '';
                 if(shouldDispatchAt.isBefore(now)) {
-                    let tardyHint = `Activity ${dAct.activityId} queued at ${dAct.createdAt.format('YYYY-MM-DD HH:mm:ssZ')} for ${dAct.delay.humanize()} is now LATE`;
+                    let tardyHint = `Activity ${dAct.activity.id} queued at ${dAct.createdAt.format('YYYY-MM-DD HH:mm:ssZ')} for ${dAct.delay.humanize()} is now LATE`;
                     if(dAct.tardyTolerant === true) {
                         tardyHint += ` but was configured as ALWAYS 'tardy tolerant' so will be dispatched immediately`;
                     } else if(dAct.tardyTolerant === false) {
                         tardyHint += ` and was not configured as 'tardy tolerant' so will be dropped`;
                         this.logger.warn(tardyHint);
-                        await this.removeDelayedActivity(dAct.id);
+                        toRemove.push(dAct.id);
                         continue;
                     } else {
                         // see if its within tolerance
@@ -426,7 +443,7 @@ export class SubredditResources {
                         if(latest.isBefore(now)) {
                             tardyHint += ` and IS NOT within tardy tolerance of ${dAct.tardyTolerant.humanize()} of planned dispatch time so will be dropped`;
                             this.logger.warn(tardyHint);
-                            await this.removeDelayedActivity(dAct.id);
+                            toRemove.push(dAct.id);
                             continue;
                         } else {
                             tardyHint += `but is within tardy tolerance of ${dAct.tardyTolerant.humanize()} of planned dispatch time so will be dispatched immediately`;
@@ -439,27 +456,115 @@ export class SubredditResources {
                 try {
                     this.delayedItems.push(await dAct.toActivityDispatch(this.client))
                 } catch (e) {
-                    this.logger.warn(new ErrorWithCause(`Unable to add Activity ${dAct.activityId} from database delayed activities to in-app delayed activities queue`, {cause: e}));
+                    this.logger.warn(new ErrorWithCause(`Unable to add Activity ${dAct.activity.id} from database delayed activities to in-app delayed activities queue`, {cause: e}));
                 }
             }
+            if(toRemove.length > 0) {
+                await this.removeDelayedActivity(toRemove);
+            }
         }
     }
 
     async addDelayedActivity(data: ActivityDispatch) {
-        const dEntity = await this.dispatchedActivityRepo.save(new DispatchedEntity({...data, manager: this.managerEntity}));
+        // TODO merge this with getActivity or something...
+        if(asComment(data.activity)) {
+            const existingSub =  await this.activityRepo.findOneBy({_id: data.activity.link_id});
+            if(existingSub === null) {
+                const sub = await this.getActivity(new Submission({name: data.activity.link_id}, this.client, false));
+                await this.activityRepo.save(await Activity.fromSnoowrapActivity(sub, {db: this.database}));
+            }
+        }
+        const dEntity = await this.dispatchedActivityRepo.save(new DispatchedEntity({...data, manager: this.managerEntity, activity: await Activity.fromSnoowrapActivity(data.activity, {db: this.database})}));
         data.id = dEntity.id;
         this.delayedItems.push(data);
     }
 
     async removeDelayedActivity(val?: string | string[]) {
-        if(val === undefined) {
-            await this.dispatchedActivityRepo.delete({manager: {id: this.managerEntity.id}});
-            this.delayedItems = [];
-        } else {
+        let dispatched: DispatchedEntity[] = [];
+        const where: FindOptionsWhere<DispatchedEntity> = {
+            manager: {
+                id: this.managerEntity.id
+            }
+        };
+
+        if(val !== undefined) {
             const ids = typeof val === 'string' ? [val] : val;
-            await this.dispatchedActivityRepo.delete(ids);
-            this.delayedItems = this.delayedItems.filter(x => !ids.includes(x.id));
+            where.id = In(ids);
         }
+
+        dispatched = await this.dispatchedActivityRepo.find({
+            where,
+            relations: {
+                manager: true,
+                activity: {
+                    actionedEvents: true,
+                    submission: {
+                        actionedEvents: true
+                    }
+                }
+            }
+        });
+
+        const actualDispatchedIds = dispatched.map(x => x.id);
+        this.logger.debug(`${actualDispatchedIds.length} marked for deletion`, {leaf: 'Delayed Activities'});
+
+        // get potential activities to delete
+        // but only include activities that don't have any actionedEvents
+        let activityIdsToDelete = Array.from(dispatched.reduce((acc, curr) => {
+            if(curr.activity.actionedEvents === null || curr.activity.actionedEvents.length === 0) {
+                acc.add(curr.activity.id);
+            }
+            if(curr.activity.submission !== undefined && curr.activity.submission !== null) {
+                if(curr.activity.submission.actionedEvents === null || curr.activity.submission.actionedEvents.length === 0) {
+                    acc.add(curr.activity.submission.id);
+                }
+            }
+            return acc;
+        }, new Set<string>()));
+        const rawActCount = activityIdsToDelete.length;
+        let activeActCount = 0;
+
+        // if we have any potential activities to delete we now need to get any dispatched actions that reference these activities
+        // that are NOT the ones we are going to delete
+        if(activityIdsToDelete.length > 0) {
+            const activeDispatchedQuery = this.dispatchedActivityRepo.createQueryBuilder('dis')
+                .leftJoinAndSelect('dis.activity', 'activity')
+                .leftJoinAndSelect('activity.submission', 'submission')
+                .where(new NotBrackets((qb) => {
+                    qb.where('dis.id IN (:...currIds)', {currIds: actualDispatchedIds});
+                }))
+                .andWhere(new Brackets((qb) => {
+                    qb.where('activity._id IN (:...actMainIds)', {actMainIds: activityIdsToDelete})
+                    qb.orWhere('submission._id IN (:...actSubIds)', {actSubIds: activityIdsToDelete})
+                }));
+            //const sql = activeDispatchedQuery.getSql();
+            const activeDispatched = await activeDispatchedQuery.getMany();
+
+            // all activity ids, from the actions to delete, that are being used by dispatched actions that are NOT the ones we are going to delete
+            const activeDispatchedIds = Array.from(activeDispatched.reduce((acc, curr) => {
+                acc.add(curr.activity.id);
+                if(curr.activity.submission !== undefined && curr.activity.submission !== null) {
+                    acc.add(curr.activity.submission.id);
+                }
+                return acc;
+            }, new Set<string>()));
+            activeActCount = activeDispatchedIds.length;
+
+            // filter out any that are still in use
+            activityIdsToDelete = activityIdsToDelete.filter(x => !activeDispatchedIds.includes(x));
+        }
+
+        this.logger.debug(`Marked ${activityIdsToDelete.length} Activities created, by Delayed, for deletion (${rawActCount} w/o Events | ${activeActCount} used by other Delayed Activities)`, {leaf: 'Delayed Activities'});
+
+        if(actualDispatchedIds.length > 0) {
+            await this.dispatchedActivityRepo.delete(actualDispatchedIds);
+        } else {
+            this.logger.warn('No dispatched ids found to delete');
+        }
+        if(activityIdsToDelete.length > 0) {
+            await this.activityRepo.delete(activityIdsToDelete);
+        }
+        this.delayedItems = this.delayedItems.filter(x => !actualDispatchedIds.includes(x.id));
     }
 
     async initStats() {
@@ -1030,13 +1135,13 @@ export class SubredditResources {
         }
     }
 
-    async getAuthorActivities(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing, prefetchedActivities?: SnoowrapActivity[]): Promise<SnoowrapActivity[]> {
+    async getAuthorActivities(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing): Promise<SnoowrapActivity[]> {
 
-        const {post} = await this.getAuthorActivitiesWithFilter(user, options, customListing, prefetchedActivities);
+        const {post} = await this.getAuthorActivitiesWithFilter(user, options, customListing);
         return post;
     }
 
-    async getAuthorActivitiesWithFilter(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing, prefetchedActivities?: SnoowrapActivity[]): Promise<FetchedActivitiesResult> {
+    async getAuthorActivitiesWithFilter(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing): Promise<FetchedActivitiesResult> {
         let listFuncName: string;
         let listFunc: ListingFunc;
 
@@ -1064,21 +1169,21 @@ export class SubredditResources {
             ...(cloneDeep(options)),
         }
 
-        return await this.getActivities(user, criteriaWithDefaults, {func: listFunc, name: listFuncName}, prefetchedActivities);
+        return await this.getActivities(user, criteriaWithDefaults, {func: listFunc, name: listFuncName});
     }
 
-    async getAuthorComments(user: RedditUser, options: ActivityWindowCriteria, prefetchedActivities?: SnoowrapActivity[]): Promise<Comment[]> {
-        return await this.getAuthorActivities(user, {...options, fetch: 'comment'}, undefined, prefetchedActivities) as unknown as Promise<Comment[]>;
+    async getAuthorComments(user: RedditUser, options: ActivityWindowCriteria): Promise<Comment[]> {
+        return await this.getAuthorActivities(user, {...options, fetch: 'comment'}) as unknown as Promise<Comment[]>;
     }
 
-    async getAuthorSubmissions(user: RedditUser, options: ActivityWindowCriteria, prefetchedActivities?: SnoowrapActivity[]): Promise<Submission[]> {
+    async getAuthorSubmissions(user: RedditUser, options: ActivityWindowCriteria): Promise<Submission[]> {
         return await this.getAuthorActivities(user, {
             ...options,
             fetch: 'submission'
-        }, undefined,prefetchedActivities) as unknown as Promise<Submission[]>;
+        }) as unknown as Promise<Submission[]>;
     }
 
-    async getActivities(user: RedditUser, options: ActivityWindowCriteria, listingData: NamedListing, prefetchedActivities: SnoowrapActivity[] = []): Promise<FetchedActivitiesResult> {
+    async getActivities(user: RedditUser, options: ActivityWindowCriteria, listingData: NamedListing): Promise<FetchedActivitiesResult> {
 
         try {
 
@@ -1213,24 +1318,12 @@ export class SubredditResources {
                     }
                 }
 
-                let preFilteredPrefetchedActivities = [...prefetchedActivities];
-                if(preFilteredPrefetchedActivities.length > 0) {
-                    switch(options.fetch) {
-                        // TODO this may not work if using a custom listingFunc that does not include fetch type
-                        case 'comment':
-                            preFilteredPrefetchedActivities = preFilteredPrefetchedActivities.filter(x => asComment(x));
-                            break;
-                        case 'submission':
-                            preFilteredPrefetchedActivities = preFilteredPrefetchedActivities.filter(x => asSubmission(x));
-                            break;
-                    }
-                    preFilteredPrefetchedActivities = await this.filterListingWithHistoryOptions(preFilteredPrefetchedActivities, user, options.filterOn?.pre);
-                }
-                let unFilteredItems: SnoowrapActivity[] | undefined = [...preFilteredPrefetchedActivities];
-                pre = pre.concat(preFilteredPrefetchedActivities);
+                let unFilteredItems: SnoowrapActivity[] | undefined;
+
 
                 const { func: listingFunc } = listingData;
 
+
                 let listing = await listingFunc(getAuthorHistoryAPIOptions(options));
                 let hitEnd = false;
                 let offset = chunkSize;
@@ -1240,9 +1333,6 @@ export class SubredditResources {
                         timeOk = false;
 
                     let listSlice = listing.slice(offset - chunkSize);
-                    // filter out any from slice that were already included from prefetched list so that prefetched aren't included twice
-                    listSlice = preFilteredPrefetchedActivities.length === 0 ? listSlice : listSlice.filter(x => !preFilteredPrefetchedActivities.some(y => y.name === x.name));
-
                     let preListSlice = await this.filterListingWithHistoryOptions(listSlice, user, options.filterOn?.pre);
 
                     // its more likely the time criteria is going to be hit before the count criteria
@@ -1517,7 +1607,6 @@ export class SubredditResources {
             usernotes,
             ruleResults,
             actionResults,
-            author: (val) => this.getAuthor(val)
         });
     }
 

src/Utils/SnoowrapUtils.ts

@@ -133,7 +133,6 @@ export interface TemplateContext {
     ruleResults?: RuleResultEntity[]
     actionResults?: ActionResultEntity[]
     activity?: SnoowrapActivity
-    author?: (val: string | RedditUser) => Promise<RedditUser>
     [key: string]: any
 }
 
@@ -141,25 +140,11 @@ export const renderContent = async (template: string, data: TemplateContext = {}
     const {
         usernotes,
         ruleResults,
-        author,
         actionResults,
         activity,
         ...restContext
     } = data;
 
-    let fetchedUser: RedditUser | undefined;
-    // @ts-ignore
-    const user = async (): Promise<RedditUser> => {
-        if(fetchedUser === undefined) {
-            if(author !== undefined) {
-                // @ts-ignore
-                fetchedUser = await author(activity.author);
-            }
-        }
-        // @ts-ignore
-        return fetchedUser;
-    }
-
     let view: GenericContentTemplateData = {
         botLink: BOT_LINK,
         ...restContext
@@ -186,24 +171,10 @@ export const renderContent = async (template: string, data: TemplateContext = {}
 
         view.modmailLink = `https://www.reddit.com/message/compose?to=%2Fr%2F${subreddit}&message=${encodeURIComponent(permalink)}`;
 
-        const author: any = {
-            toString: () => getActivityAuthorName(activity.author)
-        };
-
-        if(template.includes('{{item.author.')) {
-            // @ts-ignore
-            const auth = await user();
-
-            author.age = dayjs.unix(auth.created).fromNow(true);
-            author.linkKarma = auth.link_karma;
-            author.commentKarma = auth.comment_karma;
-            author.totalKarma = auth.comment_karma + auth.link_karma;
-            author.verified = auth.has_verified_email;
-        }
-
         const templateData: any = {
             kind: activity instanceof Submission ? 'submission' : 'comment',
-            author,
+            // @ts-ignore
+            author: getActivityAuthorName(await activity.author),
             votes: activity.score,
             age: dayjs.duration(dayjs().diff(dayjs.unix(activity.created))).humanize(),
             permalink,

src/Web/Client/StorageProvider.ts

@@ -12,7 +12,6 @@ import {Logger} from "winston";
 import {WebSetting} from "../../Common/WebEntities/WebSetting";
 import {ErrorWithCause} from "pony-cause";
 import {createCacheManager} from "../../Common/Cache";
-import {MysqlDriver} from "typeorm/driver/mysql/MysqlDriver";
 
 export interface CacheManagerStoreOptions {
     prefix?: string
@@ -104,12 +103,7 @@ export class DatabaseStorageProvider extends StorageProvider {
     }
 
     createSessionStore(options?: TypeormStoreOptions): Store {
-        // https://github.com/freshgiammi-lab/connect-typeorm#implement-the-session-entity
-        // https://github.com/freshgiammi-lab/connect-typeorm/issues/8
-        // usage of LIMIT in subquery is not supported by mariadb/mysql
-        // limitSubquery: false -- turns off LIMIT usage
-        const realOptions = this.database.driver instanceof MysqlDriver ? {...options, limitSubquery: false} : options;
-        return new TypeormStore(realOptions).connect(this.clientSessionRepo)
+        return new TypeormStore(options).connect(this.clientSessionRepo)
     }
 
     async getSessionSecret(): Promise<string | undefined> {

src/Web/assets/views/status.ejs

@@ -1297,7 +1297,7 @@
                     const durationDayjs = dayjs.duration(x.duration, 'seconds');
                     const durationDisplay = durationDayjs.humanize();
                     const cancelLink = `<a href="#" data-id="${x.id}" data-subreddit="${x.subreddit}" class="delayCancel">CANCEL</a>`;
-                    return `<div>A <a href="https://reddit.com${x.permalink}">${x.submissionId !== undefined ? 'Comment' : 'Submission'}</a>${isAll ? ` in <a href="https://reddit.com${x.subreddit}">${x.subreddit}</a> ` : ''} by <a href="https://reddit.com/u/${x.author}">${x.author}</a> queued by ${x.source} at ${queuedAtDisplay} for ${durationDisplay} (dispatches ${durationUntilNow.humanize(true)}) -- ${cancelLink}</div>`;
+                    return `<div>A <a href="https://reddit.com${x.permalink}">${x.submissionId !== undefined ? 'Comment' : 'Submission'}</a> by <a href="https://reddit.com/u/${x.author}">${x.author}</a>${isAll ? `, dispatched in <a href="https://reddit.com${x.subreddit}">${x.subreddit}</a> ,` : ''} queued by ${x.source} at ${queuedAtDisplay} for ${durationDisplay} (dispatches ${durationUntilNow.humanize(true)}) -- ${cancelLink}</div>`;
                 });
                 //let sub = resp.name;
                 if(sub === 'All') {

src/util.ts

@@ -77,7 +77,6 @@ import {
     ImageHashCacheData,
     ModUserNoteLabel,
     modUserNoteLabels,
-    PollOn, pollOnTypeMapping, pollOnTypes,
     RedditEntity,
     RedditEntityType,
     RelativeDateTimeMatch,
@@ -2856,7 +2855,7 @@ export const generateSnoowrapEntityFromRedditThing = (data: RedditThing, client:
         case 'user':
             return new RedditUser({id: data.val}, client, false);
         case 'subreddit':
-            return new Subreddit({id: data.val}, client, false);
+            return new Subreddit({name: data.val}, client, false);
         case 'message':
             return new PrivateMessage({id: data.val}, client, false)
 
@@ -3089,12 +3088,3 @@ export const toStrongSharingACLConfig = (data: SharingACLConfig | string[]): Str
         exclude: (data.exclude ?? []).map(x => parseStringToRegexOrLiteralSearch(x))
     }
 }
-
-export const toPollOn = (val: string | PollOn): PollOn => {
-    const clean = val.toLowerCase().trim();
-    const pVal = pollOnTypeMapping.get(clean);
-    if(pVal !== undefined) {
-        return pVal;
-    }
-    throw new SimpleError(`'${val}' is not a valid polling source. Valid sources: ${pollOnTypes.join(' | ')}`);
-}