fix(delayed): Prevent delete call when no ids found

Make sure we delete Activities that were inserted on dispatched actions BUT ONLY if they are not used anywhere else (events or other delayed activities)

src/Common/Entities/Activity.ts

@@ -1,4 +1,4 @@
-import {Entity, Column, ManyToOne, PrimaryColumn, OneToMany, Index} from "typeorm";
+import {Entity, Column, ManyToOne, PrimaryColumn, OneToMany, Index, DataSource, JoinColumn} from "typeorm";
 import {AuthorEntity} from "./AuthorEntity";
 import {Subreddit} from "./Subreddit";
 import {CMEvent} from "./CMEvent";
@@ -6,6 +6,8 @@ import {asComment, getActivityAuthorName, parseRedditFullname, redditThingTypeTo
 import {activityReports, ActivityType, Report, SnoowrapActivity} from "../Infrastructure/Reddit";
 import {ActivityReport} from "./ActivityReport";
 import dayjs, {Dayjs} from "dayjs";
+import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients";
+import {Comment, Submission} from 'snoowrap/dist/objects';
 
 export interface ActivityEntityOptions {
     id: string
@@ -45,7 +47,7 @@ export class Activity {
     @Column({name: 'name'})
     name!: string;
 
-    @ManyToOne(type => Subreddit, sub => sub.activities, {cascade: ['insert']})
+    @ManyToOne(type => Subreddit, sub => sub.activities, {cascade: ['insert'], eager: true})
     subreddit!: Subreddit;
 
     @Column("varchar", {length: 20})
@@ -58,17 +60,18 @@ export class Activity {
     @Column("text")
     permalink!: string;
 
-    @ManyToOne(type => AuthorEntity, author => author.activities, {cascade: ['insert']})
+    @ManyToOne(type => AuthorEntity, author => author.activities, {cascade: ['insert'], eager: true})
     author!: AuthorEntity;
 
-    @OneToMany(type => CMEvent, act => act.activity) // note: we will create author property in the Photo class below
+    @OneToMany(type => CMEvent, act => act.activity)
     actionedEvents!: CMEvent[]
 
-    @ManyToOne(type => Activity, obj => obj.comments, {nullable: true})
+    @ManyToOne('Activity', 'comments', {nullable: true, cascade: ['insert']})
+    @JoinColumn({name: 'submission_id'})
     submission?: Activity;
 
-    @OneToMany(type => Activity, obj => obj.submission, {nullable: true})
-    comments!: Activity[];
+    @OneToMany('Activity', 'submission', {nullable: true})
+    comments?: Activity[];
 
     @OneToMany(type => ActivityReport, act => act.activity, {cascade: ['insert'], eager: true})
     reports: ActivityReport[] | undefined
@@ -151,10 +154,12 @@ export class Activity {
         return false;
     }
 
-    static fromSnoowrapActivity(subreddit: Subreddit, activity: SnoowrapActivity, lastKnownStateTimestamp?: dayjs.Dayjs | undefined) {
+    static async fromSnoowrapActivity(activity: SnoowrapActivity, options: fromSnoowrapOptions | undefined = {}) {
+
         let submission: Activity | undefined;
         let type: ActivityType = 'submission';
         let content: string;
+        const subreddit = await Subreddit.fromSnoowrap(activity.subreddit, options?.db);
         if(asComment(activity)) {
             type = 'comment';
             content = activity.body;
@@ -179,8 +184,30 @@ export class Activity {
             submission
         });
 
-        entity.syncReports(activity, lastKnownStateTimestamp);
+        entity.syncReports(activity, options.lastKnownStateTimestamp);
 
         return entity;
     }
+
+    toSnoowrap(client: ExtendedSnoowrap): SnoowrapActivity {
+        let act: SnoowrapActivity;
+        if(this.type === 'submission') {
+            act = new Submission({name: this.id, id: this.name}, client, false);
+            act.title = this.content;
+        } else {
+            act = new Comment({name: this.id, id: this.name}, client, false);
+            act.link_id = this.submission?.id as string;
+            act.body = this.content;
+        }
+        act.permalink = this.permalink;
+        act.subreddit = this.subreddit.toSnoowrap(client);
+        act.author = this.author.toSnoowrap(client);
+
+        return act;
+    }
+}
+
+export interface fromSnoowrapOptions {
+    lastKnownStateTimestamp?: dayjs.Dayjs | undefined
+    db?: DataSource
 }

src/Common/Entities/AuthorEntity.ts

@@ -1,5 +1,8 @@
 import {Entity, Column, PrimaryColumn, OneToMany} from "typeorm";
 import {Activity} from "./Activity";
+import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients";
+import {SnoowrapActivity} from "../Infrastructure/Reddit";
+import {RedditUser} from "snoowrap/dist/objects";
 
 @Entity({name: 'Author'})
 export class AuthorEntity {
@@ -11,11 +14,15 @@ export class AuthorEntity {
     name!: string;
 
     @OneToMany(type => Activity, act => act.author)
-    activities!: Activity[]
+    activities!: Promise<Activity[]>
 
     constructor(data?: any) {
         if(data !== undefined) {
             this.name = data.name;
         }
     }
+
+    toSnoowrap(client: ExtendedSnoowrap): RedditUser {
+        return new RedditUser({name: this.name, id: this.id}, client, false);
+    }
 }

src/Common/Entities/DispatchedEntity.ts

@@ -6,7 +6,7 @@ import {
     ManyToOne,
     PrimaryColumn,
     BeforeInsert,
-    AfterLoad
+    AfterLoad, JoinColumn
 } from "typeorm";
 import {
     ActivityDispatch
@@ -22,15 +22,15 @@ import Comment from "snoowrap/dist/objects/Comment";
 import {ColumnDurationTransformer} from "./Transformers";
 import { RedditUser } from "snoowrap/dist/objects";
 import {ActivitySourceTypes, DurationVal, NonDispatchActivitySourceValue, onExistingFoundBehavior} from "../Infrastructure/Atomic";
+import {Activity} from "./Activity";
 
 @Entity({name: 'DispatchedAction'})
 export class DispatchedEntity extends TimeAwareRandomBaseEntity {
 
-    @Column()
-    activityId!: string
-
-    @Column()
-    author!: string
+    //@ManyToOne(type => Activity, obj => obj.dispatched, {cascade: ['insert'], eager: true, nullable: false})
+    @ManyToOne(type => Activity, undefined, {cascade: ['insert'], eager: true, nullable: false})
+    @JoinColumn({name: 'activityId'})
+    activity!: Activity
 
     @Column({
         type: 'int',
@@ -82,11 +82,10 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity {
         }})
     tardyTolerant!: boolean | Duration
 
-    constructor(data?: ActivityDispatch & { manager: ManagerEntity }) {
+    constructor(data?: HydratedActivityDispatch) {
         super();
         if (data !== undefined) {
-            this.activityId = data.activity.name;
-            this.author = getActivityAuthorName(data.activity.author);
+            this.activity = data.activity;
             this.delay = data.delay;
             this.createdAt = data.queuedAt;
             this.type = data.type;
@@ -151,20 +150,7 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity {
     }
 
     async toActivityDispatch(client: ExtendedSnoowrap): Promise<ActivityDispatch> {
-        const redditThing = parseRedditFullname(this.activityId);
-        if(redditThing === undefined) {
-            throw new Error(`Could not parse reddit ID from value '${this.activityId}'`);
-        }
-        let activity: Comment | Submission;
-        if (redditThing?.type === 'comment') {
-            // @ts-ignore
-            activity = await client.getComment(redditThing.id);
-        } else {
-            // @ts-ignore
-            activity = await client.getSubmission(redditThing.id);
-        }
-        activity.author = new RedditUser({name: this.author}, client, false);
-        activity.id  = redditThing.id;
+        let activity = this.activity.toSnoowrap(client);
         return {
             id: this.id,
             queuedAt: this.createdAt,
@@ -176,8 +162,13 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity {
             cancelIfQueued: this.cancelIfQueued,
             identifier: this.identifier,
             type: this.type,
-            author: this.author,
+            author: activity.author.name,
             dryRun: this.dryRun
         }
     }
 }
+
+export interface HydratedActivityDispatch extends Omit<ActivityDispatch, 'activity'> {
+    activity: Activity
+    manager: ManagerEntity
+}

src/Common/Entities/Subreddit.ts

@@ -1,5 +1,7 @@
-import {Entity, Column, PrimaryColumn, OneToMany, Index} from "typeorm";
+import {Entity, Column, PrimaryColumn, OneToMany, Index, DataSource} from "typeorm";
 import {Activity} from "./Activity";
+import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients";
+import {Subreddit as SnoowrapSubreddit} from "snoowrap/dist/objects";
 
 export interface SubredditEntityOptions {
     id: string
@@ -25,4 +27,18 @@ export class Subreddit {
             this.name = data.name;
         }
     }
+
+    toSnoowrap(client: ExtendedSnoowrap): SnoowrapSubreddit {
+        return new SnoowrapSubreddit({display_name: this.name, name: this.id}, client, false);
+    }
+
+    static async fromSnoowrap(subreddit: SnoowrapSubreddit, db?: DataSource) {
+        if(db !== undefined) {
+           const existing = await db.getRepository(Subreddit).findOneBy({name: subreddit.display_name});
+           if(existing) {
+               return existing;
+           }
+        }
+        return new Subreddit({id: await subreddit.name, name: await subreddit.display_name});
+    }
 }

src/Common/Migrations/Database/Server/1667415256831-delayedReset.ts

@@ -0,0 +1,19 @@
+import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"
+
+export class delayedReset1667415256831 implements MigrationInterface {
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        queryRunner.connection.logger.logSchemaBuild('Truncating (removing) existing Dispatched Actions due to internal structural changes');
+        await queryRunner.clearTable('DispatchedAction');
+        await queryRunner.changeColumn('DispatchedAction', 'author', new TableColumn({
+            name: 'author',
+            type: 'varchar',
+            length: '150',
+            isNullable: true
+        }));
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+    }
+
+}

src/Subreddit/Manager.ts

@@ -975,7 +975,7 @@ export class Manager extends EventEmitter implements RunningStates {
         let shouldPersistReports = false;
 
         if (existingEntity === null) {
-            activityEntity = Activity.fromSnoowrapActivity(this.managerEntity.subreddit, activity, lastKnownStateTimestamp);
+            activityEntity = await Activity.fromSnoowrapActivity(activity, {lastKnownStateTimestamp, db: this.resources.database});
             // always persist if activity is not already persisted and any reports exist
             if (item.num_reports > 0) {
                 shouldPersistReports = true;
@@ -1189,7 +1189,7 @@ export class Manager extends EventEmitter implements RunningStates {
                                     // @ts-ignore
                                     const subProxy = await this.client.getSubmission((item as Comment).link_id);
                                     const sub = await this.resources.getActivity(subProxy);
-                                    subActivity = await this.activityRepo.save(Activity.fromSnoowrapActivity(this.managerEntity.subreddit, sub));
+                                    subActivity = await this.activityRepo.save(await Activity.fromSnoowrapActivity(sub, {db: this.resources.database}));
                                 }
                                 event.activity.submission = subActivity;
 

src/Subreddit/SubredditResources.ts

@@ -69,7 +69,16 @@ import {cacheTTLDefaults, createHistoricalDisplayDefaults,} from "../Common/defa
 import {ExtendedSnoowrap} from "../Utils/SnoowrapClients";
 import dayjs, {Dayjs} from "dayjs";
 import ImageData from "../Common/ImageData";
-import {Between, DataSource, DeleteQueryBuilder, LessThan, Repository, SelectQueryBuilder} from "typeorm";
+import {
+    Between, Brackets,
+    DataSource,
+    DeleteQueryBuilder,
+    In,
+    LessThan,
+    NotBrackets,
+    Repository,
+    SelectQueryBuilder
+} from "typeorm";
 import {CMEvent as ActionedEventEntity, CMEvent} from "../Common/Entities/CMEvent";
 import {RuleResultEntity} from "../Common/Entities/RuleResultEntity";
 import globrex from 'globrex';
@@ -162,6 +171,8 @@ import {ActivitySource} from "../Common/ActivitySource";
 import {SubredditResourceOptions} from "../Common/Subreddit/SubredditResourceInterfaces";
 import {SubredditStats} from "./Stats";
 import {CMCache} from "../Common/Cache";
+import { Activity } from '../Common/Entities/Activity';
+import {FindOptionsWhere} from "typeorm/find-options/FindOptionsWhere";
 
 export const DEFAULT_FOOTER = '\r\n*****\r\nThis action was performed by [a bot.]({{botLink}}) Mention a moderator or [send a modmail]({{modmailLink}}) if you have any ideas, questions, or concerns about this action.';
 
@@ -193,6 +204,7 @@ export class SubredditResources {
     botAccount?: string;
     dispatchedActivityRepo: Repository<DispatchedEntity>
     activitySourceRepo: Repository<ActivitySourceEntity>
+    activityRepo: Repository<Activity>
     retention?: EventRetentionPolicyRange
     managerEntity: ManagerEntity
     botEntity: Bot
@@ -229,6 +241,7 @@ export class SubredditResources {
         this.database = database;
         this.dispatchedActivityRepo = this.database.getRepository(DispatchedEntity);
         this.activitySourceRepo = this.database.getRepository(ActivitySourceEntity);
+        this.activityRepo = this.database.getRepository(Activity);
         this.retention = retention;
         //this.prefix = prefix;
         this.client = client;
@@ -404,21 +417,25 @@ export class SubredditResources {
                     }
                 },
                 relations: {
-                    manager: true
+                    manager: true,
+                    activity: {
+                        submission: true
+                    }
                 }
             });
             const now = dayjs();
+            const toRemove = [];
             for(const dAct of dispatchedActivities) {
                 const shouldDispatchAt = dAct.createdAt.add(dAct.delay.asSeconds(), 'seconds');
                 let tardyHint = '';
                 if(shouldDispatchAt.isBefore(now)) {
-                    let tardyHint = `Activity ${dAct.activityId} queued at ${dAct.createdAt.format('YYYY-MM-DD HH:mm:ssZ')} for ${dAct.delay.humanize()} is now LATE`;
+                    let tardyHint = `Activity ${dAct.activity.id} queued at ${dAct.createdAt.format('YYYY-MM-DD HH:mm:ssZ')} for ${dAct.delay.humanize()} is now LATE`;
                     if(dAct.tardyTolerant === true) {
                         tardyHint += ` but was configured as ALWAYS 'tardy tolerant' so will be dispatched immediately`;
                     } else if(dAct.tardyTolerant === false) {
                         tardyHint += ` and was not configured as 'tardy tolerant' so will be dropped`;
                         this.logger.warn(tardyHint);
-                        await this.removeDelayedActivity(dAct.id);
+                        toRemove.push(dAct.id);
                         continue;
                     } else {
                         // see if its within tolerance
@@ -426,7 +443,7 @@ export class SubredditResources {
                         if(latest.isBefore(now)) {
                             tardyHint += ` and IS NOT within tardy tolerance of ${dAct.tardyTolerant.humanize()} of planned dispatch time so will be dropped`;
                             this.logger.warn(tardyHint);
-                            await this.removeDelayedActivity(dAct.id);
+                            toRemove.push(dAct.id);
                             continue;
                         } else {
                             tardyHint += `but is within tardy tolerance of ${dAct.tardyTolerant.humanize()} of planned dispatch time so will be dispatched immediately`;
@@ -439,27 +456,115 @@ export class SubredditResources {
                 try {
                     this.delayedItems.push(await dAct.toActivityDispatch(this.client))
                 } catch (e) {
-                    this.logger.warn(new ErrorWithCause(`Unable to add Activity ${dAct.activityId} from database delayed activities to in-app delayed activities queue`, {cause: e}));
+                    this.logger.warn(new ErrorWithCause(`Unable to add Activity ${dAct.activity.id} from database delayed activities to in-app delayed activities queue`, {cause: e}));
                 }
             }
+            if(toRemove.length > 0) {
+                await this.removeDelayedActivity(toRemove);
+            }
         }
     }
 
     async addDelayedActivity(data: ActivityDispatch) {
-        const dEntity = await this.dispatchedActivityRepo.save(new DispatchedEntity({...data, manager: this.managerEntity}));
+        // TODO merge this with getActivity or something...
+        if(asComment(data.activity)) {
+            const existingSub =  await this.activityRepo.findOneBy({_id: data.activity.link_id});
+            if(existingSub === null) {
+                const sub = await this.getActivity(new Submission({name: data.activity.link_id}, this.client, false));
+                await this.activityRepo.save(await Activity.fromSnoowrapActivity(sub, {db: this.database}));
+            }
+        }
+        const dEntity = await this.dispatchedActivityRepo.save(new DispatchedEntity({...data, manager: this.managerEntity, activity: await Activity.fromSnoowrapActivity(data.activity, {db: this.database})}));
         data.id = dEntity.id;
         this.delayedItems.push(data);
     }
 
     async removeDelayedActivity(val?: string | string[]) {
-        if(val === undefined) {
-            await this.dispatchedActivityRepo.delete({manager: {id: this.managerEntity.id}});
-            this.delayedItems = [];
-        } else {
+        let dispatched: DispatchedEntity[] = [];
+        const where: FindOptionsWhere<DispatchedEntity> = {
+            manager: {
+                id: this.managerEntity.id
+            }
+        };
+
+        if(val !== undefined) {
             const ids = typeof val === 'string' ? [val] : val;
-            await this.dispatchedActivityRepo.delete(ids);
-            this.delayedItems = this.delayedItems.filter(x => !ids.includes(x.id));
+            where.id = In(ids);
         }
+
+        dispatched = await this.dispatchedActivityRepo.find({
+            where,
+            relations: {
+                manager: true,
+                activity: {
+                    actionedEvents: true,
+                    submission: {
+                        actionedEvents: true
+                    }
+                }
+            }
+        });
+
+        const actualDispatchedIds = dispatched.map(x => x.id);
+        this.logger.debug(`${actualDispatchedIds.length} marked for deletion`, {leaf: 'Delayed Activities'});
+
+        // get potential activities to delete
+        // but only include activities that don't have any actionedEvents
+        let activityIdsToDelete = Array.from(dispatched.reduce((acc, curr) => {
+            if(curr.activity.actionedEvents === null || curr.activity.actionedEvents.length === 0) {
+                acc.add(curr.activity.id);
+            }
+            if(curr.activity.submission !== undefined && curr.activity.submission !== null) {
+                if(curr.activity.submission.actionedEvents === null || curr.activity.submission.actionedEvents.length === 0) {
+                    acc.add(curr.activity.submission.id);
+                }
+            }
+            return acc;
+        }, new Set<string>()));
+        const rawActCount = activityIdsToDelete.length;
+        let activeActCount = 0;
+
+        // if we have any potential activities to delete we now need to get any dispatched actions that reference these activities
+        // that are NOT the ones we are going to delete
+        if(activityIdsToDelete.length > 0) {
+            const activeDispatchedQuery = this.dispatchedActivityRepo.createQueryBuilder('dis')
+                .leftJoinAndSelect('dis.activity', 'activity')
+                .leftJoinAndSelect('activity.submission', 'submission')
+                .where(new NotBrackets((qb) => {
+                    qb.where('dis.id IN (:...currIds)', {currIds: actualDispatchedIds});
+                }))
+                .andWhere(new Brackets((qb) => {
+                    qb.where('activity._id IN (:...actMainIds)', {actMainIds: activityIdsToDelete})
+                    qb.orWhere('submission._id IN (:...actSubIds)', {actSubIds: activityIdsToDelete})
+                }));
+            //const sql = activeDispatchedQuery.getSql();
+            const activeDispatched = await activeDispatchedQuery.getMany();
+
+            // all activity ids, from the actions to delete, that are being used by dispatched actions that are NOT the ones we are going to delete
+            const activeDispatchedIds = Array.from(activeDispatched.reduce((acc, curr) => {
+                acc.add(curr.activity.id);
+                if(curr.activity.submission !== undefined && curr.activity.submission !== null) {
+                    acc.add(curr.activity.submission.id);
+                }
+                return acc;
+            }, new Set<string>()));
+            activeActCount = activeDispatchedIds.length;
+
+            // filter out any that are still in use
+            activityIdsToDelete = activityIdsToDelete.filter(x => !activeDispatchedIds.includes(x));
+        }
+
+        this.logger.debug(`Marked ${activityIdsToDelete.length} Activities created, by Delayed, for deletion (${rawActCount} w/o Events | ${activeActCount} used by other Delayed Activities)`, {leaf: 'Delayed Activities'});
+
+        if(actualDispatchedIds.length > 0) {
+            await this.dispatchedActivityRepo.delete(actualDispatchedIds);
+        } else {
+            this.logger.warn('No dispatched ids found to delete');
+        }
+        if(activityIdsToDelete.length > 0) {
+            await this.activityRepo.delete(activityIdsToDelete);
+        }
+        this.delayedItems = this.delayedItems.filter(x => !actualDispatchedIds.includes(x.id));
     }
 
     async initStats() {

src/Web/assets/views/status.ejs

@@ -1297,7 +1297,7 @@
                     const durationDayjs = dayjs.duration(x.duration, 'seconds');
                     const durationDisplay = durationDayjs.humanize();
                     const cancelLink = `<a href="#" data-id="${x.id}" data-subreddit="${x.subreddit}" class="delayCancel">CANCEL</a>`;
-                    return `<div>A <a href="https://reddit.com${x.permalink}">${x.submissionId !== undefined ? 'Comment' : 'Submission'}</a>${isAll ? ` in <a href="https://reddit.com${x.subreddit}">${x.subreddit}</a> ` : ''} by <a href="https://reddit.com/u/${x.author}">${x.author}</a> queued by ${x.source} at ${queuedAtDisplay} for ${durationDisplay} (dispatches ${durationUntilNow.humanize(true)}) -- ${cancelLink}</div>`;
+                    return `<div>A <a href="https://reddit.com${x.permalink}">${x.submissionId !== undefined ? 'Comment' : 'Submission'}</a> by <a href="https://reddit.com/u/${x.author}">${x.author}</a>${isAll ? `, dispatched in <a href="https://reddit.com${x.subreddit}">${x.subreddit}</a> ,` : ''} queued by ${x.source} at ${queuedAtDisplay} for ${durationDisplay} (dispatches ${durationUntilNow.humanize(true)}) -- ${cancelLink}</div>`;
                 });
                 //let sub = resp.name;
                 if(sub === 'All') {

src/util.ts

@@ -2855,7 +2855,7 @@ export const generateSnoowrapEntityFromRedditThing = (data: RedditThing, client:
         case 'user':
             return new RedditUser({id: data.val}, client, false);
         case 'subreddit':
-            return new Subreddit({id: data.val}, client, false);
+            return new Subreddit({name: data.val}, client, false);
         case 'message':
             return new PrivateMessage({id: data.val}, client, false)