Merge branch 'edge'

Stop or restart real-time data (logs, stats) based on page visibility API so that client does continue to consume data when page is in the background/in a non-visible tab

Dockerfile

@@ -120,6 +120,10 @@ ENV NPM_CONFIG_LOGLEVEL debug
 # can set database to use more performant better-sqlite3 since we control everything
 ENV DB_DRIVER=better-sqlite3
 
+# NODE_ARGS are expanded after `node` command in the entrypoint IE "node {NODE_ARGS} src/index.js run"
+# by default enforce better memory mangement by limiting max long-lived GC space to 512MB
+ENV NODE_ARGS="--max_old_space_size=512"
+
 ARG webPort=8085
 ENV PORT=$webPort
 EXPOSE $PORT

docker/root/etc/services.d/node/run

@@ -2,6 +2,8 @@
 
 # used https://github.com/linuxserver/docker-plex as a template
 
+# NODE_ARGS can be passed by ENV in docker command like "docker run foxxmd/context-mod -e NODE_ARGS=--optimize_for_size"
+
 exec \
 	s6-setuidgid abc \
-	/usr/local/bin/node /app/src/index.js run
+	/usr/local/bin/node $NODE_ARGS /app/src/index.js run

docs/operator/installation.md

@@ -76,3 +76,21 @@ Be aware that Heroku's [free dyno plan](https://devcenter.heroku.com/articles/fr
 * The **Worker** dyno **will not** go to sleep but you will NOT be able to access the web interface. You can, however, still see how Cm is running by reading the logs for the dyno.
 
 If you want to use a free dyno it is recommended you perform first-time setup (bot authentication and configuration, testing, etc...) with the **Web** dyno, then SWITCH to a **Worker** dyno so it can run 24/7.
+
+# Memory Management
+
+Node exhibits [lazy GC cleanup](https://github.com/FoxxMD/context-mod/issues/90#issuecomment-1190384006) which can result in memory usage for long-running CM instances increasing to unreasonable levels. This problem does not seem to be an issue with CM itself but with Node's GC approach. The increase does not affect CM's performance and, for systems with less memory, the Node *should* limit memory usage based on total available.
+
+In practice CM uses ~130MB for a single bot, single subreddit setup. Up to ~350MB for many (10+) bots or many (20+) subreddits.
+
+If you need to reign in CM's memory usage for some reason this can be addressed by setting an upper limit for memory usage with `node` args by using either:
+
+**--max_old_space_size=**
+
+Value is megabytes. This sets an explicit limit on GC memory usage.
+
+This is set by default in the [Docker](#docker-recommended) container using the env `NODE_ARGS` to `--max_old_space_size=512`. It can be disabled by overriding the ENV.
+
+**--optimize_for_size**
+
+Tells Node to optimize for (less) memory usage rather than some performance optimizations. This option is not memory size dependent. In practice performance does not seem to be affected and it reduces (but not entirely prevents) memory increases over long periods.

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "redditcontextbot",
-  "version": "0.5.1",
+  "version": "0.11.4",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "redditcontextbot",
-      "version": "0.5.1",
+      "version": "0.11.4",
       "hasInstallScript": true,
       "license": "ISC",
       "dependencies": {

src/Action/BanAction.ts

@@ -41,12 +41,11 @@ export class BanAction extends Action {
 
     async process(item: Comment | Submission, ruleResults: RuleResultEntity[], options: runCheckOptions): Promise<ActionProcessResult> {
         const dryRun = this.getRuntimeAwareDryrun(options);
-        const content = this.message === undefined ? undefined : await this.resources.getContent(this.message, item.subreddit);
-        const renderedBody = content === undefined ? undefined : await renderContent(content, item, ruleResults, this.resources.userNotes);
+        const renderedBody = this.message === undefined ? undefined : await this.resources.renderContent(this.message, item, ruleResults);
         const renderedContent = renderedBody === undefined ? undefined : `${renderedBody}${await this.resources.generateFooter(item, this.footer)}`;
 
-        const renderedReason = this.reason === undefined ? undefined : truncate(await renderContent(this.reason, item, ruleResults, this.resources.userNotes));
-        const renderedNote = this.note === undefined ? undefined : truncate(await renderContent(this.note, item, ruleResults, this.resources.userNotes));
+        const renderedReason = this.reason === undefined ? undefined : truncate(await this.resources.renderContent(this.reason, item, ruleResults));
+        const renderedNote = this.note === undefined ? undefined : truncate(await this.resources.renderContent(this.note, item, ruleResults));
 
         const touchedEntities = [];
         let banPieces = [];
@@ -108,7 +107,6 @@ export interface BanActionConfig extends ActionConfig, Footer {
      *
      * If the length expands to more than 100 characters it will truncated with "..."
      *
-     * @maxLength 100
      * @examples ["repeat spam"]
      * */
     reason?: string
@@ -124,7 +122,6 @@ export interface BanActionConfig extends ActionConfig, Footer {
      *
      * If the length expands to more than 100 characters it will truncated with "..."
      *
-     * @maxLength 100
      * @examples ["Sock puppet for u/AnotherUser"]
      * */
     note?: string

src/Action/MessageAction.ts

@@ -50,8 +50,9 @@ export class MessageAction extends Action {
 
     async process(item: Comment | Submission, ruleResults: RuleResultEntity[], options: runCheckOptions): Promise<ActionProcessResult> {
         const dryRun = this.getRuntimeAwareDryrun(options);
-        const content = await this.resources.getContent(this.content);
-        const body = await renderContent(content, item, ruleResults, this.resources.userNotes);
+
+        const body = await this.resources.renderContent(this.content, item, ruleResults);
+        const subject = this.title === undefined ? `Concerning your ${isSubmission(item) ? 'Submission' : 'Comment'}` : await this.resources.renderContent(this.title, item, ruleResults);
 
         const footer = await this.resources.generateFooter(item, this.footer);
 
@@ -80,7 +81,7 @@ export class MessageAction extends Action {
             text: renderedContent,
             // @ts-ignore
             fromSubreddit: this.asSubreddit ? await item.subreddit.fetch() : undefined,
-            subject: this.title || `Concerning your ${isSubmission(item) ? 'Submission' : 'Comment'}`,
+            subject: subject,
         };
 
         const msgPreview = `\r\n

src/Bot/index.ts

@@ -13,7 +13,7 @@ import {
     USER
 } from "../Common/interfaces";
 import {
-    createRetryHandler, difference,
+    createRetryHandler, symmetricalDifference,
     formatNumber, getExceptionMessage, getUserAgent,
     mergeArr,
     parseBool,
@@ -456,7 +456,7 @@ class Bot {
                     return acc;
                 }
             }, []);
-            const notMatched = difference(normalizedOverrideNames, subsToRunNames);
+            const notMatched = symmetricalDifference(normalizedOverrideNames, subsToRunNames);
             if(notMatched.length > 0) {
                 this.logger.warn(`There are overrides defined for subreddits the bot is not running. Check your spelling! Overrides not matched: ${notMatched.join(', ')}`);
             }
@@ -814,10 +814,14 @@ class Bot {
     async healthLoop() {
         while (this.running) {
             await sleep(5000);
-            await this.apiHealthCheck();
+            const time = dayjs().valueOf()
+            await this.apiHealthCheck(time);
             if (!this.running) {
                 break;
             }
+            for(const m of this.subManagers) {
+                await m.writeHealthMetrics(time);
+            }
             const now = dayjs();
             if (now.isSameOrAfter(this.nextNannyCheck)) {
                 try {
@@ -857,7 +861,7 @@ class Bot {
         return`API Usage Rolling Avg: ${formatNumber(this.apiRollingAvg)}/s | Est Depletion: ${depletion} (${formatNumber(this.depletedInSecs, {toFixed: 0})} seconds)`;
     }
 
-    async apiHealthCheck() {
+    async apiHealthCheck(time?: number) {
 
         const rollingSample = this.apiSample.slice(0, 7)
         rollingSample.unshift(this.client.ratelimitRemaining);
@@ -887,6 +891,10 @@ class Bot {
                 .intField('remaining', this.client.ratelimitRemaining)
                 .stringField('nannyMod', this.nannyMode ?? 'none');
 
+            if(time !== undefined) {
+                apiMeasure.timestamp(time);
+            }
+
             if(this.apiSample.length > 1) {
                 const curr = this.apiSample[0];
                 const last = this.apiSample[1];

src/Common/interfaces.ts

@@ -1553,6 +1553,23 @@ export interface OperatorJsonConfig {
     }
 
     credentials?: ThirdPartyCredentialsJsonConfig
+
+    dev?: {
+        /**
+         * Invoke `process.memoryUsage()` on an interval and send metrics to Influx
+         *
+         * Only works if Influx config is provided
+         * */
+        monitorMemory?: boolean
+        /**
+        * Interval, in seconds, to invoke `process.memoryUsage()` at
+        *
+        * Defaults to 15 seconds
+        *
+        * @default 15
+        * */
+        monitorMemoryInterval?: number
+    };
 }
 
 export interface RequiredOperatorRedditCredentials extends RedditCredentials {
@@ -1659,6 +1676,10 @@ export interface OperatorConfig extends OperatorJsonConfig {
     databaseStatisticsDefaults: DatabaseStatisticsOperatorConfig
     bots: BotInstanceConfig[]
     credentials: ThirdPartyCredentialsJsonConfig
+    dev: {
+        monitorMemory: boolean
+        monitorMemoryInterval: number
+    }
 }
 
 export interface OperatorFileConfig {

src/ConfigBuilder.ts

@@ -1239,6 +1239,10 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig):
         } = {},
         credentials = {},
         bots = [],
+        dev: {
+            monitorMemory = false,
+            monitorMemoryInterval = 15
+        } = {},
     } = data;
 
     let cache: StrongCache;
@@ -1387,6 +1391,10 @@ export const buildOperatorConfigWithDefaults = async (data: OperatorJsonConfig):
         },
         bots: [],
         credentials,
+        dev: {
+            monitorMemory,
+            monitorMemoryInterval
+        }
     };
 
     config.bots = bots.map(x => buildBotConfig(x, config));

src/Schema/Action.json

@@ -442,7 +442,6 @@
                     "examples": [
                         "Sock puppet for u/AnotherUser"
                     ],
-                    "maxLength": 100,
                     "type": "string"
                 },
                 "reason": {
@@ -450,7 +449,6 @@
                     "examples": [
                         "repeat spam"
                     ],
-                    "maxLength": 100,
                     "type": "string"
                 }
             },

src/Schema/App.json

@@ -923,7 +923,6 @@
                     "examples": [
                         "Sock puppet for u/AnotherUser"
                     ],
-                    "maxLength": 100,
                     "type": "string"
                 },
                 "reason": {
@@ -931,7 +930,6 @@
                     "examples": [
                         "repeat spam"
                     ],
-                    "maxLength": 100,
                     "type": "string"
                 }
             },

src/Schema/Check.json

@@ -937,7 +937,6 @@
                     "examples": [
                         "Sock puppet for u/AnotherUser"
                     ],
-                    "maxLength": 100,
                     "type": "string"
                 },
                 "reason": {
@@ -945,7 +944,6 @@
                     "examples": [
                         "repeat spam"
                     ],
-                    "maxLength": 100,
                     "type": "string"
                 }
             },

src/Schema/OperatorConfig.json

@@ -2243,6 +2243,20 @@
             "$ref": "#/definitions/DatabaseStatisticsOperatorJsonConfig",
             "description": "Set defaults for the frequency time series stats are collected"
         },
+        "dev": {
+            "properties": {
+                "monitorMemory": {
+                    "description": "Invoke `process.memoryUsage()` on an interval and send metrics to Influx\n\nOnly works if Influx config is provided",
+                    "type": "boolean"
+                },
+                "monitorMemoryInterval": {
+                    "default": 15,
+                    "description": "Interval, in seconds, to invoke `process.memoryUsage()` at\n\nDefaults to 15 seconds",
+                    "type": "number"
+                }
+            },
+            "type": "object"
+        },
         "influxConfig": {
             "$ref": "#/definitions/InfluxConfig"
         },

src/Schema/Run.json

@@ -934,7 +934,6 @@
                     "examples": [
                         "Sock puppet for u/AnotherUser"
                     ],
-                    "maxLength": 100,
                     "type": "string"
                 },
                 "reason": {
@@ -942,7 +941,6 @@
                     "examples": [
                         "repeat spam"
                     ],
-                    "maxLength": 100,
                     "type": "string"
                 }
             },

src/Subreddit/Manager.ts

@@ -1763,4 +1763,32 @@ export class Manager extends EventEmitter implements RunningStates {
 
         await this.cacheManager.defaultDatabase.getRepository(ManagerEntity).save(this.managerEntity);
     }
+
+    async writeHealthMetrics(time?: number) {
+        if (this.influxClients.length > 0) {
+            const metric = new Point('managerHealth')
+                .intField('delayedActivities', this.resources !== undefined ? this.resources.delayedItems.length : 0)
+                .intField('processing', this.queue.running())
+                .intField('queued', this.queue.length())
+                .booleanField('eventsRunning', this.eventsState.state === RUNNING)
+                .booleanField('queueRunning', this.queueState.state === RUNNING)
+                .booleanField('running', this.managerState.state === RUNNING)
+                .intField('uptime', this.startedAt !== undefined ? dayjs().diff(this.startedAt, 'seconds') : 0)
+                .intField('configAge', this.lastWikiRevision === undefined ? 0 : dayjs().diff(this.lastWikiRevision, 'seconds'));
+
+            if (this.resources !== undefined) {
+                const {req, miss} = this.resources.getCacheTotals();
+                metric.intField('cacheRequests', req)
+                    .intField('cacheMisses', miss);
+            }
+
+            if (time !== undefined) {
+                metric.timestamp(time);
+            }
+
+            for (const client of this.influxClients) {
+                await client.writePoint(metric);
+            }
+        }
+    }
 }

src/Subreddit/SubredditResources.ts

@@ -4,7 +4,7 @@ import {
     activityIsDeleted, activityIsFiltered,
     activityIsRemoved,
     AuthorTypedActivitiesOptions, BOT_LINK,
-    getAuthorHistoryAPIOptions
+    getAuthorHistoryAPIOptions, renderContent
 } from "../Utils/SnoowrapUtils";
 import {map as mapAsync} from 'async';
 import winston, {Logger} from "winston";
@@ -96,7 +96,7 @@ import {CMEvent as ActionedEventEntity, CMEvent } from "../Common/Entities/CMEve
 import {RuleResultEntity} from "../Common/Entities/RuleResultEntity";
 import globrex from 'globrex';
 import {runMigrations} from "../Common/Migrations/CacheMigrationUtils";
-import {isStatusError, MaybeSeriousErrorWithCause, SimpleError} from "../Utils/Errors";
+import {CMError, isStatusError, MaybeSeriousErrorWithCause, SimpleError} from "../Utils/Errors";
 import {ErrorWithCause} from "pony-cause";
 import {ManagerEntity} from "../Common/Entities/ManagerEntity";
 import {Bot} from "../Common/Entities/Bot";
@@ -510,9 +510,15 @@ export class SubredditResources {
         this.delayedItems.push(data);
     }
 
-    async removeDelayedActivity(id: string) {
-        await this.dispatchedActivityRepo.delete(id);
-        this.delayedItems = this.delayedItems.filter(x => x.id !== id);
+    async removeDelayedActivity(val?: string | string[]) {
+        if(val === undefined) {
+            await this.dispatchedActivityRepo.delete({manager: {id: this.managerEntity.id}});
+            this.delayedItems = [];
+        } else {
+            const ids = typeof val === 'string' ? [val] : val;
+            await this.dispatchedActivityRepo.delete(ids);
+            this.delayedItems = this.delayedItems.filter(x => !ids.includes(x.id));
+        }
     }
 
     async initStats() {
@@ -773,11 +779,15 @@ export class SubredditResources {
         }
     }
 
-    async getStats() {
-        const totals = Object.values(this.stats.cache).reduce((acc, curr) => ({
+    getCacheTotals() {
+        return Object.values(this.stats.cache).reduce((acc, curr) => ({
             miss: acc.miss + curr.miss,
             req: acc.req + curr.requests,
         }), {miss: 0, req: 0});
+    }
+
+    async getStats() {
+        const totals = this.getCacheTotals();
         const cacheKeys = Object.keys(this.stats.cache);
         const res = {
             cache: {
@@ -1751,6 +1761,14 @@ export class SubredditResources {
         return wikiContent;
     }
 
+    /**
+     * Convenience method for using getContent and SnoowrapUtils@renderContent in one method
+     * */
+    async renderContent(contentStr: string, data: SnoowrapActivity, ruleResults: RuleResultEntity[] = [], usernotes?: UserNotes) {
+        const content = await this.getContent(contentStr);
+        return await renderContent(content, data, ruleResults, usernotes ?? this.userNotes);
+    }
+
     async getConfigFragment<T>(includesData: IncludesData, validateFunc?: ConfigFragmentValidationFunc): Promise<T> {
 
         const {
@@ -2268,7 +2286,7 @@ export class SubredditResources {
                             const requestedSourcesVal: string[] = !Array.isArray(itemOptVal) ? [itemOptVal] as string[] : itemOptVal as string[];
                             const requestedSources = requestedSourcesVal.map(x => strToActivitySource(x).toLowerCase());
 
-                            propResultsMap.source!.passed = criteriaPassWithIncludeBehavior(requestedSources.some(x => source.toLowerCase().includes(x)), include);
+                            propResultsMap.source!.passed = criteriaPassWithIncludeBehavior(requestedSources.some(x => source.toLowerCase().trim() === x.toLowerCase().trim()), include);
                             break;
                         }
                     case 'score':
@@ -3568,11 +3586,19 @@ export class BotResourcesManager {
     }
 
     async addPendingSubredditInvite(subreddit: string): Promise<void> {
+        if(subreddit === null || subreddit === undefined || subreddit == '') {
+            throw new CMError('Subreddit name cannot be empty');
+        }
         let subredditNames = await this.defaultCache.get(`modInvites`) as (string[] | undefined | null);
         if (subredditNames === undefined || subredditNames === null) {
             subredditNames = [];
         }
-        subredditNames.push(subreddit);
+        const cleanName = subreddit.trim();
+
+        if(subredditNames.some(x => x.trim().toLowerCase() === cleanName.toLowerCase())) {
+            throw new CMError(`An invite for the Subreddit '${subreddit}' already exists`);
+        }
+        subredditNames.push(cleanName);
         await this.defaultCache.set(`modInvites`, subredditNames, {ttl: 0});
         return;
     }
@@ -3582,7 +3608,7 @@ export class BotResourcesManager {
         if (subredditNames === undefined || subredditNames === null) {
             subredditNames = [];
         }
-        subredditNames = subredditNames.filter(x => x !== subreddit);
+        subredditNames = subredditNames.filter(x => x.toLowerCase() !== subreddit.trim().toLowerCase());
         await this.defaultCache.set(`modInvites`, subredditNames, {ttl: 0});
         return;
     }

src/Utils/SnoowrapUtils.ts

@@ -116,6 +116,9 @@ export const isSubreddit = async (subreddit: Subreddit, stateCriteria: Subreddit
     })() as boolean;
 }
 
+const renderContentCommentTruncate = truncateStringToLength(50);
+const shortTitleTruncate = truncateStringToLength(15);
+
 export const renderContent = async (template: string, data: (Submission | Comment), ruleResults: RuleResultEntity[] = [], usernotes: UserNotes) => {
     const conditional: any = {};
     if(data.can_mod_post) {
@@ -133,11 +136,13 @@ export const renderContent = async (template: string, data: (Submission | Commen
     }
     const templateData: any = {
         kind: data instanceof Submission ? 'submission' : 'comment',
-        author: await data.author.name,
+        // @ts-ignore
+        author: getActivityAuthorName(await data.author),
         votes: data.score,
         age: dayjs.duration(dayjs().diff(dayjs.unix(data.created))).humanize(),
         permalink: `https://reddit.com${data.permalink}`,
         botLink: BOT_LINK,
+        id: data.name,
         ...conditional
     }
     if (template.includes('{{item.notes')) {
@@ -159,6 +164,10 @@ export const renderContent = async (template: string, data: (Submission | Commen
     if (data instanceof Submission) {
         templateData.url = data.url;
         templateData.title = data.title;
+        templateData.shortTitle = shortTitleTruncate(data.title);
+    } else {
+        templateData.title = renderContentCommentTruncate(data.body);
+        templateData.shortTitle = shortTitleTruncate(data.body);
     }
     // normalize rule names and map context data
     // NOTE: we are relying on users to use unique names for rules. If they don't only the last rule run of kind X will have its results here

src/Web/Client/index.ts

@@ -714,7 +714,14 @@ const webClient = async (options: OperatorConfig) => {
     // botUserRouter.use([ensureAuthenticated, defaultSession, botWithPermissions, createUserToken]);
     // app.use(botUserRouter);
 
-    app.useAsync('/api/', [ensureAuthenticated, defaultSession, instanceWithPermissions, botWithPermissions(false), createUserToken], (req: express.Request, res: express.Response) => {
+    // proxy.on('proxyReq', (req) => {
+    //    logger.debug(`Got proxy request: ${req.path}`);
+    // });
+    // proxy.on('proxyRes', (proxyRes, req, res) => {
+    //     logger.debug(`Got proxy response: ${res.statusCode} for ${req.url}`);
+    // });
+
+    app.useAsync('/api/', [ensureAuthenticatedApi, defaultSession, instanceWithPermissions, botWithPermissions(false), createUserToken], (req: express.Request, res: express.Response) => {
         req.headers.Authorization = `Bearer ${req.token}`
 
         const instance = req.instance as CMInstanceInterface;
@@ -725,6 +732,10 @@ const webClient = async (options: OperatorConfig) => {
                 port: instance.url.port,
             },
             prependPath: false,
+            proxyTimeout: 11000,
+        }, (e: any) => {
+            logger.error(e);
+            res.status(500).send();
         });
     });
 

src/Web/Server/routes/authenticated/user/index.ts

@@ -20,6 +20,7 @@ import {filterResultsBuilder} from "../../../../../Utils/typeormUtils";
 import {Brackets} from "typeorm";
 import {Activity} from "../../../../../Common/Entities/Activity";
 import {RedditThing} from "../../../../../Common/Infrastructure/Reddit";
+import {CMError} from "../../../../../Utils/Errors";
 
 const commentReg = parseLinkIdentifier([COMMENT_URL_ID]);
 const submissionReg = parseLinkIdentifier([SUBMISSION_URL_ID]);
@@ -54,7 +55,17 @@ const addInvite = async (req: Request, res: Response) => {
     if (subreddit === undefined || subreddit === null || subreddit === '') {
         return res.status(400).send('subreddit must be defined');
     }
-    await req.serverBot.cacheManager.addPendingSubredditInvite(subreddit);
+    try {
+        await req.serverBot.cacheManager.addPendingSubredditInvite(subreddit);
+    } catch (e: any) {
+        if(e instanceof CMError) {
+            req.logger.warn(e);
+            return res.status(400).send(e.message);
+        } else {
+            req.logger.error(e);
+            return res.status(500).send(e.message);
+        }
+    }
     return res.status(200).send();
 };
 
@@ -194,19 +205,24 @@ const cancelDelayed = async (req: Request, res: Response) => {
     const {id} = req.query as any;
     const {name: userName} = req.user as Express.User;
 
-    if(req.manager?.resources === undefined) {
+    if (req.manager?.resources === undefined) {
         req.manager?.logger.error('Subreddit does not have delayed items!', {user: userName});
         return res.status(400).send();
     }
 
-    const delayedItem = req.manager.resources.delayedItems.find(x => x.id === id);
-    if(delayedItem === undefined) {
-        req.manager?.logger.error(`No delayed items exists with the id ${id}`, {user: userName});
-        return res.status(400).send();
+    if (id === undefined) {
+        await req.manager.resources.removeDelayedActivity();
+    } else {
+        const delayedItem = req.manager.resources.delayedItems.find(x => x.id === id);
+        if (delayedItem === undefined) {
+            req.manager?.logger.error(`No delayed items exists with the id ${id}`, {user: userName});
+            return res.status(400).send();
+        }
+
+        await req.manager.resources.removeDelayedActivity(delayedItem.id);
+        req.manager?.logger.info(`Remove Delayed Item '${delayedItem.id}'`, {user: userName});
     }
 
-    req.manager.resources.delayedItems = req.manager.resources.delayedItems.filter(x => x.id !== id);
-    req.manager?.logger.info(`Remove Delayed Item '${delayedItem.id}'`, {user: userName});
     return res.send('OK');
 };
 

src/Web/Server/routes/authenticated/user/liveStats.ts

@@ -1,13 +1,69 @@
 import {authUserCheck, botRoute, subredditRoute} from "../../../middleware";
 import {Request, Response} from "express";
 import Bot from "../../../../../Bot";
-import {boolToString, cacheStats, filterLogs, formatNumber, logSortFunc, pollingInfo} from "../../../../../util";
+import {boolToString, cacheStats, difference, filterLogs, formatNumber, logSortFunc, pollingInfo} from "../../../../../util";
 import dayjs from "dayjs";
 import {LogInfo, ResourceStats, RUNNING, STOPPED, SYSTEM} from "../../../../../Common/interfaces";
 import {Manager} from "../../../../../Subreddit/Manager";
 import winston from "winston";
 import {opStats} from "../../../../Common/util";
 import {BotStatusResponse} from "../../../../Common/interfaces";
+import deepEqual from "fast-deep-equal";
+import {DispatchedEntity} from "../../../../../Common/Entities/DispatchedEntity";
+
+const lastFullResponse: Map<string, Record<string, any>> = new Map();
+
+const mergeDeepEqual = (a: Record<any, any>, b: Record<any, any>): Record<any, any> => {
+    const delta: Record<any, any> = {};
+    for(const [k,v] of Object.entries(a)) {
+        if(typeof v === 'object' && v !== null && typeof b[k] === 'object' && b[k] !== null) {
+            const objDelta = mergeDeepEqual(v, b[k]);
+            if(Object.keys(objDelta).length > 0) {
+                delta[k] = objDelta;
+            }
+        } else if(!deepEqual(v, b[k])) {
+            delta[k] = v;
+        }
+    }
+    return delta;
+}
+
+const generateDeltaResponse = (data: Record<string, any>, hash: string, responseType: 'full' | 'delta') => {
+    let resp = data;
+    if(responseType === 'delta') {
+        const reference = lastFullResponse.get(hash);
+        if(reference === undefined) {
+            // shouldn't happen...
+            return data;
+        }
+        const delta: Record<string, any> = {};
+        for(const [k,v] of Object.entries(data)) {
+            if(!deepEqual(v, reference[k])) {
+                // on delayed items delta we will send a different data structure back with just remove/new(add)
+                if(k === 'delayedItems') {
+                    const refIds = reference[k].map((x: DispatchedEntity) => x.id);
+                    const latestIds = v.map((x: DispatchedEntity) => x.id);
+
+                    const newIds = Array.from(difference(latestIds, refIds));
+                    const newItems = v.filter((x: DispatchedEntity) => newIds.includes(x.id));
+
+                    // just need ids that should be removed on frontend
+                    const removedItems = Array.from(difference(refIds, latestIds));
+                    delta[k] = {new: newItems, removed: removedItems};
+
+                } else if(v !== null && typeof v === 'object' && reference[k] !== null && typeof reference[k] === 'object') {
+                    // for things like cache/stats we only want to delta changed properties, not the entire object
+                    delta[k] = mergeDeepEqual(v, reference[k]);
+                } else {
+                    delta[k] = v;
+                }
+            }
+        }
+        resp = delta;
+    }
+    lastFullResponse.set(hash, data);
+    return resp;
+}
 
 const liveStats = () => {
     const middleware = [
@@ -20,7 +76,9 @@ const liveStats = () => {
     {
         const bot = req.serverBot as Bot;
         const manager = req.manager;
-        
+        const responseType = req.query.type === 'delta' ? 'delta' : 'full';
+        const hash = `${bot.botName}${manager !== undefined ? `-${manager.getDisplay()}` : ''}`;
+
         if(manager === undefined) {
             // getting all
             const subManagerData: any[] = [];
@@ -213,7 +271,11 @@ const liveStats = () => {
                 },
                 ...allManagerData,
             };
-            return res.json(data);
+            const respData = generateDeltaResponse(data, hash, responseType);
+            if(Object.keys(respData).length === 0) {
+                return res.status(304).send();
+            }
+            return res.json(respData);
         } else {
             // getting specific subreddit stats
             const sd = {
@@ -282,7 +344,11 @@ const liveStats = () => {
                 }
             }
 
-            return res.json(sd);
+            const respData = generateDeltaResponse(sd, hash, responseType);
+            if(Object.keys(respData).length === 0) {
+                return res.status(304).send();
+            }
+            return res.json(respData);
         }
     }
     return [...middleware, response];

src/Web/Server/server.ts

@@ -29,13 +29,14 @@ import {authUserCheck, botRoute} from "./middleware";
 import Bot from "../../Bot";
 import addBot from "./routes/authenticated/user/addBot";
 import ServerUser from "../Common/User/ServerUser";
-import {SimpleError} from "../../Utils/Errors";
+import {CMError, SimpleError} from "../../Utils/Errors";
 import {ErrorWithCause} from "pony-cause";
 import {Manager} from "../../Subreddit/Manager";
 import {MESSAGE} from "triple-beam";
 import dayjs from "dayjs";
 import { sleep } from '../../util';
 import {Invokee} from "../../Common/Infrastructure/Atomic";
+import {Point} from "@influxdata/influxdb-client";
 
 const server = addAsync(express());
 server.use(bodyParser.json());
@@ -147,6 +148,7 @@ const rcbServer = async function (options: OperatorConfigWithFileContext) {
     server.use(passport.authenticate('jwt', {session: false}));
     server.use((req, res, next) => {
         req.botApp = app;
+        req.logger = logger;
         next();
     });
 
@@ -232,6 +234,33 @@ const rcbServer = async function (options: OperatorConfigWithFileContext) {
         });
     }
 
+    // would like to use node-memwatch for more stats but doesn't work with docker (alpine gclib?) and requires more gyp bindings, yuck
+    // https://github.com/airbnb/node-memwatch
+    const writeMemoryMetrics = async () => {
+        if (options.dev.monitorMemory) {
+            if (options.influx !== undefined) {
+                const influx = options.influx;
+                while (true) {
+                    await sleep(options.dev.monitorMemoryInterval);
+                    try {
+                        const memUsage = process.memoryUsage();
+                        await influx.writePoint(new Point('serverMemory')
+                            .intField('external', memUsage.external)
+                            .intField('rss', memUsage.rss)
+                            .intField('arrayBuffers', memUsage.arrayBuffers)
+                            .intField('heapTotal', memUsage.heapTotal)
+                            .intField('heapUsed', memUsage.heapUsed)
+                        );
+                    } catch (e: any) {
+                        logger.warn(new CMError('Error occurred while trying to collect memory metrics', {cause: e}));
+                    }
+                }
+            } else {
+                logger.warn('Cannot monitor memory because influx config was not set');
+            }
+        }
+    }
+
     server.postAsync('/init', authUserCheck(), async (req, res) => {
         logger.info(`${(req.user as Express.User).name} requested the app to be re-built. Starting rebuild now...`, {subreddit: (req.user as Express.User).name});
         await initBot('user');
@@ -285,6 +314,7 @@ const rcbServer = async function (options: OperatorConfigWithFileContext) {
 
     logger.info('Initializing database...');
     try {
+        writeMemoryMetrics();
         const dbReady = await app.initDatabase();
         if(dbReady) {
             logger.info('Initializing application...');

src/Web/assets/views/modInvites.ejs

@@ -37,7 +37,10 @@
             } else {
                 resp.json().then(data => {
                     if (data.length > 0) {
-                        document.querySelector('#noSubs').style = 'display: none;';
+                        const ns = document.querySelector('#noSubs');
+                        if(ns !== null) {
+                            document.querySelector('#noSubs').style = 'display: none;';
+                        }
                         sl.removeChild(sl.childNodes[1]);
                     }
                     for (const sub of data) {
@@ -69,7 +72,10 @@
                         document.querySelector("#error").innerHTML = t;
                     });
                 } else {
-                    document.querySelector('#noSubs').style = 'display: none;';
+                    const ns = document.querySelector('#noSubs');
+                    if(ns !== null) {
+                        document.querySelector('#noSubs').style = 'display: none;';
+                    }
                     addSubredditElement(subName);
                     subNameElm.value = '';
                 }

src/Web/assets/views/status.ejs

@@ -999,6 +999,10 @@
             });
 
         fetchPromise.then(async res => {
+            // can potentially happen if request is aborted (by immediate log cancel) before logs begin to be read
+            if(res === undefined) {
+                return;
+            }
             const reader = res.getReader();
             let keepReading = true;
             while(keepReading) {
@@ -1039,178 +1043,374 @@
                             read();*/
         }).catch((e) => {
             if(e.name !== 'AbortError') {
+                console.debug(`Non-abort error occurred while streaming logs for ${bot} ${sub}`);
                 console.error(e);
+            } else {
+                console.debug(`Log streaming for ${bot} ${sub} aborted`);
             }
         });
 
         const existing = recentlySeen.get(`${bot}.${sub}`) || {};
-        recentlySeen.set(`${bot}.${sub}`, {...existing, fetch: fetchPromise, controller});
+        recentlySeen.set(`${bot}.${sub}`, {...existing, fetch: fetchPromise, controller, streamStart: Date.now()});
     }
 
-    function updateLiveStats(resp) {
+    const delayedItemsMap = new Map();
+    let lastSeenIdentifier = null;
+
+    function updateLiveStats(resp, sub, bot, responseType) {
         let el;
-        let isAll = resp.name.toLowerCase() === 'all';
+        let isAll = sub.toLowerCase() === 'all';
         if(isAll) {
             // got all
-            el = document.querySelector(`[data-subreddit="All"][data-bot="${resp.bot}"].sub`);
+            el = document.querySelector(`[data-subreddit="All"][data-bot="${bot}"].sub`);
         } else {
             // got subreddit
-            el = document.querySelector(`[data-subreddit="${resp.name}"].sub`);
+            el = document.querySelector(`[data-subreddit="${sub}"].sub`);
         }
 
-        if(resp.system.running && el.classList.contains('offline')) {
-            el.classList.remove('offline');
-        } else if(!resp.system.running && !el.classList.contains('offline')) {
-            el.classList.add('offline');
+        const {
+            system: {
+                running
+            } = {},
+            delayedItems,
+            runningActivities,
+            queuedActivities,
+            permissions,
+            stats: {
+                historical: {
+                    eventsCheckedTotal,
+                    checksTriggeredTotal,
+                    rulesTriggeredTotal,
+                    actionsRunTotal,
+                } = {},
+            } = {},
+            checks: {
+              comments,
+              submissions
+            } = {},
+            pollingInfo,
+        } = resp;
+
+        if(running !== undefined) {
+            if(resp.system.running && el.classList.contains('offline')) {
+                el.classList.remove('offline');
+            } else if(!resp.system.running && !el.classList.contains('offline')) {
+                el.classList.add('offline');
+            }
         }
 
-        el.querySelector('.runningActivities').innerHTML = resp.runningActivities;
-        el.querySelector('.queuedActivities').innerHTML = resp.queuedActivities;
-        el.querySelector('.delayedItemsCount').innerHTML = resp.delayedItems.length;
-        el.querySelector('.delayedItemsList').innerHTML = 'No delayed Items!';
-        if(resp.delayedItems.length > 0) {
-            el.querySelector('.delayedItemsList').innerHTML = '';
-            const now = dayjs();
-            const sorted = resp.delayedItems.map(x => ({...x, queuedAtUnix: x.queuedAt, queuedAt: dayjs.unix(x.queuedAt), dispatchAt: dayjs.unix(x.queuedAt + x.duration)}));
-            sorted.sort((a, b) => {
-                return a.dispatchAt.isSameOrAfter(b.dispatchAt) ? 1 : -1
-            });
-            const delayedItemDivs = sorted.map(x => {
-                const diffUntilNow = x.dispatchAt.diff(now);
-                const durationUntilNow = dayjs.duration(diffUntilNow, 'ms');
-                const queuedAtDisplay = x.queuedAt.format('HH:mm:ss z');
-                const durationDayjs = dayjs.duration(x.duration, 'seconds');
-                const durationDisplay = durationDayjs.humanize();
-                const cancelLink = `<a href="#" data-id="${x.id}" data-subreddit="${x.subreddit}" class="delayCancel">CANCEL</a>`;
-                return `<div>A <a href="https://reddit.com${x.permalink}">${x.submissionId !== undefined ? 'Comment' : 'Submssion'}</a>${isAll ? ` in <a href="https://reddit.com${x.subreddit}">${x.subreddit}</a> ` : ''} by <a href="https://reddit.com/u/${x.author}">${x.author}</a> queued by ${x.source} at ${queuedAtDisplay} for ${durationDisplay} (dispatches ${durationUntilNow.humanize(true)}) -- ${cancelLink}</div>`;
-            });
-            el.querySelector('.delayedItemsList').insertAdjacentHTML('afterbegin', delayedItemDivs.join(''));
-            el.querySelectorAll('.delayedItemsList .delayCancel').forEach(elm => {
-                elm.addEventListener('click', e => {
-                    e.preventDefault();
-                    const id = e.target.dataset.id;
-                    const subreddit = e.target.dataset.subreddit;
-                    fetch(`/api/delayed?instance=<%= instanceId %>&bot=${resp.bot}&subreddit=${subreddit}&id=${id}`, {
+        if(runningActivities !== undefined) {
+            el.querySelector('.runningActivities').innerHTML = resp.runningActivities;
+        }
+        if(queuedActivities !== undefined) {
+            el.querySelector('.queuedActivities').innerHTML = resp.queuedActivities;
+        }
+
+        if(delayedItems !== undefined) {
+            let items = [];
+            if(responseType === 'full') {
+                delayedItemsMap.clear();
+                for(const i of delayedItems) {
+                    delayedItemsMap.set(i.id, i);
+                }
+                items = delayedItems;
+            } else {
+                for(const n of delayedItems.new) {
+                    delayedItemsMap.set(n.id, n);
+                }
+                for(const n of delayedItems.removed) {
+                    delayedItemsMap.delete(n);
+                }
+                items = Array.from(delayedItemsMap.values());
+            }
+            el.querySelector('.delayedItemsCount').innerHTML = items.length;
+            el.querySelector('.delayedItemsList').innerHTML = 'No delayed Items!';
+            if(items.length > 0) {
+                el.querySelector('.delayedItemsList').innerHTML = '';
+                const now = dayjs();
+                const sorted = items.map(x => ({...x, queuedAtUnix: x.queuedAt, queuedAt: dayjs.unix(x.queuedAt), dispatchAt: dayjs.unix(x.queuedAt + x.duration)}));
+                sorted.sort((a, b) => {
+                    return a.dispatchAt.isSameOrAfter(b.dispatchAt) ? 1 : -1
+                });
+                const delayedItemDivs = sorted.map(x => {
+                    const diffUntilNow = x.dispatchAt.diff(now);
+                    const durationUntilNow = dayjs.duration(diffUntilNow, 'ms');
+                    const queuedAtDisplay = x.queuedAt.format('HH:mm:ss z');
+                    const durationDayjs = dayjs.duration(x.duration, 'seconds');
+                    const durationDisplay = durationDayjs.humanize();
+                    const cancelLink = `<a href="#" data-id="${x.id}" data-subreddit="${x.subreddit}" class="delayCancel">CANCEL</a>`;
+                    return `<div>A <a href="https://reddit.com${x.permalink}">${x.submissionId !== undefined ? 'Comment' : 'Submission'}</a>${isAll ? ` in <a href="https://reddit.com${x.subreddit}">${x.subreddit}</a> ` : ''} by <a href="https://reddit.com/u/${x.author}">${x.author}</a> queued by ${x.source} at ${queuedAtDisplay} for ${durationDisplay} (dispatches ${durationUntilNow.humanize(true)}) -- ${cancelLink}</div>`;
+                });
+                //let sub = resp.name;
+                if(sub === 'All') {
+                    sub = sorted.reduce((acc, curr) => {
+                        if(!acc.includes(curr.subreddit)) {
+                            return acc.concat(curr.subreddit);
+                        }
+                        return acc;
+                    },[]).join(',');
+                }
+                delayedItemDivs.unshift(`<div><a href="#"data-subreddit="${sub}" class="delayCancelAll">Cancel ALL</a></div>`)
+                el.querySelector('.delayedItemsList').insertAdjacentHTML('afterbegin', delayedItemDivs.join(''));
+                el.querySelectorAll('.delayedItemsList .delayCancel').forEach(elm => {
+                    elm.addEventListener('click', e => {
+                        e.preventDefault();
+                        const id = e.target.dataset.id;
+                        const subreddit = e.target.dataset.subreddit;
+                        fetch(`/api/delayed?instance=<%= instanceId %>&bot=${bot}&subreddit=${subreddit}&id=${id}`, {
+                            method: 'DELETE'
+                        }).then((resp) => {
+                            if (!resp.ok) {
+                                console.error('Response was not OK from delay cancel');
+                            } else {
+                                console.log('Removed ok');
+                            }
+                        });
+                    });
+                });
+                el.querySelectorAll('.delayedItemsList .delayCancelAll').forEach(elm => {
+                    elm.addEventListener('click', e => {
+                        e.preventDefault();
+                        const subreddit = e.target.dataset.subreddit;
+                        deleteDelayedActivities(bot, subreddit);
+
+                        /*fetch(`/api/delayed?instance=<%= instanceId %>&bot=${bot}&subreddit=${subreddit}`, {
                         method: 'DELETE'
                     }).then((resp) => {
                         if (!resp.ok) {
-                            console.error('Response was not OK from delay cancel');
+                            console.error('Response was not OK from delay cancel ALL');
                         } else {
-                            console.log('Removed ok');
+                            console.log('Removed ALL ok');
                         }
+                    });*/
                     });
                 });
-            });
+            }
         }
 
-        el.querySelector('.allStats .eventsCount').innerHTML = resp.stats.historical.eventsCheckedTotal;
-        el.querySelector('.allStats .checksCount').innerHTML = resp.stats.historical.checksTriggeredTotal;
-        el.querySelector('.allStats .rulesCount').innerHTML = resp.stats.historical.rulesTriggeredTotal;
-        el.querySelector('.allStats .actionsCount').innerHTML = resp.stats.historical.actionsRunTotal;
+        if(eventsCheckedTotal !== undefined) {
+            el.querySelector('.allStats .eventsCount').innerHTML = resp.stats.historical.eventsCheckedTotal;
+        }
+        if(checksTriggeredTotal !== undefined) {
+            el.querySelector('.allStats .checksCount').innerHTML = resp.stats.historical.checksTriggeredTotal;
+        }
+        if(rulesTriggeredTotal !== undefined) {
+            el.querySelector('.allStats .rulesCount').innerHTML = resp.stats.historical.rulesTriggeredTotal;
+        }
+        if(actionsRunTotal !== undefined) {
+            el.querySelector('.allStats .actionsCount').innerHTML = resp.stats.historical.actionsRunTotal;
+        }
 
         if(isAll) {
             for(const elm of ['apiAvg','apiLimit','apiDepletion','nextHeartbeat', 'nextHeartbeatHuman', 'limitReset', 'limitResetHuman', 'nannyMode', 'startedAtHuman']) {
-                el.querySelector(`#${elm}`).innerHTML = resp[elm];
+                if(resp[elm] !== undefined) {
+                    el.querySelector(`#${elm}`).innerHTML = resp[elm];
+                }
+            }
+
+            if(running !== undefined) {
+                el.querySelector(`.botStatus`).innerHTML = resp.system.running ? 'ONLINE' : 'OFFLINE';
             }
-            el.querySelector(`.botStatus`).innerHTML = resp.system.running ? 'ONLINE' : 'OFFLINE';
         } else {
-            if(el.querySelector('.modPermissionsCount').innerHTML != resp.permissions.length) {
-                el.querySelector('.modPermissionsCount').innerHTML = resp.permissions.length;
-                el.querySelector('.modPermissionsList').innerHTML = '';
-                el.querySelector('.modPermissionsList').insertAdjacentHTML('afterbegin', resp.permissions.map(x => `<li class="font-mono">${x}</li>`).join(''));
+            if(permissions !== undefined) {
+                if(el.querySelector('.modPermissionsCount').innerHTML != resp.permissions.length) {
+                    el.querySelector('.modPermissionsCount').innerHTML = resp.permissions.length;
+                    el.querySelector('.modPermissionsList').innerHTML = '';
+                    el.querySelector('.modPermissionsList').insertAdjacentHTML('afterbegin', resp.permissions.map(x => `<li class="font-mono">${x}</li>`).join(''));
+                }
             }
 
             for(const elm of ['botState', 'queueState', 'eventsState']) {
-                const state = resp[elm];
-                el.querySelector(`.${elm}`).innerHTML = `${state.state}${state.causedBy === 'system' ? '' : ' (user)'}`;
+                if(resp[elm] !== undefined) {
+                    const state = resp[elm];
+                    el.querySelector(`.${elm}`).innerHTML = `${state.state}${state.causedBy === 'system' ? '' : ' (user)'}`;
+                }
             }
             for(const elm of ['startedAt', 'startedAtHuman', 'wikiLastCheck', 'wikiLastCheckHuman', 'wikiRevision', 'wikiRevisionHuman', 'validConfig', 'delayBy']) {
-                el.querySelector(`.${elm}`).innerHTML = resp[elm];
+                if(resp[elm] !== undefined) {
+                    el.querySelector(`.${elm}`).innerHTML = resp[elm];
+                }
+            }
+            if(comments !== undefined) {
+                el.querySelector(`.commentCheckCount`).innerHTML = resp.checks.comments;
+            }
+            if(submissions !== undefined) {
+                el.querySelector(`.submissionCheckCount`).innerHTML = resp.checks.submissions;
             }
-            el.querySelector(`.commentCheckCount`).innerHTML = resp.checks.comments;
-            el.querySelector(`.submissionCheckCount`).innerHTML = resp.checks.submissions;
 
-            const newInner = resp.pollingInfo.map(x => `<li>${x}</li>`).join('');
-            if(el.querySelector(`.pollingInfo`).innerHTML !== newInner) {
-                el.querySelector(`.pollingInfo`).innerHTML = newInner;
+            if(pollingInfo !== undefined) {
+                const newInner = resp.pollingInfo.map(x => `<li>${x}</li>`).join('');
+                if(el.querySelector(`.pollingInfo`).innerHTML !== newInner) {
+                    el.querySelector(`.pollingInfo`).innerHTML = newInner;
+                }
             }
         }
     }
 
-    function getLiveStats(bot, sub) {
+    function deleteDelayedActivities(bot, subredditStr, id) {
+        const subs = subredditStr.split(',');
+        fetch(`/api/delayed?instance=<%= instanceId %>&bot=${bot}&subreddit=${subs[0]}${id !== undefined ? `&id=${id}` : ''}`, {
+            method: 'DELETE'
+        }).then((resp) => {
+            if (!resp.ok) {
+                if(id === undefined) {
+                    console.error(`Response was not OK from ${subs[0]} delay ALL cancel`);
+                } else {
+                    console.error(`Response was not OK from ${subs[0]} delay cancel ${id}`);
+                }
+            } else {
+                if(id === undefined) {
+                    console.log(`Removed ALL for ${subs[0]} ok`);
+                } else {
+                    console.log(`Removed ${id} for ${subs[0]} ok`);
+                }
+                if(subs.length > 1) {
+                    deleteDelayedActivities(bot, subs.slice(1).join(','));
+                }
+            }
+        });
+    }
+
+    function getLiveStats(bot, sub, responseType = 'full') {
         console.debug(`Getting live stats for ${bot} ${sub}`)
-        return fetch(`/api/liveStats?instance=<%= instanceId %>&bot=${bot}&subreddit=${sub}`)
-            .then(response => response.json())
-            .then(resp => updateLiveStats(resp));
+        return fetch(`/api/liveStats?instance=<%= instanceId %>&bot=${bot}&subreddit=${sub}&type=${responseType}`)
+            .then(response => {
+                if(response.status === 304) {
+                    return Promise.resolve(false);
+                }
+                return response.json();
+            })
+            .then(resp => {
+                if(resp === false) {
+                    return;
+                }
+                updateLiveStats(resp, sub, bot, responseType);
+            });
+    }
+
+    function onSubVisible (bot, sub) {
+        const identifier = `${bot}.${sub}`
+        lastSeenIdentifier = identifier;
+
+        console.debug(`Focused on ${identifier}`);
+
+        let immediateCancel = [];
+        const notNew = Array.from(recentlySeen.entries()).filter(([k,v]) => k !== identifier);
+        // browsers have a default limit for number of concurrent connections
+        // which INCLUDES streaming responses (logs)
+        // so we need to keep number of idle streaming logs low to prevent browser from hanging on new requests
+        if(notNew.length > 2) {
+            notNew.sort((a, b) =>  a[1].streamStart - b[1].streamStart);
+            immediateCancel = notNew.slice(2).map(x => x[0]);
+            console.debug(`More than 2 other views are still streaming logs! Will immediately stop the oldest (skipping two earliest): ${immediateCancel.join(' , ')}`);
+        }
+
+        recentlySeen.forEach((value, key) => {
+            const {timeout, liveStatsInt, ...rest} = value;
+            if(key === identifier && timeout !== undefined) {
+
+                console.debug(`${key} Clearing unfocused timeout on own already set`);
+                clearTimeout(timeout);
+                recentlySeen.set(key, rest);
+
+            } else if(key !== identifier) {
+
+                // stop live stats for tabs we are not viewing
+                clearInterval(liveStatsInt);
+
+                if(immediateCancel.includes(key)) {
+                    const {controller} = value;
+                    if(controller !== undefined) {
+                        console.debug(`${key} Stopping logs IMMEDIATELY`);
+                        controller.abort();
+                        recentlySeen.delete(key);
+                    }
+                } else
+                    // set timeout for logs we are not viewing
+                if(timeout === undefined) {
+                    const t = setTimeout(() => {
+                        const k = key;
+                        const val = recentlySeen.get(k);
+                        if(val !== undefined) {
+                            const {controller} = val;
+                            console.debug(`${k} 15 second unfocused timeout expired, stopping log streaming`);
+                            if(controller !== undefined) {
+                                console.debug('Stopping logs');
+                                controller.abort();
+                            }
+                            recentlySeen.delete(k);
+                        }
+                    },15000);
+                    recentlySeen.set(key, {timeout: t, liveStatsInt, ...rest});
+                }
+            }
+        });
+
+        if(!recentlySeen.has(identifier)) {
+            getLogBlock(bot, sub).then(() => {
+                getStreamingLogs(sub, bot);
+            });
+        }
+
+        delayedItemsMap.clear();
+        // always get live stats for tab we just started viewing
+        getLiveStats(bot, sub).then(() => {
+            let liveStatsInt;
+            const liveStatFunc = () => {
+                // after initial live stats "full frame" only request deltas to reduce data usage
+                getLiveStats(bot, sub, 'delta').catch((err) => {
+                    console.error(err);
+                    // stop interval if live stat encounters an error
+                    clearInterval(liveStatsInt);
+                })
+            };
+            liveStatsInt = setInterval(liveStatFunc, 5000);
+            const existing = recentlySeen.get(identifier) ?? {};
+            recentlySeen.set(identifier, {...existing, bot, sub, liveStatsInt});
+        });
     }
 
     document.querySelectorAll('.sub').forEach(el => {
         const sub = el.dataset.subreddit;
         const bot = el.dataset.bot;
-        //console.log(`Focused on ${bot} ${sub}`);
-        onVisible(el, () => {
-            console.debug(`Focused on ${bot} ${sub}`);
-
-            const identifier = `${bot}.${sub}`;
-
-            recentlySeen.forEach((value, key) => {
-                const {timeout, liveStatsInt, ...rest} = value;
-                if(key === identifier && timeout !== undefined) {
-
-                    console.debug('Clearing timeout on own already set');
-                    clearTimeout(timeout);
-                    recentlySeen.set(key, rest);
-
-                } else if(key !== identifier) {
-
-                    // stop live stats for tabs we are not viewing
-                    clearInterval(liveStatsInt);
-
-                    // set timeout for logs we are not viewing
-                    if(timeout === undefined) {
-                        const t = setTimeout(() => {
-                            const k = key;
-                            const val = recentlySeen.get(k);
-                            if(val !== undefined) {
-                                const {controller} = val;
-                                console.debug(`timeout expired, stopping live data for ${k}`);
-                                if(controller !== undefined) {
-                                    console.debug('Stopping logs');
-                                    controller.abort();
-                                }
-                                // if(liveStatInt !== undefined) {
-                                //     console.log('Stopping live stats');
-                                //     clearInterval(liveStatInt);
-                                // }
-                                recentlySeen.delete(k);
-                            }
-                        },15000);
-                        recentlySeen.set(key, {timeout: t, liveStatsInt, ...rest});
-                    }
-                }
-            });
-
-            if(!recentlySeen.has(identifier)) {
-                getLogBlock(bot, sub).then(() => {
-                    getStreamingLogs(sub, bot);
-                });
-            }
-
-            // always get live stats for tab we just started viewing
-            getLiveStats(bot, sub).then(() => {
-                let liveStatsInt;
-                const liveStatFunc = () => {
-                    getLiveStats(bot, sub).catch(() => {
-                        // stop interval if live stat encounters an error
-                        clearInterval(liveStatsInt);
-                    })
-                };
-                liveStatsInt = setInterval(liveStatFunc, 5000);
-                recentlySeen.set(identifier, {liveStatsInt});
-            });
-        });
+        onVisible(el, () => onSubVisible(bot, sub));
     });
 
+    let backgroundTimeout = null;
+
+    document.addEventListener("visibilitychange", (e) => {
+        if (document.visibilityState === "hidden") {
+            console.debug(`Set 15 seconds timeout for ${lastSeenIdentifier} live data due to page not being visible`);
+            backgroundTimeout = setTimeout(() => {
+                console.debug(`Stopping live data for ${lastSeenIdentifier} due to page not being visible`);
+
+                const {liveStatsInt, controller} = recentlySeen.get(lastSeenIdentifier) ?? {};
+                if(liveStatsInt !== undefined && liveStatsInt !== null) {
+                    clearInterval(liveStatsInt);
+                }
+                if(controller !== undefined && controller !== null) {
+                    controller.abort();
+                }
+                backgroundTimeout = null;
+            }, 15000);
+        } else {
+            // cancel real-time data timeout because page is visible again
+            if(backgroundTimeout !== null) {
+                console.debug(`Cancelled live-data timeout for ${lastSeenIdentifier}`);
+                clearTimeout(backgroundTimeout);
+                backgroundTimeout = null;
+            } else if(lastSeenIdentifier !== null) {
+                // if timeout is null then it was hit
+                // and since we have a last seen this is what is visible to the user so restart live data for it
+                const {bot, sub} = recentlySeen.get(lastSeenIdentifier) ?? {};
+                if(bot !== undefined && sub !== undefined) {
+                    console.debug(`Restarting live-data for ${lastSeenIdentifier} due to page being visible`);
+                    recentlySeen.delete(lastSeenIdentifier);
+                    onSubVisible(bot, sub);
+                }
+            }
+        }
+    });
 
     var searchParams = new URLSearchParams(window.location.search);
     const shownSub = searchParams.get('sub') || 'All'

src/Web/types/express/index.d.ts

@@ -8,6 +8,7 @@ declare global {
     declare namespace Express {
         interface Request {
             botApp: App;
+            logger: Logger;
             token?: string,
             instance?: CMInstanceInterface,
             bot?: BotInstance,

src/util.ts

@@ -1776,10 +1776,26 @@ function *setMinus(A: Array<any>, B: Array<any>) {
 }
 
 
-export const difference = (a: Array<any>, b: Array<any>) => {
+/**
+ * Returns elements that both arrays do not have in common
+ */
+export const symmetricalDifference = (a: Array<any>, b: Array<any>) => {
     return Array.from(setMinus(a, b));
 }
 
+/**
+ * Returns a Set of elements from valA not in valB
+ * */
+export function difference(valA: Set<any> | Array<any>, valB: Set<any> | Array<any>) {
+    const setA = valA instanceof Set ? valA : new Set(valA);
+    const setB = valB instanceof Set ? valB : new Set(valB);
+    const _difference = new Set(setA);
+    for (const elem of setB) {
+        _difference.delete(elem);
+    }
+    return _difference;
+}
+
 // can use 'in' operator to check if object has a property with name WITHOUT TRIGGERING a snoowrap proxy to fetch
 export const isSubreddit = (value: any) => {
     try {