Merge branch 'edge'

* Only return logs for "default viewed" subreddit/bot when fetching instance status, when specified from QS
  * Greatly reduces amount of data fetched and response time
* Return logs with formatted property for non-streaming response
* Implement server live stats endpoint to return subreddit/all stats based on QS
* Use client websocket connection to return stats for currently viewed subreddit

docs/gettingStartedOperator.md

@@ -22,13 +22,14 @@ PROTIP: Using a container management tool like [Portainer.io CE](https://www.por
 
 ### [Dockerhub](https://hub.docker.com/r/foxxmd/context-mod)
 
-```
-foxxmd/context-mod:latest
-```
+An example of starting the container using the [minimum configuration](/docs/operatorConfiguration.md#minimum-config) with a [configuration file](/docs/operatorConfiguration.md#defining-configuration-via-file):
+
+* Bind the folder where the config is located on your host machine into the container `-v /host/path/folder:/config`
+* Tell CM where to find the config using an env `-e "OPERATOR_CONFIG=/config/myConfig.yaml"`
+* Expose the web interface using the container port `8085`
 
-Adding **environmental variables** to your `docker run` command will pass them through to the app EX:
 ```
-docker run -d -e "CLIENT_ID=myId" ... foxxmd/context-mod
+docker run -d -e "OPERATOR_CONFIG=/config/myConfig.yaml" -v /host/path/folder:/config -p 8085:8085 foxxmd/context-mod
 ```
 
 ### Locally
@@ -47,6 +48,12 @@ npm install
 tsc -p .
 ```
 
+An example of running CM using the [minimum configuration](/docs/operatorConfiguration.md#minimum-config) with a [configuration file](/docs/operatorConfiguration.md#defining-configuration-via-file):
+
+```bash
+node src/index.js run
+```
+
 ### [Heroku Quick Deploy](https://heroku.com/about)
 [![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://dashboard.heroku.com/new?template=https://github.com/FoxxMD/context-mod)
 

docs/operatorConfiguration.md

@@ -41,8 +41,10 @@ configuration.
 **Note:** When reading the **schema** if the variable is available at a level of configuration other than **FILE** it will be
 noted with the same symbol as above. The value shown is the default.
 
-* To load a JSON configuration (for **FILE**) **from the command line** use the `-c` cli argument EX: `node src/index.js -c /path/to/JSON/config.json`
-* To load a JSON configuration (for **FILE**) **using an environmental variable** use `OPERATOR_CONFIG` EX: `OPERATOR_CONFIG=/path/to/JSON/config.json`
+## Defining Configuration Via File
+
+* **from the command line**  use the `-c` cli argument EX: `node src/index.js -c /path/to/JSON/config.json`
+* **using an environmental variable**  use `OPERATOR_CONFIG` EX: `OPERATOR_CONFIG=/path/to/JSON/config.json`
 
 [**See the Operator Config Schema here**](https://json-schema.app/view/%23?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Fcontext-mod%2Fmaster%2Fsrc%2FSchema%2FOperatorConfig.json)
 
@@ -121,28 +123,41 @@ Below are examples of the minimum required config to run the application using a
 Using **FILE**
 <details>
 
+CM will look for a file configuration at `PROJECT_DIR/config.yaml` by default [or you can specify your own location.](#defining-configuration-via-file)
+
 YAML
 ```yaml
+operator:
+  name: YourRedditUsername
 bots:
   - credentials:
       clientId: f4b4df1c7b2
       clientSecret: 34v5q1c56ub
-      refreshToken: 34_f1w1v4
-      accessToken: p75_1c467b2
+web:
+  credentials:
+    clientId: f4b4df1c7b2
+    clientSecret: 34v5q1c56ub
 ```
 JSON
 ```json5
 {
+  "operator": {
+    "name": "YourRedditUsername"
+  },
   "bots": [
     {
       "credentials": {
         "clientId": "f4b4df1c7b2",
-        "clientSecret": "34v5q1c56ub",
-        "refreshToken": "34_f1w1v4",
-        "accessToken": "p75_1c467b2"
+        "clientSecret": "34v5q1c56ub"
       }
     }
-  ]
+  ],
+  "web": {
+    "credentials": {
+      "clientId": "f4b4df1c7b2",
+      "clientSecret": "34v5q1c56ub"
+    }
+  }
 }
 ```
 
@@ -153,10 +168,9 @@ Using **ENV** (`.env`)
 <details>
 
 ```
+OPERATOR=YourRedditUsername
 CLIENT_ID=f4b4df1c7b2
 CLIENT_SECRET=34v5q1c56ub
-REFRESH_TOKEN=34_f1w1v4
-ACCESS_TOKEN=p75_1c467b2
 ```
 
 </details>

src/Bot/index.ts

@@ -6,7 +6,7 @@ import EventEmitter from "events";
 import {
     BotInstanceConfig,
     FilterCriteriaDefaults,
-    Invokee,
+    Invokee, LogInfo,
     PAUSED,
     PollOn,
     RUNNING,
@@ -16,7 +16,7 @@ import {
 } from "../Common/interfaces";
 import {
     createRetryHandler,
-    formatNumber, getExceptionMessage,
+    formatNumber, getExceptionMessage, getUserAgent,
     mergeArr,
     parseBool,
     parseDuration, parseMatchMessage,
@@ -38,6 +38,7 @@ class Bot {
 
     client!: ExtendedSnoowrap;
     logger!: Logger;
+    logs: LogInfo[] = [];
     wikiLocation: string;
     dryRun?: true | undefined;
     running: boolean = false;
@@ -98,6 +99,7 @@ class Bot {
                 dryRun,
                 heartbeatInterval,
             },
+            userAgent,
             credentials: {
                 reddit: {
                     clientId,
@@ -151,6 +153,12 @@ class Bot {
             }
         }, mergeArr);
 
+        this.logger.stream().on('log', (log: LogInfo) => {
+            if(log.bot !== undefined && log.bot === this.getBotName() && log.subreddit === undefined) {
+                this.logs = [log, ...this.logs].slice(0, 301);
+            }
+        });
+
         let mw = maxWorkers;
         if(maxWorkers < 1) {
             this.logger.warn(`Max queue workers must be greater than or equal to 1 (Specified: ${maxWorkers})`);
@@ -166,7 +174,9 @@ class Bot {
         this.excludeSubreddits = exclude.map(parseSubredditName);
 
         let creds: any = {
-            get userAgent() { return getUserName() },
+            get userAgent() {
+                return getUserAgent(`web:contextBot:{VERSION}{FRAG}:BOT-${getBotName()}`, userAgent)
+            },
             clientId,
             clientSecret,
             refreshToken,

src/Common/defaults.ts

@@ -39,3 +39,5 @@ export const filterCriteriaDefault: FilterCriteriaDefaults = {
         ]
     }
 }
+
+export const VERSION = '0.10.12';

src/Common/interfaces.ts

@@ -1690,6 +1690,17 @@ export interface OperatorJsonConfig {
 
     bots?: BotInstanceJsonConfig[]
 
+    /**
+     * Added to the User-Agent information sent to reddit
+     *
+     * This string will be added BETWEEN version and your bot name.
+     *
+     * EX: `myBranch` => `web:contextMod:v1.0.0-myBranch:BOT-/u/MyBotUser`
+     *
+     * * ENV => `USER_AGENT`
+     * */
+    userAgent?: string
+
     /**
      * Settings for the web interface
      * */
@@ -1865,6 +1876,7 @@ export interface BotInstanceConfig extends BotInstanceJsonConfig {
         softLimit: number,
         hardLimit: number,
     }
+    userAgent?: string
 }
 
 export interface OperatorConfig extends OperatorJsonConfig {
@@ -1935,6 +1947,7 @@ export interface LogInfo {
     instance?: string
     labels?: string[]
     bot?: string
+    user?: string
 }
 
 export interface ActionResult extends ActionProcessResult {

src/ConfigBuilder.ts

@@ -698,6 +698,7 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
             stream = {},
         } = {},
         caching: opCache,
+        userAgent,
         web: {
             port = 8085,
             maxLogs = 200,
@@ -804,6 +805,7 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
             }
         },
         caching: cache,
+        userAgent,
         web: {
             port,
             caching: {
@@ -843,7 +845,8 @@ export const buildBotConfig = (data: BotInstanceJsonConfig, opConfig: OperatorCo
             actionedEventsMax: opActionedEventsMax,
             actionedEventsDefault: opActionedEventsDefault = 25,
             provider: defaultProvider,
-        } = {}
+        } = {},
+        userAgent,
     } = opConfig;
     const {
         name: botName,
@@ -984,6 +987,7 @@ export const buildBotConfig = (data: BotInstanceJsonConfig, opConfig: OperatorCo
         },
         credentials: botCreds,
         caching: botCache,
+        userAgent,
         polling: {
             shared: [...new Set(realShared)] as PollOn[],
             stagger,

src/Schema/OperatorConfig.json

@@ -1384,6 +1384,10 @@
             "$ref": "#/definitions/SnoowrapOptions",
             "description": "Set global snoowrap options as well as default snoowrap config for all bots that don't specify their own"
         },
+        "userAgent": {
+            "description": "Added to the User-Agent information sent to reddit\n\nThis string will be added BETWEEN version and your bot name.\n\nEX: `myBranch` => `web:contextMod:v1.0.0-myBranch:BOT-/u/MyBotUser`\n\n* ENV => `USER_AGENT`",
+            "type": "string"
+        },
         "web": {
             "description": "Settings for the web interface",
             "properties": {

src/Subreddit/Manager.ts

@@ -24,7 +24,7 @@ import {
     ActionedEvent,
     ActionResult,
     DEFAULT_POLLING_INTERVAL,
-    DEFAULT_POLLING_LIMIT, FilterCriteriaDefaults, Invokee,
+    DEFAULT_POLLING_LIMIT, FilterCriteriaDefaults, Invokee, LogInfo,
     ManagerOptions, ManagerStateChangeOption, ManagerStats, PAUSED,
     PollingOptionsStrong, PollOn, RUNNING, RunState, STOPPED, SYSTEM, USER
 } from "../Common/interfaces";
@@ -87,6 +87,7 @@ export class Manager extends EventEmitter {
     subreddit: Subreddit;
     client: ExtendedSnoowrap;
     logger: Logger;
+    logs: LogInfo[] = [];
     botName: string;
     pollOptions: PollingOptionsStrong[] = [];
     submissionChecks!: SubmissionCheck[];
@@ -211,6 +212,11 @@ export class Manager extends EventEmitter {
                 return getDisplay()
             }
         }, mergeArr);
+        this.logger.stream().on('log', (log: LogInfo) => {
+            if(log.subreddit !== undefined && log.subreddit === this.getDisplay()) {
+                this.logs = [log, ...this.logs].slice(0, 301);
+            }
+        });
         this.globalDryRun = dryRun;
         this.wikiLocation = wikiLocation;
         this.filterCriteriaDefaults = filterCriteriaDefaults;

src/Web/Client/CMInstance.ts

@@ -0,0 +1,148 @@
+import {URL} from "url";
+import {Logger} from "winston";
+import {BotInstance, CMInstanceInterface, CMInstanceInterface as CMInterface} from "../interfaces";
+import dayjs from 'dayjs';
+import {BotConnection, LogInfo} from "../../Common/interfaces";
+import normalizeUrl from "normalize-url";
+import {HeartbeatResponse} from "../Common/interfaces";
+import jwt from "jsonwebtoken";
+import got from "got";
+import {ErrorWithCause} from "pony-cause";
+
+export class CMInstance implements CMInterface {
+    friendly?: string;
+    operators: string[] = [];
+    operatorDisplay: string = '';
+    url: URL;
+    normalUrl: string;
+    lastCheck?: number;
+    online: boolean = false;
+    subreddits: string[] = [];
+    bots: BotInstance[] = [];
+    error?: string | undefined;
+    host: string;
+    secret: string;
+
+    logger: Logger;
+    logs: LogInfo[] = [];
+
+    constructor(options: BotConnection, logger: Logger) {
+        const {
+            host,
+            secret
+        } = options;
+
+        this.host = host;
+        this.secret = secret;
+
+        const normalized = normalizeUrl(options.host);
+        this.normalUrl = normalized;
+        this.url = new URL(normalized);
+
+        const name = this.getName;
+
+        this.logger = logger.child({
+            get instance() {
+                return name();
+            }
+        });
+
+        this.logger.stream().on('log', (log: LogInfo) => {
+            if(log.instance !== undefined && log.instance === this.getName()) {
+                this.logs = [log, ...this.logs].slice(0, 301);
+            }
+        });
+    }
+
+    getData(): CMInterface {
+        return {
+            friendly: this.getName(),
+            operators: this.operators,
+            operatorDisplay: this.operatorDisplay,
+            url: this.url,
+            normalUrl: this.normalUrl,
+            lastCheck: this.lastCheck,
+            online: this.online,
+            subreddits: this.subreddits,
+            bots: this.bots,
+            error: this.error,
+            host: this.host,
+            secret: this.secret
+        }
+    }
+
+    getName = () => {
+        if (this.friendly !== undefined) {
+            return this.friendly
+        }
+        return this.url.host;
+    }
+
+    matchesHost = (val: string) => {
+        return normalizeUrl(val) == this.normalUrl;
+    }
+
+    updateFromHeartbeat = (resp: HeartbeatResponse, otherFriendlies: string[] = []) => {
+        this.operators = resp.operators ?? [];
+        this.operatorDisplay = resp.operatorDisplay ?? '';
+
+        const fr = resp.friendly;
+        if (fr !== undefined) {
+            if (otherFriendlies.includes(fr)) {
+                this.logger.warn(`Client returned a friendly name that is not unique (${fr}), will fallback to host as friendly (${this.url.host})`);
+            } else {
+                this.friendly = fr;
+            }
+        }
+
+        this.subreddits = resp.subreddits;
+        this.bots = resp.bots.map(x => ({...x, instance: this}));
+    }
+
+    checkHeartbeat = async (force = false, otherFriendlies: string[] = []) => {
+        let shouldCheck = force;
+        if (!shouldCheck) {
+            if (this.lastCheck === undefined) {
+                shouldCheck = true;
+            } else {
+                const lastCheck = dayjs().diff(dayjs.unix(this.lastCheck), 's');
+                if (!this.online) {
+                    if (lastCheck > 15) {
+                        shouldCheck = true;
+                    }
+                } else if (lastCheck > 60) {
+                    shouldCheck = true;
+                }
+            }
+        }
+        if (shouldCheck) {
+            this.logger.debug('Starting Heartbeat check');
+            this.lastCheck = dayjs().unix();
+            const machineToken = jwt.sign({
+                data: {
+                    machine: true,
+                },
+            }, this.secret, {
+                expiresIn: '1m'
+            });
+
+            try {
+                const resp = await got.get(`${this.normalUrl}/heartbeat`, {
+                    headers: {
+                        'Authorization': `Bearer ${machineToken}`,
+                    }
+                }).json() as CMInstanceInterface;
+
+                this.online = true;
+                this.updateFromHeartbeat(resp as HeartbeatResponse, otherFriendlies);
+                this.logger.verbose(`Heartbeat detected`);
+            } catch (err: any) {
+                this.online = false;
+                this.error = err.message;
+                const badHeartbeat = new ErrorWithCause('Heartbeat response was not ok', {cause: err});
+                this.logger.error(badHeartbeat);
+            }
+        }
+    }
+
+}

src/Web/Client/index.ts

@@ -9,12 +9,12 @@ import {Strategy as CustomStrategy} from 'passport-custom';
 import {OperatorConfig, BotConnection, LogInfo} from "../../Common/interfaces";
 import {
     buildCachePrefix,
-    createCacheManager, defaultFormat, filterLogBySubreddit,
-    formatLogLineToHtml,
+    createCacheManager, defaultFormat, filterLogBySubreddit, filterLogs,
+    formatLogLineToHtml, getUserAgent,
     intersect, isLogLineMinLevel,
     LogEntry, parseInstanceLogInfoName, parseInstanceLogName, parseRedditEntity,
     parseSubredditLogName, permissions,
-    randomId, sleep, triggeredIndicator
+    randomId, replaceApplicationIdentifier, sleep, triggeredIndicator
 } from "../../util";
 import {Cache} from "cache-manager";
 import session, {Session, SessionData} from "express-session";
@@ -39,7 +39,7 @@ import DelimiterStream from 'delimiter-stream';
 import {pipeline} from 'stream/promises';
 import {defaultBotStatus} from "../Common/defaults";
 import {arrayMiddle, booleanMiddle} from "../Common/middleware";
-import {BotInstance, CMInstance} from "../interfaces";
+import {BotInstance, CMInstanceInterface} from "../interfaces";
 import { URL } from "url";
 import {MESSAGE} from "triple-beam";
 import Autolinker from "autolinker";
@@ -50,6 +50,7 @@ import {BotStatusResponse} from "../Common/interfaces";
 import {TransformableInfo} from "logform";
 import {SimpleError} from "../../Utils/Errors";
 import {ErrorWithCause} from "pony-cause";
+import {CMInstance} from "./CMInstance";
 
 const emitter = new EventEmitter();
 
@@ -106,7 +107,7 @@ interface ConnectUserObj {
     [key: string]: ConnectedUserInfo
 }
 
-const createToken = (bot: CMInstance, user?: Express.User | any, ) => {
+const createToken = (bot: CMInstanceInterface, user?: Express.User | any, ) => {
     const payload = user !== undefined ? {...user, machine: false} : {machine: true};
     return jwt.sign({
         data: payload,
@@ -117,14 +118,13 @@ const createToken = (bot: CMInstance, user?: Express.User | any, ) => {
 
 const availableLevels = ['error', 'warn', 'info', 'verbose', 'debug'];
 
-const instanceLogMap: Map<string, LogEntry[]> = new Map();
-
 const webClient = async (options: OperatorConfig) => {
     const {
         operator: {
             name,
             display,
         },
+        userAgent: uaFragment,
         web: {
             port,
             caching,
@@ -149,22 +149,18 @@ const webClient = async (options: OperatorConfig) => {
         },
     } = options;
 
+    const userAgent = getUserAgent(`web:contextBot:{VERSION}{FRAG}:dashboard`, uaFragment);
+
+    app.use((req, res, next) => {
+        res.locals.applicationIdentifier = replaceApplicationIdentifier('{VERSION}{FRAG}', uaFragment);
+        next();
+    });
+
     const webOps = operators.map(x => x.toLowerCase());
 
     const logger = getLogger({defaultLabel: 'Web', ...options.logging}, 'Web');
 
     logger.stream().on('log', (log: LogInfo) => {
-        const logEntry: LogEntry = [dayjs(log.timestamp).unix(), log];
-        const {instance: instanceLogName} = log;
-        if (instanceLogName !== undefined) {
-            const subLogs = instanceLogMap.get(instanceLogName) || [];
-            subLogs.unshift(logEntry);
-            instanceLogMap.set(instanceLogName, subLogs.slice(0, 200 + 1));
-        } else {
-            const appLogs = instanceLogMap.get('web') || [];
-            appLogs.unshift(logEntry);
-            instanceLogMap.set('web', appLogs.slice(0, 200 + 1));
-        }
         emitter.emit('log', log[MESSAGE]);
     });
 
@@ -330,7 +326,7 @@ const webClient = async (options: OperatorConfig) => {
                 userName,
             };
             if(invite.instance !== undefined) {
-                const bot = cmInstances.find(x => x.friendly === invite.instance);
+                const bot = cmInstances.find(x => x.getName() === invite.instance);
                 if(bot !== undefined) {
                     const botPayload: any = {
                         overwrite: invite.overwrite === true,
@@ -430,7 +426,7 @@ const webClient = async (options: OperatorConfig) => {
             clientId,
             clientSecret,
             token: req.isAuthenticated() && req.user?.clientData?.webOperator ? token : undefined,
-            instances: cmInstances.filter(x => req.user?.isInstanceOperator(x)).map(x => x.friendly),
+            instances: cmInstances.filter(x => req.user?.isInstanceOperator(x)).map(x => x.getName()),
         });
     });
 
@@ -544,7 +540,7 @@ const webClient = async (options: OperatorConfig) => {
                 delimiter: '\r\n',
             });
 
-            const currInstance = cmInstances.find(x => x.friendly === sessionData.botId);
+            const currInstance = cmInstances.find(x => x.getName() === sessionData.botId);
             if(currInstance !== undefined) {
                 const ac = new AbortController();
                 const options = {
@@ -557,7 +553,8 @@ const webClient = async (options: OperatorConfig) => {
                     });
 
                     if(err !== undefined) {
-                        logger.warn(`Log streaming encountered an error, trying to reconnect (retries: ${retryCount}) -- ${err.code !== undefined ? `(${err.code}) ` : ''}${err.message}`, {instance: currInstance.friendly});
+                        // @ts-ignore
+                        currInstance.logger.warn(new ErrorWithCause(`Log streaming encountered an error, trying to reconnect (retries: ${retryCount})`, {cause: err}), {user: user.name});
                     }
                     const gotStream = got.stream.get(`${currInstance.normalUrl}/logs`, {
                         retry: {
@@ -578,7 +575,7 @@ const webClient = async (options: OperatorConfig) => {
 
                     if(err !== undefined) {
                         gotStream.once('data', () => {
-                            logger.info('Streaming resumed', {subreddit: currInstance.friendly});
+                            currInstance.logger.info('Streaming resumed', {instance: currInstance.getName(), user: user.name});
                         });
                     }
 
@@ -592,7 +589,8 @@ const webClient = async (options: OperatorConfig) => {
                     // ECONNRESET
                     s.catch((err) => {
                         if(err.code !== 'ABORT_ERR' && err.code !== 'ERR_STREAM_PREMATURE_CLOSE') {
-                            logger.error(`Unexpected error, or too many retries, occurred while streaming logs -- ${err.code !== undefined ? `(${err.code}) ` : ''}${err.message}`, {instance: currInstance.friendly});
+                            // @ts-ignore
+                            currInstance.logger.error(new ErrorWithCause('Unexpected error, or too many retries, occurred while streaming logs', {cause: err}), {user: user.name});
                         }
                     });
 
@@ -640,7 +638,7 @@ const webClient = async (options: OperatorConfig) => {
         delete req.session.authBotId;
 
         const msg = 'Bot does not exist or you do not have permission to access it';
-        const instance = cmInstances.find(x => x.friendly === req.query.instance);
+        const instance = cmInstances.find(x => x.getName() === req.query.instance);
         if (instance === undefined) {
             return res.status(404).render('error', {error: msg});
         }
@@ -653,15 +651,15 @@ const webClient = async (options: OperatorConfig) => {
             return res.status(404).render('error', {error: msg});
         }
         req.instance = instance;
-        req.session.botId = instance.friendly;
+        req.session.botId = instance.getName();
         if(req.user?.canAccessInstance(instance)) {
-            req.session.authBotId = instance.friendly;
+            req.session.authBotId = instance.getName();
         }
         return next();
     }
 
 
-    const botWithPermissions = async (req: express.Request, res: express.Response, next: Function) => {
+    const botWithPermissions = (required: boolean = false, setDefault: boolean = false) => async (req: express.Request, res: express.Response, next: Function) => {
 
         const instance = req.instance;
         if(instance === undefined) {
@@ -670,28 +668,42 @@ const webClient = async (options: OperatorConfig) => {
 
         const msg = 'Bot does not exist or you do not have permission to access it';
         const botVal = req.query.bot as string;
-        if(botVal === undefined) {
+        if(botVal === undefined && required) {
             return res.status(400).render('error', {error: `"bot" param must be defined`});
         }
 
-        const botInstance = instance.bots.find(x => x.botName === botVal);
-        if(botInstance === undefined) {
-            return res.status(404).render('error', {error: msg});
+        if(botVal !== undefined || setDefault) {
+
+            let botInstance;
+            if(botVal === undefined) {
+                // find a bot they can access
+                botInstance = instance.bots.find(x => req.user?.canAccessBot(x));
+                if(botInstance !== undefined) {
+                    req.query.bot = botInstance.botName;
+                }
+            } else {
+                botInstance = instance.bots.find(x => x.botName === botVal);
+            }
+
+            if(botInstance === undefined) {
+                return res.status(404).render('error', {error: msg});
+            }
+
+            if (!req.user?.clientData?.webOperator && !req.user?.canAccessBot(botInstance)) {
+                return res.status(404).render('error', {error: msg});
+            }
+
+            if (req.params.subreddit !== undefined && !req.user?.isInstanceOperator(instance) && !req.user?.subreddits.includes(req.params.subreddit)) {
+                return res.status(404).render('error', {error: msg});
+            }
+            req.bot = botInstance;
         }
 
-        if (!req.user?.clientData?.webOperator && !req.user?.canAccessBot(botInstance)) {
-            return res.status(404).render('error', {error: msg});
-        }
-
-        if (req.params.subreddit !== undefined && !req.user?.isInstanceOperator(instance) && !req.user?.subreddits.includes(req.params.subreddit)) {
-            return res.status(404).render('error', {error: msg});
-        }
-        req.bot = botInstance;
         next();
     }
 
     const createUserToken = async (req: express.Request, res: express.Response, next: Function) => {
-        req.token = createToken(req.instance as CMInstance, req.user);
+        req.token = createToken(req.instance as CMInstanceInterface, req.user);
         next();
     }
 
@@ -715,10 +727,10 @@ const webClient = async (options: OperatorConfig) => {
     // botUserRouter.use([ensureAuthenticated, defaultSession, botWithPermissions, createUserToken]);
     // app.use(botUserRouter);
 
-    app.useAsync('/api/', [ensureAuthenticated, defaultSession, instanceWithPermissions, botWithPermissions, createUserToken], (req: express.Request, res: express.Response) => {
+    app.useAsync('/api/', [ensureAuthenticated, defaultSession, instanceWithPermissions, botWithPermissions(true), createUserToken], (req: express.Request, res: express.Response) => {
         req.headers.Authorization = `Bearer ${req.token}`
 
-        const instance = req.instance as CMInstance;
+        const instance = req.instance as CMInstanceInterface;
         return proxy.web(req, res, {
             target: {
                 protocol: instance.url.protocol,
@@ -744,17 +756,25 @@ const webClient = async (options: OperatorConfig) => {
             });
 
             if(accessibleInstance === undefined) {
-                logger.warn(`User ${user.name} is not an operator and has no subreddits in common with any *running* bot instances. If you are sure they should have common subreddits then this client may not be able to access all defined CM servers or the bot may be offline.`);
+                logger.warn(`User ${user.name} is not an operator and has no subreddits in common with any *running* bot instances. If you are sure they should have common subreddits then this client may not be able to access all defined CM servers or the bot may be offline.`, {user: user.name});
                 return res.render('noAccess');
             }
 
             return res.redirect(`/?instance=${accessibleInstance.friendly}`);
         }
-        const instance = cmInstances.find(x => x.friendly === req.query.instance);
+        const instance = cmInstances.find(x => x.getName() === req.query.instance);
         req.instance = instance;
         next();
     }
 
+    const defaultSubreddit = async (req: express.Request, res: express.Response, next: Function) => {
+        if(req.bot !== undefined && req.query.subreddit === undefined) {
+            const firstAccessibleSub = req.bot.subreddits.find(x => req.user?.isInstanceOperator(req.instance) || req.user?.subreddits.includes(x));
+            req.query.subreddit = firstAccessibleSub;
+        }
+        next();
+    }
+
     const initHeartbeat = async (req: express.Request, res: express.Response, next: Function) => {
         if(!init) {
             for(const c of clients) {
@@ -774,7 +794,7 @@ const webClient = async (options: OperatorConfig) => {
         next();
     }
 
-    app.getAsync('/', [initHeartbeat, redirectBotsNotAuthed, ensureAuthenticated, defaultSession, defaultInstance, instanceWithPermissions, createUserToken], async (req: express.Request, res: express.Response) => {
+    app.getAsync('/', [initHeartbeat, redirectBotsNotAuthed, ensureAuthenticated, defaultSession, defaultInstance, instanceWithPermissions, botWithPermissions(false, true), createUserToken], async (req: express.Request, res: express.Response) => {
 
         const user = req.user as Express.User;
         const instance = req.instance as CMInstance;
@@ -787,13 +807,13 @@ const webClient = async (options: OperatorConfig) => {
             const isBotOperator = req.user?.isInstanceOperator(curr);
             if(user?.clientData?.webOperator) {
                 // @ts-ignore
-                return acc.concat({...curr, canAccessLocation: true, isOperator: isBotOperator});
+                return acc.concat({...curr.getData(), canAccessLocation: true, isOperator: isBotOperator});
             }
             if(!isBotOperator && !req.user?.canAccessInstance(curr)) {
                 return acc;
             }
             // @ts-ignore
-            return acc.concat({...curr, canAccessLocation: isBotOperator, isOperator: isBotOperator, botId: curr.friendly});
+            return acc.concat({...curr.getData(), canAccessLocation: isBotOperator, isOperator: isBotOperator, botId: curr.getName()});
         },[]);
 
         let resp;
@@ -803,6 +823,8 @@ const webClient = async (options: OperatorConfig) => {
                     'Authorization': `Bearer ${req.token}`,
                 },
                 searchParams: {
+                    bot: req.query.bot as (string | undefined),
+                    subreddit: req.query.sub as (string | undefined) ?? 'all',
                     limit,
                     sort,
                     level,
@@ -811,14 +833,15 @@ const webClient = async (options: OperatorConfig) => {
             }).json() as any;
 
         } catch(err: any) {
-            logger.error(`Error occurred while retrieving bot information. Will update heartbeat -- ${err.message}`, {instance: instance.friendly});
-            refreshClient(clients.find(x => normalizeUrl(x.host) === instance.normalUrl) as BotConnection);
+            instance.logger.error(new ErrorWithCause(`Could not retrieve instance information. Will attempted to update heartbeat.`, {cause: err}));
+            refreshClient({host: instance.host, secret: instance.secret});
+            const isOp = req.user?.isInstanceOperator(instance);
             return res.render('offline', {
                 instances: shownInstances,
-                instanceId: (req.instance as CMInstance).friendly,
-                isOperator: req.user?.isInstanceOperator(instance),
+                instanceId: (req.instance as CMInstance).getName(),
+                isOperator: isOp,
                 // @ts-ignore
-                logs: filterLogBySubreddit(instanceLogMap, [instance.friendly], {limit, sort, level, allLogName: 'web', allLogsParser: parseInstanceLogInfoName }).get(instance.friendly),
+                logs: filterLogs((isOp ? instance.logs : instance.logs.filter(x => x.user === undefined || x.user.includes(req.user.name))), {limit, sort, level}),
                 logSettings: {
                     limitSelect: [10, 20, 50, 100, 200].map(x => `<option ${limit === x ? 'selected' : ''} class="capitalize ${limit === x ? 'font-bold' : ''}" data-value="${x}">${x}</option>`).join(' | '),
                     sortSelect: ['ascending', 'descending'].map(x => `<option ${sort === x ? 'selected' : ''} class="capitalize ${sort === x ? 'font-bold' : ''}" data-value="${x}">${x}</option>`).join(' '),
@@ -871,8 +894,8 @@ const webClient = async (options: OperatorConfig) => {
                 });
                 return {...x, subreddits: subredditsWithSimpleLogs};
             }),
-            botId: (req.instance as CMInstance).friendly,
-            instanceId: (req.instance as CMInstance).friendly,
+            botId: (req.instance as CMInstanceInterface).friendly,
+            instanceId: (req.instance as CMInstanceInterface).friendly,
             isOperator: isOp,
             system: isOp ? {
                 logs: resp.system.logs,
@@ -902,7 +925,7 @@ const webClient = async (options: OperatorConfig) => {
         });
     });
 
-    app.postAsync('/config', [ensureAuthenticatedApi, defaultSession, instanceWithPermissions, botWithPermissions], async (req: express.Request, res: express.Response) => {
+    app.postAsync('/config', [ensureAuthenticatedApi, defaultSession, instanceWithPermissions, botWithPermissions(true)], async (req: express.Request, res: express.Response) => {
         const {subreddit} = req.query as any;
         const {location, data, create = false} = req.body as any;
 
@@ -943,9 +966,9 @@ const webClient = async (options: OperatorConfig) => {
         return res.send();
     });
 
-    app.getAsync('/events', [ensureAuthenticatedApi, defaultSession, instanceWithPermissions, botWithPermissions, createUserToken], async (req: express.Request, res: express.Response) => {
+    app.getAsync('/events', [ensureAuthenticatedApi, defaultSession, instanceWithPermissions, botWithPermissions(true), createUserToken], async (req: express.Request, res: express.Response) => {
         const {subreddit} = req.query as any;
-        const resp = await got.get(`${(req.instance as CMInstance).normalUrl}/events`, {
+        const resp = await got.get(`${(req.instance as CMInstanceInterface).normalUrl}/events`, {
             headers: {
                 'Authorization': `Bearer ${req.token}`,
             },
@@ -1056,6 +1079,9 @@ const webClient = async (options: OperatorConfig) => {
         const session = socket.handshake.session as (Session & Partial<SessionData> | undefined);
         // @ts-ignore
         const user = session !== undefined ? session?.passport?.user as Express.User : undefined;
+
+        let liveInterval: any = undefined;
+
         if (session !== undefined && user !== undefined) {
             clearSockStreams(socket.id);
             socket.join(session.id);
@@ -1070,8 +1096,44 @@ const webClient = async (options: OperatorConfig) => {
             emitter.on('log', webLogListener);
             socketListeners.set(socket.id, [...(socketListeners.get(socket.id) || []), webLogListener]);
 
+            socket.on('viewing', (data) => {
+                if(user !== undefined) {
+                    const {subreddit, bot: botVal} = data;
+                    const currBot = cmInstances.find(x => x.getName() === session.botId);
+                    if(currBot !== undefined) {
+
+                        if(liveInterval !== undefined) {
+                            clearInterval(liveInterval)
+                        }
+
+                        const liveEmit = async () => {
+                            try {
+                                const resp = await got.get(`${currBot.normalUrl}/liveStats`, {
+                                    headers: {
+                                        'Authorization': `Bearer ${createToken(currBot, user)}`,
+                                    },
+                                    searchParams: {
+                                        bot: botVal,
+                                        subreddit
+                                    }
+                                });
+                                const stats = JSON.parse(resp.body);
+                                io.to(session.id).emit('liveStats', stats);
+                            } catch (err: any) {
+                                currBot.logger.error(new ErrorWithCause('Could not retrieve live stats', {cause: err}));
+                            }
+                        }
+
+                        // do an initial get
+                        liveEmit();
+                        // and then every 5 seconds after that
+                        liveInterval = setInterval(async () => await liveEmit(), 5000);
+                    }
+                }
+            });
+
             if(session.botId !== undefined) {
-                const bot = cmInstances.find(x => x.friendly === session.botId);
+                const bot = cmInstances.find(x => x.getName() === session.botId);
                 if(bot !== undefined) {
                     // web log listener for bot specifically
                     const botWebLogListener = (log: string) => {
@@ -1101,7 +1163,7 @@ const webClient = async (options: OperatorConfig) => {
                                 }).json() as object;
                                 io.to(session.id).emit('opStats', resp);
                             } catch (err: any) {
-                                logger.error(`Could not retrieve stats ${err.message}`, {instance: bot.friendly});
+                                bot.logger.error(new ErrorWithCause('Could not retrieve stats', {cause: err}));
                                 clearInterval(interval);
                             }
                         }, 5000);
@@ -1114,12 +1176,12 @@ const webClient = async (options: OperatorConfig) => {
         socket.on('disconnect', (reason) => {
             clearSockStreams(socket.id);
             clearSockListeners(socket.id);
+            clearInterval(liveInterval);
         });
     });
 
     const loopHeartbeat = async () => {
         while(true) {
-            logger.debug('Starting heartbeat check');
             for(const c of clients) {
                 await refreshClient(c);
             }
@@ -1128,7 +1190,7 @@ const webClient = async (options: OperatorConfig) => {
         }
     }
 
-    const addBot = async (bot: CMInstance, userPayload: any, botPayload: any) => {
+    const addBot = async (bot: CMInstanceInterface, userPayload: any, botPayload: any) => {
         try {
             const token = createToken(bot, userPayload);
             const resp = await got.post(`${bot.normalUrl}/bot`, {
@@ -1145,81 +1207,13 @@ const webClient = async (options: OperatorConfig) => {
     }
 
     const refreshClient = async (client: BotConnection, force = false) => {
-        const normalized = normalizeUrl(client.host);
-        const existingClientIndex = cmInstances.findIndex(x => x.normalUrl === normalized);
-        const existingClient = existingClientIndex === -1 ? undefined : cmInstances[existingClientIndex];
+        const existingClientIndex = cmInstances.findIndex(x => x.matchesHost(client.host));
+        const instance = existingClientIndex === -1 ? new CMInstance(client, logger) : cmInstances[existingClientIndex];
 
-        let shouldCheck = false;
-        if(!existingClient) {
-            shouldCheck = true;
-        } else if(force) {
-            shouldCheck = true;
-        } else  {
-            const lastCheck = dayjs().diff(dayjs.unix(existingClient.lastCheck), 's');
-            if(!existingClient.online) {
-                if(lastCheck > 15) {
-                    shouldCheck = true;
-                }
-            } else if(lastCheck > 60) {
-                shouldCheck = true;
-            }
-        }
-        if(shouldCheck)
-        {
-            const machineToken = jwt.sign({
-                data: {
-                    machine: true,
-                },
-            }, client.secret, {
-                expiresIn: '1m'
-            });
-            //let base = `${c.host}${c.port !== undefined ? `:${c.port}` : ''}`;
-            const normalized = normalizeUrl(client.host);
-            const url = new URL(normalized);
-            let botStat: CMInstance = {
-                ...client,
-                subreddits: [] as string[],
-                operators: [] as string[],
-                operatorDisplay: '',
-                online: false,
-                friendly: url.host,
-                lastCheck: dayjs().unix(),
-                normalUrl: normalized,
-                url,
-                bots: [],
-            };
-            try {
-                const resp = await got.get(`${normalized}/heartbeat`, {
-                    headers: {
-                        'Authorization': `Bearer ${machineToken}`,
-                    }
-                }).json() as CMInstance;
+        await instance.checkHeartbeat(force);
 
-                const {bots, ...restResp} = resp;
-
-                botStat = {...botStat, ...restResp, bots: bots.map(x => ({...x, instance: botStat})), online: true};
-                const sameNameIndex = cmInstances.findIndex(x => x.friendly === botStat.friendly);
-                if(sameNameIndex > -1 && sameNameIndex !== existingClientIndex) {
-                    logger.warn(`Client returned a friendly name that is not unique (${botStat.friendly}), will fallback to host as friendly (${botStat.normalUrl})`);
-                    botStat.friendly = botStat.normalUrl;
-                }
-                botStat.online = true;
-                // if(botStat.online) {
-                //     botStat.indicator = botStat.running ? 'green' : 'yellow';
-                // } else {
-                //     botStat.indicator = 'red';
-                // }
-                logger.verbose(`Heartbeat detected`, {instance: botStat.friendly});
-            } catch (err: any) {
-                botStat.error = err.message;
-                logger.error(`Heartbeat response from ${botStat.friendly} was not ok: ${err.message}`, {instance: botStat.friendly});
-            } finally {
-                if(existingClientIndex !== -1) {
-                    cmInstances.splice(existingClientIndex, 1, botStat);
-                } else {
-                    cmInstances.push(botStat);
-                }
-            }
+        if(existingClientIndex === -1) {
+            cmInstances.push(instance);
         }
     }
 }

src/Web/Common/User/ClientUser.ts

@@ -1,14 +1,14 @@
-import {BotInstance, CMInstance} from "../../interfaces";
+import {BotInstance, CMInstanceInterface} from "../../interfaces";
 import CMUser from "./CMUser";
 import {intersect, parseRedditEntity} from "../../../util";
 
-class ClientUser extends CMUser<CMInstance, BotInstance, string> {
+class ClientUser extends CMUser<CMInstanceInterface, BotInstance, string> {
 
-    isInstanceOperator(val: CMInstance): boolean {
+    isInstanceOperator(val: CMInstanceInterface): boolean {
         return val.operators.map(x=> x.toLowerCase()).includes(this.name.toLowerCase());
     }
 
-    canAccessInstance(val: CMInstance): boolean {
+    canAccessInstance(val: CMInstanceInterface): boolean {
         return this.isInstanceOperator(val) || intersect(this.subreddits, val.subreddits.map(x => parseRedditEntity(x).name)).length > 0;
     }
 

src/Web/Common/User/ServerUser.ts

@@ -1,4 +1,4 @@
-import {BotInstance, CMInstance} from "../../interfaces";
+import {BotInstance, CMInstanceInterface} from "../../interfaces";
 import CMUser from "./CMUser";
 import {intersect, parseRedditEntity} from "../../../util";
 import {App} from "../../../App";

src/Web/Common/interfaces.ts

@@ -1,5 +1,6 @@
 import {RunningState} from "../../Subreddit/Manager";
 import {LogInfo, ManagerStats} from "../../Common/interfaces";
+import {BotInstance} from "../interfaces";
 
 export interface BotStats {
     startedAtHuman: string,
@@ -73,3 +74,11 @@ export interface IUser {
     token?: string
     tokenExpiresAt?: number
 }
+
+export interface HeartbeatResponse {
+    subreddits: string[]
+    operators: string[]
+    operatorDisplay?: string
+    friendly?: string
+    bots: BotInstance[]
+}

src/Web/Server/middleware.ts

@@ -46,19 +46,24 @@ export const subredditRoute = (required = true) => async (req: Request, res: Res
     if(subreddit === undefined && !required) {
         next();
     } else {
-        //const {name: userName} = req.user as Express.User;
 
-        const manager = bot.subManagers.find(x => x.displayLabel === subreddit);
-        if (manager === undefined) {
-            return res.status(400).send('Cannot access route for subreddit you do not manage or is not run by the bot')
+        if(subreddit.toLowerCase() === 'all') {
+            next();
+        } else {
+            //const {name: userName} = req.user as Express.User;
+
+            const manager = bot.subManagers.find(x => x.displayLabel === subreddit);
+            if (manager === undefined) {
+                return res.status(400).send('Cannot access route for subreddit you do not manage or is not run by the bot')
+            }
+
+            if (!req.user?.canAccessSubreddit(bot, subreddit)) {
+                return res.status(400).send('Cannot access route for subreddit you do not manage or is not run by the bot')
+            }
+
+            req.manager = manager;
+
+            next();
         }
-
-        if (!req.user?.canAccessSubreddit(bot, subreddit)) {
-            return res.status(400).send('Cannot access route for subreddit you do not manage or is not run by the bot')
-        }
-
-        req.manager = manager;
-
-        next();
     }
 }

src/Web/Server/routes/authenticated/user/action.ts

@@ -19,7 +19,7 @@ const action = async (req: Request, res: Response) => {
 
     for (const manager of subreddits) {
         const mLogger = manager.logger;
-        mLogger.info(`/u/${userName} invoked '${action}' action for ${type} on ${manager.displayLabel}`);
+        mLogger.info(`/u/${userName} invoked '${action}' action for ${type} on ${manager.displayLabel}`, {user: userName});
         try {
             switch (action) {
                 case 'start':

src/Web/Server/routes/authenticated/user/index.ts

@@ -106,7 +106,7 @@ const action = async (req: Request, res: Response) => {
     }
 
     if (a === undefined) {
-        winston.loggers.get('app').error('Could not parse Comment or Submission ID from given URL', {subreddit: `/u/${userName}`});
+        winston.loggers.get('app').error('Could not parse Comment or Submission ID from given URL', {user: userName});
         return res.send('OK');
     } else {
         // @ts-ignore
@@ -118,15 +118,15 @@ const action = async (req: Request, res: Response) => {
         if (manager === undefined || !req.user?.canAccessSubreddit(req.serverBot, manager.subreddit.display_name)) {
             let msg = 'Activity does not belong to a subreddit you moderate or the bot runs on.';
             if (subreddit === 'All') {
-                msg = `${msg} If you want to test an Activity against a Subreddit\'s config it does not belong to then switch to that Subreddit's tab first.`
+                msg = `${msg} If you want to test an Activity against a Subreddit's config it does not belong to then switch to that Subreddit's tab first.`
             }
-            winston.loggers.get('app').error(msg, {subreddit: `/u/${userName}`});
+            winston.loggers.get('app').error(msg, {user: userName});
             return res.send('OK');
         }
 
         // will run dryrun if specified or if running activity on subreddit it does not belong to
         const dr: boolean | undefined = (dryRun || manager.subreddit.display_name !== sub) ? true : undefined;
-        manager.logger.info(`/u/${userName} running${dr === true ? ' DRY RUN ' : ' '}check on${manager.subreddit.display_name !== sub ? ' FOREIGN ACTIVITY ' : ' '}${url}`);
+        manager.logger.info(`/u/${userName} running${dr === true ? ' DRY RUN ' : ' '}check on${manager.subreddit.display_name !== sub ? ' FOREIGN ACTIVITY ' : ' '}${url}`, {user: userName, subreddit});
         await manager.runChecks(activity instanceof Submission ? 'Submission' : 'Comment', activity, {dryRun: dr, force: true})
     }
     res.send('OK');

src/Web/Server/routes/authenticated/user/liveStats.ts

@@ -0,0 +1,261 @@
+import {authUserCheck, botRoute, subredditRoute} from "../../../middleware";
+import {Request, Response} from "express";
+import Bot from "../../../../../Bot";
+import {boolToString, cacheStats, filterLogs, formatNumber, logSortFunc, pollingInfo} from "../../../../../util";
+import dayjs from "dayjs";
+import {LogInfo, ResourceStats, RUNNING, STOPPED, SYSTEM} from "../../../../../Common/interfaces";
+import {Manager} from "../../../../../Subreddit/Manager";
+import winston from "winston";
+import {opStats} from "../../../../Common/util";
+import {BotStatusResponse} from "../../../../Common/interfaces";
+
+const liveStats = () => {
+    const middleware = [
+        authUserCheck(),
+        botRoute(),
+        subredditRoute(false),
+    ]
+
+    const response = async (req: Request, res: Response) =>
+    {
+        const bot = req.serverBot as Bot;
+        const manager = req.manager;
+        
+        if(manager === undefined) {
+            // getting all
+            const subManagerData: any[] = [];
+            for (const m of req.user?.accessibleSubreddits(bot) as Manager[]) {
+                const sd = {
+                    queuedActivities: m.queue.length(),
+                    runningActivities: m.queue.running(),
+                    maxWorkers: m.queue.concurrency,
+                    subMaxWorkers: m.subMaxWorkers || bot.maxWorkers,
+                    globalMaxWorkers: bot.maxWorkers,
+                    checks: {
+                        submissions: m.submissionChecks === undefined ? 0 : m.submissionChecks.length,
+                        comments: m.commentChecks === undefined ? 0 : m.commentChecks.length,
+                    },
+                    stats: await m.getStats(),
+                }
+            }
+
+            const totalStats = subManagerData.reduce((acc, curr) => {
+                return {
+                    checks: {
+                        submissions: acc.checks.submissions + curr.checks.submissions,
+                        comments: acc.checks.comments + curr.checks.comments,
+                    },
+                    historical: {
+                        allTime: {
+                            eventsCheckedTotal: acc.historical.allTime.eventsCheckedTotal + curr.stats.historical.allTime.eventsCheckedTotal,
+                            eventsActionedTotal: acc.historical.allTime.eventsActionedTotal + curr.stats.historical.allTime.eventsActionedTotal,
+                            checksRunTotal: acc.historical.allTime.checksRunTotal + curr.stats.historical.allTime.checksRunTotal,
+                            checksFromCacheTotal: acc.historical.allTime.checksFromCacheTotal + curr.stats.historical.allTime.checksFromCacheTotal,
+                            checksTriggeredTotal: acc.historical.allTime.checksTriggeredTotal + curr.stats.historical.allTime.checksTriggeredTotal,
+                            rulesRunTotal: acc.historical.allTime.rulesRunTotal + curr.stats.historical.allTime.rulesRunTotal,
+                            rulesCachedTotal: acc.historical.allTime.rulesCachedTotal + curr.stats.historical.allTime.rulesCachedTotal,
+                            rulesTriggeredTotal: acc.historical.allTime.rulesTriggeredTotal + curr.stats.historical.allTime.rulesTriggeredTotal,
+                            actionsRunTotal: acc.historical.allTime.actionsRunTotal + curr.stats.historical.allTime.actionsRunTotal,
+                        }
+                    },
+                    maxWorkers: acc.maxWorkers + curr.maxWorkers,
+                    subMaxWorkers: acc.subMaxWorkers + curr.subMaxWorkers,
+                    globalMaxWorkers: acc.globalMaxWorkers + curr.globalMaxWorkers,
+                    runningActivities: acc.runningActivities + curr.runningActivities,
+                    queuedActivities: acc.queuedActivities + curr.queuedActivities,
+                };
+            }, {
+                checks: {
+                    submissions: 0,
+                    comments: 0,
+                },
+                historical: {
+                    allTime: {
+                        eventsCheckedTotal: 0,
+                        eventsActionedTotal: 0,
+                        checksRunTotal: 0,
+                        checksFromCacheTotal: 0,
+                        checksTriggeredTotal: 0,
+                        rulesRunTotal: 0,
+                        rulesCachedTotal: 0,
+                        rulesTriggeredTotal: 0,
+                        actionsRunTotal: 0,
+                    }
+                },
+                maxWorkers: 0,
+                subMaxWorkers: 0,
+                globalMaxWorkers: 0,
+                runningActivities: 0,
+                queuedActivities: 0,
+            });
+            const {
+                checks,
+                maxWorkers,
+                globalMaxWorkers,
+                subMaxWorkers,
+                runningActivities,
+                queuedActivities,
+                ...rest
+            } = totalStats;
+
+            let cumRaw = subManagerData.reduce((acc, curr) => {
+                Object.keys(curr.stats.cache.types as ResourceStats).forEach((k) => {
+                    acc[k].requests += curr.stats.cache.types[k].requests;
+                    acc[k].miss += curr.stats.cache.types[k].miss;
+                    // @ts-ignore
+                    acc[k].identifierAverageHit += (typeof curr.stats.cache.types[k].identifierAverageHit === 'string' ? Number.parseFloat(curr.stats.cache.types[k].identifierAverageHit) : curr.stats.cache.types[k].identifierAverageHit);
+                    acc[k].averageTimeBetweenHits += curr.stats.cache.types[k].averageTimeBetweenHits === 'N/A' ? 0 : Number.parseFloat(curr.stats.cache.types[k].averageTimeBetweenHits)
+                });
+                return acc;
+            }, cacheStats());
+            cumRaw = Object.keys(cumRaw).reduce((acc, curr) => {
+                const per = acc[curr].miss === 0 ? 0 : formatNumber(acc[curr].miss / acc[curr].requests) * 100;
+                // @ts-ignore
+                acc[curr].missPercent = `${formatNumber(per, {toFixed: 0})}%`;
+                acc[curr].identifierAverageHit = formatNumber(acc[curr].identifierAverageHit);
+                acc[curr].averageTimeBetweenHits = formatNumber(acc[curr].averageTimeBetweenHits)
+                return acc;
+            }, cumRaw);
+            const cacheReq = subManagerData.reduce((acc, curr) => acc + curr.stats.cache.totalRequests, 0);
+            const cacheMiss = subManagerData.reduce((acc, curr) => acc + curr.stats.cache.totalMiss, 0);
+            const sharedSub = subManagerData.find(x => x.stats.cache.isShared);
+            const sharedCount = sharedSub !== undefined ? sharedSub.stats.cache.currentKeyCount : 0;
+            const scopes = req.user?.isInstanceOperator(bot) ? bot.client.scope : [];
+            let allManagerData: any = {
+                name: 'All',
+                status: bot.running ? 'RUNNING' : 'NOT RUNNING',
+                indicator: bot.running ? 'green' : 'grey',
+                maxWorkers,
+                globalMaxWorkers,
+                scopes: scopes === null || !Array.isArray(scopes) ? [] : scopes,
+                subMaxWorkers,
+                runningActivities,
+                queuedActivities,
+                botState: {
+                    state: RUNNING,
+                    causedBy: SYSTEM
+                },
+                dryRun: boolToString(bot.dryRun === true),
+                checks: checks,
+                softLimit: bot.softLimit,
+                hardLimit: bot.hardLimit,
+                stats: {
+                    ...rest,
+                    cache: {
+                        currentKeyCount: sharedCount + subManagerData.reduce((acc, curr) => curr.stats.cache.isShared ? acc : acc + curr.stats.cache.currentKeyCount,0),
+                        isShared: false,
+                        totalRequests: cacheReq,
+                        totalMiss: cacheMiss,
+                        missPercent: `${formatNumber(cacheMiss === 0 || cacheReq === 0 ? 0 : (cacheMiss / cacheReq) * 100, {toFixed: 0})}%`,
+                        types: {
+                            ...cumRaw,
+                        }
+                    }
+                },
+            };
+            // if(isOperator) {
+            allManagerData.startedAt = bot.startedAt.local().format('MMMM D, YYYY h:mm A Z');
+            allManagerData.heartbeatHuman = dayjs.duration({seconds: bot.heartbeatInterval}).humanize();
+            allManagerData.heartbeat = bot.heartbeatInterval;
+            allManagerData = {...allManagerData, ...opStats(bot)};
+            //}
+
+            const botDur = dayjs.duration(dayjs().diff(bot.startedAt))
+            if (allManagerData.stats.cache.totalRequests > 0) {
+                const minutes = botDur.asMinutes();
+                if (minutes < 10) {
+                    allManagerData.stats.cache.requestRate = formatNumber((10 / minutes) * allManagerData.stats.cache.totalRequests, {
+                        toFixed: 0,
+                        round: {enable: true, indicate: true}
+                    });
+                } else {
+                    allManagerData.stats.cache.requestRate = formatNumber(allManagerData.stats.cache.totalRequests / (minutes / 10), {
+                        toFixed: 0,
+                        round: {enable: true, indicate: true}
+                    });
+                }
+            } else {
+                allManagerData.stats.cache.requestRate = 0;
+            }
+
+            const data = {
+                bot: bot.getBotName(),
+                system: {
+                    startedAt: bot.startedAt.local().format('MMMM D, YYYY h:mm A Z'),
+                    running: bot.running,
+                    error: bot.error,
+                    ...opStats(bot),
+                },
+            };
+            return res.json(data);
+        } else {
+            // getting specific subreddit stats
+            const sd = {
+                name: manager.displayLabel,
+                botState: manager.botState,
+                eventsState: manager.eventsState,
+                queueState: manager.queueState,
+                indicator: 'gray',
+                permissions: await manager.getModPermissions(),
+                queuedActivities: manager.queue.length(),
+                runningActivities: manager.queue.running(),
+                maxWorkers: manager.queue.concurrency,
+                subMaxWorkers: manager.subMaxWorkers || bot.maxWorkers,
+                globalMaxWorkers: bot.maxWorkers,
+                validConfig: boolToString(manager.validConfigLoaded),
+                configFormat: manager.wikiFormat,
+                dryRun: boolToString(manager.dryRun === true),
+                pollingInfo: manager.pollOptions.length === 0 ? ['nothing :('] : manager.pollOptions.map(pollingInfo),
+                checks: {
+                    submissions: manager.submissionChecks === undefined ? 0 : manager.submissionChecks.length,
+                    comments: manager.commentChecks === undefined ? 0 : manager.commentChecks.length,
+                },
+                wikiRevisionHuman: manager.lastWikiRevision === undefined ? 'N/A' : `${dayjs.duration(dayjs().diff(manager.lastWikiRevision)).humanize()} ago`,
+                wikiRevision: manager.lastWikiRevision === undefined ? 'N/A' : manager.lastWikiRevision.local().format('MMMM D, YYYY h:mm A Z'),
+                wikiLastCheckHuman: `${dayjs.duration(dayjs().diff(manager.lastWikiCheck)).humanize()} ago`,
+                wikiLastCheck: manager.lastWikiCheck.local().format('MMMM D, YYYY h:mm A Z'),
+                stats: await manager.getStats(),
+                startedAt: 'Not Started',
+                startedAtHuman: 'Not Started',
+                delayBy: manager.delayBy === undefined ? 'No' : `Delayed by ${manager.delayBy} sec`,
+            };
+            // TODO replace indicator data with js on client page
+            let indicator;
+            if (manager.botState.state === RUNNING && manager.queueState.state === RUNNING && manager.eventsState.state === RUNNING) {
+                indicator = 'green';
+            } else if (manager.botState.state === STOPPED && manager.queueState.state === STOPPED && manager.eventsState.state === STOPPED) {
+                indicator = 'red';
+            } else {
+                indicator = 'yellow';
+            }
+            sd.indicator = indicator;
+            if (manager.startedAt !== undefined) {
+                const dur = dayjs.duration(dayjs().diff(manager.startedAt));
+                sd.startedAtHuman = `${dur.humanize()} ago`;
+                sd.startedAt = manager.startedAt.local().format('MMMM D, YYYY h:mm A Z');
+
+                if (sd.stats.cache.totalRequests > 0) {
+                    const minutes = dur.asMinutes();
+                    if (minutes < 10) {
+                        sd.stats.cache.requestRate = formatNumber((10 / minutes) * sd.stats.cache.totalRequests, {
+                            toFixed: 0,
+                            round: {enable: true, indicate: true}
+                        });
+                    } else {
+                        sd.stats.cache.requestRate = formatNumber(sd.stats.cache.totalRequests / (minutes / 10), {
+                            toFixed: 0,
+                            round: {enable: true, indicate: true}
+                        });
+                    }
+                } else {
+                    sd.stats.cache.requestRate = 0;
+                }
+            }
+
+            return res.json(sd);
+        }
+    }
+    return [...middleware, response];
+}
+
+export default liveStats;

src/Web/Server/routes/authenticated/user/logs.ts

@@ -1,19 +1,29 @@
 import {Router} from '@awaitjs/express';
 import {Request, Response} from 'express';
-import {filterLogBySubreddit, isLogLineMinLevel, LogEntry, parseSubredditLogName} from "../../../../../util";
+import {
+    filterLogBySubreddit,
+    filterLogs,
+    isLogLineMinLevel,
+    LogEntry,
+    parseSubredditLogName
+} from "../../../../../util";
 import {Transform} from "stream";
 import winston from "winston";
 import pEvent from "p-event";
 import {getLogger} from "../../../../../Utils/loggerFactory";
 import {booleanMiddle} from "../../../../Common/middleware";
-import {authUserCheck, botRoute} from "../../../middleware";
+import {authUserCheck, botRoute, subredditRoute} from "../../../middleware";
 import {LogInfo} from "../../../../../Common/interfaces";
 import {MESSAGE} from "triple-beam";
+import {Manager} from "../../../../../Subreddit/Manager";
+import Bot from "../../../../../Bot";
 
 // TODO update logs api
-const logs = (subLogMap: Map<string, LogEntry[]>) => {
+const logs = () => {
     const middleware = [
         authUserCheck(),
+        botRoute(false),
+        subredditRoute(false),
         booleanMiddle([{
             name: 'stream',
             defaultVal: false
@@ -33,8 +43,8 @@ const logs = (subLogMap: Map<string, LogEntry[]>) => {
             try {
                 logger.stream().on('log', (log: LogInfo) => {
                     if (isLogLineMinLevel(log, level as string)) {
-                        const {subreddit: subName} = log;
-                        if (isOperator || (subName !== undefined && (realManagers.includes(subName) || subName.includes(userName)))) {
+                        const {subreddit: subName, user} = log;
+                        if (isOperator || (subName !== undefined && (realManagers.includes(subName) || (user !== undefined && user.includes(userName))))) {
                             if(streamObjects) {
                                 let obj: any = log;
                                 if(!formatted) {
@@ -52,7 +62,7 @@ const logs = (subLogMap: Map<string, LogEntry[]>) => {
                 });
                 logger.info(`${userName} from ${origin} => CONNECTED`);
                 await pEvent(req, 'close');
-                console.log('Request closed detected with "close" listener');
+                //logger.debug('Request closed detected with "close" listener');
                 res.destroy();
                 return;
             } catch (e: any) {
@@ -64,24 +74,61 @@ const logs = (subLogMap: Map<string, LogEntry[]>) => {
                 res.destroy();
             }
         } else {
-            const logs = filterLogBySubreddit(subLogMap, realManagers, {
-                level: (level as string),
-                operator: isOperator,
-                user: userName,
-                sort: sort as 'descending' | 'ascending',
-                limit: Number.parseInt((limit as string)),
-                returnType: 'object',
-            });
-            const subArr: any = [];
-            logs.forEach((v: (string|LogInfo)[], k: string) => {
-                let logs = v as LogInfo[];
-                let output: any[] = formatted ? logs : logs.map((x) => {
-                    const {[MESSAGE]: fMessage, ...rest} = x;
-                    return rest;
-                })
-                subArr.push({name: k, logs: output});
-            });
-            return res.json(subArr);
+            let bots: Bot[] = [];
+            if(req.serverBot !== undefined) {
+                bots = [req.serverBot];
+            } else {
+                bots = req.user?.accessibleBots(req.botApp.bots) as Bot[];
+            }
+
+            const allReq = req.query.subreddit !== undefined && (req.query.subreddit as string).toLowerCase() === 'all';
+
+            const botArr: any = [];
+            for(const b of bots) {
+                const managerLogs = new Map<string, LogInfo[]>();
+                const managers = req.manager !== undefined ? [req.manager] : req.user?.accessibleSubreddits(b) as Manager[];
+                for (const m of managers) {
+                    const logs = filterLogs(m.logs, {
+                        level: (level as string),
+                        // @ts-ignore
+                        sort,
+                        limit: Number.parseInt((limit as string)),
+                        returnType: 'object'
+                    }) as LogInfo[];
+                    managerLogs.set(m.getDisplay(), logs);
+                }
+                const allLogs = filterLogs([...[...managerLogs.values()].flat(), ...(req.user?.isInstanceOperator(req.botApp) ? b.logs : b.logs.filter(x => x.user === req.user?.name))], {
+                    level: (level as string),
+                    // @ts-ignore
+                    sort,
+                    limit: limit as string,
+                    returnType: 'object'
+                }) as LogInfo[];
+                const systemLogs = filterLogs(req.user?.isInstanceOperator(req.botApp) ? b.logs : b.logs.filter(x => x.user === req.user?.name), {
+                    level: (level as string),
+                    // @ts-ignore
+                    sort,
+                    limit: limit as string,
+                    returnType: 'object'
+                }) as LogInfo[];
+                botArr.push({
+                    name: b.getBotName(),
+                    system: systemLogs,
+                    all: formatted ? allLogs.map(x => {
+                        const {[MESSAGE]: fMessage, ...rest} = x;
+                        return {...rest, formatted: fMessage};
+                    }) : allLogs,
+                    subreddits: allReq ? [] : [...managerLogs.entries()].reduce((acc: any[], curr) => {
+                        const l = formatted ? curr[1].map(x => {
+                            const {[MESSAGE]: fMessage, ...rest} = x;
+                            return {...rest, formatted: fMessage};
+                            }) : curr[1];
+                        acc.push({name: curr[0], logs: l});
+                        return acc;
+                    }, [])
+                });
+            }
+            return res.json(botArr);
         }
     };
 

src/Web/Server/routes/authenticated/user/status.ts

@@ -2,26 +2,27 @@ import {Request, Response} from 'express';
 import {
     boolToString,
     cacheStats,
-    filterLogBySubreddit,
+    filterLogBySubreddit, filterLogs,
     formatNumber,
     intersect,
-    LogEntry,
+    LogEntry, logSortFunc,
     pollingInfo
 } from "../../../../../util";
 import {Manager} from "../../../../../Subreddit/Manager";
 import dayjs from "dayjs";
-import {ResourceStats, RUNNING, STOPPED, SYSTEM} from "../../../../../Common/interfaces";
+import {LogInfo, ResourceStats, RUNNING, STOPPED, SYSTEM} from "../../../../../Common/interfaces";
 import {BotStatusResponse} from "../../../../Common/interfaces";
 import winston from "winston";
 import {opStats} from "../../../../Common/util";
-import {authUserCheck, botRoute} from "../../../middleware";
+import {authUserCheck, botRoute, subredditRoute} from "../../../middleware";
 import Bot from "../../../../../Bot";
 
 const status = () => {
 
     const middleware = [
         authUserCheck(),
-        //botRoute(),
+        botRoute(false),
+        subredditRoute(false)
     ];
 
     const response = async (req: Request, res: Response) => {
@@ -32,28 +33,17 @@ const status = () => {
             sort = 'descending',
         } = req.query;
 
-        // @ts-ignore
-        const botLogMap = req.botLogs as Map<string, Map<string, LogEntry[]>>;
-        // @ts-ignore
-        const systemLogs = req.systemLogs as LogEntry[];
+        bots = req.user?.accessibleBots(req.botApp.bots) as Bot[];
 
-
-        if(req.serverBot !== undefined) {
-            bots = [req.serverBot];
-        } else {
-            bots = req.user?.accessibleBots(req.botApp.bots) as Bot[];
-        }
         const botResponses: BotStatusResponse[] = [];
         let index = 1;
         for(const b of bots) {
-            botResponses.push(await botStatResponse(b, req, botLogMap, index));
+            botResponses.push(await botStatResponse(b, req, index));
             index++;
         }
         const system: any = {};
-        if(req.user?.isInstanceOperator(req.botApp)) {
-            // @ts-ignore
-            system.logs = filterLogBySubreddit(new Map([['app', systemLogs]]), [], {level, sort, limit, operator: true}).get('all');
-        }
+        // @ts-ignore
+        system.logs = filterLogs(req.sysLogs, {level, sort, limit, user: req.user?.isInstanceOperator(req.botApp) ? undefined : req.user?.name, returnType: 'object' }) as LogInfo[];
         const response = {
             bots: botResponses,
             system: system,
@@ -61,7 +51,7 @@ const status = () => {
         return res.json(response);
     }
 
-    const botStatResponse = async (bot: Bot, req: Request, botLogMap: Map<string, Map<string, LogEntry[]>>, index: number) => {
+    const botStatResponse = async (bot: Bot, req: Request, index: number) => {
         const {
             //subreddits = [],
             //user: userVal,
@@ -71,29 +61,26 @@ const status = () => {
             lastCheck
         } = req.query;
 
-        const user = req.user?.name as string;
-
-        const logs = filterLogBySubreddit(botLogMap.get(bot.botName as string) || new Map(), req.user?.accessibleSubreddits(bot).map(x => x.displayLabel) as string[], {
-            level: (level as string),
-            operator: req.user?.isInstanceOperator(req.botApp),
-            user,
-            // @ts-ignore
-            sort,
-            limit: Number.parseInt((limit as string)),
-            returnType: 'object'
-        });
+        const allReq = req.query.subreddit !== undefined && (req.query.subreddit as string).toLowerCase() === 'all';
 
         const subManagerData = [];
         for (const m of req.user?.accessibleSubreddits(bot) as Manager[]) {
+            const logs = req.manager === undefined || allReq || req.manager.getDisplay() === m.getDisplay() ? filterLogs(m.logs, {
+                    level: (level as string),
+                    // @ts-ignore
+                    sort,
+                    limit: limit as string,
+                    returnType: 'object'
+                }) as LogInfo[]: [];
             const sd = {
                 name: m.displayLabel,
                 //linkName: s.replace(/\W/g, ''),
-                logs: logs.get(m.displayLabel) || [], // provide a default empty value in case we truly have not logged anything for this subreddit yet
+                logs: logs || [], // provide a default empty value in case we truly have not logged anything for this subreddit yet
                 botState: m.botState,
                 eventsState: m.eventsState,
                 queueState: m.queueState,
                 indicator: 'gray',
-                permissions: await m.getModPermissions(),
+                permissions: [],
                 queuedActivities: m.queue.length(),
                 runningActivities: m.queue.running(),
                 maxWorkers: m.queue.concurrency,
@@ -234,6 +221,14 @@ const status = () => {
         const sharedSub = subManagerData.find(x => x.stats.cache.isShared);
         const sharedCount = sharedSub !== undefined ? sharedSub.stats.cache.currentKeyCount : 0;
         const scopes = req.user?.isInstanceOperator(bot) ? bot.client.scope : [];
+        const allSubLogs = subManagerData.map(x => x.logs).flat().sort(logSortFunc(sort as string)).slice(0, (limit as number) + 1);
+        const allLogs = filterLogs([...allSubLogs, ...(req.user?.isInstanceOperator(req.botApp) ? bot.logs : bot.logs.filter(x => x.user === req.user?.name))], {
+            level: (level as string),
+            // @ts-ignore
+            sort,
+            limit: limit as string,
+            returnType: 'object'
+        }) as LogInfo[];
         let allManagerData: any = {
             name: 'All',
             status: bot.running ? 'RUNNING' : 'NOT RUNNING',
@@ -249,7 +244,7 @@ const status = () => {
                 causedBy: SYSTEM
             },
             dryRun: boolToString(bot.dryRun === true),
-            logs: logs.get('all'),
+            logs: allLogs,
             checks: checks,
             softLimit: bot.softLimit,
             hardLimit: bot.hardLimit,
@@ -305,7 +300,7 @@ const status = () => {
                 name: (bot.botName as string) ?? `Bot ${index}`,
                 ...opStats(bot),
             },
-            subreddits: [allManagerData, ...subManagerData],
+            subreddits: [allManagerData, ...(allReq ? subManagerData.map(({logs, ...x}) => ({...x, logs: []})) : subManagerData)],
 
         };
 

src/Web/Server/server.ts

@@ -9,11 +9,6 @@ import {Strategy as JwtStrategy, ExtractJwt} from 'passport-jwt';
 import passport from 'passport';
 import tcpUsed from 'tcp-port-used';
 
-import {
-    intersect,
-    LogEntry, parseBotLogName,
-    parseSubredditLogName
-} from "../../util";
 import {getLogger} from "../../Utils/loggerFactory";
 import LoggedError from "../../Utils/LoggedError";
 import {Invokee, LogInfo, OperatorConfigWithFileContext} from "../../Common/interfaces";
@@ -21,13 +16,13 @@ import http from "http";
 import {heartbeat} from "./routes/authenticated/applicationRoutes";
 import logs from "./routes/authenticated/user/logs";
 import status from './routes/authenticated/user/status';
+import liveStats from './routes/authenticated/user/liveStats';
 import {actionedEventsRoute, actionRoute, configRoute, configLocationRoute, deleteInviteRoute, addInviteRoute, getInvitesRoute} from "./routes/authenticated/user";
 import action from "./routes/authenticated/user/action";
 import {authUserCheck, botRoute} from "./middleware";
 import {opStats} from "../Common/util";
 import Bot from "../../Bot";
 import addBot from "./routes/authenticated/user/addBot";
-import dayjs from "dayjs";
 import ServerUser from "../Common/User/ServerUser";
 import {SimpleError} from "../../Utils/Errors";
 import {ErrorWithCause} from "pony-cause";
@@ -47,11 +42,7 @@ declare module 'express-session' {
     }
 }
 
-const subLogMap: Map<string, LogEntry[]> = new Map();
-const systemLogs: LogEntry[] = [];
-const botLogMap: Map<string, Map<string, LogEntry[]>> = new Map();
-
-const botSubreddits: Map<string, string[]> = new Map();
+let sysLogs: LogInfo[] = [];
 
 const rcbServer = async function (options: OperatorConfigWithFileContext) {
 
@@ -73,36 +64,12 @@ const rcbServer = async function (options: OperatorConfigWithFileContext) {
     const logger = getLogger({...options.logging});
 
     logger.stream().on('log', (log: LogInfo) => {
-        const logEntry: LogEntry = [dayjs(log.timestamp).unix(), log];
 
         const {bot: botName, subreddit: subName} = log;
 
-        if(botName === undefined) {
-            systemLogs.unshift(logEntry);
-            systemLogs.slice(0, 201);
-        } else {
-            const botLog = botLogMap.get(botName) || new Map();
-
-            if(subName === undefined) {
-                const appLogs = botLog.get('app') || [];
-                appLogs.unshift(logEntry);
-                botLog.set('app', appLogs.slice(0, 200 + 1));
-            } else {
-                let botSubs = botSubreddits.get(botName) || [];
-                if(app !== undefined && (botSubs.length === 0 || !botSubs.includes(subName))) {
-                    const b = app.bots.find(x => x.botName === botName);
-                    if(b !== undefined) {
-                        botSubs = b.subManagers.map(x => x.displayLabel);
-                        botSubreddits.set(botName, botSubs);
-                    }
-                }
-                if(botSubs.length === 0 || botSubs.includes(subName)) {
-                    const subLogs = botLog.get(subName) || [];
-                    subLogs.unshift(logEntry);
-                    botLog.set(subName, subLogs.slice(0, 200 + 1));
-                }
-            }
-            botLogMap.set(botName, botLog);
+        if(botName === undefined && subName === undefined) {
+            sysLogs.unshift(log);
+            sysLogs = sysLogs.slice(0, 201);
         }
     })
 
@@ -167,7 +134,7 @@ const rcbServer = async function (options: OperatorConfigWithFileContext) {
 
     server.getAsync('/heartbeat', ...heartbeat({name, display, friendly}));
 
-    server.getAsync('/logs', ...logs(subLogMap));
+    server.getAsync('/logs', ...logs());
 
     server.getAsync('/stats', [authUserCheck(), botRoute(false)], async (req: Request, res: Response) => {
         let bots: Bot[] = [];
@@ -186,13 +153,13 @@ const rcbServer = async function (options: OperatorConfigWithFileContext) {
     });
     const passLogs = async (req: Request, res: Response, next: Function) => {
         // @ts-ignore
-        req.botLogs = botLogMap;
-        // @ts-ignore
-        req.systemLogs = systemLogs;
+        req.sysLogs = sysLogs;
         next();
     }
     server.getAsync('/status', passLogs, ...status())
 
+    server.getAsync('/liveStats', ...liveStats())
+
     server.getAsync('/config', ...configRoute);
 
     server.getAsync('/config/location', ...configLocationRoute);

src/Web/assets/views/partials/footer.ejs

@@ -1,6 +1,6 @@
 <div class="py-3 flex items-center justify-around font-semibold">
     <div>
-        <a href="https://github.com/FoxxMD/context-mod">ContextMod Web</a> created by /u/FoxxMD
+        <a href="https://github.com/FoxxMD/context-mod">ContextMod <%= locals.applicationIdentifier%></a> created by /u/FoxxMD
     </div>
 </div>
 <script type="text/javascript" src="https://cdn.statuspage.io/se-v2.js"></script>
@@ -9,7 +9,6 @@
     var sp = new StatusPage.page({ page : '2kbc0d48tv3j' });
     sp.status({
         success : function(data) {
-            debugger;
             console.log(data.status.indicator);
             switch(data.status.indicator){
                 case 'minor':

src/Web/assets/views/status.ejs

@@ -164,14 +164,14 @@
                                                 <span class="has-tooltip">
                                                     <span class='tooltip rounded shadow-lg p-1 bg-gray-100 text-black -mt-2'>
                                                         <span>
-                                                            <ul class="list-inside list-disc">
+                                                            <ul class="list-inside list-disc modPermissionsList">
                                                                 <% data.permissions.forEach(function (i){ %>
                                                                     <li class="font-mono"><%= i %></li>
                                                                 <% }) %>
                                                             </ul>
                                                         </span>
                                                     </span>
-                                                    <span class="cursor-help underline" style="text-decoration-style: dotted"><%= data.permissions.length %></span>
+                                                    <span class="cursor-help underline modPermissionsCount" style="text-decoration-style: dotted"><%= data.permissions.length %></span>
                                                 </span>
                                                 <label>Slow Mode</label>
                                                 <span><%= data.delayBy %></span>
@@ -677,6 +677,48 @@
 </div>
 <%- include('partials/instanceTabJs') %>
 <%- include('partials/logSettingsJs') %>
+<script src="https://unpkg.com/autolinker@3.14.3/dist/Autolinker.min.js" crossorigin="anonymous"></script>
+<script src="https://unpkg.com/dayjs@1.10.7/dayjs.min.js" crossorigin="anonymous"></script>
+<script src="https://unpkg.com/dayjs@1.10.7/plugin/advancedFormat.js"></script>
+<script src="https://unpkg.com/dayjs@1.10.7/plugin/timezone.js"></script>
+<script>
+    dayjs.extend(window.dayjs_plugin_timezone)
+    dayjs.extend(window.dayjs_plugin_advancedFormat)
+    window.formattedTime = (short, full) => `<span class="has-tooltip"><span style="margin-top:35px" class='tooltip rounded shadow-lg p-1 bg-gray-100 text-black space-y-3 p-2 text-left'>${full}</span><span>${short}</span></span>`;
+    window.formatLogLineToHtml = (log, timestamp = undefined) => {
+        const val = typeof log === 'string' ? log : log['MESSAGE'];
+        const logContent = Autolinker.link(val, {
+            email: false,
+            phone: false,
+            mention: false,
+            hashtag: false,
+            stripPrefix: false,
+            sanitizeHtml: true,
+        })
+            .replace(/(\s*debug\s*):/i, '<span class="debug blue">$1</span>:')
+            .replace(/(\s*warn\s*):/i, '<span class="warn yellow">$1</span>:')
+            .replace(/(\s*info\s*):/i, '<span class="info green">$1</span>:')
+            .replace(/(\s*error\s*):/i, '<span class="error red">$1</span>:')
+            .replace(/(\s*verbose\s*):/i, '<span class="error purple">$1</span>:')
+            .replaceAll('\n', '<br />');
+        //.replace(HYPERLINK_REGEX, '<a target="_blank" href="$&">$&</a>');
+        let line = '';
+
+        let timestampString = timestamp;
+        if(timestamp === undefined && typeof log !== 'string') {
+            timestampString = log.timestamp;
+        }
+
+        if(timestampString !== undefined) {
+            const timeStampReplacement = formattedTime(dayjs(timestampString).format('HH:mm:ss z'), timestampString);
+            const splitLine = logContent.split(timestampString);
+            line = `<div class="logLine">${splitLine[0]}${timeStampReplacement}<span style="white-space: pre-wrap">${splitLine[1]}</span></div>`;
+        } else {
+            line = `<div style="white-space: pre-wrap" class="logLine">${logContent}</div>`
+        }
+        return line;
+    }
+</script>
 <script>
     window.sort = 'desc';
 
@@ -837,6 +879,44 @@
                 var newRelativePathQuery = window.location.pathname + '?' + searchParams.toString();
                 history.pushState(null, '', newRelativePathQuery);
             }
+            const activeSub = document.querySelector(`[data-subreddit="${subreddit}"][data-bot="${bot}"].sub`);
+            if(!activeSub.classList.contains('seen')) {
+                //firstSub.classList.add('seen');
+
+                //subreddit = firstSub.dataset.subreddit;
+                //bot = subSection.dataset.bot;
+                level = document.querySelector(`[data-subreddit="${subreddit}"] [data-type="level"]`).value;
+                sort = document.querySelector(`[data-subreddit="${subreddit}"] [data-type="sort"]`).value;
+                limitSel = document.querySelector(`[data-subreddit="${subreddit}"] [data-type="limit"]`).value;
+
+                fetch(`/api/logs?instance=<%= instanceId %>&bot=${bot}&subreddit=${subreddit}&level=${level}&sort=${sort}&limit=${limitSel}&stream=false&formatted=true`).then((resp) => {
+                    if (!resp.ok) {
+                        console.error('Response was not OK from logs GET');
+                    } else {
+                        resp.json().then((data) => {
+                            const logContainer = document.querySelector(`[data-subreddit="${subreddit}"] .logs`);
+                            const logLines = (subreddit.toLowerCase() === 'all' ? data[0].all : data[0].subreddits[0].logs).map(x => {
+                                let fString = x.formatted;
+                                if(x.bot !== undefined) {
+                                    fString = fString.replace(`~${x.bot}~ `, '');
+                                }
+                                if(x.subreddit !== undefined && subreddit !== 'All') {
+                                    fString = fString.replace(`{${x.subreddit}} `, '');
+                                }
+                                return window.formatLogLineToHtml(fString, x.timestamp)
+                            }).join('');
+                            logContainer.insertAdjacentHTML('afterbegin', logLines);
+                            activeSub.classList.add('seen');
+                        });
+                    }
+                }).catch((err) => {
+                    console.log(err);
+                });
+
+            }
+            if(window.socket !== undefined) {
+                window.socket.emit('viewing', {bot, subreddit});
+            }
         });
     });
 
@@ -864,7 +944,7 @@
         tabSelect.classList.add('font-bold', 'no-underline', 'pointer-events-none');
     }
     document.querySelectorAll('.tabSelectWrapper').forEach(el => el.classList.add('border'));
-    document.querySelector(`[data-bot="${shownBot}"][data-subreddit="${shownSub}"].sub`).classList.add('active');
+    document.querySelector(`[data-bot="${shownBot}"][data-subreddit="${shownSub}"].sub`).classList.add('active', 'seen');
     const subWrapper = document.querySelector(`ul[data-bot="${shownBot}"] [data-subreddit="${shownSub}"].tabSelectWrapper`);
     if(subWrapper !== null) {
         subWrapper.classList.remove('border');
@@ -891,6 +971,7 @@
     let socket = io({
         reconnectionAttempts: 5, // bail after 5 attempts
     });
+    window.socket = socket;
 
     // get all bots
     let bots = [];
@@ -908,6 +989,11 @@
 
     socket.on("connect", () => {
         document.body.classList.add('connected')
+
+        const shownSub = searchParams.get('sub') || 'All'
+        let shownBot = searchParams.get('bot');
+        window.socket.emit('viewing', {bot: shownBot, subreddit: shownSub});
+
         socket.on("log", data => {
             const {
                 subreddit,
@@ -942,12 +1028,12 @@
                     }
                     subLogs.forEach((logs, subKey) => {
                         // check sub exists -- may be a web log
-                        const el = document.querySelector(`[data-subreddit="${subKey}"][data-bot="${botName}"].sub`);
+                        const el = document.querySelector(`[data-subreddit="${subKey}"][data-bot="${botName}"].sub.seen`);
                         if(null !== el) {
                             const limit = Number.parseInt(document.querySelector(`[data-subreddit="${subKey}"] [data-type="limit"]`).value);
                             const logContainer = el.querySelector(`.logs`);
                             let existingLogs;
-                            if(window.sort === 'desc') {
+                            if(window.sort === 'desc' || window.sort === 'descending') {
                                 logs.forEach((l) => {
                                     logContainer.insertAdjacentHTML('afterbegin', l);
                                 })
@@ -999,6 +1085,29 @@
             }
 
         });
+        socket.on('liveStats', (resp) => {
+            let el;
+            let isAll = resp.system !== undefined;
+            if(isAll) {
+                // got all
+                el = document.querySelector(`[data-subreddit="All"][data-bot="${resp.bot}"].sub`);
+            } else {
+                // got subreddit
+                el = document.querySelector(`[data-subreddit="${resp.name}"].sub`);
+            }
+
+            if(isAll) {
+
+            } else {
+                if(el.querySelector('.modPermissionsCount').innerHTML != resp.permissions.length) {
+                    el.querySelector('.modPermissionsCount').innerHTML = resp.permissions.length;
+                    el.querySelector('.modPermissionsList').innerHTML = '';
+                    el.querySelector('.modPermissionsList').insertAdjacentHTML('afterbegin', resp.permissions.map(x => `<li class="font-mono">${x}</li>`).join(''));
+                }
+            }
+
+            console.log(resp);
+        });
     });
 
     socket.on('disconnect', () => {

src/Web/interfaces.ts

@@ -1,5 +1,6 @@
 import { URL } from "url";
 import {BotConnection} from "../Common/interfaces";
+import {Logger} from "winston";
 
 export interface BotInstance {
     botName: string
@@ -8,16 +9,16 @@ export interface BotInstance {
     subreddits: string[]
     nanny?: string
     running: boolean
-    instance: CMInstance
+    instance: CMInstanceInterface
 }
 
-export interface CMInstance extends BotConnection {
-    friendly: string
+export interface CMInstanceInterface extends BotConnection {
+    friendly?: string
     operators: string[]
     operatorDisplay: string
     url: URL,
     normalUrl: string,
-    lastCheck: number
+    lastCheck?: number
     online: boolean
     subreddits: string[]
     bots: BotInstance[]

src/Web/types/express/index.d.ts

@@ -1,6 +1,6 @@
 import {App} from "../../../App";
 import Bot from "../../../Bot";
-import {BotInstance, CMInstance} from "../../interfaces";
+import {BotInstance, CMInstanceInterface} from "../../interfaces";
 import {Manager} from "../../../Subreddit/Manager";
 import CMUser from "../../Common/User/CMUser";
 
@@ -9,7 +9,7 @@ declare global {
         interface Request {
             botApp: App;
             token?: string,
-            instance?: CMInstance,
+            instance?: CMInstanceInterface,
             bot?: BotInstance,
             serverBot: Bot,
             manager?: Manager,

src/util.ts

@@ -35,7 +35,7 @@ import {
 import { Document as YamlDocument } from 'yaml'
 import InvalidRegexError from "./Utils/InvalidRegexError";
 import {constants, promises} from "fs";
-import {cacheOptDefaults} from "./Common/defaults";
+import {cacheOptDefaults, VERSION} from "./Common/defaults";
 import cacheManager, {Cache} from "cache-manager";
 import redisStore from "cache-manager-redis-store";
 import crypto from "crypto";
@@ -1159,9 +1159,14 @@ export const formatLogLineToHtml = (log: string | LogInfo, timestamp?: string) =
         //.replace(HYPERLINK_REGEX, '<a target="_blank" href="$&">$&</a>');
     let line = '';
 
-    if(timestamp !== undefined) {
-        const timeStampReplacement = formattedTime(dayjs(timestamp).format('HH:mm:ss z'), timestamp);
-        const splitLine = logContent.split(timestamp);
+    let timestampString = timestamp;
+    if(timestamp === undefined && typeof log !== 'string') {
+        timestampString = (log as LogInfo).timestamp;
+    }
+
+    if(timestampString !== undefined) {
+        const timeStampReplacement = formattedTime(dayjs(timestampString).format('HH:mm:ss z'), timestampString);
+        const splitLine = logContent.split(timestampString);
         line = `<div class="logLine">${splitLine[0]}${timeStampReplacement}<span style="white-space: pre-wrap">${splitLine[1]}</span></div>`;
     } else {
         line = `<div style="white-space: pre-wrap" class="logLine">${logContent}</div>`
@@ -1171,7 +1176,7 @@ export const formatLogLineToHtml = (log: string | LogInfo, timestamp?: string) =
 
 export type LogEntry = [number, LogInfo];
 export interface LogOptions {
-    limit: number,
+    limit: number | string,
     level: string,
     sort: 'ascending' | 'descending',
     operator?: boolean,
@@ -1183,7 +1188,7 @@ export interface LogOptions {
 
 export const filterLogBySubreddit = (logs: Map<string, LogEntry[]>, validLogCategories: string[] = [], options: LogOptions): Map<string, (string|LogInfo)[]> => {
     const {
-        limit,
+        limit: limitVal,
         level,
         sort,
         operator = false,
@@ -1193,6 +1198,7 @@ export const filterLogBySubreddit = (logs: Map<string, LogEntry[]>, validLogCate
         returnType = 'string',
     } = options;
 
+    let limit = typeof limitVal === 'number' ? limitVal : Number.parseInt(limitVal);
     // get map of valid logs categories
     const validSubMap: Map<string, LogEntry[]> = new Map();
     for(const [k, v] of logs) {
@@ -1242,6 +1248,35 @@ export const filterLogBySubreddit = (logs: Map<string, LogEntry[]>, validLogCate
     return preparedMap;
 }
 
+export const logSortFunc = (sort: string = 'ascending') => sort === 'ascending' ? (a: LogInfo, b: LogInfo) => (dayjs(a.timestamp).isSameOrAfter(b.timestamp) ? 1 : -1) : (a: LogInfo, b: LogInfo) => (dayjs(a.timestamp).isSameOrBefore(b.timestamp) ? 1 : -1);
+
+export const filterLogs= (logs: LogInfo[], options: LogOptions): LogInfo[] | string[] => {
+    const {
+        limit: limitVal,
+        level,
+        sort,
+        operator = false,
+        user,
+        allLogsParser = parseSubredditLogInfoName,
+        allLogName = 'app',
+        returnType = 'string',
+    } = options;
+
+    let limit = typeof limitVal === 'number' ? limitVal : Number.parseInt(limitVal);
+    let leveledLogs = logs.filter(x => isLogLineMinLevel(x, level));
+    if(user !== undefined) {
+        leveledLogs = logs.filter(x => x.user !== undefined && x.user === user);
+    }
+    leveledLogs.sort(logSortFunc(sort));
+    leveledLogs = leveledLogs.slice(0, limit + 1);
+
+    if(returnType === 'string') {
+        return leveledLogs.map(x => formatLogLineToHtml(x));
+    } else {
+        return leveledLogs;
+    }
+}
+
 export const logLevels = {
     error: 0,
     warn: 1,
@@ -2073,3 +2108,11 @@ export async function* redisScanIterator(client: any, options: any = {}): AsyncI
         }
     } while (cursor !== '0');
 }
+
+export const getUserAgent = (val: string, fragment?: string) => {
+    return `${replaceApplicationIdentifier(val, fragment)} (developed by /u/FoxxMD)`;
+}
+
+export const replaceApplicationIdentifier = (val: string, fragment?: string) => {
+    return val.replace('{VERSION}', `v${VERSION}`).replace('{FRAG}', (fragment !== undefined ? `-${fragment}` : ''));
+}