Merge branch 'edge'

* Implement separate language detection functionality
* Clearer/simpler sentiment processing
* Add languageHints to help coerce low confidence language detection
* Add test cases for lang detection, sentiment detection, and sentiment tests
* Fix neutral range -- was not using normalized score range

.gitignore

@@ -383,7 +383,7 @@ dist
 **/src/**/*.js
 **/tests/**/*.js
 **/tests/**/*.map
-!src/Web/assets/public/yaml/*
+!src/Web/assets/**
 **/src/**/*.map
 /**/*.sqlite
 /**/*.bak

Dockerfile

@@ -111,7 +111,9 @@ COPY --from=build --chown=abc:abc /app /app
 
 RUN npm install --production \
     && npm cache clean --force \
-    && chown abc:abc node_modules
+    && chown abc:abc node_modules \
+    && rm -rf node_modules/ts-node \
+    && rm -rf node_modules/typescript
 
 ENV NPM_CONFIG_LOGLEVEL debug
 

package.json

@@ -5,7 +5,8 @@
   "main": "index.js",
   "scripts": {
     "test": "nyc ./node_modules/.bin/_mocha 'tests/**/*.test.ts'",
-    "build": "tsc",
+    "build": "tsc && npm run bundle-front",
+    "bundle-front": "browserify src/Web/assets/browser.js | terser --compress --mangle > src/Web/assets/public/browserBundle.js",
     "start": "node src/index.js run",
     "schema": "npm run -s schema-app & npm run -s schema-ruleset & npm run -s schema-rule & npm run -s schema-action & npm run -s schema-config",
     "schema-app": "typescript-json-schema tsconfig.json JSONConfig --out src/Schema/App.json --required --tsNodeRegister --refs --validationKeywords deprecationMessage",
@@ -29,6 +30,13 @@
   "dependencies": {
     "@awaitjs/express": "^0.8.0",
     "@googleapis/youtube": "^2.0.0",
+    "@nlpjs/core": "^4.23.4",
+    "@nlpjs/lang-de": "^4.23.4",
+    "@nlpjs/lang-en": "^4.23.4",
+    "@nlpjs/lang-es": "^4.23.4",
+    "@nlpjs/lang-fr": "^4.23.4",
+    "@nlpjs/language": "^4.22.7",
+    "@nlpjs/nlp": "^4.23.5",
     "@stdlib/regexp-regexp": "^0.0.6",
     "ajv": "^7.2.4",
     "ansi-regex": ">=5.0.1",
@@ -43,7 +51,6 @@
     "cookie-parser": "^1.3.5",
     "dayjs": "^1.10.5",
     "deepmerge": "^4.2.2",
-    "delimiter-stream": "^3.0.1",
     "ejs": "^3.1.6",
     "env-cmd": "^10.1.0",
     "es6-error": "^4.1.1",
@@ -52,16 +59,15 @@
     "express-session-cache-manager": "^1.0.2",
     "express-socket.io-session": "^1.3.5",
     "fast-deep-equal": "^3.1.3",
-    "fuse.js": "^6.4.6",
     "globrex": "^0.1.2",
     "got": "^11.8.2",
     "he": "^1.2.0",
     "http-proxy": "^1.18.1",
-    "image-size": "^1.0.0",
     "json5": "^2.2.0",
     "jsonwebtoken": "^8.5.1",
     "leven": "^3.1.0",
     "lodash": "^4.17.21",
+    "logform": "^2.4.0",
     "lru-cache": "^6.0.0",
     "migrate": "github:johsunds/node-migrate#49b0054de0a9295857aa8b8eea9a3cdeb2643913",
     "mustache": "^4.2.0",
@@ -77,9 +83,7 @@
     "patch-package": "^6.4.7",
     "pixelmatch": "^5.2.1",
     "pony-cause": "^1.1.1",
-    "pretty-print-json": "^1.0.3",
     "reflect-metadata": "^0.1.13",
-    "safe-stable-stringify": "^1.1.1",
     "snoostorm": "^1.5.2",
     "snoowrap": "^1.23.0",
     "socket.io": "^4.1.3",
@@ -89,8 +93,9 @@
     "triple-beam": "^1.3.0",
     "typeorm": "^0.3.4",
     "typeorm-logger-adaptor": "^1.1.0",
-    "typescript": "^4.3.4",
+    "vader-sentiment": "^1.1.3",
     "webhook-discord": "^3.7.7",
+    "wink-sentiment": "^5.0.2",
     "winston": "github:FoxxMD/winston#fbab8de969ecee578981c77846156c7f43b5f01e",
     "winston-daily-rotate-file": "^4.5.5",
     "winston-duplex": "^0.1.1",
@@ -106,6 +111,7 @@
     "@types/cache-manager": "^3.4.2",
     "@types/cache-manager-redis-store": "^2.0.0",
     "@types/chai": "^4.3.0",
+    "@types/chai-as-promised": "^7.1.5",
     "@types/cookie-parser": "^1.4.2",
     "@types/express": "^4.17.13",
     "@types/express-session": "^1.17.4",
@@ -130,16 +136,19 @@
     "@types/string-similarity": "^4.0.0",
     "@types/tcp-port-used": "^1.0.0",
     "@types/triple-beam": "^1.3.2",
+    "browserify": "^17.0.0",
     "chai": "^4.3.6",
+    "chai-as-promised": "^7.1.1",
     "mocha": "^9.2.1",
     "nyc": "^15.1.0",
     "source-map-support": "^0.5.21",
+    "terser": "^5.13.1",
     "ts-essentials": "^9.1.2",
     "ts-json-schema-generator": "^0.93.0",
     "ts-mockito": "^2.6.1",
     "ts-node": "^10.7.0",
     "tsconfig-paths": "^3.13.0",
-    "typescript": "^4.3.4",
+    "typescript": "^4.6.4",
     "typescript-json-schema": "~0.53"
   },
   "optionalDependencies": {

src/Action/ApproveAction.ts

@@ -7,6 +7,7 @@ import Comment from "snoowrap/dist/objects/Comment";
 import {RuleResultEntity} from "../Common/Entities/RuleResultEntity";
 import {runCheckOptions} from "../Subreddit/Manager";
 import {ActionTarget, ActionTypes} from "../Common/Infrastructure/Atomic";
+import {asComment, asSubmission} from "../util";
 
 export class ApproveAction extends Action {
 
@@ -29,22 +30,24 @@ export class ApproveAction extends Action {
         const dryRun = this.getRuntimeAwareDryrun(options);
         const touchedEntities = [];
 
-        const realTargets = item instanceof Submission ? ['self'] : this.targets;
+        const realTargets = asSubmission(item) ? ['self'] : this.targets;
+
+        let msg: string[] = [];
 
         for(const target of realTargets) {
             let targetItem = item;
-            if(target !== 'self' && item instanceof Comment) {
+            if(target !== 'self' && asComment(item)) {
                 targetItem = await this.resources.getActivity(this.client.getSubmission(item.link_id));
             }
 
             // @ts-ignore
             if (targetItem.approved) {
-                const msg = `${target === 'self' ? 'Item' : 'Comment\'s parent Submission'} is already approved`;
+                msg.push(`${target === 'self' ? 'Item' : 'Comment\'s parent Submission'} is already approved??`);
                 this.logger.warn(msg);
                 return {
                     dryRun,
                     success: false,
-                    result: msg
+                    result: msg.join('|')
                 }
             }
 
@@ -53,6 +56,9 @@ export class ApproveAction extends Action {
                 if(target !== 'self' && !(targetItem instanceof Submission)) {
                     // @ts-ignore
                     targetItem = await this.client.getSubmission((item as Comment).link_id).fetch();
+                    msg.push(`Approving parent Submission ${targetItem.name}`);
+                } else {
+                    msg.push(`Approving self ${targetItem.name}`);
                 }
                 // @ts-ignore
                 touchedEntities.push(await targetItem.approve());
@@ -70,6 +76,7 @@ export class ApproveAction extends Action {
         }
 
         return {
+            result: msg.join(' | '),
             dryRun,
             success: true,
             touchedEntities

src/Action/CancelDispatchAction.ts

@@ -71,12 +71,7 @@ export class CancelDispatchAction extends Action {
                 } else {
                     matchedDispatchIdentifier = this.identifiers.filter(x => x !== null).includes(x.identifier);
                 }
-                const matched = matchedId && matchedDispatchIdentifier;
-                if(matched && x.processing) {
-                    this.logger.debug(`Cannot remove ${isSubmission(x.activity) ? 'Submission' : 'Comment'} ${x.activity.name} because it is currently processing`);
-                    return false;
-                }
-                return matched;
+                return matchedId && matchedDispatchIdentifier;
             });
             let cancelCrit;
             if (this.identifiers === undefined) {

src/Common/Entities/ActionPremise.ts

@@ -79,6 +79,7 @@ export class ActionPremise extends TimeAwareRandomBaseEntity  {
             this.active = data.active ?? true;
             this.configHash = objectHash.sha1(data.config);
             this.manager = data.manager;
+            this.managerId = data.manager.id;
             this.name = data.name;
 
             const {

src/Common/Entities/DispatchedEntity.ts

@@ -152,6 +152,9 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity {
 
     async toActivityDispatch(client: ExtendedSnoowrap): Promise<ActivityDispatch> {
         const redditThing = parseRedditFullname(this.activityId);
+        if(redditThing === undefined) {
+            throw new Error(`Could not parse reddit ID from value '${this.activityId}'`);
+        }
         let activity: Comment | Submission;
         if (redditThing?.type === 'comment') {
             // @ts-ignore
@@ -161,12 +164,12 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity {
             activity = await client.getSubmission(redditThing.id);
         }
         activity.author = new RedditUser({name: this.author}, client, false);
+        activity.id  = redditThing.id;
         return {
             id: this.id,
             queuedAt: this.createdAt,
             activity,
             delay: this.delay,
-            processing: false,
             action: this.action,
             goto: this.goto,
             onExistingFound: this.onExistingFound,

src/Common/Entities/RulePremise.ts

@@ -83,6 +83,7 @@ export class RulePremise extends TimeAwareRandomBaseEntity {
             this.active = data.active ?? true;
             this.configHash = objectHash.sha1(data.config);
             this.manager = data.manager;
+            this.managerId = data.manager.id;
             this.name = data.name;
 
             const {

src/Common/ImageData.ts

@@ -2,7 +2,6 @@ import fetch from "node-fetch";
 import {Submission} from "snoowrap/dist/objects";
 import {URL} from "url";
 import {absPercentDifference, getSharpAsync, isValidImageURL} from "../util";
-import sizeOf from "image-size";
 import {Sharp} from "sharp";
 import {blockhash} from "./blockhash/blockhash";
 import {SimpleError} from "../Utils/Errors";

src/Common/Infrastructure/Atomic.ts

@@ -186,3 +186,46 @@ export type ActionTypes =
     | 'dispatch'
     | 'cancelDispatch'
     | 'contributor';
+
+/**
+ * Test the calculated VADER sentiment (compound) score for an Activity using this comparison. Can be either a numerical or natural language
+ *
+ * Sentiment values range from extremely negative to extremely positive in a numerical range of -1 to +1:
+ *
+ * * -0.6 => extremely negative
+ * * -0.3 => very negative
+ * * -0.1 => negative
+ * *    0 => neutral
+ * *  0.1 => positive
+ * *  0.3 => very positive
+ * *  0.6 => extremely positive
+ *
+ * The below examples are all equivocal. You can use either set of values as the value for `sentiment` (numerical comparisons or natural langauge)
+ *
+ * * `>= 0.1` = `is positive`
+ * * `<= 0.3` = `is very negative`
+ * * `< 0.1` = `is not positive`
+ * * `> -0.3` = `is not very negative`
+ *
+ * Special case:
+ *
+ * * `is neutral` equates to `> -0.1 and < 0.1`
+ * * `is not neutral` equates to `< -0.1 or > 0.1`
+ *
+ * ContextMod uses a normalized, weighted average from these sentiment tools:
+ *
+ * * NLP.js (english, french, german, and spanish) https://github.com/axa-group/nlp.js/blob/master/docs/v3/sentiment-analysis.md
+ * * (english only) vaderSentiment-js https://github.com/vaderSentiment/vaderSentiment-js/
+ * * (english only) wink-sentiment https://github.com/winkjs/wink-sentiment
+ *
+ * More about the sentiment algorithms used:
+ * * VADER https://github.com/cjhutto/vaderSentiment
+ * * AFINN http://corpustext.com/reference/sentiment_afinn.html
+ * * Senticon https://ieeexplore.ieee.org/document/8721408
+ * * Pattern https://github.com/clips/pattern
+ * * wink https://github.com/winkjs/wink-sentiment
+ *
+ * @pattern ((>|>=|<|<=)\s*(-?\d?\.?\d+))|((not)?\s*(very|extremely)?\s*(positive|neutral|negative))
+ * @examples ["is negative", "> 0.2"]
+ * */
+export type VaderSentimentComparison = string;

src/Common/Infrastructure/Comparisons.ts

@@ -1,15 +1,64 @@
 import {StringOperator} from "./Atomic";
 import {Duration} from "dayjs/plugin/duration";
+import InvalidRegexError from "../../Utils/InvalidRegexError";
 
 export interface DurationComparison {
     operator: StringOperator,
     duration: Duration
 }
 
-export interface GenericComparison {
+export interface GenericComparison extends HasDisplayText {
     operator: StringOperator,
     value: number,
     isPercent: boolean,
     extra?: string,
     displayText: string,
 }
+
+export interface HasDisplayText {
+    displayText: string
+}
+
+export interface RangedComparison extends HasDisplayText {
+    range: [number, number]
+    not: boolean
+}
+
+export const asGenericComparison = (val: any): val is GenericComparison => {
+    return typeof val === 'object' && 'value' in val;
+}
+
+export const GENERIC_VALUE_COMPARISON = /^\s*(?<opStr>>|>=|<|<=)\s*(?<value>-?\d?\.?\d+)(?<extra>\s+.*)*$/
+export const GENERIC_VALUE_COMPARISON_URL = 'https://regexr.com/60dq4';
+export const parseGenericValueComparison = (val: string): GenericComparison => {
+    const matches = val.match(GENERIC_VALUE_COMPARISON);
+    if (matches === null) {
+        throw new InvalidRegexError(GENERIC_VALUE_COMPARISON, val, GENERIC_VALUE_COMPARISON_URL)
+    }
+    const groups = matches.groups as any;
+
+    return {
+        operator: groups.opStr as StringOperator,
+        value: Number.parseFloat(groups.value),
+        isPercent: false,
+        extra: groups.extra,
+        displayText: `${groups.opStr} ${groups.value}`
+    }
+}
+const GENERIC_VALUE_PERCENT_COMPARISON = /^\s*(?<opStr>>|>=|<|<=)\s*(?<value>\d+)\s*(?<percent>%?)(?<extra>.*)$/
+const GENERIC_VALUE_PERCENT_COMPARISON_URL = 'https://regexr.com/60a16';
+export const parseGenericValueOrPercentComparison = (val: string): GenericComparison => {
+    const matches = val.match(GENERIC_VALUE_PERCENT_COMPARISON);
+    if (matches === null) {
+        throw new InvalidRegexError(GENERIC_VALUE_PERCENT_COMPARISON, val, GENERIC_VALUE_PERCENT_COMPARISON_URL)
+    }
+    const groups = matches.groups as any;
+
+    return {
+        operator: groups.opStr as StringOperator,
+        value: Number.parseFloat(groups.value),
+        isPercent: groups.percent !== '',
+        extra: groups.extra,
+        displayText: `${groups.opStr} ${groups.value}${groups.percent === undefined || groups.percent === '' ? '' : '%'}`
+    }
+}

src/Common/LangaugeProcessing.ts

@@ -0,0 +1,493 @@
+import {containerBootstrap} from '@nlpjs/core';
+import {Language, LanguageGuess, LanguageType} from '@nlpjs/language';
+import {Nlp} from '@nlpjs/nlp';
+import {SentimentIntensityAnalyzer} from 'vader-sentiment';
+import wink from 'wink-sentiment';
+import {SnoowrapActivity} from "./Infrastructure/Reddit";
+import {
+    asGenericComparison,
+    GenericComparison,
+    parseGenericValueComparison,
+    RangedComparison
+} from "./Infrastructure/Comparisons";
+import {asSubmission, between, comparisonTextOp, formatNumber} from "../util";
+import {CMError, MaybeSeriousErrorWithCause} from "../Utils/Errors";
+import InvalidRegexError from "../Utils/InvalidRegexError";
+import {StringOperator} from "./Infrastructure/Atomic";
+import {LangEs} from "@nlpjs/lang-es";
+import {LangDe} from "@nlpjs/lang-de";
+import {LangEn} from "@nlpjs/lang-en";
+import {LangFr} from "@nlpjs/lang-fr";
+
+export type SentimentAnalysisType = 'vader' | 'afinn' | 'senticon' | 'pattern' | 'wink';
+
+export const sentimentQuantifier = {
+    'extremely negative': -0.6,
+    'very negative': -0.3,
+    'negative': -0.1,
+    'neutral': 0,
+    'positive': 0.1,
+    'very positive': 0.3,
+    'extremely positive': 0.6,
+}
+
+export const sentimentQuantifierRanges = [
+    {
+        range: [Number.MIN_SAFE_INTEGER, -0.6],
+        quant: 'extremely negative'
+    },
+    {
+        range: [-0.6, -0.3],
+        quant: 'very negative'
+    },
+    {
+        range: [-0.3, -0.1],
+        quant: 'negative'
+    },
+    {
+        range: [-0.1, 0.1],
+        quant: 'neutral'
+    },
+    {
+        range: [0.1, 0.3],
+        quant: 'positive'
+    },
+    {
+        range: [0.3, 0.6],
+        quant: 'very positive'
+    },
+    {
+        range: [0.6, Number.MAX_SAFE_INTEGER],
+        quant: 'extremely positive'
+    }
+]
+
+const scoreToSentimentText = (val: number) => {
+    for (const segment of sentimentQuantifierRanges) {
+        if (between(val, segment.range[0], segment.range[1], false, true)) {
+            return segment.quant;
+        }
+    }
+    throw new Error('should not hit this!');
+}
+
+export interface SentimentResult {
+    comparative: number
+    type: SentimentAnalysisType
+    sentiment: string
+    weight: number
+    tokens: number
+    matchedTokens?: number,
+    usableResult: true | string
+}
+
+export interface StringSentiment {
+    results: SentimentResult[]
+    score: number
+    scoreWeighted: number
+    sentiment: string
+    sentimentWeighted: string
+    guessedLanguage: LanguageGuessResult
+    usedLanguage: LanguageType
+    usableScore: boolean
+    reason?: string
+}
+
+export interface ActivitySentiment extends StringSentiment {
+    activity: SnoowrapActivity
+}
+
+export interface StringSentimentTestResult extends StringSentiment {
+    passes: boolean
+    test: GenericComparison | RangedComparison
+}
+
+export interface ActivitySentimentTestResult extends StringSentimentTestResult {
+    activity: SnoowrapActivity
+}
+
+export interface ActivitySentimentOptions {
+    testOn?: ('title' | 'body')[]
+    /**
+     * Make the analyzer assume a language if it cannot determine one itself.
+     *
+     * This is very useful for the analyzer when it is parsing short pieces of content. For example, if you know your subreddit is majority english speakers this will make the analyzer return "neutral" sentiment instead of "not detected language".
+     *
+     * Defaults to 'en'
+     *
+     * @example ["en"]
+     * @default en
+     * */
+    defaultLanguage?: string | null | false
+
+    /**
+     * Helps the analyzer coerce a low confidence language guess into a known-used languages in two ways:
+     *
+     * If the analyzer's
+     *   * *best* guess is NOT one of these
+     *     * but it did guess one of these
+     *     * and its guess is above requiredLanguageConfidence score then use the hinted language instead of best guess
+     *   * OR text content is very short (4 words or less)
+     *     * and the best guess was below the requiredLanguageConfidence score
+     *     * and none of guesses was a hinted language then use the defaultLanguage
+     *
+     * Defaults to popular romance languages: ['en', 'es', 'de', 'fr']
+     *
+     * @example [["en", "es", "de", "fr"]]
+     * @default ["en", "es", "de", "fr"]
+     * */
+    languageHints?: string[]
+
+    /**
+     * Required confidence to use a guessed language as the best guess. Score from 0 to 1.
+     *
+     * Defaults to 0.9
+     *
+     * @example [0.9]
+     * @default 0.9
+     * */
+    requiredLanguageConfidence?: number
+}
+
+export type SentimentCriteriaTest = GenericComparison | RangedComparison;
+
+export const availableSentimentLanguages = ['en', 'es', 'de', 'fr'];
+
+export const textComparison = /(?<not>not)?\s*(?<modifier>very|extremely)?\s*(?<sentiment>positive|neutral|negative)/i;
+
+export const parseTextToNumberComparison = (val: string): RangedComparison | GenericComparison => {
+
+    let genericError: Error | undefined;
+    try {
+        return parseGenericValueComparison(val);
+    } catch (e) {
+        genericError = e as Error;
+        // now try text match
+    }
+
+    const matches = val.match(textComparison);
+    if (matches === null) {
+        const textError = new InvalidRegexError(textComparison, val);
+        throw new CMError(`Sentiment value did not match a valid numeric comparison or valid text: \n ${genericError.message} \n ${textError.message}`);
+    }
+    const groups = matches.groups as any;
+
+    const negate = groups.not !== undefined && groups.not !== '';
+
+    if (groups.sentiment === 'neutral') {
+        if (negate) {
+            return {
+                displayText: 'not neutral (not -0.1 to 0.1)',
+                range: [-0.1, 0.1],
+                not: true,
+            }
+        }
+        return {
+            displayText: 'is neutral (-0.1 to 0.1)',
+            range: [-0.1, 0.1],
+            not: false
+        }
+    }
+
+    const compoundSentimentText = `${groups.modifier !== undefined && groups.modifier !== '' ? `${groups.modifier} ` : ''}${groups.sentiment}`.toLocaleLowerCase();
+    // @ts-ignore
+    const numericVal = sentimentQuantifier[compoundSentimentText] as number;
+    if (numericVal === undefined) {
+        throw new CMError(`Sentiment given did not match any known phrases: '${compoundSentimentText}'`);
+    }
+
+    let operator: StringOperator;
+    if (negate) {
+        operator = numericVal > 0 ? '<' : '>';
+    } else {
+        operator = numericVal > 0 ? '>=' : '<=';
+    }
+
+    return {
+        operator,
+        value: numericVal,
+        isPercent: false,
+        displayText: `is${negate ? ' not ' : ' '}${compoundSentimentText} (${operator} ${numericVal})`
+    }
+}
+
+let nlp: Nlp;
+let container: any;
+
+const bootstrapNlp = async () => {
+
+    container = await containerBootstrap();
+    container.use(Language);
+    container.use(Nlp);
+    container.use(LangEs);
+    container.use(LangDe);
+    container.use(LangEn);
+    container.use(LangFr);
+    nlp = container.get('nlp');
+    nlp.settings.autoSave = false;
+    nlp.addLanguage('en');
+    nlp.addLanguage('es');
+    nlp.addLanguage('de');
+    nlp.addLanguage('fr');
+    nlp.nluManager.guesser.processExtraSentences();
+    await nlp.train();
+}
+
+export const getNlp = async () => {
+    if (nlp === undefined) {
+        await bootstrapNlp();
+    }
+
+    return nlp;
+}
+
+export const getActivityContent = (item: SnoowrapActivity, options?: ActivitySentimentOptions): string => {
+    const {
+        testOn = ['body', 'title'],
+    } = options || {};
+
+    // determine what content we are testing
+    let contents: string[] = [];
+    if (asSubmission(item)) {
+        for (const l of testOn) {
+            switch (l) {
+                case 'title':
+                    contents.push(item.title);
+                    break;
+                case 'body':
+                    if (item.is_self) {
+                        contents.push(item.selftext);
+                    }
+                    break;
+            }
+        }
+    } else {
+        contents.push(item.body)
+    }
+
+    return contents.join(' ');
+}
+
+export const getLanguageTypeFromValue = async (val: string): Promise<LanguageType> => {
+
+    if (nlp === undefined) {
+        await bootstrapNlp();
+    }
+
+    const langObj = container.get('Language') as Language;
+
+    const cleanVal = val.trim().toLocaleLowerCase();
+
+    const foundLang = Object.values(langObj.languagesAlpha2).find(x => x.alpha2 === cleanVal || x.alpha3 === cleanVal || x.name.toLocaleLowerCase() === cleanVal);
+    if (foundLang === undefined) {
+        throw new MaybeSeriousErrorWithCause(`Could not find Language with identifier '${val}'`, {isSerious: false});
+    }
+    const {alpha2, alpha3, name: language} = foundLang;
+    return {
+        alpha2,
+        alpha3,
+        language
+    };
+}
+
+export interface LanguageGuessResult {
+    bestGuess: LanguageGuess
+    guesses: LanguageGuess[]
+    requiredConfidence: number
+    sparse: boolean
+    language: LanguageType
+    usedDefault: boolean
+}
+
+export const getContentLanguage = async (content: string, options?: ActivitySentimentOptions): Promise<LanguageGuessResult> => {
+
+    const {
+        defaultLanguage = 'en',
+        requiredLanguageConfidence = 0.9,
+        languageHints = availableSentimentLanguages
+    } = options || {};
+
+    if (nlp === undefined) {
+        await bootstrapNlp();
+    }
+
+    const spaceNormalizedTokens = content.trim().split(' ').filter(x => x !== ''.trim());
+
+    const lang = container.get('Language') as Language;
+    // would like to improve this https://github.com/axa-group/nlp.js/issues/761
+    const guesses = lang.guess(content, null, 4);
+    let bestLang = guesses[0];
+    const shortContent = spaceNormalizedTokens.length <= 4;
+
+    const altBest = languageHints.includes(bestLang.alpha2) ? undefined : guesses.find(x => x.score >= requiredLanguageConfidence && languageHints.includes(x.alpha2));
+
+    // coerce best guess into a supported language that has a good enough confidence
+    if(!shortContent && altBest !== undefined) {
+        bestLang = altBest;
+    }
+
+    let usedLang: LanguageType = bestLang;
+    let usedDefault = false;
+
+    if (typeof defaultLanguage === 'string' && (bestLang.score < requiredLanguageConfidence || (shortContent && !languageHints.includes(bestLang.alpha2)))) {
+        usedLang = await getLanguageTypeFromValue(defaultLanguage);
+        usedDefault = true;
+    }
+
+    return {
+        guesses,
+        bestGuess: bestLang,
+        requiredConfidence: requiredLanguageConfidence,
+        sparse: shortContent,
+        language: usedLang,
+        usedDefault
+    }
+}
+
+export const getActivitySentiment = async (item: SnoowrapActivity, options?: ActivitySentimentOptions): Promise<ActivitySentiment> => {
+
+    const result = await getStringSentiment(getActivityContent(item, options), options);
+
+    return {
+        ...result,
+        activity: item
+    }
+}
+
+export const getStringSentiment = async (contentStr: string, options?: ActivitySentimentOptions): Promise<StringSentiment> => {
+
+    const langResult = await getContentLanguage(contentStr, options);
+
+    let usedLanguage: LanguageType = langResult.language;
+
+    const spaceNormalizedTokens = contentStr.trim().split(' ').filter(x => x !== ''.trim());
+
+    const results: SentimentResult[] = [];
+
+    const nlpResult = await nlp.process(langResult.language.alpha2, contentStr);
+
+    results.push({
+        comparative: nlpResult.sentiment.average,
+        type: nlpResult.sentiment.type as SentimentAnalysisType,
+        sentiment: scoreToSentimentText(nlpResult.sentiment.average),
+        weight: 1,
+        matchedTokens: nlpResult.sentiment.numHits,
+        tokens: nlpResult.sentiment.numWords,
+        usableResult: availableSentimentLanguages.includes(langResult.language.alpha2) ? true : (nlpResult.sentiment.numHits / nlpResult.sentiment.numWords) >= 0.5 ? true : `${langResult.sparse ? 'Content was too short to guess language' : 'Unsupported language'} and less than 50% of tokens matched`,
+    });
+
+    // only run vader/wink if either
+    //
+    // * content was short which means we aren't confident on language guess
+    // * OR language is english (guessed or explicitly set as language fallback by user due to low confidence)
+    //
+    if (langResult.sparse || langResult.language.alpha2 === 'en') {
+
+        // neg post neu are ratios of *recognized* tokens in the content
+        // when neu is close to 1 its either extremely neutral or no tokens were recognized
+        const vaderScore = SentimentIntensityAnalyzer.polarity_scores(contentStr);
+        const vaderRes: SentimentResult = {
+            comparative: vaderScore.compound,
+            type: 'vader',
+            sentiment: scoreToSentimentText(vaderScore.compound),
+            // may want to weight higher in the future...
+            weight: 1,
+            tokens: spaceNormalizedTokens.length,
+            usableResult: langResult.language.alpha2 === 'en' ? true : (vaderScore.neu < 0.5 ? true : `Unable to guess language and unable to determine if more than 50% of tokens are negative or not matched`)
+        };
+        results.push(vaderRes);
+
+        const winkScore = wink(contentStr);
+        const matchedTokens = winkScore.tokenizedPhrase.filter(x => x.score !== undefined);
+        const matchedMeaningfulTokens = winkScore.tokenizedPhrase.filter(x => x.tag === 'word' || x.tag === 'emoji');
+        // normalizedScore is range of -5 to +5 -- convert to -1 to +1
+        const winkAdjusted = (winkScore.normalizedScore * 2) / 10;
+        const winkRes: SentimentResult = {
+            comparative: winkAdjusted,
+            type: 'wink',
+            sentiment: scoreToSentimentText(winkAdjusted),
+            weight: 1,
+            matchedTokens: matchedTokens.length,
+            tokens: winkScore.tokenizedPhrase.length,
+            usableResult: langResult.language.alpha2 === 'en' ? true : ((matchedTokens.length / matchedMeaningfulTokens.length) > 0.5 ? true : 'Unable to guess language and less than 50% of tokens matched')
+        };
+        results.push(winkRes);
+
+        if ((vaderRes.usableResult == true || winkRes.usableResult === true) && usedLanguage.alpha2 !== 'en') {
+            // since we are confident enough to use one of these then we are assuming language is mostly english
+            usedLanguage = await getLanguageTypeFromValue('en');
+        }
+    }
+
+    const score = results.reduce((acc, curr) => acc + curr.comparative, 0) / results.length;
+    const sentiment = scoreToSentimentText(score);
+
+    const weightSum = results.reduce((acc, curr) => acc + curr.weight, 0);
+    const weightedScores = results.reduce((acc, curr) => acc + (curr.weight * curr.comparative), 0);
+    const weightedScore = weightedScores / weightSum;
+    const weightedSentiment = scoreToSentimentText(weightedScore);
+
+    const actSentResult: StringSentiment = {
+        results,
+        score,
+        sentiment,
+        scoreWeighted: weightedScore,
+        sentimentWeighted: weightedSentiment,
+        guessedLanguage: langResult,
+        usedLanguage,
+        usableScore: results.filter(x => x.usableResult === true).length > 0,
+    }
+
+    if (!actSentResult.usableScore) {
+        if (actSentResult.guessedLanguage.sparse) {
+            actSentResult.reason = 'Content may be supported language but was too short to guess accurately and no algorithm matched enough tokens to be considered confident.';
+        } else {
+            actSentResult.reason = 'Unsupported language'
+        }
+    }
+
+    return actSentResult;
+}
+
+export const testActivitySentiment = async (item: SnoowrapActivity, criteria: SentimentCriteriaTest, options?: ActivitySentimentOptions): Promise<ActivitySentimentTestResult> => {
+    const sentimentResult = await getActivitySentiment(item, options);
+
+    const testResult = testSentiment(sentimentResult, criteria);
+
+    return {
+        ...testResult,
+        activity: item
+    }
+}
+
+export const testSentiment = (sentimentResult: StringSentiment, criteria: SentimentCriteriaTest): StringSentimentTestResult => {
+
+    if (!sentimentResult.usableScore) {
+        return {
+            passes: false,
+            test: criteria,
+            ...sentimentResult,
+        }
+    }
+
+    if (asGenericComparison(criteria)) {
+        return {
+            passes: comparisonTextOp(sentimentResult.scoreWeighted, criteria.operator, criteria.value),
+            test: criteria,
+            ...sentimentResult,
+        }
+    } else {
+        if (criteria.not) {
+            return {
+                passes: sentimentResult.scoreWeighted < criteria.range[0] || sentimentResult.scoreWeighted > criteria.range[1],
+                test: criteria,
+                ...sentimentResult,
+            }
+        }
+        return {
+            passes: sentimentResult.scoreWeighted >= criteria.range[0] || sentimentResult.scoreWeighted <= criteria.range[1],
+            test: criteria,
+            ...sentimentResult,
+        }
+    }
+}

src/Common/Typings/support.d.ts

@@ -3,9 +3,11 @@ declare module 'snoowrap/dist/errors' {
     export interface InvalidUserError extends Error {
 
     }
+
     export interface NoCredentialsError extends Error {
 
     }
+
     export interface InvalidMethodCallError extends Error {
 
     }
@@ -26,9 +28,138 @@ declare module 'snoowrap/dist/errors' {
 }
 
 declare module 'winston-null' {
-  import TransportStream from "winston-transport";
+    import TransportStream from "winston-transport";
 
     export class NullTransport extends TransportStream {
 
     }
 }
+
+declare module '@nlpjs/*' {
+
+    declare interface SentimentResult {
+        score: number,
+        average: number,
+        numWords: number,
+        numHits: number,
+        type: string,
+        language: string
+    }
+
+    declare interface NLPSentimentResult extends Omit<SentimentResult, 'language'> {
+        vote: string
+        locale: string
+    }
+
+
+    declare module '@nlpjs/language' {
+
+        export interface LanguageType {
+            alpha3: string,
+            alpha2: string,
+            language: string,
+        }
+
+        export interface LanguageObj {
+            alpha3: string,
+            alpha2: string,
+            name: string,
+        }
+
+        export interface LanguageGuess extends LanguageType {
+            score: number
+        }
+
+        export class Language {
+            guess(val: string, allowedList?: string[] | null, limit?: number): LanguageGuess[];
+
+            guessBest(val: string, allowedList?: string[] | null): LanguageGuess;
+
+            /**
+             * Key is alpha2 lang IE en es de fr
+             * */
+            languagesAlpha2: Record<string, LanguageObj>;
+            /**
+             * Key is alpha3 lang IE eng spa deu fra
+             * */
+            languagesAlpha3: Record<string, LanguageObj>;
+        }
+    }
+
+    declare module '@nlpjs/sentiment' {
+
+        declare interface SentimentPipelineResult {
+            utterance: string
+            locale: string
+            settings: { tag: string }
+            tokens: string[]
+            sentiment: SentimentResult
+        }
+
+        declare interface SentimentPipelineInput {
+            utterance: string
+            locale: string
+
+            [key: string]: any
+        }
+
+        export class SentimentAnalyzer {
+            constructor(settings?: { language?: string }, container?: any)
+
+            container: any
+
+            process(srcInput: SentimentPipelineInput, settings?: object): Promise<SentimentPipelineResult>
+        }
+    }
+
+    declare module '@nlpjs/nlp' {
+
+        declare interface NlpResult {
+            locale: string
+            language: string
+            languageGuessed: boolean
+            sentiment: NLPSentimentResult
+        }
+
+        export class Nlp {
+            settings: any;
+            nluManager: any;
+
+            constructor(settings?: { language?: string }, container?: any)
+
+            // locale language languageGuessed sentiment
+            process(locale: string, utterance?: string, srcContext?: object, settings?: object): Promise<NlpResult>
+            addLanguage(locale: string)
+            train(): Promise<any>;
+
+        }
+    }
+
+    declare module '@nlpjs/lang-es' {
+        export const LangEs: any
+    }
+    declare module '@nlpjs/lang-en' {
+        export const LangEn: any
+    }
+    declare module '@nlpjs/lang-de' {
+        export const LangDe: any
+    }
+    declare module '@nlpjs/lang-fr' {
+        export const LangFr: any
+    }
+    declare module '@nlpjs/nlu' {
+        export const Nlu: any
+    }
+
+    declare module '@nlpjs/core' {
+        export const Container: any
+        export const containerBootstrap: any
+    }
+}
+
+
+declare module 'wink-sentiment' {
+    function sentiment(phrase: string): { score: number, normalizedScore: number, tokenizedPhrase: any[] };
+
+    export default sentiment;
+}

src/Common/Typings/vader-sentiment.d.ts

@@ -0,0 +1,50 @@
+declare module 'vader-sentiment' {
+    export const REGEX_REMOVE_PUNCTUATION: RegExp;
+    export const B_INCR: number;
+    export const B_DECR: number;
+    export const C_INCR: number;
+    export const N_SCALER: number;
+    export const PUNC_LIST: string[];
+    export const NEGATE: string[];
+    export const BOOSTER_DICT: Record<string, number>;
+    export const SPECIAL_CASE_IDIOMS: Record<string, number>;
+
+    export interface Scores {
+        neg: number
+        neu: number
+        pos: number
+        compound: number
+    }
+
+    export function negated(input_words: string[], include_nt: boolean = true): boolean;
+    export function normalize(score: number, alpha: number): number;
+    export function allcap_differential(words: string[]): boolean;
+    export function scalar_inc_dec(word: string, valence: number, is_cap_diff: boolean): number
+    export function is_upper_function(word: string): boolean
+
+    export class SentiText {
+        public text: string;
+        public words_and_emoticons: string[];
+        public is_cap_diff: boolean;
+
+        constructor(text: string);
+
+        get_words_plus_punc(): Record<string, string>;
+        get_words_and_emoticons(): string[];
+    }
+
+    export class SentimentIntensityAnalyzer {
+
+        static polarity_scores(text: string): Scores;
+        static sentiment_valence(valence: number, sentiText: SentiText, item: string, index: number, sentiments: number[]);
+        static least_check(valence: number, words_and_emoticons: string[], index: number): number;
+        static but_check(words_and_emoticons: string[], sentiments: number[]): number[]
+        static idioms_check(valence: number, words_and_emoticons: string[], index: number): number;
+        static never_check(valence: number, words_and_emoticons: string[], start_i: number, index: number): number
+        static punctuation_emphasis(sum_s: any, text: string);
+        static amplify_ep(text: string): number;
+        static amplify_qm(text: string): number;
+        static sift_sentiment_scores(sentiments: number[]): number[];
+        static score_valence(sentiments: number[], text: string): Scores;
+    }
+}

src/Common/defaults.ts

@@ -32,4 +32,4 @@ export const filterCriteriaDefault: FilterCriteriaDefaults = {
 export const defaultDataDir = path.resolve(__dirname, '../..');
 export const defaultConfigFilenames = ['config.json', 'config.yaml'];
 
-export const VERSION = '0.10.12';
+export const VERSION = '0.11.1';

src/Common/interfaces.ts

@@ -1676,6 +1676,7 @@ export interface LogInfo {
     labels?: string[]
     bot?: string
     user?: string
+    transport?: string[]
 }
 
 export interface ActionResult extends ActionProcessResult {
@@ -1942,7 +1943,6 @@ export interface ActivityDispatch extends Omit<ActivityDispatchConfig, 'delay'|
     author: string
     delay: Duration
     tardyTolerant?: boolean | Duration
-    processing: boolean
     action?: string
     type: ActivitySourceTypes
     dryRun?: boolean

src/Common/types.ts

@@ -18,7 +18,8 @@ import {RepostRuleJSONConfig} from "../Rule/RepostRule";
 import {DispatchActionJson} from "../Action/DispatchAction";
 import {CancelDispatchActionJson} from "../Action/CancelDispatchAction";
 import {ContributorActionJson} from "../Action/ContributorAction";
+import {SentimentRuleJSONConfig} from "../Rule/SentimentRule";
 
-export type RuleObjectJsonTypes = RecentActivityRuleJSONConfig | RepeatActivityJSONConfig | AuthorRuleJSONConfig | AttributionJSONConfig | HistoryJSONConfig | RegexRuleJSONConfig | RepostRuleJSONConfig
+export type RuleObjectJsonTypes = RecentActivityRuleJSONConfig | RepeatActivityJSONConfig | AuthorRuleJSONConfig | AttributionJSONConfig | HistoryJSONConfig | RegexRuleJSONConfig | RepostRuleJSONConfig | SentimentRuleJSONConfig
 
 export type ActionJson = CommentActionJson | FlairActionJson | ReportActionJson | LockActionJson | RemoveActionJson | ApproveActionJson | BanActionJson | UserNoteActionJson | MessageActionJson | UserFlairActionJson | DispatchActionJson | CancelDispatchActionJson | ContributorActionJson | string;

src/Rule/AttributionRule.ts

@@ -13,7 +13,6 @@ import {
     comparisonTextOp, convertSubredditsRawToStrong,
     FAIL,
     formatNumber, getActivitySubredditName, isActivityWindowConfig, isSubmission,
-    parseGenericValueOrPercentComparison,
     parseSubredditName,
     PASS, windowConfigToWindowCriteria
 } from "../util";
@@ -27,6 +26,7 @@ import {
     HistoryFiltersOptions
 } from "../Common/Infrastructure/ActivityWindow";
 import {FilterOptions} from "../Common/Infrastructure/Filters/FilterShapes";
+import {parseGenericValueOrPercentComparison} from "../Common/Infrastructure/Comparisons";
 
 
 export interface AttributionCriteria {

src/Rule/HistoryRule.ts

@@ -11,7 +11,7 @@ import {
     comparisonTextOp,
     FAIL,
     formatNumber, getActivitySubredditName, historyFilterConfigToOptions, isSubmission,
-    parseGenericValueOrPercentComparison, parseSubredditName,
+    parseSubredditName,
     PASS,
     percentFromString, removeUndefinedKeys, toStrongSubredditState, windowConfigToWindowCriteria
 } from "../util";
@@ -20,6 +20,7 @@ import {SubredditCriteria} from "../Common/Infrastructure/Filters/FilterCriteria
 import {CompareValueOrPercent} from "../Common/Infrastructure/Atomic";
 import {ActivityWindowConfig, ActivityWindowCriteria} from "../Common/Infrastructure/ActivityWindow";
 import {ErrorWithCause} from "pony-cause";
+import {parseGenericValueOrPercentComparison} from "../Common/Infrastructure/Comparisons";
 
 export interface CommentThresholdCriteria extends ThresholdCriteria {
     /**

src/Rule/RecentActivityRule.ts

@@ -19,7 +19,7 @@ import {
     isSubmission,
     isValidImageURL,
     objectToStringSummary,
-    parseGenericValueOrPercentComparison, parseRedditEntity,
+    parseRedditEntity,
     parseStringToRegex,
     parseSubredditName,
     parseUsableLinkIdentifier,
@@ -41,6 +41,7 @@ import {
     SubredditCriteria
 } from "../Common/Infrastructure/Filters/FilterCriteria";
 import {ActivityWindow, ActivityWindowConfig} from "../Common/Infrastructure/ActivityWindow";
+import {parseGenericValueOrPercentComparison} from "../Common/Infrastructure/Comparisons";
 
 const parseLink = parseUsableLinkIdentifier();
 

src/Rule/RegexRule.ts

@@ -3,8 +3,7 @@ import {Comment} from "snoowrap";
 import Submission from "snoowrap/dist/objects/Submission";
 import {
     asSubmission,
-    comparisonTextOp, FAIL, isExternalUrlSubmission, isSubmission, parseGenericValueComparison,
-    parseGenericValueOrPercentComparison, parseRegex, parseStringToRegex,
+    comparisonTextOp, FAIL, isExternalUrlSubmission, isSubmission, parseRegex, parseStringToRegex,
     PASS, triggeredIndicator, windowConfigToWindowCriteria
 } from "../util";
 import {
@@ -14,6 +13,7 @@ import dayjs from 'dayjs';
 import {SimpleError} from "../Utils/Errors";
 import {JoinOperands} from "../Common/Infrastructure/Atomic";
 import {ActivityWindowConfig} from "../Common/Infrastructure/ActivityWindow";
+import {parseGenericValueComparison, parseGenericValueOrPercentComparison} from "../Common/Infrastructure/Comparisons";
 
 export interface RegexCriteria {
     /**

src/Rule/RepeatActivityRule.ts

@@ -8,7 +8,6 @@ import {
     getActivitySubredditName, isActivityWindowConfig,
     isExternalUrlSubmission,
     isRedditMedia,
-    parseGenericValueComparison,
     parseSubredditName,
     parseUsableLinkIdentifier as linkParser,
     PASS,
@@ -23,7 +22,6 @@ import {
 } from "../Common/interfaces";
 import Submission from "snoowrap/dist/objects/Submission";
 import dayjs from "dayjs";
-import Fuse from 'fuse.js'
 import {StrongSubredditCriteria, SubredditCriteria} from "../Common/Infrastructure/Filters/FilterCriteria";
 import {
     ActivityWindow,
@@ -31,6 +29,7 @@ import {
     ActivityWindowCriteria,
     HistoryFiltersOptions
 } from "../Common/Infrastructure/ActivityWindow";
+import {parseGenericValueComparison} from "../Common/Infrastructure/Comparisons";
 
 const parseUsableLinkIdentifier = linkParser();
 

src/Rule/RepostRule.ts

@@ -6,8 +6,7 @@ import {
     compareDurationValue,
     comparisonTextOp,
     FAIL, formatNumber,
-    isRepostItemResult, parseDurationComparison, parseGenericValueComparison,
-    parseUsableLinkIdentifier,
+    isRepostItemResult, parseDurationComparison, parseUsableLinkIdentifier,
     PASS, searchAndReplace, stringSameness, triggeredIndicator, windowConfigToWindowCriteria, wordCount
 } from "../util";
 import {
@@ -18,13 +17,12 @@ import {
 } from "../Common/interfaces";
 import objectHash from "object-hash";
 import {getAttributionIdentifier} from "../Utils/SnoowrapUtils";
-import Fuse from "fuse.js";
-import leven from "leven";
 import {YoutubeClient, commentsAsRepostItems} from "../Utils/ThirdParty/YoutubeClient";
 import dayjs from "dayjs";
 import {rest} from "lodash";
 import {CompareValue, DurationComparor, JoinOperands, SearchFacetType} from "../Common/Infrastructure/Atomic";
 import {ActivityWindow, ActivityWindowConfig} from "../Common/Infrastructure/ActivityWindow";
+import {parseGenericValueComparison} from "../Common/Infrastructure/Comparisons";
 
 const parseYtIdentifier = parseUsableLinkIdentifier();
 

src/Rule/RuleFactory.ts

@@ -10,6 +10,7 @@ import {SubredditResources} from "../Subreddit/SubredditResources";
 import Snoowrap from "snoowrap";
 import {RepostRule, RepostRuleJSONConfig} from "./RepostRule";
 import {StructuredFilter} from "../Common/Infrastructure/Filters/FilterShapes";
+import {SentimentRule, SentimentRuleJSONConfig} from "./SentimentRule";
 
 export function ruleFactory
 (config: StructuredRuleJson, logger: Logger, subredditName: string, resources: SubredditResources, client: Snoowrap): Rule {
@@ -37,7 +38,10 @@ export function ruleFactory
         case 'repost':
             cfg = config as StructuredFilter<RepostRuleJSONConfig>;
             return new RepostRule({...cfg, logger, subredditName, resources, client});
+        case 'sentiment':
+            cfg = config as StructuredFilter<SentimentRuleJSONConfig>;
+            return new SentimentRule({...cfg, logger, subredditName, resources, client});
         default:
-            throw new Error('rule "kind" was not recognized.');
+            throw new Error(`Rule with kind '${config.kind}' was not recognized.`);
     }
 }

src/Rule/SentimentRule.ts

@@ -0,0 +1,247 @@
+import {Rule, RuleJSONConfig, RuleOptions} from "./index";
+import {Comment} from "snoowrap";
+import Submission from "snoowrap/dist/objects/Submission";
+import {
+    comparisonTextOp, formatNumber,
+    triggeredIndicator, windowConfigToWindowCriteria
+} from "../util";
+
+import dayjs from 'dayjs';
+import {map as mapAsync} from 'async';
+import {
+    GenericComparison,
+    parseGenericValueOrPercentComparison,
+    RangedComparison
+} from "../Common/Infrastructure/Comparisons";
+import {ActivityWindowConfig, ActivityWindowCriteria} from "../Common/Infrastructure/ActivityWindow";
+import {VaderSentimentComparison} from "../Common/Infrastructure/Atomic";
+import {RuleResult} from "../Common/interfaces";
+import {SnoowrapActivity} from "../Common/Infrastructure/Reddit";
+import {
+    ActivitySentimentOptions,
+    ActivitySentimentTestResult,
+    parseTextToNumberComparison,
+    testActivitySentiment
+} from "../Common/LangaugeProcessing";
+
+export class SentimentRule extends Rule {
+
+    sentimentVal: string;
+    sentiment: GenericComparison | RangedComparison;
+
+    historical?: HistoricalSentiment;
+
+    testOn: ('title' | 'body')[]
+
+    constructor(options: SentimentRuleOptions) {
+        super(options);
+
+        this.sentimentVal = options.sentiment;
+        this.sentiment = parseTextToNumberComparison(options.sentiment);
+        this.testOn = options.testOn ?? ['title', 'body'];
+
+        if(options.historical !== undefined) {
+            const {
+                window,
+                sentiment: historicalSentiment = this.sentimentVal,
+                mustMatchCurrent = false,
+                totalMatching = '> 0',
+            } = options.historical
+
+            this.historical = {
+                sentiment: parseTextToNumberComparison(historicalSentiment),
+                sentimentVal: historicalSentiment,
+                window: windowConfigToWindowCriteria(window),
+                mustMatchCurrent,
+                totalMatching: parseGenericValueOrPercentComparison(totalMatching),
+            };
+        }
+    }
+
+    getKind(): string {
+        return 'sentiment';
+    }
+
+    getSpecificPremise(): object {
+        return {
+            sentiment: this.sentiment,
+        }
+    }
+
+    protected async process(item: Submission | Comment): Promise<[boolean, RuleResult]> {
+
+        let ogResult = await this.testActivity(item, this.sentiment);
+        let historicResults: ActivitySentimentTestResult[] | undefined;
+
+        if(this.historical !== undefined && (!this.historical.mustMatchCurrent || ogResult.passes)) {
+            const {
+                sentiment = this.sentiment,
+                window,
+            } = this.historical;
+            const history = await this.resources.getAuthorActivities(item.author, window);
+
+            historicResults = await mapAsync(history, async (x: SnoowrapActivity) => await this.testActivity(x, sentiment)); // history.map(x => this.testActivity(x, sentiment));
+        }
+
+
+
+        const logSummary: string[] = [];
+
+        const sentimentTest = this.sentiment.displayText;
+        const historicalSentimentTest = this.historical !== undefined ? this.historical.sentiment.displayText : undefined;
+
+        let triggered = false;
+        let averageScore: number;
+        let averageWindowScore: number | undefined;
+        let humanWindow: string | undefined;
+        let historicalPassed: string | undefined;
+        let totalMatchingText: string | undefined;
+
+        if(historicResults === undefined) {
+            triggered = ogResult.passes;
+            averageScore = ogResult.scoreWeighted;
+            logSummary.push(`${triggeredIndicator(triggered)} Current Activity Sentiment '${ogResult.sentiment} (${ogResult.scoreWeighted})' ${triggered ? 'PASSED' : 'DID NOT PASS'} sentiment test '${sentimentTest}'`);
+            if(!triggered && this.historical !== undefined && this.historical.mustMatchCurrent) {
+                logSummary.push(`Did not check Historical because 'mustMatchCurrent' is true`);
+            }
+        } else {
+
+            const {
+                totalMatching,
+                sentiment,
+            } = this.historical as HistoricalSentiment;
+
+            totalMatchingText = totalMatching.displayText;
+            const allResults = historicResults
+            const passed = allResults.filter(x => x.passes);
+            averageScore = passed.reduce((acc, curr) => acc + curr.scoreWeighted,0) / passed.length;
+            averageWindowScore = allResults.reduce((acc, curr) => acc + curr.scoreWeighted,0) / allResults.length;
+
+            const firstActivity = allResults[0].activity;
+            const lastActivity = allResults[allResults.length - 1].activity;
+
+            const humanRange = dayjs.duration(dayjs(firstActivity.created_utc * 1000).diff(dayjs(lastActivity.created_utc * 1000))).humanize();
+
+            humanWindow = `${allResults.length} Activities (${humanRange})`;
+
+            const {operator, value, isPercent} = totalMatching;
+            if(isPercent) {
+                const passPercentVal = passed.length/allResults.length
+                triggered = comparisonTextOp(passPercentVal, operator, (value/100));
+                historicalPassed = `${passed.length} (${formatNumber(passPercentVal)}%)`;
+            } else {
+                triggered = comparisonTextOp(passed.length, operator, value);
+                historicalPassed = `${passed.length}`;
+            }
+            logSummary.push(`${triggeredIndicator(triggered)} ${historicalPassed} historical activities of ${humanWindow} passed sentiment test '${sentiment.displayText}' which ${triggered ? 'MET' : 'DID NOT MEET'} threshold '${totalMatching.displayText}'`);
+        }
+
+        const result = logSummary.join(' || ');
+        this.logger.verbose(result);
+
+        return Promise.resolve([triggered, this.getResult(triggered, {
+            result,
+            data: {
+                results: {
+                    triggered,
+                    sentimentTest,
+                    historicalSentimentTest,
+                    averageScore,
+                    averageWindowScore,
+                    window: humanWindow,
+                    totalMatching: totalMatchingText
+                }
+            }
+        })]);
+    }
+
+    protected async testActivity(a: (Submission | Comment), criteria: GenericComparison | RangedComparison): Promise<ActivitySentimentTestResult> {
+        return await testActivitySentiment(a, criteria, {testOn: this.testOn});
+    }
+}
+
+/**
+ * Test the Sentiment of Activities from the Author history
+ *
+ * If this is defined then the `totalMatching` threshold must pass for the Rule to trigger
+ *
+ * If `sentiment` is defined here it overrides the top-level `sentiment` value
+ *
+ * */
+interface HistoricalSentimentConfig {
+    window: ActivityWindowConfig
+
+    sentiment?: VaderSentimentComparison
+
+    /**
+     * When `true` the original Activity being checked MUST match desired sentiment before the Rule considers any history
+     *
+     * @default false
+     * */
+    mustMatchCurrent?: boolean
+
+    /**
+     * A string containing a comparison operator and a value to compare Activities from history that pass the given `sentiment` comparison
+     *
+     * The syntax is `(< OR > OR <= OR >=) <number>[percent sign]`
+     *
+     * * EX `> 12`  => greater than 12 activities passed given `sentiment` comparison
+     * * EX `<= 10%` => less than 10% of all Activities from history passed given `sentiment` comparison
+     *
+     * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(%?)(.*)$
+     * @default "> 0"
+     * @examples ["> 0","> 10%"]
+     * */
+    totalMatching: string
+}
+
+interface HistoricalSentiment extends Omit<HistoricalSentimentConfig, 'sentiment' | 'window' | 'totalMatching'> {
+    sentiment: GenericComparison | RangedComparison,
+    sentimentVal: string
+    window: ActivityWindowCriteria
+    totalMatching: GenericComparison
+}
+
+interface SentimentConfig extends ActivitySentimentOptions {
+
+    sentiment: VaderSentimentComparison
+
+    /**
+     * Test the Sentiment of Activities from the Author history
+     *
+     * If this is defined then the `totalMatching` threshold must pass for the Rule to trigger
+     *
+     * If `sentiment` is defined here it overrides the top-level `sentiment` value
+     *
+     * */
+    historical?: HistoricalSentimentConfig
+
+    /**
+     * Which content from an Activity to test for `sentiment` against
+     *
+     * Only used if the Activity being tested is a Submission -- Comments are only tested against their body
+     *
+     * If more than one type of content is specified then all text is tested together as one string
+     *
+     * @default ["title", "body"]
+     * */
+    testOn?: ('title' | 'body')[]
+}
+
+export interface SentimentRuleOptions extends SentimentConfig, RuleOptions {
+}
+
+/**
+ * Test the calculated VADER sentiment for an Activity to determine if the text context is negative, neutral, or positive in tone.
+ *
+ * More about VADER Sentiment: https://github.com/cjhutto/vaderSentiment
+ *
+ * */
+export interface SentimentRuleJSONConfig extends SentimentConfig, RuleJSONConfig {
+    /**
+     * @examples ["sentiment"]
+     * */
+    kind: 'sentiment'
+}
+
+export default SentimentRule;

src/Rule/index.ts

@@ -185,7 +185,7 @@ export interface RuleJSONConfig extends IRule {
      * The kind of rule to run
      * @examples ["recentActivity", "repeatActivity", "author", "attribution", "history"]
      */
-    kind: 'recentActivity' | 'repeatActivity' | 'author' | 'attribution' | 'history' | 'regex' | 'repost'
+    kind: 'recentActivity' | 'repeatActivity' | 'author' | 'attribution' | 'history' | 'regex' | 'repost' | 'sentiment'
 }
 
 

src/Schema/App.json

@@ -1581,6 +1581,9 @@
                             {
                                 "$ref": "#/definitions/RepostRuleJSONConfig"
                             },
+                            {
+                                "$ref": "#/definitions/SentimentRuleJSONConfig"
+                            },
                             {
                                 "$ref": "#/definitions/RuleSetJson"
                             },
@@ -2914,6 +2917,60 @@
             },
             "type": "object"
         },
+        "HistoricalSentimentConfig": {
+            "description": "Test the Sentiment of Activities from the Author history\n\nIf this is defined then the `totalMatching` threshold must pass for the Rule to trigger\n\nIf `sentiment` is defined here it overrides the top-level `sentiment` value",
+            "properties": {
+                "mustMatchCurrent": {
+                    "default": false,
+                    "description": "When `true` the original Activity being checked MUST match desired sentiment before the Rule considers any history",
+                    "type": "boolean"
+                },
+                "sentiment": {
+                    "description": "Test the calculated VADER sentiment (compound) score for an Activity using this comparison. Can be either a numerical or natural language\n\nSentiment values range from extremely negative to extremely positive in a numerical range of -1 to +1:\n\n* -0.6 => extremely negative\n* -0.3 => very negative\n* -0.1 => negative\n*    0 => neutral\n*  0.1 => positive\n*  0.3 => very positive\n*  0.6 => extremely positive\n\nThe below examples are all equivocal. You can use either set of values as the value for `sentiment` (numerical comparisons or natural langauge)\n\n* `>= 0.1` = `is positive`\n* `<= 0.3` = `is very negative`\n* `< 0.1` = `is not positive`\n* `> -0.3` = `is not very negative`\n\nSpecial case:\n\n* `is neutral` equates to `> -0.1 and < 0.1`\n* `is not neutral` equates to `< -0.1 or > 0.1`\n\nContextMod uses a normalized, weighted average from these sentiment tools:\n\n* NLP.js (english, french, german, and spanish) https://github.com/axa-group/nlp.js/blob/master/docs/v3/sentiment-analysis.md\n* (english only) vaderSentiment-js https://github.com/vaderSentiment/vaderSentiment-js/\n* (english only) wink-sentiment https://github.com/winkjs/wink-sentiment\n\nMore about the sentiment algorithms used:\n* VADER https://github.com/cjhutto/vaderSentiment\n* AFINN http://corpustext.com/reference/sentiment_afinn.html\n* Senticon https://ieeexplore.ieee.org/document/8721408\n* Pattern https://github.com/clips/pattern\n* wink https://github.com/winkjs/wink-sentiment",
+                    "examples": [
+                        "is negative",
+                        "> 0.2"
+                    ],
+                    "pattern": "((>|>=|<|<=)\\s*(-?\\d?\\.?\\d+))|((not)?\\s*(very|extremely)?\\s*(positive|neutral|negative))",
+                    "type": "string"
+                },
+                "totalMatching": {
+                    "default": "> 0",
+                    "description": "A string containing a comparison operator and a value to compare Activities from history that pass the given `sentiment` comparison\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 12`  => greater than 12 activities passed given `sentiment` comparison\n* EX `<= 10%` => less than 10% of all Activities from history passed given `sentiment` comparison",
+                    "examples": [
+                        "> 0",
+                        "> 10%"
+                    ],
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "window": {
+                    "anyOf": [
+                        {
+                            "$ref": "#/definitions/DurationObject"
+                        },
+                        {
+                            "$ref": "#/definitions/FullActivityWindowConfig"
+                        },
+                        {
+                            "type": [
+                                "string",
+                                "number"
+                            ]
+                        }
+                    ],
+                    "description": "A value to define the range of Activities to retrieve.\n\nAcceptable values:\n\n**`ActivityWindowCriteria` object**\n\nAllows specify multiple range properties and more specific behavior\n\n**A `number` of Activities to retrieve**\n\n* EX `100` => 100 Activities\n\n*****\n\nAny of the below values that specify the amount of time to subtract from `NOW` to create a time range IE `NOW <---> [duration] ago`\n\nAcceptable values:\n\n**A `string` consisting of a value and a [Day.js](https://day.js.org/docs/en/durations/creating#list-of-all-available-units) time UNIT**\n\n* EX `9 days` => Range is `NOW <---> 9 days ago`\n\n**A [Day.js](https://day.js.org/docs/en/durations/creating) `object`**\n\n* EX `{\"days\": 90, \"minutes\": 15}` => Range is `NOW <---> 90 days and 15 minutes ago`\n\n**An [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) `string`**\n\n* EX `PT15M` => 15 minutes => Range is `NOW <----> 15 minutes ago`",
+                    "examples": [
+                        "90 days"
+                    ]
+                }
+            },
+            "required": [
+                "totalMatching",
+                "window"
+            ],
+            "type": "object"
+        },
         "HistoryCriteria": {
             "description": "Criteria will only trigger if ALL present thresholds (comment, submission, total) are met",
             "properties": {
@@ -4864,6 +4921,9 @@
                             {
                                 "$ref": "#/definitions/RepostRuleJSONConfig"
                             },
+                            {
+                                "$ref": "#/definitions/SentimentRuleJSONConfig"
+                            },
                             {
                                 "type": "string"
                             }
@@ -5107,6 +5167,149 @@
             ],
             "type": "object"
         },
+        "SentimentRuleJSONConfig": {
+            "description": "Test the calculated VADER sentiment for an Activity to determine if the text context is negative, neutral, or positive in tone.\n\nMore about VADER Sentiment: https://github.com/cjhutto/vaderSentiment",
+            "properties": {
+                "authorIs": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "anyOf": [
+                                    {
+                                        "$ref": "#/definitions/AuthorCriteria"
+                                    },
+                                    {
+                                        "$ref": "#/definitions/NamedCriteria<AuthorCriteria>"
+                                    },
+                                    {
+                                        "type": "string"
+                                    }
+                                ]
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "$ref": "#/definitions/FilterOptionsJson<AuthorCriteria>"
+                        }
+                    ],
+                    "description": "If present then these Author criteria are checked before running the Check. If criteria fails then the Check will fail."
+                },
+                "defaultLanguage": {
+                    "anyOf": [
+                        {
+                            "enum": [
+                                false
+                            ],
+                            "type": "boolean"
+                        },
+                        {
+                            "type": [
+                                "null",
+                                "string"
+                            ]
+                        }
+                    ],
+                    "default": "en",
+                    "description": "Make the analyzer assume a language if it cannot determine one itself.\n\nThis is very useful for the analyzer when it is parsing short pieces of content. For example, if you know your subreddit is majority english speakers this will make the analyzer return \"neutral\" sentiment instead of \"not detected language\".\n\nDefaults to 'en'"
+                },
+                "historical": {
+                    "$ref": "#/definitions/HistoricalSentimentConfig",
+                    "description": "Test the Sentiment of Activities from the Author history\n\nIf this is defined then the `totalMatching` threshold must pass for the Rule to trigger\n\nIf `sentiment` is defined here it overrides the top-level `sentiment` value"
+                },
+                "itemIs": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "anyOf": [
+                                    {
+                                        "$ref": "#/definitions/SubmissionState"
+                                    },
+                                    {
+                                        "$ref": "#/definitions/CommentState"
+                                    },
+                                    {
+                                        "$ref": "#/definitions/NamedCriteria<TypedActivityState>"
+                                    },
+                                    {
+                                        "type": "string"
+                                    }
+                                ]
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "$ref": "#/definitions/FilterOptionsJson<TypedActivityState>"
+                        }
+                    ],
+                    "description": "A list of criteria to test the state of the `Activity` against before running the check.\n\nIf any set of criteria passes the Check will be run. If the criteria fails then the Check will fail.\n\n* @examples [[{\"over_18\": true, \"removed': false}]]"
+                },
+                "kind": {
+                    "description": "The kind of rule to run",
+                    "enum": [
+                        "sentiment"
+                    ],
+                    "examples": [
+                        "sentiment"
+                    ],
+                    "type": "string"
+                },
+                "languageHints": {
+                    "default": [
+                        "en",
+                        "es",
+                        "de",
+                        "fr"
+                    ],
+                    "description": "Helps the analyzer coerce a low confidence language guess into a known-used languages in two ways:\n\nIf the analyzer's\n  * *best* guess is NOT one of these\n    * but it did guess one of these\n    * and its guess is above requiredLanguageConfidence score then use the hinted language instead of best guess\n  * OR text content is very short (4 words or less)\n    * and the best guess was below the requiredLanguageConfidence score\n    * and none of guesses was a hinted language then use the defaultLanguage\n\nDefaults to popular romance languages: ['en', 'es', 'de', 'fr']",
+                    "items": {
+                        "type": "string"
+                    },
+                    "type": "array"
+                },
+                "name": {
+                    "description": "An optional, but highly recommended, friendly name for this rule. If not present will default to `kind`.\n\nCan only contain letters, numbers, underscore, spaces, and dashes\n\nname is used to reference Rule result data during Action content templating. See CommentAction or ReportAction for more details.",
+                    "examples": [
+                        "myNewRule"
+                    ],
+                    "pattern": "^[a-zA-Z]([\\w -]*[\\w])?$",
+                    "type": "string"
+                },
+                "requiredLanguageConfidence": {
+                    "default": 0.9,
+                    "description": "Required confidence to use a guessed language as the best guess. Score from 0 to 1.\n\nDefaults to 0.9",
+                    "type": "number"
+                },
+                "sentiment": {
+                    "description": "Test the calculated VADER sentiment (compound) score for an Activity using this comparison. Can be either a numerical or natural language\n\nSentiment values range from extremely negative to extremely positive in a numerical range of -1 to +1:\n\n* -0.6 => extremely negative\n* -0.3 => very negative\n* -0.1 => negative\n*    0 => neutral\n*  0.1 => positive\n*  0.3 => very positive\n*  0.6 => extremely positive\n\nThe below examples are all equivocal. You can use either set of values as the value for `sentiment` (numerical comparisons or natural langauge)\n\n* `>= 0.1` = `is positive`\n* `<= 0.3` = `is very negative`\n* `< 0.1` = `is not positive`\n* `> -0.3` = `is not very negative`\n\nSpecial case:\n\n* `is neutral` equates to `> -0.1 and < 0.1`\n* `is not neutral` equates to `< -0.1 or > 0.1`\n\nContextMod uses a normalized, weighted average from these sentiment tools:\n\n* NLP.js (english, french, german, and spanish) https://github.com/axa-group/nlp.js/blob/master/docs/v3/sentiment-analysis.md\n* (english only) vaderSentiment-js https://github.com/vaderSentiment/vaderSentiment-js/\n* (english only) wink-sentiment https://github.com/winkjs/wink-sentiment\n\nMore about the sentiment algorithms used:\n* VADER https://github.com/cjhutto/vaderSentiment\n* AFINN http://corpustext.com/reference/sentiment_afinn.html\n* Senticon https://ieeexplore.ieee.org/document/8721408\n* Pattern https://github.com/clips/pattern\n* wink https://github.com/winkjs/wink-sentiment",
+                    "examples": [
+                        "is negative",
+                        "> 0.2"
+                    ],
+                    "pattern": "((>|>=|<|<=)\\s*(-?\\d?\\.?\\d+))|((not)?\\s*(very|extremely)?\\s*(positive|neutral|negative))",
+                    "type": "string"
+                },
+                "testOn": {
+                    "default": [
+                        "title",
+                        "body"
+                    ],
+                    "description": "Which content from an Activity to test for `sentiment` against\n\nOnly used if the Activity being tested is a Submission -- Comments are only tested against their body\n\nIf more than one type of content is specified then all text is tested together as one string",
+                    "items": {
+                        "enum": [
+                            "body",
+                            "title"
+                        ],
+                        "type": "string"
+                    },
+                    "type": "array"
+                }
+            },
+            "required": [
+                "kind",
+                "sentiment"
+            ],
+            "type": "object"
+        },
         "SubmissionCheckJson": {
             "properties": {
                 "actions": {
@@ -5326,6 +5529,9 @@
                             {
                                 "$ref": "#/definitions/RepostRuleJSONConfig"
                             },
+                            {
+                                "$ref": "#/definitions/SentimentRuleJSONConfig"
+                            },
                             {
                                 "$ref": "#/definitions/RuleSetJson"
                             },

src/Schema/Rule.json

@@ -22,6 +22,9 @@
         {
             "$ref": "#/definitions/RepostRuleJSONConfig"
         },
+        {
+            "$ref": "#/definitions/SentimentRuleJSONConfig"
+        },
         {
             "type": "string"
         }
@@ -1297,6 +1300,60 @@
             },
             "type": "object"
         },
+        "HistoricalSentimentConfig": {
+            "description": "Test the Sentiment of Activities from the Author history\n\nIf this is defined then the `totalMatching` threshold must pass for the Rule to trigger\n\nIf `sentiment` is defined here it overrides the top-level `sentiment` value",
+            "properties": {
+                "mustMatchCurrent": {
+                    "default": false,
+                    "description": "When `true` the original Activity being checked MUST match desired sentiment before the Rule considers any history",
+                    "type": "boolean"
+                },
+                "sentiment": {
+                    "description": "Test the calculated VADER sentiment (compound) score for an Activity using this comparison. Can be either a numerical or natural language\n\nSentiment values range from extremely negative to extremely positive in a numerical range of -1 to +1:\n\n* -0.6 => extremely negative\n* -0.3 => very negative\n* -0.1 => negative\n*    0 => neutral\n*  0.1 => positive\n*  0.3 => very positive\n*  0.6 => extremely positive\n\nThe below examples are all equivocal. You can use either set of values as the value for `sentiment` (numerical comparisons or natural langauge)\n\n* `>= 0.1` = `is positive`\n* `<= 0.3` = `is very negative`\n* `< 0.1` = `is not positive`\n* `> -0.3` = `is not very negative`\n\nSpecial case:\n\n* `is neutral` equates to `> -0.1 and < 0.1`\n* `is not neutral` equates to `< -0.1 or > 0.1`\n\nContextMod uses a normalized, weighted average from these sentiment tools:\n\n* NLP.js (english, french, german, and spanish) https://github.com/axa-group/nlp.js/blob/master/docs/v3/sentiment-analysis.md\n* (english only) vaderSentiment-js https://github.com/vaderSentiment/vaderSentiment-js/\n* (english only) wink-sentiment https://github.com/winkjs/wink-sentiment\n\nMore about the sentiment algorithms used:\n* VADER https://github.com/cjhutto/vaderSentiment\n* AFINN http://corpustext.com/reference/sentiment_afinn.html\n* Senticon https://ieeexplore.ieee.org/document/8721408\n* Pattern https://github.com/clips/pattern\n* wink https://github.com/winkjs/wink-sentiment",
+                    "examples": [
+                        "is negative",
+                        "> 0.2"
+                    ],
+                    "pattern": "((>|>=|<|<=)\\s*(-?\\d?\\.?\\d+))|((not)?\\s*(very|extremely)?\\s*(positive|neutral|negative))",
+                    "type": "string"
+                },
+                "totalMatching": {
+                    "default": "> 0",
+                    "description": "A string containing a comparison operator and a value to compare Activities from history that pass the given `sentiment` comparison\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 12`  => greater than 12 activities passed given `sentiment` comparison\n* EX `<= 10%` => less than 10% of all Activities from history passed given `sentiment` comparison",
+                    "examples": [
+                        "> 0",
+                        "> 10%"
+                    ],
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "window": {
+                    "anyOf": [
+                        {
+                            "$ref": "#/definitions/DurationObject"
+                        },
+                        {
+                            "$ref": "#/definitions/FullActivityWindowConfig"
+                        },
+                        {
+                            "type": [
+                                "string",
+                                "number"
+                            ]
+                        }
+                    ],
+                    "description": "A value to define the range of Activities to retrieve.\n\nAcceptable values:\n\n**`ActivityWindowCriteria` object**\n\nAllows specify multiple range properties and more specific behavior\n\n**A `number` of Activities to retrieve**\n\n* EX `100` => 100 Activities\n\n*****\n\nAny of the below values that specify the amount of time to subtract from `NOW` to create a time range IE `NOW <---> [duration] ago`\n\nAcceptable values:\n\n**A `string` consisting of a value and a [Day.js](https://day.js.org/docs/en/durations/creating#list-of-all-available-units) time UNIT**\n\n* EX `9 days` => Range is `NOW <---> 9 days ago`\n\n**A [Day.js](https://day.js.org/docs/en/durations/creating) `object`**\n\n* EX `{\"days\": 90, \"minutes\": 15}` => Range is `NOW <---> 90 days and 15 minutes ago`\n\n**An [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) `string`**\n\n* EX `PT15M` => 15 minutes => Range is `NOW <----> 15 minutes ago`",
+                    "examples": [
+                        "90 days"
+                    ]
+                }
+            },
+            "required": [
+                "totalMatching",
+                "window"
+            ],
+            "type": "object"
+        },
         "HistoryCriteria": {
             "description": "Criteria will only trigger if ALL present thresholds (comment, submission, total) are met",
             "properties": {
@@ -2721,6 +2778,149 @@
             ],
             "type": "object"
         },
+        "SentimentRuleJSONConfig": {
+            "description": "Test the calculated VADER sentiment for an Activity to determine if the text context is negative, neutral, or positive in tone.\n\nMore about VADER Sentiment: https://github.com/cjhutto/vaderSentiment",
+            "properties": {
+                "authorIs": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "anyOf": [
+                                    {
+                                        "$ref": "#/definitions/AuthorCriteria"
+                                    },
+                                    {
+                                        "$ref": "#/definitions/NamedCriteria<AuthorCriteria>"
+                                    },
+                                    {
+                                        "type": "string"
+                                    }
+                                ]
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "$ref": "#/definitions/FilterOptionsJson<AuthorCriteria>"
+                        }
+                    ],
+                    "description": "If present then these Author criteria are checked before running the Check. If criteria fails then the Check will fail."
+                },
+                "defaultLanguage": {
+                    "anyOf": [
+                        {
+                            "enum": [
+                                false
+                            ],
+                            "type": "boolean"
+                        },
+                        {
+                            "type": [
+                                "null",
+                                "string"
+                            ]
+                        }
+                    ],
+                    "default": "en",
+                    "description": "Make the analyzer assume a language if it cannot determine one itself.\n\nThis is very useful for the analyzer when it is parsing short pieces of content. For example, if you know your subreddit is majority english speakers this will make the analyzer return \"neutral\" sentiment instead of \"not detected language\".\n\nDefaults to 'en'"
+                },
+                "historical": {
+                    "$ref": "#/definitions/HistoricalSentimentConfig",
+                    "description": "Test the Sentiment of Activities from the Author history\n\nIf this is defined then the `totalMatching` threshold must pass for the Rule to trigger\n\nIf `sentiment` is defined here it overrides the top-level `sentiment` value"
+                },
+                "itemIs": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "anyOf": [
+                                    {
+                                        "$ref": "#/definitions/SubmissionState"
+                                    },
+                                    {
+                                        "$ref": "#/definitions/CommentState"
+                                    },
+                                    {
+                                        "$ref": "#/definitions/NamedCriteria<TypedActivityState>"
+                                    },
+                                    {
+                                        "type": "string"
+                                    }
+                                ]
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "$ref": "#/definitions/FilterOptionsJson<TypedActivityState>"
+                        }
+                    ],
+                    "description": "A list of criteria to test the state of the `Activity` against before running the check.\n\nIf any set of criteria passes the Check will be run. If the criteria fails then the Check will fail.\n\n* @examples [[{\"over_18\": true, \"removed': false}]]"
+                },
+                "kind": {
+                    "description": "The kind of rule to run",
+                    "enum": [
+                        "sentiment"
+                    ],
+                    "examples": [
+                        "sentiment"
+                    ],
+                    "type": "string"
+                },
+                "languageHints": {
+                    "default": [
+                        "en",
+                        "es",
+                        "de",
+                        "fr"
+                    ],
+                    "description": "Helps the analyzer coerce a low confidence language guess into a known-used languages in two ways:\n\nIf the analyzer's\n  * *best* guess is NOT one of these\n    * but it did guess one of these\n    * and its guess is above requiredLanguageConfidence score then use the hinted language instead of best guess\n  * OR text content is very short (4 words or less)\n    * and the best guess was below the requiredLanguageConfidence score\n    * and none of guesses was a hinted language then use the defaultLanguage\n\nDefaults to popular romance languages: ['en', 'es', 'de', 'fr']",
+                    "items": {
+                        "type": "string"
+                    },
+                    "type": "array"
+                },
+                "name": {
+                    "description": "An optional, but highly recommended, friendly name for this rule. If not present will default to `kind`.\n\nCan only contain letters, numbers, underscore, spaces, and dashes\n\nname is used to reference Rule result data during Action content templating. See CommentAction or ReportAction for more details.",
+                    "examples": [
+                        "myNewRule"
+                    ],
+                    "pattern": "^[a-zA-Z]([\\w -]*[\\w])?$",
+                    "type": "string"
+                },
+                "requiredLanguageConfidence": {
+                    "default": 0.9,
+                    "description": "Required confidence to use a guessed language as the best guess. Score from 0 to 1.\n\nDefaults to 0.9",
+                    "type": "number"
+                },
+                "sentiment": {
+                    "description": "Test the calculated VADER sentiment (compound) score for an Activity using this comparison. Can be either a numerical or natural language\n\nSentiment values range from extremely negative to extremely positive in a numerical range of -1 to +1:\n\n* -0.6 => extremely negative\n* -0.3 => very negative\n* -0.1 => negative\n*    0 => neutral\n*  0.1 => positive\n*  0.3 => very positive\n*  0.6 => extremely positive\n\nThe below examples are all equivocal. You can use either set of values as the value for `sentiment` (numerical comparisons or natural langauge)\n\n* `>= 0.1` = `is positive`\n* `<= 0.3` = `is very negative`\n* `< 0.1` = `is not positive`\n* `> -0.3` = `is not very negative`\n\nSpecial case:\n\n* `is neutral` equates to `> -0.1 and < 0.1`\n* `is not neutral` equates to `< -0.1 or > 0.1`\n\nContextMod uses a normalized, weighted average from these sentiment tools:\n\n* NLP.js (english, french, german, and spanish) https://github.com/axa-group/nlp.js/blob/master/docs/v3/sentiment-analysis.md\n* (english only) vaderSentiment-js https://github.com/vaderSentiment/vaderSentiment-js/\n* (english only) wink-sentiment https://github.com/winkjs/wink-sentiment\n\nMore about the sentiment algorithms used:\n* VADER https://github.com/cjhutto/vaderSentiment\n* AFINN http://corpustext.com/reference/sentiment_afinn.html\n* Senticon https://ieeexplore.ieee.org/document/8721408\n* Pattern https://github.com/clips/pattern\n* wink https://github.com/winkjs/wink-sentiment",
+                    "examples": [
+                        "is negative",
+                        "> 0.2"
+                    ],
+                    "pattern": "((>|>=|<|<=)\\s*(-?\\d?\\.?\\d+))|((not)?\\s*(very|extremely)?\\s*(positive|neutral|negative))",
+                    "type": "string"
+                },
+                "testOn": {
+                    "default": [
+                        "title",
+                        "body"
+                    ],
+                    "description": "Which content from an Activity to test for `sentiment` against\n\nOnly used if the Activity being tested is a Submission -- Comments are only tested against their body\n\nIf more than one type of content is specified then all text is tested together as one string",
+                    "items": {
+                        "enum": [
+                            "body",
+                            "title"
+                        ],
+                        "type": "string"
+                    },
+                    "type": "array"
+                }
+            },
+            "required": [
+                "kind",
+                "sentiment"
+            ],
+            "type": "object"
+        },
         "SubmissionState": {
             "description": "Different attributes a `Submission` can be in. Only include a property if you want to check it.",
             "examples": [

src/Schema/RuleSet.json

@@ -1271,6 +1271,60 @@
             },
             "type": "object"
         },
+        "HistoricalSentimentConfig": {
+            "description": "Test the Sentiment of Activities from the Author history\n\nIf this is defined then the `totalMatching` threshold must pass for the Rule to trigger\n\nIf `sentiment` is defined here it overrides the top-level `sentiment` value",
+            "properties": {
+                "mustMatchCurrent": {
+                    "default": false,
+                    "description": "When `true` the original Activity being checked MUST match desired sentiment before the Rule considers any history",
+                    "type": "boolean"
+                },
+                "sentiment": {
+                    "description": "Test the calculated VADER sentiment (compound) score for an Activity using this comparison. Can be either a numerical or natural language\n\nSentiment values range from extremely negative to extremely positive in a numerical range of -1 to +1:\n\n* -0.6 => extremely negative\n* -0.3 => very negative\n* -0.1 => negative\n*    0 => neutral\n*  0.1 => positive\n*  0.3 => very positive\n*  0.6 => extremely positive\n\nThe below examples are all equivocal. You can use either set of values as the value for `sentiment` (numerical comparisons or natural langauge)\n\n* `>= 0.1` = `is positive`\n* `<= 0.3` = `is very negative`\n* `< 0.1` = `is not positive`\n* `> -0.3` = `is not very negative`\n\nSpecial case:\n\n* `is neutral` equates to `> -0.1 and < 0.1`\n* `is not neutral` equates to `< -0.1 or > 0.1`\n\nContextMod uses a normalized, weighted average from these sentiment tools:\n\n* NLP.js (english, french, german, and spanish) https://github.com/axa-group/nlp.js/blob/master/docs/v3/sentiment-analysis.md\n* (english only) vaderSentiment-js https://github.com/vaderSentiment/vaderSentiment-js/\n* (english only) wink-sentiment https://github.com/winkjs/wink-sentiment\n\nMore about the sentiment algorithms used:\n* VADER https://github.com/cjhutto/vaderSentiment\n* AFINN http://corpustext.com/reference/sentiment_afinn.html\n* Senticon https://ieeexplore.ieee.org/document/8721408\n* Pattern https://github.com/clips/pattern\n* wink https://github.com/winkjs/wink-sentiment",
+                    "examples": [
+                        "is negative",
+                        "> 0.2"
+                    ],
+                    "pattern": "((>|>=|<|<=)\\s*(-?\\d?\\.?\\d+))|((not)?\\s*(very|extremely)?\\s*(positive|neutral|negative))",
+                    "type": "string"
+                },
+                "totalMatching": {
+                    "default": "> 0",
+                    "description": "A string containing a comparison operator and a value to compare Activities from history that pass the given `sentiment` comparison\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 12`  => greater than 12 activities passed given `sentiment` comparison\n* EX `<= 10%` => less than 10% of all Activities from history passed given `sentiment` comparison",
+                    "examples": [
+                        "> 0",
+                        "> 10%"
+                    ],
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "window": {
+                    "anyOf": [
+                        {
+                            "$ref": "#/definitions/DurationObject"
+                        },
+                        {
+                            "$ref": "#/definitions/FullActivityWindowConfig"
+                        },
+                        {
+                            "type": [
+                                "string",
+                                "number"
+                            ]
+                        }
+                    ],
+                    "description": "A value to define the range of Activities to retrieve.\n\nAcceptable values:\n\n**`ActivityWindowCriteria` object**\n\nAllows specify multiple range properties and more specific behavior\n\n**A `number` of Activities to retrieve**\n\n* EX `100` => 100 Activities\n\n*****\n\nAny of the below values that specify the amount of time to subtract from `NOW` to create a time range IE `NOW <---> [duration] ago`\n\nAcceptable values:\n\n**A `string` consisting of a value and a [Day.js](https://day.js.org/docs/en/durations/creating#list-of-all-available-units) time UNIT**\n\n* EX `9 days` => Range is `NOW <---> 9 days ago`\n\n**A [Day.js](https://day.js.org/docs/en/durations/creating) `object`**\n\n* EX `{\"days\": 90, \"minutes\": 15}` => Range is `NOW <---> 90 days and 15 minutes ago`\n\n**An [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) `string`**\n\n* EX `PT15M` => 15 minutes => Range is `NOW <----> 15 minutes ago`",
+                    "examples": [
+                        "90 days"
+                    ]
+                }
+            },
+            "required": [
+                "totalMatching",
+                "window"
+            ],
+            "type": "object"
+        },
         "HistoryCriteria": {
             "description": "Criteria will only trigger if ALL present thresholds (comment, submission, total) are met",
             "properties": {
@@ -2695,6 +2749,149 @@
             ],
             "type": "object"
         },
+        "SentimentRuleJSONConfig": {
+            "description": "Test the calculated VADER sentiment for an Activity to determine if the text context is negative, neutral, or positive in tone.\n\nMore about VADER Sentiment: https://github.com/cjhutto/vaderSentiment",
+            "properties": {
+                "authorIs": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "anyOf": [
+                                    {
+                                        "$ref": "#/definitions/AuthorCriteria"
+                                    },
+                                    {
+                                        "$ref": "#/definitions/NamedCriteria<AuthorCriteria>"
+                                    },
+                                    {
+                                        "type": "string"
+                                    }
+                                ]
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "$ref": "#/definitions/FilterOptionsJson<AuthorCriteria>"
+                        }
+                    ],
+                    "description": "If present then these Author criteria are checked before running the Check. If criteria fails then the Check will fail."
+                },
+                "defaultLanguage": {
+                    "anyOf": [
+                        {
+                            "enum": [
+                                false
+                            ],
+                            "type": "boolean"
+                        },
+                        {
+                            "type": [
+                                "null",
+                                "string"
+                            ]
+                        }
+                    ],
+                    "default": "en",
+                    "description": "Make the analyzer assume a language if it cannot determine one itself.\n\nThis is very useful for the analyzer when it is parsing short pieces of content. For example, if you know your subreddit is majority english speakers this will make the analyzer return \"neutral\" sentiment instead of \"not detected language\".\n\nDefaults to 'en'"
+                },
+                "historical": {
+                    "$ref": "#/definitions/HistoricalSentimentConfig",
+                    "description": "Test the Sentiment of Activities from the Author history\n\nIf this is defined then the `totalMatching` threshold must pass for the Rule to trigger\n\nIf `sentiment` is defined here it overrides the top-level `sentiment` value"
+                },
+                "itemIs": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "anyOf": [
+                                    {
+                                        "$ref": "#/definitions/SubmissionState"
+                                    },
+                                    {
+                                        "$ref": "#/definitions/CommentState"
+                                    },
+                                    {
+                                        "$ref": "#/definitions/NamedCriteria<TypedActivityState>"
+                                    },
+                                    {
+                                        "type": "string"
+                                    }
+                                ]
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "$ref": "#/definitions/FilterOptionsJson<TypedActivityState>"
+                        }
+                    ],
+                    "description": "A list of criteria to test the state of the `Activity` against before running the check.\n\nIf any set of criteria passes the Check will be run. If the criteria fails then the Check will fail.\n\n* @examples [[{\"over_18\": true, \"removed': false}]]"
+                },
+                "kind": {
+                    "description": "The kind of rule to run",
+                    "enum": [
+                        "sentiment"
+                    ],
+                    "examples": [
+                        "sentiment"
+                    ],
+                    "type": "string"
+                },
+                "languageHints": {
+                    "default": [
+                        "en",
+                        "es",
+                        "de",
+                        "fr"
+                    ],
+                    "description": "Helps the analyzer coerce a low confidence language guess into a known-used languages in two ways:\n\nIf the analyzer's\n  * *best* guess is NOT one of these\n    * but it did guess one of these\n    * and its guess is above requiredLanguageConfidence score then use the hinted language instead of best guess\n  * OR text content is very short (4 words or less)\n    * and the best guess was below the requiredLanguageConfidence score\n    * and none of guesses was a hinted language then use the defaultLanguage\n\nDefaults to popular romance languages: ['en', 'es', 'de', 'fr']",
+                    "items": {
+                        "type": "string"
+                    },
+                    "type": "array"
+                },
+                "name": {
+                    "description": "An optional, but highly recommended, friendly name for this rule. If not present will default to `kind`.\n\nCan only contain letters, numbers, underscore, spaces, and dashes\n\nname is used to reference Rule result data during Action content templating. See CommentAction or ReportAction for more details.",
+                    "examples": [
+                        "myNewRule"
+                    ],
+                    "pattern": "^[a-zA-Z]([\\w -]*[\\w])?$",
+                    "type": "string"
+                },
+                "requiredLanguageConfidence": {
+                    "default": 0.9,
+                    "description": "Required confidence to use a guessed language as the best guess. Score from 0 to 1.\n\nDefaults to 0.9",
+                    "type": "number"
+                },
+                "sentiment": {
+                    "description": "Test the calculated VADER sentiment (compound) score for an Activity using this comparison. Can be either a numerical or natural language\n\nSentiment values range from extremely negative to extremely positive in a numerical range of -1 to +1:\n\n* -0.6 => extremely negative\n* -0.3 => very negative\n* -0.1 => negative\n*    0 => neutral\n*  0.1 => positive\n*  0.3 => very positive\n*  0.6 => extremely positive\n\nThe below examples are all equivocal. You can use either set of values as the value for `sentiment` (numerical comparisons or natural langauge)\n\n* `>= 0.1` = `is positive`\n* `<= 0.3` = `is very negative`\n* `< 0.1` = `is not positive`\n* `> -0.3` = `is not very negative`\n\nSpecial case:\n\n* `is neutral` equates to `> -0.1 and < 0.1`\n* `is not neutral` equates to `< -0.1 or > 0.1`\n\nContextMod uses a normalized, weighted average from these sentiment tools:\n\n* NLP.js (english, french, german, and spanish) https://github.com/axa-group/nlp.js/blob/master/docs/v3/sentiment-analysis.md\n* (english only) vaderSentiment-js https://github.com/vaderSentiment/vaderSentiment-js/\n* (english only) wink-sentiment https://github.com/winkjs/wink-sentiment\n\nMore about the sentiment algorithms used:\n* VADER https://github.com/cjhutto/vaderSentiment\n* AFINN http://corpustext.com/reference/sentiment_afinn.html\n* Senticon https://ieeexplore.ieee.org/document/8721408\n* Pattern https://github.com/clips/pattern\n* wink https://github.com/winkjs/wink-sentiment",
+                    "examples": [
+                        "is negative",
+                        "> 0.2"
+                    ],
+                    "pattern": "((>|>=|<|<=)\\s*(-?\\d?\\.?\\d+))|((not)?\\s*(very|extremely)?\\s*(positive|neutral|negative))",
+                    "type": "string"
+                },
+                "testOn": {
+                    "default": [
+                        "title",
+                        "body"
+                    ],
+                    "description": "Which content from an Activity to test for `sentiment` against\n\nOnly used if the Activity being tested is a Submission -- Comments are only tested against their body\n\nIf more than one type of content is specified then all text is tested together as one string",
+                    "items": {
+                        "enum": [
+                            "body",
+                            "title"
+                        ],
+                        "type": "string"
+                    },
+                    "type": "array"
+                }
+            },
+            "required": [
+                "kind",
+                "sentiment"
+            ],
+            "type": "object"
+        },
         "SubmissionState": {
             "description": "Different attributes a `Submission` can be in. Only include a property if you want to check it.",
             "examples": [
@@ -3016,6 +3213,9 @@
                     {
                         "$ref": "#/definitions/RepostRuleJSONConfig"
                     },
+                    {
+                        "$ref": "#/definitions/SentimentRuleJSONConfig"
+                    },
                     {
                         "type": "string"
                     }

src/Subreddit/Manager.ts

@@ -229,6 +229,8 @@ export class Manager extends EventEmitter implements RunningStates {
     rulesUniqueRollingAvg: number = 0;
     actionedEvents: ActionedEvent[] = [];
 
+    delayedQueueInterval: any;
+
     processEmitter: EventEmitter = new EventEmitter();
 
     activityRepo!: Repository<Activity>;
@@ -272,12 +274,11 @@ export class Manager extends EventEmitter implements RunningStates {
             return {
                 id: x.id,
                 activityId: x.activity.name,
-                permalink: x.activity.permalink,
+                permalink: x.activity.permalink, // TODO construct this without having to fetch activity
                 submissionId: asComment(x.activity) ? x.activity.link_id : undefined,
                 author: x.author,
                 queuedAt: x.queuedAt.unix(),
-                durationMilli: x.delay.asSeconds(),
-                duration: x.delay.humanize(),
+                duration: x.delay.asSeconds(),
                 source: `${x.action}${x.identifier !== undefined ? ` (${x.identifier})` : ''}`,
                 subreddit: this.subreddit.display_name_prefixed
             }
@@ -399,6 +400,45 @@ export class Manager extends EventEmitter implements RunningStates {
             }
         })(this), 10000);
 
+        this.delayedQueueInterval = setInterval((function(self) {
+            return function() {
+                if(!self.queue.paused && self.resources !== undefined) {
+                    let index = 0;
+                    let anyQueued = false;
+                    for(const ar of self.resources.delayedItems) {
+                        if(ar.queuedAt.add(ar.delay).isSameOrBefore(dayjs())) {
+                            anyQueued = true;
+                            self.logger.info(`Activity ${ar.activity.name} dispatched at ${ar.queuedAt.format('HH:mm:ss z')} (delayed for ${ar.delay.humanize()}) is now being queued.`, {leaf: 'Delayed Activities'});
+                            self.firehose.push({
+                                activity: ar.activity,
+                                options: {
+                                    refresh: true,
+                                    // @ts-ignore
+                                    source: ar.identifier === undefined ? ar.type : `${ar.type}:${ar.identifier}`,
+                                    initialGoto: ar.goto,
+                                    activitySource: {
+                                        id: ar.id,
+                                        queuedAt: ar.queuedAt,
+                                        delay: ar.delay,
+                                        action: ar.action,
+                                        goto: ar.goto,
+                                        identifier: ar.identifier,
+                                        type: ar.type
+                                    },
+                                    dryRun: ar.dryRun,
+                                }
+                            });
+                            self.resources.removeDelayedActivity(ar.id);
+                        }
+                        index++;
+                    }
+                    if(!anyQueued) {
+                        self.logger.debug('No Activities ready to queue', {leaf: 'Delayed Activities'});
+                    }
+                }
+            }
+        })(this), 5000); // every 5 seconds
+
         this.processEmitter.on('notify', (payload: NotificationEventPayload) => {
            this.notificationManager.handle(payload.type, payload.title, payload.body, payload.causedBy, payload.logLevel);
         });
@@ -449,7 +489,7 @@ export class Manager extends EventEmitter implements RunningStates {
             //
             // if we insert the same item again because it is currently being processed AND THEN we get the item AGAIN we only want to update the newest meta
             // so search the array backwards to get the neweset only
-            const queuedItemIndex = findLastIndex(this.queuedItemsMeta, x => x.id === task.activity.id);
+            const queuedItemIndex = findLastIndex(this.queuedItemsMeta, x => x.id === task.activity.name);
             if(queuedItemIndex !== -1) {
                 const itemMeta = this.queuedItemsMeta[queuedItemIndex];
                 let msg = `Item ${itemMeta.id} is already ${itemMeta.state}.`;
@@ -458,11 +498,11 @@ export class Manager extends EventEmitter implements RunningStates {
                     this.queuedItemsMeta.splice(queuedItemIndex, 1, {...itemMeta, shouldRefresh: true});
                 } else {
                     this.logger.debug(`${msg} Re-queuing item but will also refresh data before processing.`);
-                    this.queuedItemsMeta.push({id: task.activity.id, shouldRefresh: true, state: 'queued'});
+                    this.queuedItemsMeta.push({id: task.activity.name, shouldRefresh: true, state: 'queued'});
                     this.queue.push(task);
                 }
             } else {
-                this.queuedItemsMeta.push({id: task.activity.id, shouldRefresh: false, state: 'queued'});
+                this.queuedItemsMeta.push({id: task.activity.name, shouldRefresh: false, state: 'queued'});
                 this.queue.push(task);
             }
 
@@ -493,40 +533,6 @@ export class Manager extends EventEmitter implements RunningStates {
         , 1);
     }
 
-    protected async startDelayQueue() {
-        while(this.queueState.state === RUNNING) {
-            let index = 0;
-            for(const ar of this.resources.delayedItems) {
-                if(!ar.processing && ar.queuedAt.add(ar.delay).isSameOrBefore(dayjs())) {
-                    this.logger.info(`Delayed Activity ${ar.activity.name} is being queued.`);
-                    await this.firehose.push({
-                        activity: ar.activity,
-                        options: {
-                            refresh: true,
-                            // @ts-ignore
-                            source: ar.identifier === undefined ? ar.type : `${ar.type}:${ar.identifier}`,
-                            initialGoto: ar.goto,
-                            activitySource: {
-                                id: ar.id,
-                                queuedAt: ar.queuedAt,
-                                delay: ar.delay,
-                                action: ar.action,
-                                goto: ar.goto,
-                                identifier: ar.identifier,
-                                type: ar.type
-                            },
-                            dryRun: ar.dryRun,
-                        }
-                    });
-                    this.resources.delayedItems.splice(index, 1, {...ar, processing: true});
-                }
-                index++;
-            }
-            // sleep for 5 seconds
-            await sleep(5000);
-        }
-    }
-
     protected generateQueue(maxWorkers: number) {
         if (maxWorkers > 1) {
             this.logger.warn(`Setting max queue workers above 1 (specified: ${maxWorkers}) may have detrimental effects to log readability and api usage. Consult the documentation before using this advanced/experimental feature.`);
@@ -538,7 +544,7 @@ export class Manager extends EventEmitter implements RunningStates {
                     await sleep(this.delayBy * 1000);
                 }
 
-                const queuedItemIndex = this.queuedItemsMeta.findIndex(x => x.id === task.activity.id);
+                const queuedItemIndex = this.queuedItemsMeta.findIndex(x => x.id === task.activity.name);
                 try {
                     const itemMeta = this.queuedItemsMeta[queuedItemIndex];
                     this.queuedItemsMeta.splice(queuedItemIndex, 1, {...itemMeta, state: 'processing'});
@@ -551,9 +557,6 @@ export class Manager extends EventEmitter implements RunningStates {
                 } finally {
                     // always remove item meta regardless of success or failure since we are done with it meow
                     this.queuedItemsMeta.splice(queuedItemIndex, 1);
-                    if(task.options.activitySource?.id !== undefined) {
-                        await this.resources.removeDelayedActivity(task.options.activitySource?.id);
-                    }
                 }
             }
             , maxWorkers);
@@ -875,7 +878,6 @@ export class Manager extends EventEmitter implements RunningStates {
         const checkType = isSubmission(activity) ? 'Submission' : 'Comment';
         let item = activity,
             runtimeShouldRefresh = false;
-        const itemId = await item.id;
 
         const {
             delayUntil,
@@ -885,6 +887,14 @@ export class Manager extends EventEmitter implements RunningStates {
             force = false,
         } = options;
 
+        if(refresh) {
+            this.logger.verbose(`Refreshed data`);
+            // @ts-ignore
+            item = await activity.refresh();
+        }
+
+        const itemId = await item.id;
+
         if(await this.resources.hasRecentSelf(item)) {
             let recentMsg = `Found in Activities recently (last ${this.resources.selfTTL} seconds) modified/created by this bot`;
             if(force) {
@@ -963,7 +973,6 @@ export class Manager extends EventEmitter implements RunningStates {
                             delay: dayjs.duration(remaining, 'seconds'),
                             id: 'notUsed',
                             queuedAt: dayjs(),
-                            processing: false,
                             activity,
                             author: getActivityAuthorName(activity.author),
                         });
@@ -977,7 +986,7 @@ export class Manager extends EventEmitter implements RunningStates {
             }
             // refresh signal from firehose if activity was ingested multiple times before processing or re-queued while processing
             // want to make sure we have the most recent data
-            if(runtimeShouldRefresh || refresh) {
+            if(runtimeShouldRefresh) {
                 this.logger.verbose(`Refreshed data`);
                 // @ts-ignore
                 item = await activity.refresh();
@@ -1348,7 +1357,6 @@ export class Manager extends EventEmitter implements RunningStates {
                 state: RUNNING,
                 causedBy
             }
-            this.startDelayQueue();
             if(!suppressNotification) {
                 this.notificationManager.handle('runStateChanged', 'Queue Started', reason, causedBy);
             }

src/Subreddit/Streams.ts

@@ -1,11 +1,12 @@
 import {Poll, SnooStormOptions} from "snoostorm"
-import Snoowrap, {Listing} from "snoowrap";
+import Snoowrap, {Listing, RedditContent} from "snoowrap";
 import {EventEmitter} from "events";
 import {PollConfiguration} from "snoostorm/out/util/Poll";
 import {DEFAULT_POLLING_INTERVAL} from "../Common/interfaces";
 import {mergeArr, parseDuration, random} from "../util";
 import { Logger } from "winston";
 import {ErrorWithCause} from "pony-cause";
+import dayjs, {Dayjs as DayjsObj} from "dayjs";
 
 type Awaitable<T> = Promise<T> | T;
 
@@ -16,13 +17,15 @@ interface RCBPollingOptions<T> extends SnooStormOptions {
     name?: string,
     processed?: Set<T[keyof T]>
     label?: string
+    dateCutoff?: boolean
 }
 
 interface RCBPollConfiguration<T> extends PollConfiguration<T>,RCBPollingOptions<T> {
     get: () => Promise<Listing<T>>
+    dateCutoff: boolean
 }
 
-export class SPoll<T extends object> extends Poll<T> {
+export class SPoll<T extends RedditContent<object>> extends Poll<T> {
     identifier: keyof T;
     getter: () => Promise<Listing<T>>;
     frequency;
@@ -31,6 +34,8 @@ export class SPoll<T extends object> extends Poll<T> {
     // -- that is, we don't want to emit the items we immediately fetch on a fresh poll start since they existed "before" polling started
     newStart: boolean = true;
     enforceContinuity: boolean;
+    useDateCutoff: boolean;
+    dateCutoff?: DayjsObj;
     randInterval?: { clear: () => void };
     name: string = 'Reddit Stream';
     logger: Logger;
@@ -47,7 +52,8 @@ export class SPoll<T extends object> extends Poll<T> {
             name,
             subreddit,
             label = 'Polling',
-            processed
+            processed,
+            dateCutoff,
         } = options;
         this.subreddit = subreddit;
         this.name = name !== undefined ? name : this.name;
@@ -56,6 +62,7 @@ export class SPoll<T extends object> extends Poll<T> {
         this.getter = get;
         this.frequency = frequency;
         this.enforceContinuity = enforceContinuity;
+        this.useDateCutoff = dateCutoff;
 
         // if we pass in processed on init the intention is to "continue" from where the previous stream left off
         // WITHOUT new start behavior
@@ -80,7 +87,7 @@ export class SPoll<T extends object> extends Poll<T> {
                     // but only continue iterating if stream enforces continuity and we've only seen new items so far
                     while(page === 1 || (self.enforceContinuity && !self.newStart && !anyAlreadySeen)) {
                         if(page !== 1) {
-                            self.logger.debug(`Did not find any already seen activities and continuity is enforced. This probably means there were more new items than 1 api call can return. Fetching next page (${page})...`);
+                            self.logger.debug(`Did not find any already seen Activities and continuity is enforced. This probably means there were more new Activities than 1 api call can return. Fetching next page (page ${page})...`);
                             // @ts-ignore
                             batch = await batch.fetchMore({amount: 100});
                         }
@@ -95,24 +102,67 @@ export class SPoll<T extends object> extends Poll<T> {
                                 continue;
                             }
 
-                            // Emit for new items and add it to the list
+                            // add new item to list and set as processed
                             newItems.push(item);
                             self.processed.add(id);
-                            // but don't emit on new start since we are "buffering" already existing activities
-                            if(!self.newStart) {
-                                self.emit("item", item);
-                            }
                         }
                         page++;
                     }
-                    const newItemMsg = `Found ${newItems.length} new items out of ${batch.length} returned`;
+
                     if(self.newStart) {
-                        self.logger.debug(`${newItemMsg} but will ignore all on first start.`);
+
+                        self.logger.debug(`Found ${newItems.length} unseen Activities out of ${batch.length} returned, but will ignore all on first start.`);
                         self.emit("listing", []);
+
+                        if(self.useDateCutoff && self.dateCutoff === undefined) {
+                            self.logger.debug('Cutoff date should be used for filtering unseen Activities but none was set. Will determine date based on newest Activity returned from first polling results.');
+                            if(newItems.length === 0) {
+                                // no items found, cutoff is now
+                                self.dateCutoff = dayjs();
+                                self.logger.debug(`Cutoff date set to NOW (${self.dateCutoff.format('YYYY-MM-DD HH:mm:ssZ')}) since no unseen Activities returned. Unseen Activities will only be returned if newer than this date.`);
+                            } else {
+                                // set cutoff date for new items from the newest items found
+                                const sorted = [...newItems];
+                                sorted.sort((a, z) => z.created_utc - a.created_utc);
+                                self.dateCutoff = dayjs.unix(sorted[0].created_utc);
+                                self.logger.debug(`Cutoff date set to newest unseen Activity found, ${self.dateCutoff.format('YYYY-MM-DD HH:mm:ssZ')}. Unseen Activities will only be returned if newer than this date.`);
+                            }
+                        }
+
                     } else {
-                        self.logger.debug(newItemMsg);
+
+                        // applies mostly (only?) to 'unmoderated' polling
+                        //
+                        // scenario:
+                        // * polling unmoderated for many subreddits and unmoderated has not been clearing out for awhile so it has many (100's) of items
+                        // * a moderator, or CM, iterates through list and actions items so the list is shorter
+                        // * CM polling unmoderated and finds "unseen" items that don't appear in unprocessed list
+                        //
+                        // these "unseen" are OLDER than the "newest" seen items we have got from polling because CM only got the first page of unmoderated items
+                        // so now CM emits them as "new" and CM starts processing them. If it continues to process them then more and more 'unseen old' items continue to appear in stream,
+                        // creating a feedback loop where CM eventually processes the entire backlog of unmoderated items
+                        //
+                        // this is UNWANTED behavior. CM should only ever process items added to polling sources after it starts monitoring them.
+                        //
+                        // to address this we use a cutoff date determined from the newest activity returned from the first polling call (or current datetime if none returned)
+                        // then we make sure any 'new' items (unseen by CM) are newer than this cutoff date
+                        //
+                        // -- this is the default behavior for all polling sources except modqueue. See comments on that class below for why.
+                        const unixCutoff = self.useDateCutoff && self.dateCutoff !== undefined ? self.dateCutoff.unix() : undefined;
+                        const validNewItems = unixCutoff === undefined || newItems.length === 0 ? newItems : newItems.filter(x => x.created_utc >= unixCutoff);
+
+                        if(validNewItems.length !== newItems.length && self.dateCutoff !== undefined) {
+                            self.logger.warn(`${newItems.length - validNewItems.length} unseen Activities were created before cutoff date (${self.dateCutoff.format('YYYY-MM-DD HH:mm:ssZ')}) and have been filtered out.`);
+                        }
+                        self.logger.debug(`Found ${validNewItems.length} valid, unseen Activities out of ${batch.length} returned`);
+
+                        // only emit if not new start since we are "buffering" already existing activities
+                        for(const item of validNewItems) {
+                            self.emit('item', item);
+                        }
+
                         // Emit the new listing of all new items
-                        self.emit("listing", newItems);
+                        self.emit("listing", validNewItems);
                     }
                     // no longer new start on n+1 interval
                     self.newStart = false;
@@ -146,6 +196,7 @@ export class SPoll<T extends object> extends Poll<T> {
         this.logger.debug(msg);
         this.running = false;
         this.newStart = true;
+        this.dateCutoff = undefined;
         super.end();
     }
 }
@@ -159,6 +210,7 @@ export class UnmoderatedStream extends SPoll<Snoowrap.Submission | Snoowrap.Comm
             get: async () => client.getSubreddit(options.subreddit).getUnmoderated(options),
             identifier: "id",
             name: 'Unmoderated',
+            dateCutoff: true,
             ...options,
         });
     }
@@ -173,6 +225,9 @@ export class ModQueueStream extends SPoll<Snoowrap.Submission | Snoowrap.Comment
             get: async () => client.getSubreddit(options.subreddit).getModqueue(options),
             identifier: "id",
             name: 'Modqueue',
+            // cannot use cutoff date since 'new' items in this list are based on when they were reported, not when the item was created
+            // and unfortunately there is no way to use that "reported at" time since reddit doesn't include it in the returned items
+            dateCutoff: false,
             ...options,
         });
     }
@@ -187,6 +242,7 @@ export class SubmissionStream extends SPoll<Snoowrap.Submission | Snoowrap.Comme
             get: async () => client.getNew(options.subreddit, options),
             identifier: "id",
             name: 'Submission',
+            dateCutoff: true,
             ...options,
         });
     }
@@ -201,6 +257,7 @@ export class CommentStream extends SPoll<Snoowrap.Submission | Snoowrap.Comment>
             get: async () => client.getNewComments(options.subreddit, options),
             identifier: "id",
             name: 'Comment',
+            dateCutoff: true,
             ...options,
         });
     }

src/Subreddit/SubredditResources.ts

@@ -37,8 +37,6 @@ import {
     mergeArr,
     parseDurationComparison,
     parseExternalUrl,
-    parseGenericValueComparison,
-    parseGenericValueOrPercentComparison,
     parseRedditEntity,
     parseStringToRegex,
     parseWikiContext,
@@ -145,6 +143,7 @@ import {
 } from "../Common/Infrastructure/Reddit";
 import {AuthorCritPropHelper} from "../Common/Infrastructure/Filters/AuthorCritPropHelper";
 import {NoopLogger} from "../Utils/loggerFactory";
+import {parseGenericValueComparison, parseGenericValueOrPercentComparison} from "../Common/Infrastructure/Comparisons";
 
 export const DEFAULT_FOOTER = '\r\n*****\r\nThis action was performed by [a bot.]({{botLink}}) Mention a moderator or [send a modmail]({{modmailLink}}) if you any ideas, questions, or concerns about this action.';
 
@@ -440,6 +439,7 @@ export class SubredditResources {
             const now = dayjs();
             for(const dAct of dispatchedActivities) {
                 const shouldDispatchAt = dAct.createdAt.add(dAct.delay.asSeconds(), 'seconds');
+                let tardyHint = '';
                 if(shouldDispatchAt.isBefore(now)) {
                     let tardyHint = `Activity ${dAct.activityId} queued at ${dAct.createdAt.format('YYYY-MM-DD HH:mm:ssZ')} for ${dAct.delay.humanize()} is now LATE`;
                     if(dAct.tardyTolerant === true) {
@@ -453,7 +453,8 @@ export class SubredditResources {
                         // see if its within tolerance
                         const latest = shouldDispatchAt.add(dAct.tardyTolerant);
                         if(latest.isBefore(now)) {
-                            tardyHint += `and IS NOT within tardy tolerance of ${dAct.tardyTolerant.humanize()} of planned dispatch time so will be dropped`;
+                            tardyHint += ` and IS NOT within tardy tolerance of ${dAct.tardyTolerant.humanize()} of planned dispatch time so will be dropped`;
+                            this.logger.warn(tardyHint);
                             await this.removeDelayedActivity(dAct.id);
                             continue;
                         } else {
@@ -461,8 +462,14 @@ export class SubredditResources {
                         }
                     }
                 }
-                // TODO make this less api heavy
-                this.delayedItems.push(await dAct.toActivityDispatch(this.client))
+                if(tardyHint !== '') {
+                    this.logger.warn(tardyHint);
+                }
+                try {
+                    this.delayedItems.push(await dAct.toActivityDispatch(this.client))
+                } catch (e) {
+                    this.logger.warn(new ErrorWithCause(`Unable to add Activity ${dAct.activityId} from database delayed activities to in-app delayed activities queue`, {cause: e}));
+                }
             }
         }
     }
@@ -475,7 +482,7 @@ export class SubredditResources {
 
     async removeDelayedActivity(id: string) {
         await this.dispatchedActivityRepo.delete(id);
-        this.delayedItems.filter(x => x.id !== id);
+        this.delayedItems = this.delayedItems.filter(x => x.id !== id);
     }
 
     async initStats() {
@@ -742,7 +749,7 @@ export class SubredditResources {
             req: acc.req + curr.requests,
         }), {miss: 0, req: 0});
         const cacheKeys = Object.keys(this.stats.cache);
-        return {
+        const res = {
             cache: {
                 // TODO could probably combine these two
                 totalRequests: totals.req,
@@ -770,24 +777,29 @@ export class SubredditResources {
 
                     if(acc[curr].requestTimestamps.length > 1) {
                         // calculate average time between request
-                        const diffData = acc[curr].requestTimestamps.reduce((acc, curr: number) => {
-                            if(acc.last === 0) {
-                                acc.last = curr;
-                                return acc;
+                        const diffData = acc[curr].requestTimestamps.reduce((accTimestampData, curr: number) => {
+                            if(accTimestampData.last === 0) {
+                                accTimestampData.last = curr;
+                                return accTimestampData;
                             }
-                            acc.diffs.push(curr - acc.last);
-                            acc.last = curr;
-                            return acc;
+                            accTimestampData.diffs.push(curr - accTimestampData.last);
+                            accTimestampData.last = curr;
+                            return accTimestampData;
                         },{last: 0, diffs: [] as number[]});
                         const avgDiff = diffData.diffs.reduce((acc, curr) => acc + curr, 0) / diffData.diffs.length;
 
                         acc[curr].averageTimeBetweenHits = formatNumber(avgDiff/1000);
                     }
 
+                    const {requestTimestamps, identifierRequestCount, ...rest} = acc[curr];
+                    // @ts-ignore
+                    acc[curr] = rest;
+
                     return acc;
-                }, Promise.resolve(this.stats.cache))
+                }, Promise.resolve({...this.stats.cache}))
             }
         }
+        return res;
     }
 
     setLogger(logger: Logger) {
@@ -906,39 +918,43 @@ export class SubredditResources {
                 return await item.fetch();
             }
         } catch (err: any) {
-            this.logger.error('Error while trying to fetch a cached activity', err);
-            throw err.logged;
+            throw new ErrorWithCause('Error while trying to fetch a cached Activity', {cause: err});
         }
     }
 
     // @ts-ignore
     public async setActivity(item: Submission | Comment, tryToFetch = true)
     {
-        let hash = '';
-        if(this.submissionTTL !== false && isSubmission(item)) {
-            hash = `sub-${item.name}`;
-            if(tryToFetch && item instanceof Submission) {
-                // @ts-ignore
-                const itemToCache = await item.fetch();
-                await this.cache.set(hash, itemToCache, {ttl: this.submissionTTL});
-                return itemToCache;
-            } else {
-                // @ts-ignore
-                await this.cache.set(hash, item, {ttl: this.submissionTTL});
-                return item;
-            }
-        } else if(this.commentTTL !== false){
-            hash = `comm-${item.name}`;
-            if(tryToFetch && item instanceof Comment) {
-                // @ts-ignore
-                const itemToCache = await item.fetch();
-                await this.cache.set(hash, itemToCache, {ttl: this.commentTTL});
-                return itemToCache;
-            } else {
-                // @ts-ignore
-                await this.cache.set(hash, item, {ttl: this.commentTTL});
-                return item;
+        try {
+            let hash = '';
+            if (this.submissionTTL !== false && isSubmission(item)) {
+                hash = `sub-${item.name}`;
+                if (tryToFetch && item instanceof Submission) {
+                    // @ts-ignore
+                    const itemToCache = await item.fetch();
+                    await this.cache.set(hash, itemToCache, {ttl: this.submissionTTL});
+                    return itemToCache;
+                } else {
+                    // @ts-ignore
+                    await this.cache.set(hash, item, {ttl: this.submissionTTL});
+                    return item;
+                }
+            } else if (this.commentTTL !== false) {
+                hash = `comm-${item.name}`;
+                if (tryToFetch && item instanceof Comment) {
+                    // @ts-ignore
+                    const itemToCache = await item.fetch();
+                    await this.cache.set(hash, itemToCache, {ttl: this.commentTTL});
+                    return itemToCache;
+                } else {
+                    // @ts-ignore
+                    await this.cache.set(hash, item, {ttl: this.commentTTL});
+                    return item;
+                }
             }
+            return item;
+        } catch (e) {
+            throw new ErrorWithCause('Error occurred while trying to add Activity to cache', {cause: e});
         }
     }
 
@@ -1015,7 +1031,7 @@ export class SubredditResources {
                 return subreddit as Subreddit;
             }
         } catch (err: any) {
-            this.logger.error('Error while trying to fetch a cached activity', err);
+            this.logger.error('Error while trying to fetch a cached subreddit', err);
             throw err.logged;
         }
     }

src/Utils/databaseUtils.ts

@@ -3,6 +3,7 @@ import {MysqlConnectionOptions} from "typeorm/driver/mysql/MysqlConnectionOption
 import {MongoConnectionOptions} from "typeorm/driver/mongodb/MongoConnectionOptions";
 import {PostgresConnectionOptions} from "typeorm/driver/postgres/PostgresConnectionOptions";
 import {resolve, parse as parsePath} from 'path';
+// https://stackoverflow.com/questions/49618719/why-does-typeorm-need-reflect-metadata
 import "reflect-metadata";
 import {DataSource} from "typeorm";
 import {castToBool, fileOrDirectoryIsWriteable, mergeArr, resolvePath} from "../util";

src/Web/Client/StorageProvider.ts

@@ -88,7 +88,8 @@ export class CacheStorageProvider extends StorageProvider {
 
     constructor(caching: CacheOptions & StorageProviderOptions) {
         super(caching);
-        this.cache = createCacheManager({...caching, prefix: buildCachePrefix(['web'])}) as Cache;
+        const {logger, invitesMaxAge, loggerLabels, ...restCache } = caching;
+        this.cache = createCacheManager({...restCache, prefix: buildCachePrefix(['web'])}) as Cache;
         this.logger.debug('Using CACHE');
         if (caching.store === 'none') {
             this.logger.warn(`Using 'none' as cache provider means no one will be able to access the interface since sessions will never be persisted!`);

src/Web/Client/index.ts

@@ -29,8 +29,6 @@ import session, {Session, SessionData} from "express-session";
 import Snoowrap, {Subreddit} from "snoowrap";
 import {getLogger} from "../../Utils/loggerFactory";
 import EventEmitter from "events";
-import stream, {Readable, Writable, Transform} from "stream";
-import winston from "winston";
 import tcpUsed from "tcp-port-used";
 import http from "http";
 import jwt from 'jsonwebtoken';
@@ -39,13 +37,6 @@ import got from 'got';
 import sharedSession from "express-socket.io-session";
 import dayjs from "dayjs";
 import httpProxy from 'http-proxy';
-import normalizeUrl from 'normalize-url';
-import GotRequest from "got/dist/source/core";
-import {prettyPrintJson} from "pretty-print-json";
-// @ts-ignore
-import DelimiterStream from 'delimiter-stream';
-import {pipeline} from 'stream/promises';
-import {defaultBotStatus} from "../Common/defaults";
 import {arrayMiddle, booleanMiddle} from "../Common/middleware";
 import {BotInstance, CMInstanceInterface} from "../interfaces";
 import { URL } from "url";
@@ -54,8 +45,6 @@ import Autolinker from "autolinker";
 import path from "path";
 import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients";
 import ClientUser from "../Common/User/ClientUser";
-import {BotStatusResponse, InviteData} from "../Common/interfaces";
-import {TransformableInfo} from "logform";
 import {SimpleError} from "../../Utils/Errors";
 import {ErrorWithCause} from "pony-cause";
 import {CMInstance} from "./CMInstance";
@@ -65,8 +54,6 @@ import { ActionPremise } from "../../Common/Entities/ActionPremise";
 import {CacheStorageProvider, DatabaseStorageProvider} from "./StorageProvider";
 import {nanoid} from "nanoid";
 import {MigrationService} from "../../Common/MigrationService";
-import {WebSetting} from "../../Common/WebEntities/WebSetting";
-import {CheckResultEntity} from "../../Common/Entities/CheckResultEntity";
 import {RuleResultEntity} from "../../Common/Entities/RuleResultEntity";
 import {RuleSetResultEntity} from "../../Common/Entities/RuleSetResultEntity";
 import { PaginationAwareObject } from "../Common/util";
@@ -606,106 +593,10 @@ const webClient = async (options: OperatorConfig) => {
     const cmInstances: CMInstance[] = [];
     let init = false;
     const formatter = defaultFormat();
-    const formatTransform = formatter.transform as (info: TransformableInfo, opts?: any) => TransformableInfo;
 
     let server: http.Server,
         io: SocketServer;
 
-    const startLogStream = (sessionData: Session & Partial<SessionData>, user: Express.User) => {
-        // @ts-ignore
-        const sessionId = sessionData.id as string;
-        
-        if(connectedUsers[sessionId] !== undefined) {
-
-            const delim = new DelimiterStream({
-                delimiter: '\r\n',
-            });
-
-            const currInstance = cmInstances.find(x => x.getName() === sessionData.botId);
-            if(currInstance !== undefined) {
-                const ac = new AbortController();
-                const options = {
-                    signal: ac.signal,
-                };
-
-                const retryFn = (retryCount = 0, err: any = undefined) => {
-                    const delim = new DelimiterStream({
-                        delimiter: '\r\n',
-                    });
-
-                    if(err !== undefined) {
-                        // @ts-ignore
-                        currInstance.logger.warn(new ErrorWithCause(`Log streaming encountered an error, trying to reconnect (retries: ${retryCount})`, {cause: err}), {user: user.name});
-                    }
-                    const gotStream = got.stream.get(`${currInstance.normalUrl}/logs`, {
-                        retry: {
-                            limit: 5,
-                        },
-                        headers: {
-                            'Authorization': `Bearer ${createToken(currInstance, user)}`,
-                        },
-                        searchParams: {
-                            limit: sessionData.limit,
-                            sort: sessionData.sort,
-                            level: sessionData.level,
-                            stream: true,
-                            streamObjects: true,
-                            formatted: false,
-                        }
-                    });
-
-                    if(err !== undefined) {
-                        gotStream.once('data', () => {
-                            currInstance.logger.info('Streaming resumed', {instance: currInstance.getName(), user: user.name});
-                        });
-                    }
-
-                    gotStream.retryCount = retryCount;
-                    const s = pipeline(
-                        gotStream,
-                        delim,
-                        options
-                    ) as Promise<void>;
-
-                    // ECONNRESET
-                    s.catch((err) => {
-                        if(err.code !== 'ABORT_ERR' && err.code !== 'ERR_STREAM_PREMATURE_CLOSE') {
-                            // @ts-ignore
-                            currInstance.logger.error(new ErrorWithCause('Unexpected error, or too many retries, occurred while streaming logs', {cause: err}), {user: user.name});
-                        }
-                    });
-
-
-                    delim.on('data', (c: any) => {
-                        const logObj = JSON.parse(c) as LogInfo;
-                        let subredditMessage;
-                        let allMessage;
-                        if(logObj.subreddit !== undefined) {
-                            const {subreddit, bot, ...rest} = logObj
-                            // @ts-ignore
-                            subredditMessage = formatLogLineToHtml(formatter.transform(rest)[MESSAGE], rest.timestamp);
-                        }
-                        if(logObj.bot !== undefined) {
-                            const {bot, ...rest} = logObj
-                            // @ts-ignore
-                            allMessage = formatLogLineToHtml(formatter.transform(rest)[MESSAGE], rest.timestamp);
-                        }
-                        // @ts-ignore
-                        let formattedMessage = formatLogLineToHtml(formatter.transform(logObj)[MESSAGE], logObj.timestamp);
-                        io.to(sessionId).emit('log', {...logObj, subredditMessage, allMessage, formattedMessage});
-                    });
-
-                    gotStream.once('retry', retryFn);
-                }
-
-                retryFn();
-
-                return ac;
-            }
-            return undefined;
-        }
-    }
-
     try {
         server = await app.listen(port);
         io = new SocketServer(server);
@@ -965,29 +856,7 @@ const webClient = async (options: OperatorConfig) => {
 
         res.render('status', {
             instances: shownInstances,
-            bots: resp.bots.map((x: BotStatusResponse) => {
-                const {subreddits = []} = x;
-                const subredditsWithSimpleLogs = subreddits.map(y => {
-                    let transformedLogs: string[];
-                    if(y.name === 'All') {
-                        // only need to remove bot name here
-                        transformedLogs = (y.logs as LogInfo[]).map((z: LogInfo) => {
-                           const {bot, ...rest} = z;
-                           // @ts-ignore
-                           return formatLogLineToHtml(formatter.transform(rest)[MESSAGE] as string, rest.timestamp);
-                        });
-                    } else {
-                        transformedLogs = (y.logs as LogInfo[]).map((z: LogInfo) => {
-                            const {bot, subreddit, ...rest} = z;
-                            // @ts-ignore
-                            return formatLogLineToHtml(formatter.transform(rest)[MESSAGE] as string, rest.timestamp);
-                        });
-                    }
-                    y.logs = transformedLogs;
-                    return y;
-                });
-                return {...x, subreddits: subredditsWithSimpleLogs};
-            }),
+            bots: resp.bots,
             botId: (req.instance as CMInstance).getName(),
             instanceId: (req.instance as CMInstance).getName(),
             isOperator: isOp,
@@ -1398,42 +1267,6 @@ const webClient = async (options: OperatorConfig) => {
             clearSockStreams(socket.id);
             socket.join(session.id);
 
-            socket.on('viewing', (data) => {
-                if(user !== undefined) {
-                    const {subreddit, bot: botVal} = data;
-                    const currBot = cmInstances.find(x => x.getName() === session.botId);
-                    if(currBot !== undefined) {
-
-                        if(liveInterval !== undefined) {
-                            clearInterval(liveInterval)
-                        }
-
-                        const liveEmit = async () => {
-                            try {
-                                const resp = await got.get(`${currBot.normalUrl}/liveStats`, {
-                                    headers: {
-                                        'Authorization': `Bearer ${createToken(currBot, user)}`,
-                                    },
-                                    searchParams: {
-                                        bot: botVal,
-                                        subreddit
-                                    }
-                                });
-                                const stats = JSON.parse(resp.body);
-                                io.to(session.id).emit('liveStats', stats);
-                            } catch (err: any) {
-                                currBot.logger.error(new ErrorWithCause('Could not retrieve live stats', {cause: err}));
-                            }
-                        }
-
-                        // do an initial get
-                        liveEmit();
-                        // and then every 5 seconds after that
-                        liveInterval = setInterval(async () => await liveEmit(), 5000);
-                    }
-                }
-            });
-
             if(session.botId !== undefined) {
                 const bot = cmInstances.find(x => x.getName() === session.botId);
                 if(bot !== undefined) {
@@ -1456,12 +1289,8 @@ const webClient = async (options: OperatorConfig) => {
 
                     // only setup streams if the user can actually access them (not just a web operator)
                     if(session.authBotId !== undefined) {
-                        // streaming logs and stats from client
+                        // streaming stats from client
                         const newStreams: (AbortController | NodeJS.Timeout)[] = [];
-                        const ac = startLogStream(session, user);
-                        if(ac !== undefined) {
-                            newStreams.push(ac);
-                        }
                         const interval = setInterval(async () => {
                             try {
                                 const resp = await got.get(`${bot.normalUrl}/stats`, {

src/Web/Server/routes/authenticated/user/liveStats.ts

@@ -136,6 +136,9 @@ const liveStats = () => {
                 acc[curr].missPercent = `${formatNumber(per, {toFixed: 0})}%`;
                 acc[curr].identifierAverageHit = formatNumber(acc[curr].identifierAverageHit);
                 acc[curr].averageTimeBetweenHits = formatNumber(acc[curr].averageTimeBetweenHits)
+
+                delete acc[curr].requestTimestamps;
+                delete acc[curr].identifierRequestCount;
                 return acc;
             }, cumRaw);
             const cacheReq = subManagerData.reduce((acc, curr) => acc + curr.stats.cache.totalRequests, 0);

src/Web/Server/routes/authenticated/user/logs.ts

@@ -27,6 +27,15 @@ const logs = () => {
         booleanMiddle([{
             name: 'stream',
             defaultVal: false
+        }, {
+            name: 'formatted',
+            defaultVal: true,
+        }, {
+            name: 'transports',
+            defaultVal: false
+        }, {
+            name: 'streamObjects',
+            defaultVal: false
         }])
     ];
 
@@ -37,20 +46,56 @@ const logs = () => {
         const userName = req.user?.name as string;
         const isOperator = req.user?.isInstanceOperator(req.botApp);
         const realManagers = req.botApp.bots.map(x => req.user?.accessibleSubreddits(x).map(x => x.displayLabel)).flat() as string[];
-        const {level = 'verbose', stream, limit = 200, sort = 'descending', streamObjects = false, formatted = true} = req.query;
+        const {level = 'verbose', stream, limit = 200, sort = 'descending', streamObjects = false, formatted: formattedVal = true, transports: transportsVal = false} = req.query;
+
+        const formatted = formattedVal as boolean;
+        const transports = transportsVal as boolean;
+
+        let bots: Bot[] = [];
+        if(req.serverBot !== undefined) {
+            bots = [req.serverBot];
+        } else {
+            bots = req.user?.accessibleBots(req.botApp.bots) as Bot[];
+        }
+
+        let managers: Manager[] = [];
+
+        if(req.manager !== undefined) {
+            managers = [req.manager];
+        } else {
+            for(const b of bots) {
+                managers = managers.concat(req.user?.accessibleSubreddits(b) as Manager[]);
+            }
+        }
+
+        //const allReq = req.query.subreddit !== undefined && (req.query.subreddit as string).toLowerCase() === 'all';
+
         if (stream) {
+
+            const requestedManagers = managers.map(x => x.displayLabel);
+            const requestedBots = bots.map(x => x.botName);
+
             const origin = req.header('X-Forwarded-For') ?? req.header('host');
             try {
                 logger.stream().on('log', (log: LogInfo) => {
                     if (isLogLineMinLevel(log, level as string)) {
-                        const {subreddit: subName, user} = log;
-                        if (isOperator || (subName !== undefined && (realManagers.includes(subName) || (user !== undefined && user.includes(userName))))) {
+                        const {subreddit: subName, bot, user} = log;
+                        let canAccess = false;
+                        if(user !== undefined && user.includes(userName)) {
+                            canAccess = true;
+                        } else if(subName !== undefined || bot !== undefined) {
+                            if(subName === undefined) {
+                                canAccess = requestedBots.includes(bot);
+                            } else {
+                                canAccess = requestedManagers.includes(subName);
+                            }
+                        } else if(isOperator) {
+                            canAccess = true;
+                        }
+
+                        if (canAccess) {
                             if(streamObjects) {
-                                let obj: any = log;
-                                if(!formatted) {
-                                    const {[MESSAGE]: fMessage, ...rest} = log;
-                                    obj = rest;
-                                }
+                                let obj: any = transformLog(log, {formatted, transports});
                                 res.write(`${JSON.stringify(obj)}\r\n`);
                             } else if(formatted) {
                                 res.write(`${log[MESSAGE]}\r\n`)
@@ -74,12 +119,6 @@ const logs = () => {
                 res.destroy();
             }
         } else {
-            let bots: Bot[] = [];
-            if(req.serverBot !== undefined) {
-                bots = [req.serverBot];
-            } else {
-                bots = req.user?.accessibleBots(req.botApp.bots) as Bot[];
-            }
 
             const allReq = req.query.subreddit !== undefined && (req.query.subreddit as string).toLowerCase() === 'all';
 
@@ -114,15 +153,9 @@ const logs = () => {
                 botArr.push({
                     name: b.getBotName(),
                     system: systemLogs,
-                    all: formatted ? allLogs.map(x => {
-                        const {[MESSAGE]: fMessage, ...rest} = x;
-                        return {...rest, formatted: fMessage};
-                    }) : allLogs,
+                    all: allLogs.map(x => transformLog(x, {formatted, transports })),
                     subreddits: allReq ? [] : [...managerLogs.entries()].reduce((acc: any[], curr) => {
-                        const l = formatted ? curr[1].map(x => {
-                            const {[MESSAGE]: fMessage, ...rest} = x;
-                            return {...rest, formatted: fMessage};
-                            }) : curr[1];
+                        const l = curr[1].map(x => transformLog(x, {formatted, transports }));
                         acc.push({name: curr[0], logs: l});
                         return acc;
                     }, [])
@@ -135,4 +168,23 @@ const logs = () => {
     return [...middleware, response];
 }
 
+const transformLog = (obj: LogInfo, options: { formatted: boolean, transports: boolean }) => {
+    const {
+        [MESSAGE]: fMessage,
+        transport,
+        //@ts-ignore
+        name, // name is the name of the last transport
+        ...rest
+    } = obj;
+    const transformed: any = rest;
+    if (options.formatted) {
+        transformed.formatted = fMessage;
+    }
+    if (options.transports) {
+        transformed.transport = transport;
+        transformed.name = name;
+    }
+    return transformed;
+}
+
 export default logs;

src/Web/Server/routes/authenticated/user/status.ts

@@ -66,13 +66,13 @@ const status = () => {
 
         const subManagerData = [];
         for (const m of req.user?.accessibleSubreddits(bot) as Manager[]) {
-            const logs = req.manager === undefined || allReq || req.manager.getDisplay() === m.getDisplay() ? filterLogs(m.logs, {
-                    level: (level as string),
-                    // @ts-ignore
-                    sort,
-                    limit: limit as string,
-                    returnType: 'object'
-                }) as LogInfo[]: [];
+            // const logs = req.manager === undefined || allReq || req.manager.getDisplay() === m.getDisplay() ? filterLogs(m.logs, {
+            //         level: (level as string),
+            //         // @ts-ignore
+            //         sort,
+            //         limit: limit as string,
+            //         returnType: 'object'
+            //     }) as LogInfo[]: [];
 
             let retention = 'Unknown';
             if (m.resources !== undefined) {
@@ -89,7 +89,7 @@ const status = () => {
             const sd = {
                 name: m.displayLabel,
                 //linkName: s.replace(/\W/g, ''),
-                logs: logs || [], // provide a default empty value in case we truly have not logged anything for this subreddit yet
+                logs: [], // provide a default empty value in case we truly have not logged anything for this subreddit yet
                 botState: m.managerState,
                 eventsState: m.eventsState,
                 queueState: m.queueState,
@@ -237,14 +237,14 @@ const status = () => {
         const sharedSub = subManagerData.find(x => x.stats.cache.isShared);
         const sharedCount = sharedSub !== undefined ? sharedSub.stats.cache.currentKeyCount : 0;
         const scopes = req.user?.isInstanceOperator(bot) ? bot.client.scope : [];
-        const allSubLogs = subManagerData.map(x => x.logs).flat().sort(logSortFunc(sort as string)).slice(0, (limit as number) + 1);
-        const allLogs = filterLogs([...allSubLogs, ...(req.user?.isInstanceOperator(req.botApp) ? bot.logs : bot.logs.filter(x => x.user === req.user?.name))], {
-            level: (level as string),
-            // @ts-ignore
-            sort,
-            limit: limit as string,
-            returnType: 'object'
-        }) as LogInfo[];
+        //const allSubLogs = subManagerData.map(x => x.logs).flat().sort(logSortFunc(sort as string)).slice(0, (limit as number) + 1);
+        // const allLogs = filterLogs([...allSubLogs, ...(req.user?.isInstanceOperator(req.botApp) ? bot.logs : bot.logs.filter(x => x.user === req.user?.name))], {
+        //     level: (level as string),
+        //     // @ts-ignore
+        //     sort,
+        //     limit: limit as string,
+        //     returnType: 'object'
+        // }) as LogInfo[];
         let allManagerData: any = {
             name: 'All',
             status: bot.running ? 'RUNNING' : 'NOT RUNNING',
@@ -261,7 +261,7 @@ const status = () => {
                 causedBy: SYSTEM
             },
             dryRun: boolToString(bot.dryRun === true),
-            logs: allLogs,
+            logs: [],
             checks: checks,
             softLimit: bot.softLimit,
             hardLimit: bot.hardLimit,

src/Web/assets/browser.js

@@ -0,0 +1,5 @@
+const logform = require('logform');
+const tripleBeam = require('triple-beam');
+
+window.format = logform.format;
+window.beam = tripleBeam;

src/Web/assets/public/TextEncoderStream.js

@@ -0,0 +1,160 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Polyfill for TextEncoderStream and TextDecoderStream
+
+(function() {
+    'use strict';
+
+    if (typeof self.TextEncoderStream === 'function' &&
+        typeof self.TextDecoderStream === 'function') {
+        // The constructors exist. Assume that they work and don't replace them.
+        return;
+    }
+
+    if (typeof self.TextEncoder !== 'function') {
+        throw new ReferenceError('TextEncoder implementation required');
+    }
+
+    if (typeof self.TextDecoder !== 'function') {
+        throw new ReferenceError('TextDecoder implementation required');
+    }
+
+    // These symbols end up being different for every realm, so mixing objects
+    // created in one realm with methods created in another fails.
+    const codec = Symbol('codec');
+    const transform = Symbol('transform');
+
+    class TextEncoderStream {
+        constructor() {
+            this[codec] = new TextEncoder();
+            this[transform] =
+                new TransformStream(new TextEncodeTransformer(this[codec]));
+        }
+    }
+
+    class TextDecoderStream {
+        constructor(label = undefined, options = undefined) {
+            this[codec] = new TextDecoder(label, options);
+            this[transform] = new TransformStream(
+                new TextDecodeTransformer(this[codec]));
+        }
+    }
+
+    // ECMAScript class syntax will create getters that are non-enumerable, but we
+    // need them to be enumerable in WebIDL-style, so we add them manually.
+    // "readable" and "writable" are always delegated to the TransformStream
+    // object. Properties specified in |properties| are delegated to the
+    // underlying TextEncoder or TextDecoder.
+    function addDelegatingProperties(prototype, properties) {
+        for (const transformProperty of ['readable', 'writable']) {
+            addGetter(prototype, transformProperty, function() {
+                return this[transform][transformProperty];
+            });
+        }
+
+        for (const codecProperty of properties) {
+            addGetter(prototype, codecProperty, function() {
+                return this[codec][codecProperty];
+            });
+        }
+    }
+
+    function addGetter(prototype, property, getter) {
+        Object.defineProperty(prototype, property,
+            {
+                configurable: true,
+                enumerable: true,
+                get: getter
+            });
+    }
+
+    addDelegatingProperties(TextEncoderStream.prototype, ['encoding']);
+    addDelegatingProperties(TextDecoderStream.prototype,
+        ['encoding', 'fatal', 'ignoreBOM']);
+
+    class TextEncodeTransformer {
+        constructor() {
+            this._encoder = new TextEncoder();
+            this._carry = undefined;
+        }
+
+        transform(chunk, controller) {
+            chunk = String(chunk);
+            if (this._carry !== undefined) {
+                chunk = this._carry + chunk;
+                this._carry = undefined;
+            }
+            const terminalCodeUnit = chunk.charCodeAt(chunk.length - 1);
+            if (terminalCodeUnit >= 0xD800 && terminalCodeUnit < 0xDC00) {
+                this._carry = chunk.substring(chunk.length - 1);
+                chunk = chunk.substring(0, chunk.length - 1);
+            }
+            const encoded = this._encoder.encode(chunk);
+            if (encoded.length > 0) {
+                controller.enqueue(encoded);
+            }
+        }
+
+        flush(controller) {
+            if (this._carry !== undefined) {
+                controller.enqueue(this._encoder.encode(this._carry));
+                this._carry = undefined;
+            }
+        }
+    }
+
+    class TextDecodeTransformer {
+        constructor(decoder) {
+            this._decoder = new TextDecoder(decoder.encoding, {
+                fatal: decoder.fatal,
+                ignoreBOM: decoder.ignoreBOM
+            });
+        }
+
+        transform(chunk, controller) {
+            const decoded = this._decoder.decode(chunk, {stream: true});
+            if (decoded != '') {
+                controller.enqueue(decoded);
+            }
+        }
+
+        flush(controller) {
+            // If {fatal: false} is in options (the default), then the final call to
+            // decode() can produce extra output (usually the unicode replacement
+            // character 0xFFFD). When fatal is true, this call is just used for its
+            // side-effect of throwing a TypeError exception if the input is
+            // incomplete.
+            var output = this._decoder.decode();
+            if (output !== '') {
+                controller.enqueue(output);
+            }
+        }
+    }
+
+    function exportAs(name, value) {
+        // Make it stringify as [object <name>] rather than [object Object].
+        value.prototype[Symbol.toStringTag] = name;
+        Object.defineProperty(self, name,
+            {
+                configurable: true,
+                enumerable: false,
+                writable: true,
+                value
+            });
+    }
+
+    exportAs('TextEncoderStream', TextEncoderStream);
+    exportAs('TextDecoderStream', TextDecoderStream);
+})();

src/Web/assets/public/logUtils.js

@@ -0,0 +1,120 @@
+const SPLAT = window.beam.SPLAT;
+const {combine, printf, timestamp, label, splat, errors} = window.format;
+
+window.formattedTime = (short, full) => `<span class="has-tooltip"><span style="margin-top:35px" class='tooltip rounded shadow-lg p-1 bg-gray-100 text-black space-y-3 p-2 text-left'>${full}</span><span>${short}</span></span>`;
+window.formatLogLineToHtml = (log, timestamp = undefined) => {
+    const val = typeof log === 'string' ? log : log[window.beam.MESSAGE];
+    const logContent = Autolinker.link(val, {
+        email: false,
+        phone: false,
+        mention: false,
+        hashtag: false,
+        stripPrefix: false,
+        sanitizeHtml: true,
+    })
+        .replace(/(\s*debug\s*):/i, '<span class="debug blue">$1</span>:')
+        .replace(/(\s*warn\s*):/i, '<span class="warn yellow">$1</span>:')
+        .replace(/(\s*info\s*):/i, '<span class="info green">$1</span>:')
+        .replace(/(\s*error\s*):/i, '<span class="error red">$1</span>:')
+        .replace(/(\s*verbose\s*):/i, '<span class="error purple">$1</span>:')
+        .replaceAll('\n', '<br />');
+    //.replace(HYPERLINK_REGEX, '<a target="_blank" href="$&">$&</a>');
+    let line;
+
+    let timestampString = timestamp;
+    if(timestamp === undefined && typeof log !== 'string') {
+        timestampString = log.timestamp;
+    }
+
+    if(timestampString !== undefined) {
+        const timeStampReplacement = formattedTime(dayjs(timestampString).format('HH:mm:ss z'), timestampString);
+        const splitLine = logContent.split(timestampString);
+        line = `<div class="logLine">${splitLine[0]}${timeStampReplacement}<span style="white-space: pre-wrap">${splitLine[1]}</span></div>`;
+    } else {
+        line = `<div style="white-space: pre-wrap" class="logLine">${logContent}</div>`
+    }
+    return line;
+}
+
+window.formatNumber = (val, options) => {
+    const {
+        toFixed = 2,
+        defaultVal = null,
+        prefix = '',
+        suffix = '',
+        round,
+    } = options || {};
+    let parsedVal = typeof val === 'number' ? val : Number.parseFloat(val);
+    if (Number.isNaN(parsedVal)) {
+        return defaultVal;
+    }
+    let prefixStr = prefix;
+    const {enable = false, indicate = true, type = 'round'} = round || {};
+    if (enable && !Number.isInteger(parsedVal)) {
+        switch (type) {
+            case 'round':
+                parsedVal = Math.round(parsedVal);
+                break;
+            case 'ceil':
+                parsedVal = Math.ceil(parsedVal);
+                break;
+            case 'floor':
+                parsedVal = Math.floor(parsedVal);
+        }
+        if (indicate) {
+            prefixStr = `~${prefix}`;
+        }
+    }
+    const localeString = parsedVal.toLocaleString(undefined, {
+        minimumFractionDigits: toFixed,
+        maximumFractionDigits: toFixed,
+    });
+    return `${prefixStr}${localeString}${suffix}`;
+};
+logFormatter = printf(({
+                                   level,
+                                   message,
+                                   labels = ['App'],
+                                   subreddit,
+                                   bot,
+                                   instance,
+                                   leaf,
+                                   itemId,
+                                   timestamp,
+                                   durationMs,
+                                   // @ts-ignore
+                                   [SPLAT]: splatObj,
+                                   stack,
+                                   ...rest
+                               }) => {
+    let stringifyValue = splatObj !== undefined ? JSON.stringify(splatObj) : '';
+    let msg = message;
+    let stackMsg = '';
+    if (stack !== undefined) {
+        const stackArr = stack.split('\n');
+        const stackTop = stackArr[0];
+        const cleanedStack = stackArr
+            .slice(1) // don't need actual error message since we are showing it as msg
+            .join('\n'); // rejoin with newline to preserve formatting
+        stackMsg = `\n${cleanedStack}`;
+        if (msg === undefined || msg === null || typeof message === 'object') {
+            msg = stackTop;
+        } else {
+            stackMsg = `\n${stackTop}${stackMsg}`
+        }
+    }
+
+    let nodes = labels;
+    if (leaf !== null && leaf !== undefined && !nodes.includes(leaf)) {
+        nodes.push(leaf);
+    }
+    const labelContent = `${nodes.map((x) => `[${x}]`).join(' ')}`;
+
+    return `${timestamp} ${level.padEnd(7)}: ${instance !== undefined ? `|${instance}| ` : ''}${bot !== undefined ? `~${bot}~ ` : ''}${subreddit !== undefined ? `{${subreddit}} ` : ''}${labelContent} ${msg}${durationMs !== undefined ? ` Elapsed: ${durationMs}ms (${window.formatNumber(durationMs/1000)}s) ` : ''}${stringifyValue !== '' ? ` ${stringifyValue}` : ''}${stackMsg}`;
+});
+
+window.formatLog = (logObj) => {
+    const formatted = logFormatter.transform(logObj);
+    const html = window.formatLogLineToHtml(formatted);
+    return {...formatted, html};
+}

src/Web/assets/views/status.ejs

@@ -672,41 +672,10 @@
     dayjs.extend(window.dayjs_plugin_duration)
     dayjs.extend(window.dayjs_plugin_relativeTime)
     dayjs.extend(window.dayjs_plugin_isSameOrAfter)
-    window.formattedTime = (short, full) => `<span class="has-tooltip"><span style="margin-top:35px" class='tooltip rounded shadow-lg p-1 bg-gray-100 text-black space-y-3 p-2 text-left'>${full}</span><span>${short}</span></span>`;
-    window.formatLogLineToHtml = (log, timestamp = undefined) => {
-        const val = typeof log === 'string' ? log : log['MESSAGE'];
-        const logContent = Autolinker.link(val, {
-            email: false,
-            phone: false,
-            mention: false,
-            hashtag: false,
-            stripPrefix: false,
-            sanitizeHtml: true,
-        })
-            .replace(/(\s*debug\s*):/i, '<span class="debug blue">$1</span>:')
-            .replace(/(\s*warn\s*):/i, '<span class="warn yellow">$1</span>:')
-            .replace(/(\s*info\s*):/i, '<span class="info green">$1</span>:')
-            .replace(/(\s*error\s*):/i, '<span class="error red">$1</span>:')
-            .replace(/(\s*verbose\s*):/i, '<span class="error purple">$1</span>:')
-            .replaceAll('\n', '<br />');
-        //.replace(HYPERLINK_REGEX, '<a target="_blank" href="$&">$&</a>');
-        let line = '';
-
-        let timestampString = timestamp;
-        if(timestamp === undefined && typeof log !== 'string') {
-            timestampString = log.timestamp;
-        }
-
-        if(timestampString !== undefined) {
-            const timeStampReplacement = formattedTime(dayjs(timestampString).format('HH:mm:ss z'), timestampString);
-            const splitLine = logContent.split(timestampString);
-            line = `<div class="logLine">${splitLine[0]}${timeStampReplacement}<span style="white-space: pre-wrap">${splitLine[1]}</span></div>`;
-        } else {
-            line = `<div style="white-space: pre-wrap" class="logLine">${logContent}</div>`
-        }
-        return line;
-    }
 </script>
+<script src="public/TextEncoderStream.js"></script>
+<script src="public/browserBundle.js"></script>
+<script src="public/logUtils.js"></script>
 <script>
     window.sort = 'desc';
 
@@ -878,46 +847,363 @@
                 history.pushState(null, '', newRelativePathQuery);
             }
             const activeSub = document.querySelector(`[data-subreddit="${subreddit}"][data-bot="${bot}"].sub`);
-            if(!activeSub.classList.contains('seen')) {
-                //firstSub.classList.add('seen');
-
-                //subreddit = firstSub.dataset.subreddit;
-                //bot = subSection.dataset.bot;
-                level = document.querySelector(`[data-subreddit="${subreddit}"] [data-type="level"]`).value;
-                sort = document.querySelector(`[data-subreddit="${subreddit}"] [data-type="sort"]`).value;
-                limitSel = document.querySelector(`[data-subreddit="${subreddit}"] [data-type="limit"]`).value;
-
-                fetch(`/api/logs?instance=<%= instanceId %>&bot=${bot}&subreddit=${subreddit}&level=${level}&sort=${sort}&limit=${limitSel}&stream=false&formatted=true`).then((resp) => {
-                    if (!resp.ok) {
-                        console.error('Response was not OK from logs GET');
-                    } else {
-                        resp.json().then((data) => {
-                            const logContainer = document.querySelector(`[data-subreddit="${subreddit}"] .logs`);
-                            const logLines = (subreddit.toLowerCase() === 'all' ? data[0].all : data[0].subreddits[0].logs).map(x => {
-                                let fString = x.formatted;
-                                if(x.bot !== undefined) {
-                                    fString = fString.replace(`~${x.bot}~ `, '');
-                                }
-                                if(x.subreddit !== undefined && subreddit !== 'All') {
-                                    fString = fString.replace(`{${x.subreddit}} `, '');
-                                }
-                                return window.formatLogLineToHtml(fString, x.timestamp)
-                            }).join('');
-                            logContainer.insertAdjacentHTML('afterbegin', logLines);
-                            activeSub.classList.add('seen');
-                        });
-                    }
-                }).catch((err) => {
-                    console.log(err);
-                });
-
-            }
             if(window.socket !== undefined) {
                 window.socket.emit('viewing', {bot, subreddit});
             }
         });
     });
 
+    let recentlySeen = new Map();
+
+
+    function getLogBlock(bot, subreddit) {
+
+        console.debug(`Getting initial logs for ${bot} ${subreddit}`);
+
+        level = document.querySelector(`[data-subreddit="${subreddit}"] [data-type="level"]`).value;
+        sort = document.querySelector(`[data-subreddit="${subreddit}"] [data-type="sort"]`).value;
+        limitSel = document.querySelector(`[data-subreddit="${subreddit}"] [data-type="limit"]`).value;
+
+        return new Promise((resolve, reject) => {
+            fetch(`/api/logs?instance=<%= instanceId %>&bot=${bot}&subreddit=${subreddit}&level=${level}&sort=${sort}&limit=${limitSel}&stream=false&formatted=false`).then((resp) => {
+                if (!resp.ok) {
+                    console.error('Response was not OK from logs GET');
+                    reject('Response was not OK from logs GET');
+                } else {
+                    resp.json().then((data) => {
+                        const logContainer = document.querySelector(`.sub[data-bot="${bot}"] .logs[data-subreddit="${subreddit}"]`);
+                        const logLines = (subreddit.toLowerCase() === 'all' ? data[0].all : data[0].subreddits[0].logs).map(x => {
+                            const logObj = window.formatLog(x);
+                            let fString = logObj[window.beam.MESSAGE];
+                            if(x.bot !== undefined) {
+                                fString = fString.replace(`~${x.bot}~ `, '');
+                            }
+                            if(x.subreddit !== undefined && subreddit !== 'All') {
+                                fString = fString.replace(`{${x.subreddit}} `, '');
+                            }
+                            return window.formatLogLineToHtml(fString, x.timestamp)
+                        }).join('');
+                        logContainer.insertAdjacentHTML('afterbegin', logLines);
+                        console.debug(`Done with initial logs for ${bot} ${subreddit}`);
+                        resolve();
+                        //activeSub.classList.add('seen');
+                    });
+                }
+            }).catch((err) => {
+                console.log(err);
+                reject(err);
+            });
+        });
+    }
+
+    // https://stackoverflow.com/a/66394121/1469797
+    function onVisible(element, callback) {
+        new IntersectionObserver((entries, observer) => {
+            entries.forEach(entry => {
+                if(entry.intersectionRatio > 0) {
+                    callback(element);
+                    //observer.disconnect();
+                }
+            });
+        }).observe(element);
+    }
+
+    function getStreamingLogs(sub, bot) {
+
+        console.debug(`Getting stream for ${bot} ${sub}`);
+
+        level = document.querySelector(`[data-subreddit="${sub}"] [data-type="level"]`).value;
+        sort = document.querySelector(`[data-subreddit="${sub}"] [data-type="sort"]`).value;
+        limitSel = document.querySelector(`[data-subreddit="${sub}"] [data-type="limit"]`).value;
+
+        const logContainer = document.querySelector(`.sub[data-bot="${bot}"] .logs[data-subreddit="${sub}"]`);
+
+        let textBuffer = '';
+
+        var controller = new AbortController();
+        var signal = controller.signal;
+
+        let lastFlush;
+        let bufferTimeout;
+
+        let bufferedLogs = [];
+
+        const formattedMsg = (x) => {
+            const logObj = window.formatLog(x);
+            let fString = logObj[window.beam.MESSAGE];
+            if(x.bot !== undefined) {
+                fString = fString.replace(`~${x.bot}~ `, '');
+            }
+            if(x.subreddit !== undefined && sub !== 'All') {
+                fString = fString.replace(`{${x.subreddit}} `, '');
+            }
+            return window.formatLogLineToHtml(fString, x.timestamp);
+        }
+
+        const flushLogs = () => {
+            let existingLogs;
+
+            //const el = document.querySelector(`[data-subreddit="${sub}"][data-bot="${bot}"].sub`);
+            //const logContainer = el.querySelector(`.logs`);
+
+            if(window.sort === 'desc' || window.sort === 'descending') {
+                bufferedLogs.forEach((l) => {
+                    logContainer.insertAdjacentHTML('afterbegin', formattedMsg(l));
+                })
+                existingLogs = Array.from(logContainer.querySelectorAll(`.logLine`));
+                logContainer.replaceChildren(...existingLogs.slice(0, limitSel));
+            } else {
+                bufferedLogs.forEach((l) => {
+                    logContainer.insertAdjacentHTML('beforeend', formattedMsg(l));
+                    existingLogs = Array.from(logContainer.querySelectorAll(`.logLine`));
+                    const overLimit = limitSel - existingLogs.length;
+                    logContainer.replaceChildren(...existingLogs.slice(overLimit -1, limitSel));
+                })
+            }
+            lastFlush = Date.now();
+            bufferedLogs = [];
+        }
+
+        const fetchPromise = fetch(`/api/logs?instance=<%= instanceId %>&bot=${bot}&subreddit=${sub}&level=${level}&sort=${sort}&limit=${limitSel}&stream=true&streamObjects=true&formatted=false`, {signal})
+            .then(response => response.body)
+            .then(rs =>
+                rs.pipeThrough(new TextDecoderStream())
+                    .pipeThrough(new TransformStream({
+                        transform(chunk, controller) {
+                            textBuffer += chunk;
+                            const lines = textBuffer.split('\n');
+                            for (const line of lines.slice(0, -1)) {
+                                controller.enqueue(line);
+                            }
+                            textBuffer = lines.slice(-1)[0];
+                        },
+                        flush(controller) {
+                            if (textBuffer) {
+                                controller.enqueue(textBuffer);
+                            }
+                        }
+                    }))
+
+                    // Parse JSON objects
+                    .pipeThrough(new TransformStream({
+                        transform(line, controller) {
+                            if (line) {
+                                controller.enqueue(
+                                    JSON.parse(line)
+                                );
+                            }
+                        }
+                    }))
+            ).catch((e) => {
+                console.warn(e);
+            });
+
+        fetchPromise.then(async res => {
+            const reader = res.getReader();
+            let keepReading = true;
+            while(keepReading) {
+                const {done, value} = await reader.read();
+                if(done) {
+                    keepReading = false;
+                    console.debug('done');
+                }
+                if(value) {
+                    //console.log(`((Logged For ${bot} ${sub})) ${value.message}`);
+
+                    bufferedLogs.push(value);
+
+                    if(lastFlush !== undefined && bufferTimeout !== undefined && ((Date.now() - lastFlush)/1000) > 3) {
+                        //console.log('Immediate flush');
+                        clearTimeout(bufferTimeout);
+                        bufferTimeout = undefined;
+                        flushLogs();
+                    } else {
+                        //console.log('Using timeout');
+                        clearTimeout(bufferTimeout);
+                        bufferTimeout = setTimeout(() => {flushLogs();}, 1000);
+                    }
+                }
+            }
+            /*                function read() {
+                                reader.read().then(({done, value}) => {
+                                    if(done) {
+                                        console.log('done');
+                                        return;
+                                    }
+                                    if(value) {
+                                        console.log(value);
+                                        read();
+                                    }
+                                });
+                            }
+                            read();*/
+        }).catch((e) => {
+            if(e.name !== 'AbortError') {
+                console.error(e);
+            }
+        });
+
+        const existing = recentlySeen.get(`${bot}.${sub}`) || {};
+        recentlySeen.set(`${bot}.${sub}`, {...existing, fetch: fetchPromise, controller});
+    }
+
+    function updateLiveStats(resp) {
+        let el;
+        let isAll = resp.name.toLowerCase() === 'all';
+        if(isAll) {
+            // got all
+            el = document.querySelector(`[data-subreddit="All"][data-bot="${resp.bot}"].sub`);
+        } else {
+            // got subreddit
+            el = document.querySelector(`[data-subreddit="${resp.name}"].sub`);
+        }
+
+        if(resp.system.running && el.classList.contains('offline')) {
+            el.classList.remove('offline');
+        } else if(!resp.system.running && !el.classList.contains('offline')) {
+            el.classList.add('offline');
+        }
+
+        el.querySelector('.runningActivities').innerHTML = resp.runningActivities;
+        el.querySelector('.queuedActivities').innerHTML = resp.queuedActivities;
+        el.querySelector('.delayedItemsCount').innerHTML = resp.delayedItems.length;
+        el.querySelector('.delayedItemsList').innerHTML = 'No delayed Items!';
+        if(resp.delayedItems.length > 0) {
+            el.querySelector('.delayedItemsList').innerHTML = '';
+            const now = dayjs();
+            const sorted = resp.delayedItems.map(x => ({...x, queuedAtUnix: x.queuedAt, queuedAt: dayjs.unix(x.queuedAt), dispatchAt: dayjs.unix(x.queuedAt + x.duration)}));
+            sorted.sort((a, b) => {
+                return a.dispatchAt.isSameOrAfter(b.dispatchAt) ? 1 : -1
+            });
+            const delayedItemDivs = sorted.map(x => {
+                const diffUntilNow = x.dispatchAt.diff(now);
+                const durationUntilNow = dayjs.duration(diffUntilNow, 'ms');
+                const queuedAtDisplay = x.queuedAt.format('HH:mm:ss z');
+                const durationDayjs = dayjs.duration(x.duration, 'seconds');
+                const durationDisplay = durationDayjs.humanize();
+                const cancelLink = `<a href="#" data-id="${x.id}" data-subreddit="${x.subreddit}" class="delayCancel">CANCEL</a>`;
+                return `<div>A <a href="https://reddit.com${x.permalink}">${x.submissionId !== undefined ? 'Comment' : 'Submssion'}</a>${isAll ? ` in <a href="https://reddit.com${x.subreddit}">${x.subreddit}</a> ` : ''} by <a href="https://reddit.com/u/${x.author}">${x.author}</a> queued by ${x.source} at ${queuedAtDisplay} for ${durationDisplay} (dispatches ${durationUntilNow.humanize(true)}) -- ${cancelLink}</div>`;
+            });
+            el.querySelector('.delayedItemsList').insertAdjacentHTML('afterbegin', delayedItemDivs.join(''));
+            el.querySelectorAll('.delayedItemsList .delayCancel').forEach(elm => {
+                elm.addEventListener('click', e => {
+                    e.preventDefault();
+                    const id = e.target.dataset.id;
+                    const subreddit = e.target.dataset.subreddit;
+                    fetch(`/api/delayed?instance=<%= instanceId %>&bot=${resp.bot}&subreddit=${subreddit}&id=${id}`, {
+                        method: 'DELETE'
+                    }).then((resp) => {
+                        if (!resp.ok) {
+                            console.error('Response was not OK from delay cancel');
+                        } else {
+                            console.log('Removed ok');
+                        }
+                    });
+                });
+            });
+        }
+
+        el.querySelector('.allStats .eventsCount').innerHTML = resp.stats.historical.eventsCheckedTotal;
+        el.querySelector('.allStats .checksCount').innerHTML = resp.stats.historical.checksTriggeredTotal;
+        el.querySelector('.allStats .rulesCount').innerHTML = resp.stats.historical.rulesTriggeredTotal;
+        el.querySelector('.allStats .actionsCount').innerHTML = resp.stats.historical.actionsRunTotal;
+
+        if(isAll) {
+            for(const elm of ['apiAvg','apiLimit','apiDepletion','nextHeartbeat', 'nextHeartbeatHuman', 'limitReset', 'limitResetHuman', 'nannyMode', 'startedAtHuman']) {
+                el.querySelector(`#${elm}`).innerHTML = resp[elm];
+            }
+            el.querySelector(`.botStatus`).innerHTML = resp.system.running ? 'ONLINE' : 'OFFLINE';
+        } else {
+            if(el.querySelector('.modPermissionsCount').innerHTML != resp.permissions.length) {
+                el.querySelector('.modPermissionsCount').innerHTML = resp.permissions.length;
+                el.querySelector('.modPermissionsList').innerHTML = '';
+                el.querySelector('.modPermissionsList').insertAdjacentHTML('afterbegin', resp.permissions.map(x => `<li class="font-mono">${x}</li>`).join(''));
+            }
+
+            for(const elm of ['botState', 'queueState', 'eventsState']) {
+                const state = resp[elm];
+                el.querySelector(`.${elm}`).innerHTML = `${state.state}${state.causedBy === 'system' ? '' : ' (user)'}`;
+            }
+            for(const elm of ['startedAt', 'startedAtHuman', 'wikiLastCheck', 'wikiLastCheckHuman', 'wikiRevision', 'wikiRevisionHuman', 'validConfig', 'delayBy']) {
+                el.querySelector(`.${elm}`).innerHTML = resp[elm];
+            }
+            el.querySelector(`.commentCheckCount`).innerHTML = resp.checks.comments;
+            el.querySelector(`.submissionCheckCount`).innerHTML = resp.checks.submissions;
+
+            const newInner = resp.pollingInfo.map(x => `<li>${x}</li>`).join('');
+            if(el.querySelector(`.pollingInfo`).innerHTML !== newInner) {
+                el.querySelector(`.pollingInfo`).innerHTML = newInner;
+            }
+        }
+    }
+
+    function getLiveStats(bot, sub) {
+        console.debug(`Getting live stats for ${bot} ${sub}`)
+        const fetchPromise = fetch(`/api/liveStats?instance=<%= instanceId %>&bot=${bot}&subreddit=${sub}`)
+            .then(response => response.json())
+            .then(resp => updateLiveStats(resp));
+    }
+
+    document.querySelectorAll('.sub').forEach(el => {
+        const sub = el.dataset.subreddit;
+        const bot = el.dataset.bot;
+        //console.log(`Focused on ${bot} ${sub}`);
+        onVisible(el, () => {
+            console.debug(`Focused on ${bot} ${sub}`);
+
+            const identifier = `${bot}.${sub}`;
+
+            recentlySeen.forEach((value, key) => {
+                const {timeout, liveStatsInt, ...rest} = value;
+                if(key === identifier && timeout !== undefined) {
+
+                    console.debug('Clearing timeout on own already set');
+                    clearTimeout(timeout);
+                    recentlySeen.set(key, rest);
+
+                } else if(key !== identifier) {
+
+                    // stop live stats for tabs we are not viewing
+                    clearInterval(liveStatsInt);
+
+                    // set timeout for logs we are not viewing
+                    if(timeout === undefined) {
+                        const t = setTimeout(() => {
+                            const k = key;
+                            const val = recentlySeen.get(k);
+                            if(val !== undefined) {
+                                const {controller} = val;
+                                console.debug(`timeout expired, stopping live data for ${k}`);
+                                if(controller !== undefined) {
+                                    console.debug('Stopping logs');
+                                    controller.abort();
+                                }
+                                // if(liveStatInt !== undefined) {
+                                //     console.log('Stopping live stats');
+                                //     clearInterval(liveStatInt);
+                                // }
+                                recentlySeen.delete(k);
+                            }
+                        },15000);
+                        recentlySeen.set(key, {timeout: t, liveStatsInt, ...rest});
+                    }
+                }
+            });
+
+            if(!recentlySeen.has(identifier)) {
+                getLogBlock(bot, sub).then(() => {
+                    getStreamingLogs(sub, bot);
+                });
+            }
+
+            // always get live stats for tab we just started viewing
+            getLiveStats(bot, sub);
+            const liveStatsInt = setInterval(() => getLiveStats(bot, sub), 5000);
+            recentlySeen.set(identifier, {liveStatsInt});
+
+        });
+    });
+
 
     var searchParams = new URLSearchParams(window.location.search);
     const shownSub = searchParams.get('sub') || 'All'
@@ -992,85 +1278,9 @@
         let shownBot = searchParams.get('bot');
         window.socket.emit('viewing', {bot: shownBot, subreddit: shownSub});
 
-        socket.on("log", data => {
-            const {
-                subreddit,
-                bot,
-                subredditMessage,
-                allMessage,
-                formattedMessage
-            } = data;
-            if(bot === undefined && subreddit === undefined) {
-                const sys = bufferedBot.get('system');
-                if(sys !== undefined) {
-                    sys.set('All', sys.get('All').concat(formattedMessage));
-                    bufferedBot.set('system', sys);
-                }
-            }
-            if(bot !== undefined) {
-                bufferedBot.set('All', bufferedBot.get('All').concat(allMessage));
+        // TODO web logging
+        // socket.on('log')
 
-                const buffBot = bufferedBot.get(bot) || newBufferedLogs();
-                buffBot.set('All', buffBot.get('All').concat(allMessage));
-                if (subreddit !== undefined) {
-                    buffBot.set(subreddit, (buffBot.get(subreddit) || []).concat(subredditMessage));
-                }
-                bufferedBot.set(bot, buffBot);
-            }
-
-
-            const flushLogs = () => {
-                bufferedBot.forEach((subLogs, botName) => {
-                    if(botName === 'All') {
-                        return;
-                    }
-                    subLogs.forEach((logs, subKey) => {
-                        // check sub exists -- may be a web log
-                        const el = document.querySelector(`[data-subreddit="${subKey}"][data-bot="${botName}"].sub.seen`);
-                        if(null !== el) {
-                            const limit = Number.parseInt(document.querySelector(`[data-subreddit="${subKey}"] [data-type="limit"]`).value);
-                            const logContainer = el.querySelector(`.logs`);
-                            let existingLogs;
-                            if(window.sort === 'desc' || window.sort === 'descending') {
-                                logs.forEach((l) => {
-                                    logContainer.insertAdjacentHTML('afterbegin', l);
-                                })
-                                existingLogs = Array.from(el.querySelectorAll(`.logs .logLine`));
-                                logContainer.replaceChildren(...existingLogs.slice(0, limit));
-                            } else {
-                                logs.forEach((l) => {
-                                    logContainer.insertAdjacentHTML('beforeend', l);
-                                    existingLogs = Array.from(el.querySelectorAll(`.logs .logLine`));
-                                    const overLimit = limit - existingLogs.length;
-                                    logContainer.replaceChildren(...existingLogs.slice(overLimit -1, limit));
-                                })
-                            }
-                        }
-                    });
-                });
-                lastFlush = Date.now();
-                bufferedBot = newBufferedBot();
-                //bufferedLogs = newBufferedLogs();
-                //console.log('Flushed Logs');
-            }
-
-            if(lastFlush !== undefined && bufferTimeout !== undefined && ((Date.now() - lastFlush)/1000) > 3) {
-                //console.log('Immediate flush');
-                clearTimeout(bufferTimeout);
-                bufferTimeout = undefined;
-                flushLogs();
-            } else {
-                //console.log('Using timeout');
-                clearTimeout(bufferTimeout);
-                bufferTimeout = setTimeout(() => {flushLogs();}, 1000);
-            }
-        });
-        socket.on("logClear", data => {
-            data.forEach((obj) => {
-                const n = obj.name === 'all' ? 'All' : obj.name;
-                document.querySelector(`[data-subreddit="${n}"].logs`).innerHTML = obj.logs;
-            })
-        });
         const subIndicators = ['red', 'green', 'yellow'];
         socket.on('opStats', (resp) => {
             for(const b of resp) {
@@ -1100,94 +1310,6 @@
             }
 
         });
-        socket.on('liveStats', (resp) => {
-            let el;
-            let isAll = resp.name.toLowerCase() === 'all';
-            if(isAll) {
-                // got all
-                el = document.querySelector(`[data-subreddit="All"][data-bot="${resp.bot}"].sub`);
-            } else {
-                // got subreddit
-                el = document.querySelector(`[data-subreddit="${resp.name}"].sub`);
-            }
-
-            if(resp.system.running && el.classList.contains('offline')) {
-                el.classList.remove('offline');
-            } else if(!resp.system.running && !el.classList.contains('offline')) {
-                el.classList.add('offline');
-            }
-
-            el.querySelector('.runningActivities').innerHTML = resp.runningActivities;
-            el.querySelector('.queuedActivities').innerHTML = resp.queuedActivities;
-            el.querySelector('.delayedItemsCount').innerHTML = resp.delayedItems.length;
-            el.querySelector('.delayedItemsList').innerHTML = 'No delayed Items!';
-            if(resp.delayedItems.length > 0) {
-                el.querySelector('.delayedItemsList').innerHTML = '';
-                const now = dayjs();
-                const sorted = resp.delayedItems.map(x => ({...x, dispatchAt: dayjs.unix(x.queuedAt + (x.durationMilli))}));
-                sorted.sort((a, b) => {
-                    return a.dispatchAt.isSameOrAfter(b.dispatchAt) ? 1 : -1
-                });
-                const delayedItemDivs = sorted.map(x => {
-                    const diffUntilNow = x.dispatchAt.diff(now)
-                    const durationUntilNow = dayjs.duration(diffUntilNow);
-                    const cancelLink = `<a href="#" data-id="${x.id}" data-subreddit="${x.subreddit}" class="delayCancel">CANCEL</a>`;
-                    return `<div>A <a href="https://reddit.com${x.permalink}">${x.submissionId !== undefined ? 'Comment' : 'Submssion'}</a>${isAll ? ` in <a href="https://reddit.com${x.subreddit}">${x.subreddit}</a> ` : ''} by <a href="https://reddit.com/u/${x.author}">${x.author}</a> queued by ${x.source} at ${dayjs.unix(x.queuedAt).format('HH:mm:ss z')} for ${x.duration} (dispatches in ${durationUntilNow.humanize()}) -- ${cancelLink}</div>`;
-                });
-                el.querySelector('.delayedItemsList').insertAdjacentHTML('afterbegin', delayedItemDivs.join(''));
-                el.querySelectorAll('.delayedItemsList .delayCancel').forEach(elm => {
-                    elm.addEventListener('click', e => {
-                        e.preventDefault();
-                        const id = e.target.dataset.id;
-                        const subreddit = e.target.dataset.subreddit;
-                        fetch(`/api/delayed?instance=<%= instanceId %>&bot=${resp.bot}&subreddit=${subreddit}&id=${id}`, {
-                            method: 'DELETE'
-                        }).then((resp) => {
-                            if (!resp.ok) {
-                                console.error('Response was not OK from delay cancel');
-                            } else {
-                                console.log('Removed ok');
-                            }
-                        });
-                    });
-                });
-            }
-
-            el.querySelector('.allStats .eventsCount').innerHTML = resp.stats.historical.eventsCheckedTotal;
-            el.querySelector('.allStats .checksCount').innerHTML = resp.stats.historical.checksTriggeredTotal;
-            el.querySelector('.allStats .rulesCount').innerHTML = resp.stats.historical.rulesTriggeredTotal;
-            el.querySelector('.allStats .actionsCount').innerHTML = resp.stats.historical.actionsRunTotal;
-
-            if(isAll) {
-                for(const elm of ['apiAvg','apiLimit','apiDepletion','nextHeartbeat', 'nextHeartbeatHuman', 'limitReset', 'limitResetHuman', 'nannyMode', 'startedAtHuman']) {
-                    el.querySelector(`#${elm}`).innerHTML = resp[elm];
-                }
-                el.querySelector(`.botStatus`).innerHTML = resp.system.running ? 'ONLINE' : 'OFFLINE';
-            } else {
-                if(el.querySelector('.modPermissionsCount').innerHTML != resp.permissions.length) {
-                    el.querySelector('.modPermissionsCount').innerHTML = resp.permissions.length;
-                    el.querySelector('.modPermissionsList').innerHTML = '';
-                    el.querySelector('.modPermissionsList').insertAdjacentHTML('afterbegin', resp.permissions.map(x => `<li class="font-mono">${x}</li>`).join(''));
-                }
-
-                for(const elm of ['botState', 'queueState', 'eventsState']) {
-                    const state = resp[elm];
-                    el.querySelector(`.${elm}`).innerHTML = `${state.state}${state.causedBy === 'system' ? '' : ' (user)'}`;
-                }
-                for(const elm of ['startedAt', 'startedAtHuman', 'wikiLastCheck', 'wikiLastCheckHuman', 'wikiRevision', 'wikiRevisionHuman', 'validConfig', 'delayBy']) {
-                    el.querySelector(`.${elm}`).innerHTML = resp[elm];
-                }
-                el.querySelector(`.commentCheckCount`).innerHTML = resp.checks.comments;
-                el.querySelector(`.submissionCheckCount`).innerHTML = resp.checks.submissions;
-
-                const newInner = resp.pollingInfo.map(x => `<li>${x}</li>`).join('');
-                if(el.querySelector(`.pollingInfo`).innerHTML !== newInner) {
-                    el.querySelector(`.pollingInfo`).innerHTML = newInner;
-                }
-            }
-
-            //console.log(resp);
-        });
     });
 
     socket.on('disconnect', () => {

src/util.ts

@@ -1,5 +1,4 @@
 import winston, {Logger} from "winston";
-import jsonStringify from 'safe-stable-stringify';
 import dayjs, {Dayjs, OpUnitType} from 'dayjs';
 import {Duration} from 'dayjs/plugin/duration.js';
 import Ajv from "ajv";
@@ -78,7 +77,7 @@ import {
     StatisticFrequencyOption,
     StringOperator
 } from "./Common/Infrastructure/Atomic";
-import {DurationComparison, GenericComparison} from "./Common/Infrastructure/Comparisons";
+import {DurationComparison} from "./Common/Infrastructure/Comparisons";
 import {
     AuthorOptions,
     FilterCriteriaDefaults,
@@ -239,7 +238,7 @@ export const defaultFormat = (defaultLabel = 'App') => printf(({
                                                                    stack,
                                                                    ...rest
                                                                }) => {
-    let stringifyValue = splatObj !== undefined ? jsonStringify(splatObj) : '';
+    let stringifyValue = splatObj !== undefined ? JSON.stringify(splatObj) : '';
     let msg = message;
     let stackMsg = '';
     if (stack !== undefined) {
@@ -258,7 +257,7 @@ export const defaultFormat = (defaultLabel = 'App') => printf(({
     }
 
     let nodes = labels;
-    if (leaf !== null && leaf !== undefined) {
+    if (leaf !== null && leaf !== undefined && !nodes.includes(leaf)) {
         nodes.push(leaf);
     }
     const labelContent = `${nodes.map((x: string) => `[${x}]`).join(' ')}`;
@@ -724,42 +723,6 @@ export const comparisonTextOp = (val1: number, strOp: string, val2: number): boo
     }
 }
 
-const GENERIC_VALUE_COMPARISON = /^\s*(?<opStr>>|>=|<|<=)\s*(?<value>\d+)(?<extra>\s+.*)*$/
-const GENERIC_VALUE_COMPARISON_URL = 'https://regexr.com/60dq4';
-export const parseGenericValueComparison = (val: string): GenericComparison => {
-    const matches = val.match(GENERIC_VALUE_COMPARISON);
-    if (matches === null) {
-        throw new InvalidRegexError(GENERIC_VALUE_COMPARISON, val, GENERIC_VALUE_COMPARISON_URL)
-    }
-    const groups = matches.groups as any;
-
-    return {
-        operator: groups.opStr as StringOperator,
-        value: Number.parseFloat(groups.value),
-        isPercent: false,
-        extra: groups.extra,
-        displayText: `${groups.opStr} ${groups.value}`
-    }
-}
-
-const GENERIC_VALUE_PERCENT_COMPARISON = /^\s*(?<opStr>>|>=|<|<=)\s*(?<value>\d+)\s*(?<percent>%?)(?<extra>.*)$/
-const GENERIC_VALUE_PERCENT_COMPARISON_URL = 'https://regexr.com/60a16';
-export const parseGenericValueOrPercentComparison = (val: string): GenericComparison => {
-    const matches = val.match(GENERIC_VALUE_PERCENT_COMPARISON);
-    if (matches === null) {
-        throw new InvalidRegexError(GENERIC_VALUE_PERCENT_COMPARISON, val, GENERIC_VALUE_PERCENT_COMPARISON_URL)
-    }
-    const groups = matches.groups as any;
-
-    return {
-        operator: groups.opStr as StringOperator,
-        value: Number.parseFloat(groups.value),
-        isPercent: groups.percent !== '',
-        extra: groups.extra,
-        displayText: `${groups.opStr} ${groups.value}${groups.percent === undefined || groups.percent === '' ? '': '%'}`
-    }
-}
-
 export const dateComparisonTextOp = (val1: Dayjs, strOp: StringOperator, val2: Dayjs, granularity?: OpUnitType): boolean => {
     switch (strOp) {
         case '>':
@@ -2666,7 +2629,6 @@ export const activityDispatchConfigToDispatch = (config: ActivityDispatchConfig,
         delay: parseDurationValToDuration(config.delay),
         tardyTolerant: tolerantVal,
         queuedAt: dayjs().utc(),
-        processing: false,
         id: nanoid(16),
         activity,
         action,
@@ -2793,3 +2755,21 @@ export const filterByTimeRequirement = (satisfiedEndtime: Dayjs, listSlice: Snoo
 
     return [truncatedItems.length !== listSlice.length, truncatedItems]
 }
+
+export const between = (val: number, a: number, b: number, inclusiveMin: boolean = false, inclusiveMax: boolean = false): boolean => {
+    var min = Math.min(a, b),
+        max = Math.max(a, b);
+
+    if(!inclusiveMin && !inclusiveMax) {
+        return val > min && val < max;
+    }
+    if(inclusiveMin && inclusiveMax) {
+        return val >= min && val <= max;
+    }
+    if(inclusiveMin) {
+        return val >= min && val < max;
+    }
+
+    // inclusive max
+    return val > min && val <= max;
+}

tests/languageProcessing.test.ts

@@ -0,0 +1,342 @@
+import {describe, it} from 'mocha';
+import chai,{assert} from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+import {
+    getContentLanguage,
+    getLanguageTypeFromValue,
+    getStringSentiment, parseTextToNumberComparison,
+    testSentiment
+} from "../src/Common/LangaugeProcessing";
+import {GenericComparison, RangedComparison} from "../src/Common/Infrastructure/Comparisons";
+
+chai.use(chaiAsPromised);
+
+const longNeutralEnglish = "This is a normal english sentence without emotion";
+const longNeutralEnglish2 = 'I am neutral on the current subject';
+const longNeutralEnglish3 = 'The midterms were an election that happened';
+const longNegativeEnglish = "I hate when idiots drive their bad cars terribly. 😡";
+const longPositiveEnglish = "We love to be happy and laugh on this wonderful, amazing day";
+
+const shortIndistinctEnglish = "metal gear";
+const shortIndistinctEnglish2 = "idk hole ref";
+
+const shortPositiveEnglish = "haha fun";
+const shortNegativeEnglish = "fuck you";
+const shortSlangPositiveEnglish = "lol lmao";
+const shortSlangNegativeEnglish = "get fuked";
+
+const longIndonesian = "setiap kali scroll mesti nampak dia nie haih";
+const shortIndonesian = "Saya bangga saya rasis";
+const shortPolish = 'Dobry wieczór';
+const longRussian = 'Чит на золото для аватарии без скачивания бесплатно';
+const longItalian = 'Sembra ormai passato un secolo, visto che gli anime sono praticamente scomparsi dalla televisione.';
+
+const shortRomanian = 'Tu știi unde sta?';
+const longRomanian = 'Deci , daca aveti chef de un mic protest , va astept la aceste coordonate';
+
+const longFrench = "J’approuve et à ce moment là ça se soigne plus malheureusement";
+
+const longSpanish = "La segunda parece una mezcla entre una convención de fanáticos de los monster truck y un vertedero.";
+const longPositiveSpanish = 'me encanta esta hermosa cancion';
+const longPositiveSpanish2 = 'Increíble muy divertido gracias por compartir';
+
+const longGerman = "bin mir auch sicher, dass zb mein 65er halb so viel wiegt wie ein kasten Bier";
+
+const shortEmojiNegative = "France 😫 😞 :(";
+const shortEmojiPositive = "France 😂 😄 😁";
+
+describe('Language Detection', function () {
+
+    describe('Derives language from user input', async function () {
+        it('gets from valid, case-insensitive alpha2', async function () {
+            const lang = await getLanguageTypeFromValue('eN');
+            assert.equal(lang.alpha2, 'en');
+        });
+        it('gets from valid, case-insensitive alpha3', async function () {
+            const lang = await getLanguageTypeFromValue('eNg');
+            assert.equal(lang.alpha2, 'en');
+        });
+        it('gets from valid, case-insensitive language name', async function () {
+            const lang = await getLanguageTypeFromValue('EnGlIsH');
+            assert.equal(lang.alpha2, 'en');
+        });
+
+        it('throws on invalid value', function () {
+            assert.isRejected(getLanguageTypeFromValue('pofdsfa'))
+        });
+    })
+
+    describe('Recognizes the language in moderately long content well', function () {
+        it('should recognize english', async function () {
+            const lang = await getContentLanguage(longPositiveEnglish);
+            assert.equal(lang.language.alpha2, 'en');
+            assert.isFalse(lang.usedDefault);
+            assert.isAtLeast(lang.bestGuess.score, 0.9);
+        });
+        it('should recognize french', async function () {
+            const lang = await getContentLanguage(longFrench);
+            assert.equal(lang.language.alpha2, 'fr');
+            assert.isFalse(lang.usedDefault);
+            assert.isAtLeast(lang.bestGuess.score, 0.9);
+        });
+        it('should recognize spanish', async function () {
+            const lang = await getContentLanguage(longSpanish);
+            assert.equal(lang.language.alpha2, 'es');
+            assert.isFalse(lang.usedDefault);
+            assert.isAtLeast(lang.bestGuess.score, 0.9);
+        });
+        it('should recognize german', async function () {
+            const lang = await getContentLanguage(longGerman);
+            assert.equal(lang.language.alpha2, 'de');
+            assert.isFalse(lang.usedDefault);
+            assert.isAtLeast(lang.bestGuess.score, 0.9);
+        });
+        it('should recognize indonesian', async function () {
+            const lang = await getContentLanguage(longIndonesian);
+            assert.equal(lang.language.alpha2, 'id');
+            assert.isFalse(lang.usedDefault);
+            assert.isAtLeast(lang.bestGuess.score, 0.9);
+        });
+    });
+
+    describe('Correctly handles short content classification', function () {
+        it('uses default language', async function () {
+
+            for (const content of [shortIndistinctEnglish, shortIndistinctEnglish2, shortIndonesian]) {
+                const lang = await getContentLanguage(content);
+                assert.equal(lang.language.alpha2, 'en', content);
+                assert.isTrue(lang.usedDefault, content);
+            }
+        });
+
+        it('uses best guess when default language is not provided', async function () {
+
+            for (const content of [shortIndistinctEnglish, shortIndistinctEnglish2, shortIndonesian]) {
+                const lang = await getContentLanguage(content, {defaultLanguage: false});
+                assert.isFalse(lang.usedDefault);
+            }
+        });
+    });
+});
+
+describe('Sentiment', function() {
+
+    describe('Is conservative when no default language is used for short content', function() {
+
+        it('should return unusable result for short, ambiguous english content', async function() {
+            for(const content of [shortIndistinctEnglish, shortIndistinctEnglish2]) {
+                const res = await getStringSentiment(content, {defaultLanguage: false});
+                assert.isFalse(res.usableScore);
+            }
+        });
+
+        it('should return unusable result for short, non-english content', async function() {
+            for(const content of [shortIndonesian, shortPolish, shortRomanian]) {
+                const res = await getStringSentiment(content, {defaultLanguage: false});
+                assert.isFalse(res.usableScore);
+            }
+        });
+
+    });
+
+    describe('Is conservative when language confidence is high for unsupported languages', function() {
+
+        it('should return unusable result for long, non-english content', async function() {
+            for(const content of [longIndonesian, longRussian, longItalian, longRomanian]) {
+                const res = await getStringSentiment(content);
+                assert.isFalse(res.usableScore, content);
+            }
+        });
+    });
+
+    describe('vader/wink supersedes low confidence language guess', function() {
+
+        it('should return usable result when valid words found by vader/wink', async function() {
+            for(const content of [shortPositiveEnglish,shortNegativeEnglish]) {
+                const res = await getStringSentiment(content, {defaultLanguage: false});
+                assert.isTrue(res.usableScore);
+            }
+        });
+
+        it('should return usable result when valid slang found by vader/wink', async function() {
+            for(const content of [shortSlangPositiveEnglish,shortSlangNegativeEnglish]) {
+                const res = await getStringSentiment(content, {defaultLanguage: false});
+                assert.isTrue(res.usableScore);
+            }
+        });
+
+        it('should return usable result when valid emojis found by vader/wink', async function() {
+            for(const content of [shortEmojiPositive,shortEmojiNegative]) {
+                const res = await getStringSentiment(content, {defaultLanguage: false});
+                assert.isTrue(res.usableScore);
+            }
+        });
+    })
+
+    describe('Detects correct sentiment', function() {
+
+        describe('In English', function() {
+
+            it('should detect positive sentiment', async function() {
+                for(const content of [shortEmojiPositive,longPositiveEnglish, shortPositiveEnglish, shortSlangPositiveEnglish]) {
+                    const res = await getStringSentiment(content);
+                    assert.isTrue(res.usableScore);
+                    assert.isAtLeast(res.scoreWeighted, 0.1);
+                }
+            });
+
+            it('should detect negative sentiment', async function() {
+                for(const content of [shortEmojiNegative,longNegativeEnglish, shortNegativeEnglish, shortSlangNegativeEnglish]) {
+                    const res = await getStringSentiment(content);
+                    assert.isTrue(res.usableScore);
+                    assert.isAtMost(res.scoreWeighted, -0.1);
+                }
+            });
+
+            it('should detect neutral sentiment', async function() {
+                for(const content of [longNeutralEnglish, longNeutralEnglish2, longNeutralEnglish3]) {
+                    const res = await getStringSentiment(content);
+                    assert.isTrue(res.usableScore, content);
+                    assert.isAtMost(res.scoreWeighted, 0.1, content);
+                    assert.isAtLeast(res.scoreWeighted, -0.1, content);
+                }
+            });
+
+            it('should detect neutral sentiment for short content when english is default language', async function() {
+                for(const content of [shortIndistinctEnglish, shortIndistinctEnglish2, shortPolish]) {
+                    const res = await getStringSentiment(content);
+                    assert.isTrue(res.usableScore);
+                    assert.isAtMost(res.scoreWeighted, 0.1, content);
+                    assert.isAtLeast(res.scoreWeighted, -0.1, content);
+                }
+            });
+        });
+
+        describe('In Spanish', function() {
+            it('should detect positive ', async function() {
+                for(const content of [longPositiveSpanish, longPositiveSpanish2]) {
+                    const res = await getStringSentiment(content);
+                    assert.isTrue(res.usableScore, longPositiveSpanish2);
+                    assert.isAtLeast(res.scoreWeighted, 0.1, longPositiveSpanish2);
+                }
+            });
+        });
+    });
+
+    describe('Testing', function () {
+
+        describe('Parsing user input to comparison', function() {
+
+            it(`parses 'is neutral'`, function() {
+                const res = parseTextToNumberComparison('is neutral') as RangedComparison;
+                assert.deepEqual(res.range, [-0.1, 0.1]);
+                assert.isFalse(res.not);
+            });
+
+            it(`parses 'is not neutral'`, function() {
+                const res = parseTextToNumberComparison('is not neutral') as RangedComparison;
+                assert.deepEqual(res.range, [-0.1, 0.1]);
+                assert.isTrue(res.not);
+            });
+
+            it(`parses 'is positive'`, function() {
+                const res = parseTextToNumberComparison('is positive') as GenericComparison;
+                assert.equal(res.operator, '>=');
+                assert.equal(res.value, 0.1);
+            });
+
+            it(`parses 'is very positive'`, function() {
+                const res = parseTextToNumberComparison('is very positive') as GenericComparison;
+                assert.equal(res.operator, '>=');
+                assert.equal(res.value, 0.3);
+            });
+
+            it(`parses 'is extremely positive'`, function() {
+                const res = parseTextToNumberComparison('is extremely positive') as GenericComparison;
+                assert.equal(res.operator, '>=');
+                assert.equal(res.value, 0.6);
+            });
+
+            it(`parses 'is negative'`, function() {
+                const res = parseTextToNumberComparison('is negative') as GenericComparison;
+                assert.equal(res.operator, '<=');
+                assert.equal(res.value, -0.1);
+            });
+
+            it(`parses 'is very negative'`, function() {
+                const res = parseTextToNumberComparison('is very negative') as GenericComparison;
+                assert.equal(res.operator, '<=');
+                assert.equal(res.value, -0.3);
+            });
+
+            it(`parses 'is extremely negative'`, function() {
+                const res = parseTextToNumberComparison('is extremely negative') as GenericComparison;
+                assert.equal(res.operator, '<=');
+                assert.equal(res.value, -0.6);
+            });
+
+            it(`parses negative negations`, function() {
+                const res = parseTextToNumberComparison('is not extremely negative') as GenericComparison;
+                assert.equal(res.operator, '>');
+                assert.equal(res.value, -0.6);
+            });
+
+            it(`parses positive negations`, function() {
+                const res = parseTextToNumberComparison('is not positive') as GenericComparison;
+                assert.equal(res.operator, '<');
+                assert.equal(res.value, 0.1);
+            });
+
+        });
+
+        it('should fail test if score is unusable', async function() {
+
+            const comparison = parseTextToNumberComparison('is positive');
+
+            for(const content of [shortIndistinctEnglish, shortIndistinctEnglish2, shortPolish, longRomanian]) {
+                const sentimentResult = await getStringSentiment(content, {defaultLanguage: false});
+
+                const testResult = testSentiment(sentimentResult, comparison);
+                assert.isFalse(testResult.passes);
+            }
+        });
+
+        it('should handle generic comparisons', async function() {
+
+            const comparison = parseTextToNumberComparison('is positive');
+
+            for(const content of [shortEmojiPositive,longPositiveEnglish, shortPositiveEnglish, shortSlangPositiveEnglish]) {
+                const sentimentResult = await getStringSentiment(content, {defaultLanguage: false});
+
+                const testResult = testSentiment(sentimentResult, comparison);
+                assert.isTrue(testResult.passes);
+            }
+        });
+
+        it('should handle ranged comparisons', async function() {
+
+            const comparison = parseTextToNumberComparison('is neutral');
+
+            for(const content of [longNeutralEnglish, longNeutralEnglish2, longNeutralEnglish3]) {
+                const sentimentResult = await getStringSentiment(content, {defaultLanguage: false});
+
+                const testResult = testSentiment(sentimentResult, comparison);
+                assert.isTrue(testResult.passes);
+            }
+        });
+
+        it('should handle negated ranged comparisons', async function() {
+
+            const comparison = parseTextToNumberComparison('is not neutral');
+
+            for(const content of [longPositiveEnglish, longPositiveSpanish, longNegativeEnglish]) {
+                const sentimentResult = await getStringSentiment(content, {defaultLanguage: false});
+
+                const testResult = testSentiment(sentimentResult, comparison);
+                assert.isTrue(testResult.passes, content);
+            }
+        });
+    });
+});
+
+

tests/utils.test.ts

@@ -4,12 +4,15 @@ import {
     COMMENT_URL_ID,
     parseDuration,
     parseDurationComparison,
-    parseGenericValueComparison,
-    parseGenericValueOrPercentComparison, parseLinkIdentifier,
+    parseLinkIdentifier,
     parseRedditEntity, removeUndefinedKeys, SUBMISSION_URL_ID
 } from "../src/util";
 import dayjs from "dayjs";
 import dduration, {DurationUnitType} from 'dayjs/plugin/duration.js';
+import {
+    parseGenericValueComparison,
+    parseGenericValueOrPercentComparison
+} from "../src/Common/Infrastructure/Comparisons";
 
 
 describe('Non-temporal Comparison Operations', function () {
@@ -150,7 +153,7 @@ describe('Parsing Reddit Entity strings', function () {
 describe('Config Parsing', function () {
     describe('Deep pruning of undefined keys on config objects', function () {
         it('removes undefined keys from objects', function () {
-            const obj = {
+            const obj: {keyA: string, keyB: string, keyC?: string } = {
                 keyA: 'foo',
                 keyB: 'bar',
                 keyC: undefined
@@ -166,7 +169,7 @@ describe('Config Parsing', function () {
             assert.isUndefined(removeUndefinedKeys(obj))
         })
         it('ignores arrays', function () {
-            const obj = {
+            const obj: { keyA?: string, keyB: string, keyC: any[] } = {
                 keyA: undefined,
                 keyB: 'bar',
                 keyC: ['foo', 'bar']