feat: Add dns lookup caching

Reduces dns traffic by 95%

docs/operator/configuration.md

@@ -36,7 +36,7 @@ configuration.
 * **FILE** -- Values specified in a YAML/JSON configuration file using the structure [in the schema](https://json-schema.app/view/%23?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Fcontext-mod%2Fmaster%2Fsrc%2FSchema%2FOperatorConfig.json)
   * When reading the **schema** if the variable is available at a level of configuration other than **FILE** it will be
     noted with the same symbol as above. The value shown is the default.
-* **ARG** -- Values specified as CLI arguments to the program (see [ClI Usage](#cli-usage) below)
+* **ARG** -- Values specified as CLI arguments to the program (see [CLI Usage](#cli-usage) below)
 
 ## File Configuration (Recommended)
 

package-lock.json

@@ -31,6 +31,7 @@
         "body-parser": "^1.19.0",
         "cache-manager": "^3.4.4",
         "cache-manager-redis-store": "^2.0.0",
+        "cacheable-lookup": "^6.1.0",
         "command-exists": "^1.2.9",
         "commander": "^8.0.0",
         "comment-json": "^4.1.1",
@@ -2395,9 +2396,9 @@
       }
     },
     "node_modules/cacheable-lookup": {
-      "version": "5.0.4",
-      "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz",
-      "integrity": "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==",
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-6.1.0.tgz",
+      "integrity": "sha512-KJ/Dmo1lDDhmW2XDPMo+9oiy/CeqosPguPCrgcVzKyZrL6pM1gU2GmPY/xo6OQPTUaA/c0kwHuywB4E6nmT9ww==",
       "engines": {
         "node": ">=10.6.0"
       }
@@ -4421,6 +4422,14 @@
         "url": "https://github.com/sindresorhus/got?sponsor=1"
       }
     },
+    "node_modules/got/node_modules/cacheable-lookup": {
+      "version": "5.0.4",
+      "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz",
+      "integrity": "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==",
+      "engines": {
+        "node": ">=10.6.0"
+      }
+    },
     "node_modules/graceful-fs": {
       "version": "4.2.10",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz",
@@ -12324,9 +12333,9 @@
       }
     },
     "cacheable-lookup": {
-      "version": "5.0.4",
-      "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz",
-      "integrity": "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA=="
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-6.1.0.tgz",
+      "integrity": "sha512-KJ/Dmo1lDDhmW2XDPMo+9oiy/CeqosPguPCrgcVzKyZrL6pM1gU2GmPY/xo6OQPTUaA/c0kwHuywB4E6nmT9ww=="
     },
     "cacheable-request": {
       "version": "7.0.2",
@@ -13900,6 +13909,13 @@
         "lowercase-keys": "^2.0.0",
         "p-cancelable": "^2.0.0",
         "responselike": "^2.0.0"
+      },
+      "dependencies": {
+        "cacheable-lookup": {
+          "version": "5.0.4",
+          "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz",
+          "integrity": "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA=="
+        }
       }
     },
     "graceful-fs": {

package.json

@@ -53,6 +53,7 @@
     "body-parser": "^1.19.0",
     "cache-manager": "^3.4.4",
     "cache-manager-redis-store": "^2.0.0",
+    "cacheable-lookup": "^6.1.0",
     "command-exists": "^1.2.9",
     "commander": "^8.0.0",
     "comment-json": "^4.1.1",

src/Action/ModNoteAction.ts

@@ -57,8 +57,18 @@ export class ModNoteAction extends Action {
             // nothing to do!
             noteCheckResult = 'existingNoteCheck=false so no existing note checks were performed.';
         } else {
+            const contextualCheck = {...this.existingNoteCheck};
+            let contextualNotes: string[] | undefined = undefined;
+            if(this.existingNoteCheck.note !== undefined && this.existingNoteCheck.note !== null) {
+                contextualNotes = [];
+                const notes = Array.isArray(this.existingNoteCheck.note) ? this.existingNoteCheck.note : [this.existingNoteCheck.note];
+                for(const n of notes) {
+                    contextualNotes.push((await this.renderContent(n, item, ruleResults, actionResults) as string))
+                }
+                contextualCheck.note = contextualNotes;
+            }
             const noteCheckCriteriaResult = await this.resources.isAuthor(item, {
-                modActions: [this.existingNoteCheck]
+                modActions: [contextualCheck]
             });
             noteCheckPassed = noteCheckCriteriaResult.passed;
             const {details} = buildFilterCriteriaSummary(noteCheckCriteriaResult);

src/Action/UserNoteAction.ts

@@ -54,8 +54,18 @@ export class UserNoteAction extends Action {
             // nothing to do!
             noteCheckResult = 'existingNoteCheck=false so no existing note checks were performed.';
         } else {
+            const contextualCheck = {...this.existingNoteCheck};
+            let contextualNotes: string[] | undefined = undefined;
+            if(this.existingNoteCheck.note !== undefined && this.existingNoteCheck.note !== null) {
+                contextualNotes = [];
+                const notes = Array.isArray(this.existingNoteCheck.note) ? this.existingNoteCheck.note : [this.existingNoteCheck.note];
+                for(const n of notes) {
+                    contextualNotes.push((await this.renderContent(n, item, ruleResults, actionResults) as string))
+                }
+                contextualCheck.note = contextualNotes;
+            }
             const noteCheckCriteriaResult = await this.resources.isAuthor(item, {
-                userNotes: [this.existingNoteCheck]
+                userNotes: [contextualCheck]
             });
             noteCheckPassed = noteCheckCriteriaResult.passed;
             const {details} = buildFilterCriteriaSummary(noteCheckCriteriaResult);

src/Rule/HistoryRule.ts

@@ -311,21 +311,24 @@ export class HistoryRule extends Rule {
 
         let criteriaMet = false;
         let failCriteriaResult: string = '';
+
+        const criteriaResultsSummary = criteriaResults.map(x => this.generateResultDataFromCriteria(x, true).result).join(this.condition === 'OR' ? ' OR ' : ' AND ');
+
         if (this.condition === 'OR') {
             criteriaMet = criteriaResults.some(x => x.triggered);
             if(!criteriaMet) {
-                failCriteriaResult = `${FAIL} No criteria was met`;
+                failCriteriaResult = `${FAIL} No criteria was met => ${criteriaResultsSummary}`;
             }
         } else {
             criteriaMet = criteriaResults.every(x => x.triggered);
             if(!criteriaMet) {
                 if(criteriaResults.some(x => x.triggered)) {
                     const met = criteriaResults.filter(x => x.triggered);
-                    failCriteriaResult = `${FAIL} ${met.length} out of ${criteriaResults.length} criteria met but Rule required all be met. Set log level to debug to see individual results`;
+                    failCriteriaResult = `${FAIL} ${met.length} out of ${criteriaResults.length} criteria met but Rule required all be met => ${criteriaResultsSummary}`;
                     const results = criteriaResults.map(x => this.generateResultDataFromCriteria(x, true));
                     this.logger.debug(`\r\n ${results.map(x => x.result).join('\r\n')}`);
                 } else {
-                    failCriteriaResult = `${FAIL} No criteria was met`;
+                    failCriteriaResult = `${FAIL} No criteria was met => ${criteriaResultsSummary}`;
                 }
             }
         }
@@ -335,8 +338,8 @@ export class HistoryRule extends Rule {
             const refCriteriaResults = criteriaResults.find(x => x.triggered);
             const resultData = this.generateResultDataFromCriteria(refCriteriaResults);
 
-            this.logger.verbose(`${PASS} ${resultData.result}`);
-            return Promise.resolve([true, this.getResult(true, resultData)]);
+            this.logger.verbose(`${PASS} ${criteriaResultsSummary}`);
+            return Promise.resolve([true, this.getResult(true, {data: resultData.data, result: criteriaResultsSummary})]);
         } else {
             // log failures for easier debugging
             for(const res of criteriaResults) {

src/Rule/MHSRule.ts

@@ -241,6 +241,8 @@ export class MHSRule extends Rule {
             res = await this.callMHS(content);
             if(res.response.toLowerCase() === 'success') {
                 await this.resources.cache.set(key, res, {ttl: this.resources.ttl.wikiTTL});
+            } else if(res.response.toLowerCase().includes('authentication failure')) {
+                throw new CMError(`MHS Request failed with Authentication failure. You most likely need to generate a new API key.`);
             }
             return res;
         }

src/Subreddit/ModNotes/ModNote.ts

@@ -167,7 +167,15 @@ export class ModNote {
             if (referenceItem === undefined) {
                 throw new CMError('Criteria wants to check if mod note references activity but not activity was given.');
             }
-            const isCurrentActivity = this.action.actedOn !== undefined && referenceItem !== undefined && this.action.actedOn.name === referenceItem.name;
+            let isCurrentActivity = false;
+            if(referenceItem !== undefined) {
+                if(this.action.actedOn !== undefined) {
+                    isCurrentActivity = this.action.actedOn.name === referenceItem.name;
+                }
+                if(isCurrentActivity === false && this.note !== undefined && this.note.actedOn !== undefined) {
+                    isCurrentActivity = this.note.actedOn.name === referenceItem.name;
+                }
+            }
             if ((referencesCurrentActivity === true && !isCurrentActivity) || (referencesCurrentActivity === false && isCurrentActivity)) {
                 return false;
             }

src/Subreddit/ModNotes/ModUserNote.ts

@@ -33,9 +33,19 @@ export class ModUserNote {
     }
 
     toRaw(): ModUserNoteRaw {
+        let id = undefined;
+        if(this.actedOn !== undefined) {
+            if(this.actedOn instanceof Submission) {
+                id = `t3_${this.actedOn.id}`;
+            } else if(this.actedOn instanceof Comment) {
+                id = `t1_${this.actedOn.id}`;
+            } else if(this.actedOn instanceof RedditUser) {
+                id = `t2_${this.actedOn.id}`;
+            }
+        }
         return {
             note: this.note,
-            reddit_id: this.actedOn !== undefined ? this.actedOn.id : undefined,
+            reddit_id: id,
             label: this.label
         }
     }

src/Subreddit/SubredditResources.ts

@@ -1287,7 +1287,7 @@ export class SubredditResources {
                     }
                     preFilteredPrefetchedActivities = await this.filterListingWithHistoryOptions(preFilteredPrefetchedActivities, user, options.filterOn?.pre);
                 }
-                let unFilteredItems: SnoowrapActivity[] | undefined = [...preFilteredPrefetchedActivities];
+                let unFilteredItems: SnoowrapActivity[] | undefined = undefined;
                 pre = pre.concat(preFilteredPrefetchedActivities);
 
                 const { func: listingFunc } = listingData;
@@ -1346,7 +1346,7 @@ export class SubredditResources {
 
                     if(satisfiedPreEndtime !== undefined || satisfiedPreCount !== undefined) {
                         if(unFilteredItems === undefined) {
-                            unFilteredItems = [];
+                            unFilteredItems = [...preFilteredPrefetchedActivities];
                         }
                         // window has pre filtering, need to check if fallback max would be hit
                         if(satisfiedPreEndtime !== undefined) {

src/index.ts

@@ -1,6 +1,9 @@
 import winston from 'winston';
 import 'winston-daily-rotate-file';
 import dayjs from 'dayjs';
+import http from 'http';
+import https from 'https';
+import CacheableLookup from 'cacheable-lookup';
 import utc from 'dayjs/plugin/utc.js';
 import advancedFormat from 'dayjs/plugin/advancedFormat';
 import tz from 'dayjs/plugin/timezone';
@@ -9,7 +12,6 @@ import relTime from 'dayjs/plugin/relativeTime.js';
 import sameafter from 'dayjs/plugin/isSameOrAfter.js';
 import samebefore from 'dayjs/plugin/isSameOrBefore.js';
 import weekOfYear from 'dayjs/plugin/weekOfYear.js';
-import {Manager} from "./Subreddit/Manager";
 import {Command, Argument} from 'commander';
 
 import {
@@ -40,6 +42,17 @@ dayjs.extend(tz);
 dayjs.extend(advancedFormat);
 dayjs.extend(weekOfYear);
 
+const cacheable = new CacheableLookup({
+    // cache dns entries for 60 seconds
+    maxTtl: 60,
+    // fallback to node lookup for 10 minutes in the event of a failure for 10 minutes
+    fallbackDuration: 600
+});
+
+// replace node native request agents, globally, so they used cached dns lookup
+cacheable.install(http.globalAgent);
+cacheable.install(https.globalAgent);
+
 const commentReg = parseLinkIdentifier([COMMENT_URL_ID]);
 const submissionReg = parseLinkIdentifier([SUBMISSION_URL_ID]);