Fix heartbeat interval

* Separate manager instantiation from configuration flow so config can be reloaded
* Move wiki page parsing into manager for better encapsulation
* Check for wiki revision date on heartbeat and on checks if older than one minute
* Catch config parsing issues and retry on next heartbeat

.idea/redditcontextbot.iml

@@ -5,7 +5,9 @@
       <excludeFolder url="file://$MODULE_DIR$/temp" />
       <excludeFolder url="file://$MODULE_DIR$/.tmp" />
       <excludeFolder url="file://$MODULE_DIR$/tmp" />
+      <excludeFolder url="file://$MODULE_DIR$/src/logs" />
     </content>
+    <content url="file://$MODULE_DIR$/node_modules" />
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>

Dockerfile

@@ -24,4 +24,4 @@ RUN mkdir -p $log_dir
 VOLUME $log_dir
 ENV LOG_DIR=$log_dir
 
-CMD [ "node", "src/index.js" ]
+CMD [ "node", "src/index.js", "run" ]

README.md

@@ -19,17 +19,23 @@ Some feature highlights:
 * Simple rule-action behavior can be combined to create any level of complexity in behavior
 * One instance can handle managing many subreddits (as many as it has moderator permissions in!)
 * Per-subreddit configuration is handled by JSON stored in the subreddit wiki
-* Any text-based actions (comment, submission, message, etc...) can be configured via a wiki page or raw text in JSON
+* Any text-based actions (comment, submission, message, usernotes, etc...) can be configured via a wiki page or raw text in JSON
 * All text-based actions support [mustache](https://mustache.github.io) templating
 * History-based rules support multiple "valid window" types -- [ISO 8601 Durations](https://en.wikipedia.org/wiki/ISO_8601#Durations), [Day.js Durations](https://day.js.org/docs/en/durations/creating), and submission/comment count limits.
-* All rules support skipping behavior based on author criteria -- name, css flair/text, and moderator status
-* Docker container support *(coming soon...)*
+* Checks/Rules support skipping behavior based on:
+  * author criteria (name, css flair/text, moderator status, and [Toolbox User Notes](https://www.reddit.com/r/toolbox/wiki/docs/usernotes))
+  * Activity state (removed, locked, distinguished, etc.)
+* Rules and Actions support named references so you write rules/actions once and reference them anywhere
+* User-configurable global/subreddit-level API caching
+* Support for [Toolbox User Notes](https://www.reddit.com/r/toolbox/wiki/docs/usernotes) as criteria or Actions (writing notes)
+* Docker container support
 
-## Table of Contents
+# Table of Contents
 
 * [How It Works](#how-it-works)
 * [Installation](#installation)
 * [Configuration](#configuration)
+  * [Examples](#examples)
 * [Usage](#usage)
 
 ### How It Works
@@ -78,88 +84,120 @@ Adding [**environmental variables**](#usage) to your `docker run` command will p
 docker run -e "CLIENT_ID=myId" ... foxxmd/reddit-context-bot
 ```
 
+### [Heroku Quick Deploy](https://heroku.com/about)
+[![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://dashboard.heroku.com/new?template=https://github.com/FoxxMD/reddit-context-bot)
+
+
 ## Configuration
 
-Context Bot's [configuration schema](/src/Schema/App.json) conforms to [JSON Schema](https://json-schema.org/) Draft 7.
+Context Bot's configuration can be written in JSON, [JSON5](https://json5.org/) or YAML. It's [schema](/src/Schema/App.json) conforms to [JSON Schema Draft 7](https://json-schema.org/).
 
 I suggest using [Atlassian JSON Schema Viewer](https://json-schema.app/start) ([direct link](https://json-schema.app/view/%23?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Freddit-context-bot%2Fmaster%2Fsrc%2FSchema%2FApp.json)) so you can view all documentation while also interactively writing and validating your config! From there you can drill down into any object, see its requirements, view an example JSON document, and live-edit your configuration on the right-hand side.
 
-### Example Config
+### Examples
 
-Below is a configuration fulfilling the example given at the start of this readme:
+Read through the [Examples](/examples) section for a thorough introduction to all the **Rules**, in-depth concepts, and sample configuration files.
 
-```json
+### Action Templating
+
+Actions that can submit text (Report, Comment) will have their `content` values run through a [Mustache Template](https://mustache.github.io/). This means you can insert data generated by Rules into your text before the Action is performed.
+
+See here for a [cheatsheet](https://gist.github.com/FoxxMD/d365707cf99fdb526a504b8b833a5b78) and [here](https://www.tsmean.com/articles/mustache/the-ultimate-mustache-tutorial/) for a more thorough tutorial.
+
+All Actions with `content` have access to this data:
+
+```json5
 {
-  "checks": [
-    {
-      "name": "repeatSpam",
-      "kind": "submission",
-      "rules": [
-        {
-          "kind": "repeatSubmission",
-          "gapAllowance": 2,
-          "threshold": 10
-        }
-      ],
-      "actions": [
-        {
-          "kind": "remove"
-        },
-        {
-          "kind": "comment",
-          "content": "Thank you for your submission but we do not allow mass crossposting. Your submission has been removed",
-          "distingish": true
-        }
-      ]
+    item: {
+        kind: 'string', // the type of item (comment/submission)
+        author: 'string', // name of the item author (reddit user)
+        permalink: 'string', // a url to the item
+        url: 'string', // if the item is a Submission then its URL (external for link type submission, reddit link for self-posts)
+        title: 'string', // if the item is a Submission, then the title of the Submission,
+        botLink: 'string' // a link to the bot's FAQ
     },
-    {
-      "name": "selfPromoActivity",
-      "kind": "submission",
-      "rules": [
-        {
-          "kind": "recentActivity",
-          "thresholds": [
-            {
-              "subreddits": [
-                "YouTubeSubscribeBoost",
-                "AdvertiseYourVideos"
-              ]
-            }
-          ]
-        }
-      ],
-      "actions": [
-        {
-          "kind": "report",
-          "content": "CB: self-promo"
-        },
-        {
-          "kind": "comment",
-          "content": "wiki:botconfig/contextbot/reportSelfPromo",
-          "distingish": true
-        }
-      ]
+    rules: {
+        // contains all rules that were run and are accessible using the name, lowercased, with all spaces/dashes/underscores removed
     }
-  ]
 }
 
 ```
 
+The properties of `rules` are accessible using the name, lower-cased, with all spaces/dashes/underscores. If no name is given `kind` is used as `name` Example:
+
+```
+"rules": [
+  {
+    "name": "My Custom-Recent Activity Rule", // mycustomrecentactivityrule
+    "kind": "recentActivity"
+  },
+  {
+    // name = repeatsubmission
+    "kind": "repeatActivity",
+  }
+]
+```
+
+**To see what data is available for individual Rules [consult the schema](#configuration) for each Rule.**
+
+#### Quick Templating Tutorial
+
+<details>
+
+As a quick example for how you will most likely be using templating -- wrapping a variable in curly brackets, `{{variable}}`, will cause the variable value to be rendered instead of the brackets:
+```
+myVariable = 50;
+myOtherVariable = "a text fragment"
+template = "This is my template, the variable is {{myVariable}}, my other variable is {{myOtherVariable}}, and that's it!";
+
+console.log(Mustache.render(template, {myVariable});
+// will render...
+"This is my template, the variable is 50, my other variable is a text fragment, and that's it!";
+```
+
+**Note: When accessing an object or its properties you must use dot notation**
+```
+const item = {
+aProperty: 'something',
+anotherObject: {
+bProperty: 'something else'
+}
+}
+const content = "My content will render the property {{item.aProperty}} like this, and another nested property {{item.anotherObject.bProperty}} like this."
+```
+</details>
+
 ## Usage
 
-`npm run start [list,of,subreddits] [...--options]`
+```
+Usage: index [options] [command]
 
-CLI options take precedence over environmental variables
+Options:
+  -c, --clientId <id>                          Client ID for your Reddit application (default: process.env.CLIENT_ID)
+  -e, --clientSecret <secret>                  Client Secret for your Reddit application (default: process.env.CLIENT_SECRET)
+  -a, --accessToken <token>                    Access token retrieved from authenticating an account with your Reddit Application (default: process.env.ACCESS_TOKEN)
+  -r, --refreshToken <token>                   Refresh token retrieved from authenticating an account with your Reddit Application (default: process.env.REFRESH_TOKEN)
+  -s, --subreddits <list...>                   List of subreddits to run on. Bot will run on all subs it has access to if not defined (default: process.env.SUBREDDITS (comma-seperated))
+  -d, --logDir <dir>                           Absolute path to directory to store rotated logs in (default: process.env.LOG_DIR || process.cwd()/logs)
+  -l, --logLevel <level>                       Log level (default: process.env.LOG_LEVEL || info)
+  -w, --wikiConfig <path>                      Relative url to contextbot wiki page EX https://reddit.com/r/subreddit/wiki/<path> (default: process.env.WIKI_CONFIG || 'botconfig/contextbot')
+  --snooDebug                                  Set Snoowrap to debug (default: process.env.SNOO_DEBUG || false)
+  --authorTTL <ms>                             Set the TTL (ms) for the Author Activities shared cache (default: process.env.AUTHOR_TTL || 10000)
+  --heartbeat <s>                              Interval, in seconds, between heartbeat logs. Set to 0 to disable (default: process.env.HEARTBEAT || 300)
+  --apiLimitWarning <remaining>                When API limit remaining (600/10min) is lower than this value log statements for limit will be raised to WARN level (default: process.env.API_REMAINING || 250)
+  --dryRun                                     Set dryRun=true for all checks/actions on all subreddits (overrides any existing) (default: process.env.DRYRUN)
+  --disableCache                               Disable caching for all subreddits (default: process.env.DISABLE_CACHE || false)
+  -h, --help                                   display help for command
 
-| CLI              | Environmental Variable | Required | Description                                                                                                                      |
-|------------------|------------------------|----------|----------------------------------------------------------------------------------------------------------------------------------|
-| [First Argument] |                        | No       | Comma-deliminated list of subreddits to run on if you don't want to run all the account has access to.                           |
-| --clientId       | CLIENT_ID              | **Yes**  | Your reddit application client id                                                                                                |
-| --clientSecret   | CLIENT_SECRET          | **Yes**  | Your reddit application client secret                                                                                            |
-| --accessToken    | ACCESS_TOKEN           | **Yes**  | A valid access token retrieved from completing the oauth flow for a user with your application.                                  |
-| --refreshToken   | REFRESH_TOKEN          | **Yes**  | A valid refresh token retrieved from completing the oauth flow for a user with your application.                                 |
-| --logDir         | LOG_DIR                | No       | The absolute path to where logs should be stored. use `false` to turn off log files. Defaults to `CWD/logs`                      |
-| --logLevel       | LOG_LEVEL              | No       | The minimum level to log at. Uses [Winston Log Levels](https://github.com/winstonjs/winston#logging-levels). Defaults to `info`  |
+Commands:
+  run                                          Runs bot normally
+  check [options] <activityIdentifier> [type]  Run check(s) on a specific activity
+  unmoderated [options] <subreddits...>        Run checks on all unmoderated activity in the modqueue
+  help [command]                               display help for command
+
+```
+
+### Logging
 
 ### Reddit App??
 
@@ -192,7 +230,8 @@ Visit https://not-an-aardvark.github.io/reddit-oauth-helper/
     * report
     * submit
     * wikiread
-* Click **Generate tokens*, you will get a popup asking you to approve access (or login) -- **the account you approve access with is the account that Bot will control.**
+    * wikiedit (if you are using Toolbox User Notes)
+* Click **Generate tokens**, you will get a popup asking you to approve access (or login) -- **the account you approve access with is the account that Bot will control.**
 * After approving an **Access Token** and **Refresh Token** will be shown at the bottom of the page. Write these down. 
   
 You should now have all the information you need to start the bot.

app.json

@@ -0,0 +1,33 @@
+{
+  "name": "Reddit Context Bot",
+  "description": "An event-based, reddit moderation bot built on top of snoowrap and written in typescript",
+  "repository": "https://github.com/FoxxMD/reddit-context-bot",
+  "stack": "container",
+  "env": {
+    "CLIENT_ID": {
+      "description": "Client ID for your Reddit application",
+      "value": "",
+      "required": true
+    },
+    "CLIENT_SECRET": {
+      "description": "Client Secret for your Reddit application",
+      "value": "",
+      "required": true
+    },
+    "REFRESH_TOKEN": {
+      "description": "Refresh token retrieved from authenticating an account with your Reddit Application",
+      "value": "",
+      "required": true
+    },
+    "ACCESS_TOKEN": {
+      "description": "Access token retrieved from authenticating an account with your Reddit Application",
+      "value": "",
+      "required": true
+    },
+    "WIKI_CONFIG": {
+      "description": "Relative url to contextbot wiki page EX https://reddit.com/r/subreddit/wiki/<path>",
+      "value": "botconfig/contextbot",
+      "required": false
+    }
+  }
+}

examples/README.md

@@ -0,0 +1,43 @@
+# Examples
+
+This directory contains example of valid, ready-to-go configurations for Context Bot for the purpose of:
+
+* showcasing what the bot can do
+* providing best practices for writing your configuration
+* providing generally useful configurations **that can be used immediately** or as a jumping-off point for your configuration
+
+### Creating Your Configuration
+
+#### Get the raw contents of the configuration
+
+* In a new tab open the github page for the configuration you want ([example](/examples/repeatActivity/crosspostSpamming.json5))
+* Click the **Raw** button...keep this tab open and move on to the next step
+
+#### Edit your wiki configuration
+
+* Visit the wiki page of the subreddit you want the bot to moderate
+    * Using default bot settings this will be `https://old.reddit.com/r/YOURSUBERDDIT/wiki/botconfig/contextbot`
+    * If the page does not exist create it, otherwise click **Edit**
+* Copy-paste the configuration into the wiki text box
+    * In the previous tab you opened (for the configuration) **Select All** (Ctrl+A), then **Copy**
+    * On the wiki page **Paste** into the text box
+* Save the edited wiki page
+* Ensure the wiki page visibility is restricted
+    * On the wiki page click **settings** (**Page settings** in new reddit)
+    * Check the box for **Only mods may edit and view** and then **save**
+    
+### Examples Overview
+
+* Rules
+  * [Attribution](/examples/attribution)
+  * [Recent Activity](/examples/recentActivity)
+  * [Repeat Activity](/examples/repeatActivity)
+  * [History](/examples/history)
+  * [Author](/examples/author)
+* [Toolbox User Notes](/examples/userNotes)
+* [Advanced Concepts](/examples/advancedConcepts)
+  * [Rule Sets](/examples/advancedConcepts/ruleSets.json5)
+  * [Name Rules](/examples/advancedConcepts/ruleNameReuse.json5)
+  * [Check Ordering](/examples/advancedConcepts)
+* Subreddit-ready examples
+  * Coming soon...

examples/advancedConcepts/README.md

@@ -0,0 +1,56 @@
+### Named Rules
+
+See [ruleNameReuse.json5](/examples/advancedConcepts/ruleNameReuse.json5)
+
+### Check Order
+
+Checks are run in the order they appear in your configuration, therefore you should place your highest requirement/severe action checks at the top and lowest requirement/moderate actions at the bottom.
+
+This is so that if an Activity warrants a more serious reaction that Check is triggered first rather than having a lower requirement check with less severe actions triggered and causing all subsequent Checks to be skipped.
+
+* Attribution >50% AND Repeat Activity 8x AND Recent Activity in 2 subs => remove submission + ban
+* Attribution >20% AND Repeat Activity 4x AND Recent Activity in 5 subs => remove submission + flair user restricted
+* Attribution >20% AND Repeat Activity 2x => remove submission
+* Attribution >20% AND History comments <30% => remove submission
+* Attribution >15% => report
+* Repeat Activity 2x => report
+* Recent Activity in 3 subs => report
+* Author not vetted => flair new user submission
+
+### Rule Sets
+
+The `rules` array on a `Checks` can contain both `Rule` objects and `RuleSet` objects.
+
+A **Rule Set** is a "nested" set of `Rule` objects with a passing condition specified. These allow you to create more complex trigger behavior by combining multiple rules. 
+
+See **[ruleSets.json5](/examples/advancedConcepts/ruleSets.json5)** for a complete example as well as consulting the [schema](https://json-schema.app/view/%23%2Fdefinitions%2FRuleSetJson?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Freddit-context-bot%2Fmaster%2Fsrc%2FSchema%2FApp.json).
+
+### Rule Order
+
+The ordering of your Rules within a Check/RuleSet can have an impact on Check performance (speed) as well as API usage.
+
+Consider these three rules:
+
+* Rule A -- Recent Activity => 3 subreddits => last 15 submissions
+* Rule B -- Repeat Activity => last 3 days
+* Rule C -- Attribution => >10% => last 90 days or 300 submissions
+
+The first two rules are lightweight in their requirements -- Rule A can be completed in 1 API call, Rule B potentially completed in 1 Api call. 
+
+However, depending on how active the Author is, Rule C will take *at least* 3 API calls just to get all activities (Reddit limit 100 items per call).
+
+If the Check is using `AND` condition for its rules (default) then if either Rule A or Rule B fail then Rule C will never run. This means 3 API calls never made plus the time waiting for each to return.
+
+**It is therefore advantageous to list your lightweight Rules first in each Check.**
+
+### API Caching
+
+Context bot implements some basic caching functionality for **Author Activities** and wiki pages (on Comment/Report Actions).
+
+**Author Activities** are cached for a subreddit-configurable amount of time (10 seconds by default). A cached activities set can be re-used if the **window on a Rule is identical to the window on another Rule**.
+
+This means that when possible you should re-use window values. 
+
+IE If you want to check an Author's Activities for a time range try to always use **7 Days** or always use **50 Items** for absolute counts.
+
+Re-use will result in less API calls and faster Check times.

examples/advancedConcepts/ruleNameReuse.json5

@@ -0,0 +1,79 @@
+{
+  "checks": [
+    {
+      "name": "Auto Remove SP Karma",
+      "description": "Remove submission because author has self-promo >10% and posted in karma subs recently",
+      "kind": "submission",
+      "rules": [
+        // named rules can be referenced at any point in the configuration (where they occur does not matter)
+        // and can be used in any Check
+        // Note: rules do not transfer between subreddit configurations
+        "freekarmasub",
+        {
+          "name": "attr10all",
+          "kind": "attribution",
+          "criteria": [
+            {
+              "threshold": "10%",
+              "window": {
+                "days": 90
+              }
+            },
+            {
+              "threshold": "10%",
+              "window": 100
+            }
+          ],
+        }
+      ],
+      "actions": [
+        {
+          "kind": "remove"
+        },
+        {
+          "kind": "comment",
+          "content": "Your submission was removed because you are over reddit's threshold for self-promotion and recently posted this content in a karma sub"
+        }
+      ]
+    },
+    {
+      "name": "Free Karma On Submission Alert",
+      "description": "Check if author has posted this submission in 'freekarma' subreddits",
+      "kind": "submission",
+      "rules": [
+        {
+          // rules can be re-used throughout a configuration by referencing them by name
+          //
+          // The rule name itself can only contain spaces, hyphens and underscores
+          // The value used to reference it will have all of these removed, and lower-cased
+          //
+          // so to reference this rule use the value 'freekarmasub'
+          "name": "Free_Karma-SUB",
+          "kind": "recentActivity",
+          "lookAt": "submissions",
+          "useSubmissionAsReference":true,
+          "thresholds": [
+            {
+              "totalCount": 1,
+              "subreddits": [
+                "DeFreeKarma",
+                "FreeKarma4U",
+                "FreeKarma4You",
+                "upvote"
+              ]
+            }
+          ],
+          "window": {
+            "days": 7
+          }
+        }
+      ],
+      "actions": [
+        {
+          "kind": "report",
+          "content": "Submission posted {{rules.freekarmasub.totalCount}} times in karma {{rules.freekarmasub.subCount}} subs over {{rules.freekarmasub.window}}: {{rules.freekarmasub.subSummary}}"
+        }
+      ]
+    },
+  ]
+}

examples/advancedConcepts/ruleSets.json5

@@ -0,0 +1,99 @@
+{
+  "checks": [
+    {
+      "name": "Self Promo All or low comment",
+      "description": "SP >10% of all activities or >10% of submissions with low comment engagement",
+      "kind": "submission",
+      "rules": [
+        {
+          // this attribution rule is looking at all activities
+          //
+          // we want want this one rule to trigger the check because >10% of all activity (submission AND comments) is a good requirement
+          "name": "attr10all",
+          "kind": "attribution",
+          "criteria": [
+            {
+              "threshold": "10%",
+              "window": {
+                "days": 90
+              }
+            },
+            {
+              "threshold": "10%",
+              "window": 100
+            }
+          ],
+        },
+        {
+          // this is a **Rule Set**
+          //
+          // it is made up of "nested" rules with a pass condition (AND/OR)
+          // if the nested rules pass the condition then the Rule Set triggers the Check
+          //
+          // AND = all nested rules must be triggered to make the Rule Set trigger
+          // AND = any of the nested Rules will be the Rule Set trigger
+          "condition": "AND",
+          // in this check we use an Attribution >10% on ONLY submissions, which is a lower requirement then the above attribution rule
+          // and combine it with a History rule looking for low comment engagement
+          // to make a "higher" requirement Rule Set our of two low requirement Rules
+          "rules": [
+            {
+              "name": "attr20sub",
+              "kind": "attribution",
+              "criteria": [
+                {
+                  "threshold": "10%",
+                  "thresholdOn": "submissions",
+                  "window": {
+                    "days": 90
+                  }
+                },
+                {
+                  "threshold": "10%",
+                  "thresholdOn": "submissions",
+                  "window": 100
+                }
+              ],
+              "lookAt": "media"
+            },
+            {
+              "name": "lowOrOpComm",
+              "kind": "history",
+              "criteriaJoin": "OR",
+              "criteria": [
+                {
+                  "window": {
+                    "days": 90
+                  },
+                  "comment": {
+                    "threshold": "50%",
+                    "condition": "<"
+                  }
+                },
+                {
+                  "window": {
+                    "days": 90
+                  },
+                  "comment": {
+                    "asOp": true,
+                    "threshold": "40%",
+                    "condition": ">"
+                  }
+                }
+              ]
+            }
+          ]
+        }
+      ],
+      "actions": [
+        {
+          "kind": "remove"
+        },
+        {
+          "kind": "comment",
+          "content": "Your submission was removed because you are over reddit's threshold for self-promotion or exhibit low comment engagement"
+        }
+      ]
+    },
+  ]
+}

examples/attribution/README.md

@@ -0,0 +1,14 @@
+# Attribution
+
+The **Attribution** rule will aggregate an Author's content Attribution (youtube channels, twitter, website domains, etc.) and can check on their totals or percentages of all Activities over a time period:
+* Total # of attributions 
+* As percentage of all Activity or only Submissions
+* Look at all domains or only media (youtube, vimeo, etc.)
+* Include self posts (by reddit domain) or not
+
+Consult the [schema](https://json-schema.app/view/%23/%23%2Fdefinitions%2FCheckJson/%23%2Fdefinitions%2FAttributionJSONConfig?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Freddit-context-bot%2Fmaster%2Fsrc%2FSchema%2FApp.json) for a complete reference of the rule's properties.
+
+### Examples
+
+* [Self Promotion as percentage of all Activities](/examples/attribution/redditSelfPromoAll.json5) - Check if Author is submitting much more than they comment.
+* [Self Promotion as percentage of Submissions](/examples/attribution/redditSelfPromoSubmissionsOnly.json5) - Check if any of Author's aggregated submission origins are >10% of their submissions

examples/attribution/redditSelfPromoAll.json5

@@ -0,0 +1,41 @@
+{
+  "checks": [
+    {
+      "name": "Self Promo Activities",
+      "description": "Check if any of Author's aggregated submission origins are >10% of entire history",
+      // check will run on a new submission in your subreddit and look at the Author of that submission
+      "kind": "submission",
+      "rules": [
+        {
+          "name": "attr10all",
+          "kind": "attribution",
+          // criteria defaults to OR -- so either of these criteria will trigger the rule
+          "criteria": [
+            {
+              // threshold can be a percent or an absolute number
+              "threshold": "10%",
+              // The default is "all" -- calculate percentage of entire history (submissions & comments)
+              // "thresholdOn": "all",
+
+              // look at last 90 days of Author's activities (comments and submissions)
+              "window": {
+                "days": 90
+              }
+            },
+            {
+              "threshold": "10%",
+              // look at Author's last 100 activities (comments and submissions)
+              "window": 100
+            }
+          ],
+        }
+      ],
+      "actions": [
+        {
+          "kind": "report",
+          "content": "{{rules.attr10all.largestPercent}}% of {{rules.attr10all.activityTotal}} items over {{rules.attr10all.window}}"
+        }
+      ]
+    }
+  ]
+}

examples/attribution/redditSelfPromoSubmissionsOnly.json5

@@ -0,0 +1,42 @@
+{
+  "checks": [
+    {
+      "name": "Self Promo Submissions",
+      "description": "Check if any of Author's aggregated submission origins are >10% of their submissions",
+      // check will run on a new submission in your subreddit and look at the Author of that submission
+      "kind": "submission",
+      "rules": [
+        {
+          "name": "attr10sub",
+          "kind": "attribution",
+          // criteria defaults to OR -- so either of these criteria will trigger the rule
+          "criteria": [
+            {
+              // threshold can be a percent or an absolute number
+              "threshold": "10%",
+              // calculate percentage of submissions, rather than entire history (submissions & comments)
+              "thresholdOn": "submissions",
+
+              // look at last 90 days of Author's activities (comments and submissions)
+              "window": {
+                "days": 90
+              }
+            },
+            {
+              "threshold": "10%",
+              "thresholdOn": "submissions",
+              // look at Author's last 100 activities (comments and submissions)
+              "window": 100
+            }
+          ],
+        }
+      ],
+      "actions": [
+        {
+          "kind": "report",
+          "content": "{{rules.attr10sub.largestPercent}}% of {{rules.attr10sub.activityTotal}} items over {{rules.attr10sub.window}}"
+        }
+      ]
+    }
+  ]
+}

examples/author/README.md

@@ -0,0 +1,38 @@
+# Author
+
+## Rule
+
+The **Author** rule triggers if any [AuthorCriteria](https://json-schema.app/view/%23%2Fdefinitions%2FAuthorCriteria?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Freddit-context-bot%2Fmaster%2Fsrc%2FSchema%2FApp.json) from a list are either **included** or **excluded**, depending on which property you put them in.
+
+**AuthorCriteria** that can be checked:
+* name (u/userName)
+* author's subreddit flair text
+* author's subreddit flair css
+* author's subreddit mod status
+* [Toolbox User Notes](/examples/userNotes)
+
+The Author **Rule** is best used in conjunction with other Rules to short-circuit a Check based on who the Author is. It is easier to use a Rule to do this then to write **author filters** for every Rule (and makes Rules more re-useable).
+
+Consult the [schema](https://json-schema.app/view/%23%2Fdefinitions%2FAuthorRuleJSONConfig?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Freddit-context-bot%2Fmaster%2Fsrc%2FSchema%2FApp.json) for a complete reference of the rule's properties.
+
+### Examples
+
+* Basic examples
+    * [Flair new user Submission](/examples/author/flairNewUserSubmission.json5) - If the Author does not have the `vet` flair then flair the Submission with `New User`
+    * [Flair vetted user Submission](/examples/author/flairNewUserSubmission.json5) - If the Author does have the `vet` flair then flair the Submission with `Vetted`
+* Used with other Rules
+    * [Ignore vetted user](/examples/author/flairNewUserSubmission.json5) - Short-circuit the Check if the Author has the `vet` flair
+    
+## Filter
+
+All **Rules** and **Checks** have an optional `authorIs` property that takes an [AuthorOptions](https://json-schema.app/view/%23%2Fdefinitions%2FAuthorOptions?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Freddit-context-bot%2Fmaster%2Fsrc%2FSchema%2FApp.json) object. 
+
+**This property works the same as the Author Rule except that:**
+* On **Rules** if all criteria fail the Rule is **skipped.** 
+  * If a Rule is skipped **it does not fail or pass** and so does not affect the outcome of the Check.
+  * However, if all Rules on a Check are skipped the Check will fail.
+* On **Checks** if all criteria fail the Check **fails**.
+
+### Examples
+
+* [Skip recent activity check based on author](/examples/author/authorFilter.json5) - Skip a Recent Activity check for a set of subreddits if the Author of the Submission has any set of flairs.

examples/author/authorFilter.json5

@@ -0,0 +1,73 @@
+{
+  "checks": [
+    {
+      "name": "Karma/Meme Sub Activity",
+      "description": "Report on karma sub activity or meme sub activity if user isn't a memelord",
+      // check will run on a new submission in your subreddit and look at the Author of that submission
+      "kind": "submission",
+      "rules": [
+        {
+          "name": "freekarma",
+          "kind": "recentActivity",
+          "lookAt": "submissions",
+          "thresholds": [
+            {
+              "totalCount": 1,
+              "subreddits": [
+                "DeFreeKarma",
+                "FreeKarma4U",
+              ]
+            }
+          ],
+          "window": {
+            "days": 7
+          }
+        },
+        {
+          "name": "noobmemer",
+          "kind": "recentActivity",
+          // authors filter will be checked before a rule is run. If anything passes then the Rule is skipped -- it is not failed or triggered.
+          // if *all* Rules for a Check are skipped due to authors filter then the Check will fail
+          "authorIs": {
+            // each property (include/exclude) can contain multiple AuthorCriteria
+            // if any AuthorCriteria passes its test the Rule is skipped
+            //
+            // for an AuthorCriteria to pass all properties present on it must pass
+            //
+            // if "include" is present it will always run and exclude will be skipped
+            // "include:" []
+            "exclude": [
+              // for this to pass the Author of the Submission must not have the flair "Supreme Memer" and have the name "user1" or "user2"
+              {
+                "flairText": ["Supreme Memer"],
+                "names": ["user1","user2"]
+              },
+              {
+                // for this to pass the Author of the Submission must not have the flair "Decent Memer"
+                "flairText": ["Decent Memer"]
+              }
+            ]
+          },
+          "lookAt": "submissions",
+          "thresholds": [
+            {
+              "totalCount": 1,
+              "subreddits": [
+                "dankmemes",
+              ]
+            }
+          ],
+          "window": {
+            "days": 7
+          }
+        }
+      ],
+      "actions": [
+        {
+          "kind": "report",
+          "content": "Author has posted in free karma sub, or in /r/dankmemes and does not have meme flair in this subreddit"
+        }
+      ]
+    }
+  ]
+}

examples/author/flairNewUserSubmission.json5

@@ -0,0 +1,29 @@
+{
+  "checks": [
+    {
+      "name": "Flair New User Sub",
+      "description": "Flair submission as sketchy if user does not have vet flair",
+      // check will run on a new submission in your subreddit and look at the Author of that submission
+      "kind": "submission",
+      "rules": [
+        {
+          "name": "newflair",
+          "kind": "author",
+          // rule will trigger if Author does not have "vet" flair text
+          "exclude": [
+            {
+              "flairText": ["vet"]
+            }
+          ]
+        }
+      ],
+      "actions": [
+        {
+          "kind": "flair",
+          "text": "New User",
+          "css": "orange"
+        }
+      ]
+    }
+  ]
+}

examples/author/flairVettedUserSubmission.json5

@@ -0,0 +1,29 @@
+{
+  "checks": [
+    {
+      "name": "Flair Vetted User Submission",
+      "description": "Flair submission as Approved if user has vet flair",
+      // check will run on a new submission in your subreddit and look at the Author of that submission
+      "kind": "submission",
+      "rules": [
+        {
+          "name": "newflair",
+          "kind": "author",
+          // rule will trigger if Author has "vet" flair text
+          "include": [
+            {
+              "flairText": ["vet"]
+            }
+          ]
+        }
+      ],
+      "actions": [
+        {
+          "kind": "flair",
+          "text": "Vetted",
+          "css": "green"
+        }
+      ]
+    }
+  ]
+}

examples/author/ignoreVettedUser.json5

@@ -0,0 +1,81 @@
+{
+  "checks": [
+    {
+      "name": "non-vetted karma/meme activity",
+      "description": "Report if Author has SP and has recent karma/meme sub activity and isn't vetted",
+      // check will run on a new submission in your subreddit and look at the Author of that submission
+      "kind": "submission",
+      "rules": [
+        {
+          // The Author Rule is best used in conjunction with other Rules --
+          // instead of having to write an AuthorFilter for every Rule where you want to skip it based on Author criteria
+          // you can write one Author Rule and make it fail on the required criteria
+          // so that the check fails and Actions don't run
+          "name": "nonvet",
+          "kind": "author",
+          "exclude": [
+            {
+              "flairText": ["vet"]
+            }
+          ]
+        },
+        {
+          "name": "attr10",
+          "kind": "attribution",
+          "criteria": [
+            {
+              "threshold": "10%",
+              "window": {
+                "days": 90
+              }
+            },
+            {
+              "threshold": "10%",
+              "window": 100
+            }
+          ],
+        },
+        {
+          "name": "freekarma",
+          "kind": "recentActivity",
+          "lookAt": "submissions",
+          "thresholds": [
+            {
+              "totalCount": 1,
+              "subreddits": [
+                "DeFreeKarma",
+                "FreeKarma4U",
+              ]
+            }
+          ],
+          "window": {
+            "days": 7
+          }
+        },
+        {
+          "name": "memes",
+          "kind": "recentActivity",
+          "lookAt": "submissions",
+          "thresholds": [
+            {
+              "totalCount": 3,
+              "subreddits": [
+                "dankmemes",
+              ]
+            }
+          ],
+          "window": {
+            "days": 7
+          }
+        }
+      ],
+      // will NOT run if the Author for this Submission has the flair "vet"
+      "actions": [
+        {
+          "kind": "report",
+          "content": "Author has posted in free karma or meme subs recently"
+        }
+      ]
+    }
+  ]
+}

examples/history/README.md

@@ -0,0 +1,13 @@
+# History
+
+The **History** rule can check an Author's submission/comment statistics over a time period:
+* Submission total or percentage of All Activity
+* Comment total or percentage of all Activity
+* Comments made as OP (commented in their own Submission) total or percentage of all Comments
+
+Consult the [schema](https://json-schema.app/view/%23%2Fdefinitions%2FHistoryJSONConfig?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Freddit-context-bot%2Fmaster%2Fsrc%2FSchema%2FApp.json) for a complete reference of the rule's properties.
+
+### Examples
+
+* [Low Comment Engagement](/examples/history/lowEngagement.json5) - Check if Author is submitting much more than they comment.
+* [OP Comment Engagement](/examples/history/opOnlyEngagement.json5) - Check if Author is mostly engaging only in their own content

examples/history/lowEngagement.json5

@@ -0,0 +1,35 @@
+{
+  "checks": [
+    {
+      "name": "Low Comment Engagement",
+      "description": "Check if Author is submitting much more than they comment",
+      // check will run on a new submission in your subreddit and look at the Author of that submission
+      "kind": "submission",
+      "rules": [
+        {
+          "name": "lowComm",
+          "kind": "history",
+          "criteria": [
+            {
+              // look at last 90 days of Author's activities
+              "window": {
+                "days": 90
+              },
+              // trigger if less than 30% of their activities in this time period are comments
+              "comment": {
+                "threshold": "30%",
+                "condition": "<"
+              }
+            },
+          ]
+        }
+      ],
+      "actions": [
+        {
+          "kind": "report",
+          "content": "Low engagement: comments were {{rules.lowcomm.commentPercent}} of {{rules.lowcomm.activityTotal}} over {{rules.lowcomm.window}}"
+        }
+      ]
+    }
+  ]
+}

examples/history/opOnlyEngagement.json5

@@ -0,0 +1,36 @@
+{
+  "checks": [
+    {
+      "name": "Engaging Own Content Only",
+      "description": "Check if Author is mostly engaging in their own content only",
+      // check will run on a new submission in your subreddit and look at the Author of that submission
+      "kind": "submission",
+      "rules": [
+        {
+          "name": "opOnly",
+          "kind": "history",
+          "criteria": [
+            {
+              // look at last 90 days of Author's activities
+              "window": {
+                "days": 90
+              },
+              // trigger if less than 30% of their activities in this time period are comments
+              "comment": {
+                "asOp": true,
+                "threshold": "60%",
+                "condition": ">"
+              }
+            },
+          ]
+        }
+      ],
+      "actions": [
+        {
+          "kind": "report",
+          "content": "Selfish OP: {{rules.oponly.opPercent}} of {{rules.oponly.commentTotal}} comments over {{rules.oponly.window}} are as OP"
+        }
+      ]
+    }
+  ]
+}

examples/recentActivity/README.md

@@ -0,0 +1,10 @@
+# Recent Activity
+
+The **Recent Activity** rule can check if an Author has made any Submissions/Comments in a list of defined Subreddits.
+
+Consult the [schema](https://json-schema.app/view/%23%2Fdefinitions%2FRecentActivityRuleJSONConfig?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Freddit-context-bot%2Fmaster%2Fsrc%2FSchema%2FApp.json) for a complete reference of the rule's properties.
+
+### Examples
+
+* [Free Karma Subreddits](/examples/recentActivity/freeKarma.json5) - Check if the Author has recently posted in any "free karma" subreddits
+* [Submission in Free Karma Subreddits](/examples/recentActivity/freeKarmaOnSubmission.json5) - Check if the Author has posted the Submission this check is running on in any "free karma" subreddits recently

examples/recentActivity/freeKarma.json5

@@ -0,0 +1,41 @@
+{
+  "checks": [
+    {
+      "name": "Free Karma Alert",
+      "description": "Check if author has posted in 'freekarma' subreddits",
+      // check will run on a new submission in your subreddit and look at the Author of that submission
+      "kind": "submission",
+      "rules": [
+        {
+          "name": "freekarma",
+          "kind": "recentActivity",
+          // when `lookAt` is not present this rule will look for submissions and comments
+          // lookAt: "submissions"
+          // lookAt: "comments"
+          "thresholds": [
+            {
+              // for all subreddits, if the number of activities (sub/comment) is equal to or greater than 1 then the rule is triggered
+              "totalCount": 1,
+              "subreddits": [
+                "DeFreeKarma",
+                "FreeKarma4U",
+                "FreeKarma4You",
+                "upvote"
+              ]
+            }
+          ],
+          // will look at all of the Author's activities in the last 7 days
+          "window": {
+            "days": 7
+          }
+        }
+      ],
+      "actions": [
+        {
+          "kind": "report",
+          "content": "{{rules.freekarma.totalCount}} activities in karma {{rules.freekarma.subCount}} subs over {{rules.freekarma.window}}: {{rules.freekarma.subSummary}}"
+        }
+      ]
+    }
+  ]
+}

examples/recentActivity/freeKarmaOnSubmission.json5

@@ -0,0 +1,43 @@
+{
+  "checks": [
+    {
+      "name": "Free Karma On Submission Alert",
+      "description": "Check if author has posted this submission in 'freekarma' subreddits",
+      // check will run on a new submission in your subreddit and look at the Author of that submission
+      "kind": "submission",
+      "rules": [
+        {
+          "name": "freekarmasub",
+          "kind": "recentActivity",
+          // rule will only look at Author's submissions in these subreddits
+          "lookAt": "submissions",
+          // rule will only look at Author's submissions in these subreddits that have the same content (link) as the submission this event was made on
+          // In simpler terms -- rule will only check to see if the same link the author just posted is also posted in these subreddits
+          "useSubmissionAsReference":true,
+          "thresholds": [
+            {
+              // for all subreddits, if the number of activities (sub/comment) is equal to or greater than 1 then the rule is triggered
+              "totalCount": 1,
+              "subreddits": [
+                "DeFreeKarma",
+                "FreeKarma4U",
+                "FreeKarma4You",
+                "upvote"
+              ]
+            }
+          ],
+          // look at all of the Author's submissions in the last 7 days
+          "window": {
+            "days": 7
+          }
+        }
+      ],
+      "actions": [
+        {
+          "kind": "report",
+          "content": "Submission posted {{rules.freekarmasub.totalCount}} times in karma {{rules.freekarmasub.subCount}} subs over {{rules.freekarmasub.window}}: {{rules.freekarmasub.subSummary}}"
+        }
+      ]
+    }
+  ]
+}

examples/repeatActivity/README.md

@@ -0,0 +1,8 @@
+# Repeat Activity
+
+The **Repeat Activity** rule will check for patterns of repetition in an Author's Submission/Comment history. Consult the [schema](https://json-schema.app/view/%23%2Fdefinitions%2FRepeatActivityJSONConfig?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Freddit-context-bot%2Fmaster%2Fsrc%2FSchema%2FApp.json) for a complete reference of the rule's properties.
+
+### Examples
+
+* [Crosspost Spamming](/examples/repeatActivity/crosspostSpamming.json5) - Check if an Author is spamming their Submissions across multiple subreddits
+* [Burst-posting](/examples/repeatActivity/burstPosting.json5) - Check if Author is crossposting their Submissions in short bursts

examples/repeatActivity/burstPosting.json5

@@ -0,0 +1,32 @@
+{
+  "checks": [
+    {
+      "name": "Burstpost Spam",
+      "description": "Check if Author is crossposting in short bursts",
+      // check will run on a new submission in your subreddit and look at the Author of that submission
+      "kind": "submission",
+      "rules": [
+        {
+          "name": "burstpost",
+          "kind": "repeatActivity",
+          // will only look at Submissions in Author's history that contain the same content (link) as the Submission this check was initiated by
+          "useSubmissionAsReference": true,
+          // the number of non-repeat activities (submissions or comments) to ignore between repeat submissions
+          "gapAllowance": 3,
+          // if the Author has posted this Submission 6 times, ignoring 3 non-repeat activities between each repeat, then this rule will trigger
+          "threshold": 6,
+          // look at all of the Author's submissions in the last 7 days
+          "window": {
+            "days": 7
+          }
+        }
+      ],
+      "actions": [
+        {
+          "kind": "report",
+          "content": "Author has burst-posted this link {{rules.burstpost.largestRepeat}} times over {{rules.burstpost.window}}"
+        }
+      ]
+    }
+  ]
+}

examples/repeatActivity/crosspostSpamming.json5

@@ -0,0 +1,30 @@
+{
+  "checks": [
+    {
+      "name": "Crosspost Spam",
+      "description": "Check if Author is spamming Submissions across subreddits",
+      // check will run on a new submission in your subreddit and look at the Author of that submission
+      "kind": "submission",
+      "rules": [
+        {
+          "name": "xpostspam",
+          "kind": "repeatActivity",
+          // will only look at Submissions in Author's history that contain the same content (link) as the Submission this check was initiated by
+          "useSubmissionAsReference": true,
+          // if the Author has posted this Submission 5 times consecutively then this rule will trigger
+          "threshold": 5,
+          // look at all of the Author's submissions in the last 7 days
+          "window": {
+            "days": 7
+          }
+        }
+      ],
+      "actions": [
+        {
+          "kind": "report",
+          "content": "Author has posted this link {{rules.xpostspam.largestRepeat}} times over {{rules.xpostspam.window}}"
+        }
+      ]
+    }
+  ]
+}

examples/userNotes/README.md

@@ -0,0 +1,26 @@
+# [Toolbox](https://www.reddit.com/r/toolbox/wiki/docs) [User Notes](https://www.reddit.com/r/toolbox/wiki/docs/usernotes)
+
+Context Bot supports reading and writing [User Notes](https://www.reddit.com/r/toolbox/wiki/docs/usernotes) for the [Toolbox](https://www.reddit.com/r/toolbox/wiki/docs) extension.
+
+**You must have Toolbox setup for your subreddit and at least one User Note created before you can use User Notes related features on Context Bot.** 
+
+[Click here for the Toolbox Quickstart Guide](https://www.reddit.com/r/toolbox/wiki/docs/quick_start)
+
+## Filter
+
+User Notes are an additional criteria on [AuthorCriteria](https://json-schema.app/view/%23%2Fdefinitions%2FAuthorCriteria?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Freddit-context-bot%2Fmaster%2Fsrc%2FSchema%2FApp.json) that can be used alongside other Author properties for both [filtering rules and in the AuthorRule.](/examples/author/)
+
+Consult the [schema](https://json-schema.app/view/%23%2Fdefinitions%2FUserNoteCriteria?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Freddit-context-bot%2Fmaster%2Fsrc%2FSchema%2FApp.json) for a complete reference of the **UserNoteCriteria** object that can be used in AuthorCriteria.
+
+### Examples
+
+* [Do not tag user with Good User note](/examples/userNotes/usernoteFilter.json5)
+
+## Action
+
+A User Note can also be added to the Author of a Submission or Comment with the [UserNoteAction.](https://json-schema.app/view/%23%2Fdefinitions%2FUserNoteActionJson?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Freddit-context-bot%2Fmaster%2Fsrc%2FSchema%2FApp.json)
+
+
+### Examples
+
+* [Add note on user doing self promotion](/examples/userNotes/usernoteSP.json5)

examples/userNotes/usernoteFilter.json5

@@ -0,0 +1,47 @@
+{
+  "checks": [
+    {
+      "name": "Self Promo Activities",
+      "description": "Tag SP only if user does not have good contributor user note",
+      // check will run on a new submission in your subreddit and look at the Author of that submission
+      "kind": "submission",
+      "rules": [
+        {
+          "name": "attr10all",
+          "kind": "attribution",
+          "author": {
+            "exclude": [
+              {
+                // the key of the usernote type to look for https://github.com/toolbox-team/reddit-moderator-toolbox/wiki/Subreddit-Wikis%3A-usernotes#working-with-note-types
+                // rule will not run if current usernote on Author is of type 'gooduser'
+                "type": "gooduser"
+              }
+            ]
+          },
+          "criteria": [
+            {
+              "threshold": "10%",
+              "window": {
+                "days": 90
+              }
+            },
+            {
+              "threshold": "10%",
+              "window": 100
+            }
+          ],
+        }
+      ],
+      "actions": [
+        {
+          "kind": "usernote",
+          // the key of usernote type
+          // https://github.com/toolbox-team/reddit-moderator-toolbox/wiki/Subreddit-Wikis%3A-usernotes#working-with-note-types
+          "type": "spamwarn",
+          // content is mustache templated as usual
+          "content": "Self Promotion: {{rules.attr10all.refDomainTitle}} {{rules.attr10sub.largestPercent}}%"
+        }
+      ]
+    }
+  ]
+}

examples/userNotes/usernoteSP.json5

@@ -0,0 +1,38 @@
+{
+  "checks": [
+    {
+      "name": "Self Promo Activities",
+      "description": "Check if any of Author's aggregated submission origins are >10% of entire history",
+      // check will run on a new submission in your subreddit and look at the Author of that submission
+      "kind": "submission",
+      "rules": [
+        {
+          "name": "attr10all",
+          "kind": "attribution",
+          "criteria": [
+            {
+              "threshold": "10%",
+              "window": {
+                "days": 90
+              }
+            },
+            {
+              "threshold": "10%",
+              "window": 100
+            }
+          ],
+        }
+      ],
+      "actions": [
+        {
+          "kind": "usernote",
+          // the key of usernote type
+          // https://github.com/toolbox-team/reddit-moderator-toolbox/wiki/Subreddit-Wikis%3A-usernotes#working-with-note-types
+          "type": "spamwarn",
+          // content is mustache templated as usual
+          "content": "Self Promotion: {{rules.attr10all.refDomainTitle}} {{rules.attr10sub.largestPercent}}%"
+        }
+      ]
+    }
+  ]
+}

heroku.yml

@@ -0,0 +1,3 @@
+build:
+  docker:
+    worker: Dockerfile

package-lock.json

@@ -8,24 +8,39 @@
       "version": "1.0.0",
       "license": "ISC",
       "dependencies": {
-        "ajv": "^6.12.6",
+        "ajv": "^7.2.4",
+        "commander": "^7.2.0",
         "dayjs": "^1.10.5",
         "es6-error": "^4.1.1",
         "fast-deep-equal": "^3.1.3",
-        "minimist": "^1.2.5",
+        "he": "^1.2.0",
+        "js-yaml": "^4.1.0",
+        "json5": "^2.2.0",
+        "memory-cache": "^0.2.0",
         "mustache": "^4.2.0",
+        "node-fetch": "^2.6.1",
+        "object-hash": "^2.2.0",
         "p-event": "^4.2.0",
+        "pako": "^0.2.6",
         "safe-stable-stringify": "^1.1.1",
         "snoostorm": "^1.5.2",
         "snoowrap": "^1.23.0",
-        "winston": "FoxxMD/winston#9639da027cd4f3b46b055b0193f240639ef53409",
-        "winston-daily-rotate-file": "^4.5.5"
+        "typescript": "^4.3.4",
+        "winston": "FoxxMD/winston#fbab8de969ecee578981c77846156c7f43b5f01e",
+        "winston-daily-rotate-file": "^4.5.5",
+        "zlib": "^1.0.5"
       },
       "devDependencies": {
         "@tsconfig/node14": "^1.0.0",
+        "@types/he": "^1.1.1",
+        "@types/js-yaml": "^4.0.1",
+        "@types/memory-cache": "^0.2.1",
         "@types/minimist": "^1.2.1",
         "@types/mustache": "^4.1.1",
         "@types/node": "^15.6.1",
+        "@types/node-fetch": "^2.5.10",
+        "@types/object-hash": "^2.1.0",
+        "@types/pako": "^1.0.1",
         "ts-auto-guard": "*",
         "ts-json-schema-generator": "^0.93.0",
         "typescript-json-schema": "^0.50.1"
@@ -58,12 +73,12 @@
       }
     },
     "node_modules/@nodelib/fs.scandir": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.4.tgz",
-      "integrity": "sha512-33g3pMJk3bg5nXbL/+CY6I2eJDzZAni49PfJnL5fghPTggPvBd/pFNSgJsdAgWptuFu7qq/ERvOYFlhvsLTCKA==",
+      "version": "2.1.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
+      "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
       "dev": true,
       "dependencies": {
-        "@nodelib/fs.stat": "2.0.4",
+        "@nodelib/fs.stat": "2.0.5",
         "run-parallel": "^1.1.9"
       },
       "engines": {
@@ -71,21 +86,21 @@
       }
     },
     "node_modules/@nodelib/fs.stat": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.4.tgz",
-      "integrity": "sha512-IYlHJA0clt2+Vg7bccq+TzRdJvv19c2INqBSsoOLp1je7xjtr7J26+WXR72MCdvU9q1qTzIWDfhMf+DRvQJK4Q==",
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
+      "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
       "dev": true,
       "engines": {
         "node": ">= 8"
       }
     },
     "node_modules/@nodelib/fs.walk": {
-      "version": "1.2.6",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.6.tgz",
-      "integrity": "sha512-8Broas6vTtW4GIXTAHDoE32hnN2M5ykgCpWGbuXHQ15vEMqr23pB76e/GZcYsZCHALv50ktd24qhEyKr6wBtow==",
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.7.tgz",
+      "integrity": "sha512-BTIhocbPBSrRmHxOAJFtR18oLhxTtAFDAvL8hY1S3iU8k+E60W/YFs4jrixGzQjMpF4qPXxIQHcjVD9dz1C2QA==",
       "dev": true,
       "dependencies": {
-        "@nodelib/fs.scandir": "2.1.4",
+        "@nodelib/fs.scandir": "2.1.5",
         "fastq": "^1.6.0"
       },
       "engines": {
@@ -106,10 +121,23 @@
         "typescript": "~4.1.3"
       }
     },
+    "node_modules/@ts-morph/common/node_modules/typescript": {
+      "version": "4.1.6",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.1.6.tgz",
+      "integrity": "sha512-pxnwLxeb/Z5SP80JDRzVjh58KsM6jZHRAOtTpS7sXLS4ogXNKC9ANxHHZqLLeVHZN35jCtI4JdmLLbLiC1kBow==",
+      "dev": true,
+      "bin": {
+        "tsc": "bin/tsc",
+        "tsserver": "bin/tsserver"
+      },
+      "engines": {
+        "node": ">=4.2.0"
+      }
+    },
     "node_modules/@tsconfig/node14": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.0.tgz",
-      "integrity": "sha512-RKkL8eTdPv6t5EHgFKIVQgsDapugbuOptNd9OOunN/HAkzmmTnZELx1kNCK0rSdUYGmiFMM3rRQMAWiyp023LQ==",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.1.tgz",
+      "integrity": "sha512-509r2+yARFfHHE7T6Puu2jjkoycftovhXRqW328PDXTVGKihlb1P8Z9mMZH04ebyajfRY7dedfGynlrFHJUQCg==",
       "dev": true
     },
     "node_modules/@types/command-line-args": {
@@ -124,6 +152,18 @@
       "integrity": "sha512-/xUgezxxYePeXhg5S04hUjxG9JZi+rJTs1+4NwpYPfSaS7BeDa6tVJkH6lN9Cb6rl8d24Fi2uX0s0Ngg2JT6gg==",
       "dev": true
     },
+    "node_modules/@types/he": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@types/he/-/he-1.1.1.tgz",
+      "integrity": "sha512-jpzrsR1ns0n3kyWt92QfOUQhIuJGQ9+QGa7M62rO6toe98woQjnsnzjdMtsQXCdvjjmqjS2ZBCC7xKw0cdzU+Q==",
+      "dev": true
+    },
+    "node_modules/@types/js-yaml": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.1.tgz",
+      "integrity": "sha512-xdOvNmXmrZqqPy3kuCQ+fz6wA0xU5pji9cd1nDrflWaAWtYLLGk5ykW0H6yg5TVyehHP1pfmuuSaZkhP+kspVA==",
+      "dev": true
+    },
     "node_modules/@types/json-schema": {
       "version": "7.0.7",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.7.tgz",
@@ -136,6 +176,12 @@
       "integrity": "sha512-bpcvu/MKHHeYX+qeEN8GE7DIravODWdACVA1ctevD8CN24RhPZIKMn9ntfAsrvLfSX3cR5RrBKAbYm9bGs0A+Q==",
       "dev": true
     },
+    "node_modules/@types/memory-cache": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/@types/memory-cache/-/memory-cache-0.2.1.tgz",
+      "integrity": "sha512-6rmz3bMqJNkj0HIN3AMhOfRN+JhnxxTULeBkwgilfnspjABtKG6ig8mbIzkOjtmiRw+WG1B56z+BO6htGz3IBw==",
+      "dev": true
+    },
     "node_modules/@types/minimatch": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.4.tgz",
@@ -155,9 +201,45 @@
       "dev": true
     },
     "node_modules/@types/node": {
-      "version": "15.12.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.0.tgz",
-      "integrity": "sha512-+aHJvoCsVhO2ZCuT4o5JtcPrCPyDE3+1nvbDprYes+pPkEsbjH7AGUCNtjMOXS0fqH14t+B7yLzaqSz92FPWyw==",
+      "version": "15.12.2",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.2.tgz",
+      "integrity": "sha512-zjQ69G564OCIWIOHSXyQEEDpdpGl+G348RAKY0XXy9Z5kU9Vzv1GMNnkar/ZJ8dzXB3COzD9Mo9NtRZ4xfgUww==",
+      "dev": true
+    },
+    "node_modules/@types/node-fetch": {
+      "version": "2.5.10",
+      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.10.tgz",
+      "integrity": "sha512-IpkX0AasN44hgEad0gEF/V6EgR5n69VEqPEgnmoM8GsIGro3PowbWs4tR6IhxUTyPLpOn+fiGG6nrQhcmoCuIQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*",
+        "form-data": "^3.0.0"
+      }
+    },
+    "node_modules/@types/node-fetch/node_modules/form-data": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
+      "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
+      "dev": true,
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
+    "node_modules/@types/object-hash": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@types/object-hash/-/object-hash-2.1.0.tgz",
+      "integrity": "sha512-RW3VRiuQIMo5PJ4Q1IwBtdLHL/t8ACpzUY40norN9ejE6CUBwKetmSxJnITJ0NlzN/ymF1nvPvlpvegtns7yOg==",
+      "dev": true
+    },
+    "node_modules/@types/pako": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@types/pako/-/pako-1.0.1.tgz",
+      "integrity": "sha512-GdZbRSJ3Cv5fiwT6I0SQ3ckeN2PWNqxd26W9Z2fCK1tGrrasGy4puvNFtnddqH9UJFMQYXxEuuB7B8UK+LLwSg==",
       "dev": true
     },
     "node_modules/@types/strip-bom": {
@@ -173,13 +255,13 @@
       "dev": true
     },
     "node_modules/ajv": {
-      "version": "6.12.6",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
-      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+      "version": "7.2.4",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-7.2.4.tgz",
+      "integrity": "sha512-nBeQgg/ZZA3u3SYxyaDvpvDtgZ/EZPF547ARgZBrG9Bhu1vKDwAIjtIf+sDtJUKa2zOcEbmRLBRSyMraS/Oy1A==",
       "dependencies": {
         "fast-deep-equal": "^3.1.1",
-        "fast-json-stable-stringify": "^2.0.0",
-        "json-schema-traverse": "^0.4.1",
+        "json-schema-traverse": "^1.0.0",
+        "require-from-string": "^2.0.2",
         "uri-js": "^4.2.2"
       },
       "funding": {
@@ -214,6 +296,11 @@
       "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==",
       "dev": true
     },
+    "node_modules/argparse": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
+      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
+    },
     "node_modules/array-back": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz",
@@ -488,7 +575,6 @@
       "version": "7.2.0",
       "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz",
       "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==",
-      "dev": true,
       "engines": {
         "node": ">= 10"
       }
@@ -788,6 +874,26 @@
         "node": ">=6"
       }
     },
+    "node_modules/har-validator/node_modules/ajv": {
+      "version": "6.12.6",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+      "dependencies": {
+        "fast-deep-equal": "^3.1.1",
+        "fast-json-stable-stringify": "^2.0.0",
+        "json-schema-traverse": "^0.4.1",
+        "uri-js": "^4.2.2"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/epoberezkin"
+      }
+    },
+    "node_modules/har-validator/node_modules/json-schema-traverse": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
+    },
     "node_modules/harmony-reflect": {
       "version": "1.6.2",
       "resolved": "https://registry.npmjs.org/harmony-reflect/-/harmony-reflect-1.6.2.tgz",
@@ -802,6 +908,14 @@
         "node": ">=4"
       }
     },
+    "node_modules/he": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
+      "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==",
+      "bin": {
+        "he": "bin/he"
+      }
+    },
     "node_modules/http-signature": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
@@ -953,6 +1067,17 @@
       "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
       "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
     },
+    "node_modules/js-yaml": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
+      "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+      "dependencies": {
+        "argparse": "^2.0.1"
+      },
+      "bin": {
+        "js-yaml": "bin/js-yaml.js"
+      }
+    },
     "node_modules/jsbn": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
@@ -964,9 +1089,9 @@
       "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
     },
     "node_modules/json-schema-traverse": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
-      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
+      "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="
     },
     "node_modules/json-stable-stringify": {
       "version": "1.0.1",
@@ -982,6 +1107,20 @@
       "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
       "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
     },
+    "node_modules/json5": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.0.tgz",
+      "integrity": "sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA==",
+      "dependencies": {
+        "minimist": "^1.2.5"
+      },
+      "bin": {
+        "json5": "lib/cli.js"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/jsonify": {
       "version": "0.0.0",
       "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz",
@@ -1041,6 +1180,11 @@
       "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
       "dev": true
     },
+    "node_modules/memory-cache": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/memory-cache/-/memory-cache-0.2.0.tgz",
+      "integrity": "sha1-eJCwHVLADI68nVM+H46xfjA0hxo="
+    },
     "node_modules/merge2": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
@@ -1151,6 +1295,14 @@
         "mustache": "bin/mustache"
       }
     },
+    "node_modules/node-fetch": {
+      "version": "2.6.1",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
+      "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==",
+      "engines": {
+        "node": "4.x || >=6.0.0"
+      }
+    },
     "node_modules/oauth-sign": {
       "version": "0.9.0",
       "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
@@ -1217,6 +1369,11 @@
         "node": ">=8"
       }
     },
+    "node_modules/pako": {
+      "version": "0.2.6",
+      "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.6.tgz",
+      "integrity": "sha1-PgxUg1O4WaucgAX6xwa91sevUF8="
+    },
     "node_modules/path-is-absolute": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
@@ -1391,6 +1548,14 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/require-from-string": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
+      "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/reusify": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
@@ -1710,19 +1875,6 @@
         "node": ">=10.0.0"
       }
     },
-    "node_modules/ts-json-schema-generator/node_modules/typescript": {
-      "version": "4.3.2",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.2.tgz",
-      "integrity": "sha512-zZ4hShnmnoVnAHpVHWpTcxdv7dWP60S2FsydQLV8V5PbS3FifjWFFRiHSWpDJahly88PRyV5teTSLoq4eG7mKw==",
-      "dev": true,
-      "bin": {
-        "tsc": "bin/tsc",
-        "tsserver": "bin/tsserver"
-      },
-      "engines": {
-        "node": ">=4.2.0"
-      }
-    },
     "node_modules/ts-morph": {
       "version": "9.1.0",
       "resolved": "https://registry.npmjs.org/ts-morph/-/ts-morph-9.1.0.tgz",
@@ -1789,10 +1941,9 @@
       "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
     },
     "node_modules/typescript": {
-      "version": "4.1.5",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.1.5.tgz",
-      "integrity": "sha512-6OSu9PTIzmn9TCDiovULTnET6BgXtDYL4Gg4szY+cGsc3JP1dQL8qvE8kShTRx1NIw4Q9IBHlwODjkjWEtMUyA==",
-      "dev": true,
+      "version": "4.3.4",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz",
+      "integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew==",
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
@@ -1820,9 +1971,9 @@
       }
     },
     "node_modules/typescript-json-schema/node_modules/@types/node": {
-      "version": "14.17.2",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.2.tgz",
-      "integrity": "sha512-sld7b/xmFum66AAKuz/rp/CUO8+98fMpyQ3SBfzzBNGMd/1iHBTAg9oyAvcYlAj46bpc74r91jSw2iFdnx29nw==",
+      "version": "14.17.3",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.3.tgz",
+      "integrity": "sha512-e6ZowgGJmTuXa3GyaPbTGxX17tnThl2aSSizrFthQ7m9uLGZBXiGhgE55cjRZTF5kjZvYn9EOPOMljdjwbflxw==",
       "dev": true
     },
     "node_modules/typescript-json-schema/node_modules/typescript": {
@@ -1898,8 +2049,8 @@
     },
     "node_modules/winston": {
       "version": "3.3.3",
-      "resolved": "git+ssh://git@github.com/FoxxMD/winston.git#9639da027cd4f3b46b055b0193f240639ef53409",
-      "integrity": "sha512-StxHu2puJAl2Ky8mXitI2nQ7lDNT5PPS4cnTj+2FF4orQSKVRIGhulMkRpoAtk9Z40QuhkgNRfCmBGQXa30BZQ==",
+      "resolved": "git+ssh://git@github.com/FoxxMD/winston.git#fbab8de969ecee578981c77846156c7f43b5f01e",
+      "integrity": "sha512-OnunfctuocKBmG0uzkBkVYGSW13VYmzglQPwj4ZFOBEtV2e2ECqe65eCaYRYHSvKynGc8T5hiaEWAVR2hVj+Yg==",
       "license": "MIT",
       "dependencies": {
         "@dabh/diagnostics": "^2.0.2",
@@ -2110,6 +2261,15 @@
       "engines": {
         "node": ">=6"
       }
+    },
+    "node_modules/zlib": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/zlib/-/zlib-1.0.5.tgz",
+      "integrity": "sha1-bnyXL8NxxkWmr7A6sUdp3vEU/MA=",
+      "hasInstallScript": true,
+      "engines": {
+        "node": ">=0.2.0"
+      }
     }
   },
   "dependencies": {
@@ -2134,28 +2294,28 @@
       }
     },
     "@nodelib/fs.scandir": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.4.tgz",
-      "integrity": "sha512-33g3pMJk3bg5nXbL/+CY6I2eJDzZAni49PfJnL5fghPTggPvBd/pFNSgJsdAgWptuFu7qq/ERvOYFlhvsLTCKA==",
+      "version": "2.1.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
+      "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
       "dev": true,
       "requires": {
-        "@nodelib/fs.stat": "2.0.4",
+        "@nodelib/fs.stat": "2.0.5",
         "run-parallel": "^1.1.9"
       }
     },
     "@nodelib/fs.stat": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.4.tgz",
-      "integrity": "sha512-IYlHJA0clt2+Vg7bccq+TzRdJvv19c2INqBSsoOLp1je7xjtr7J26+WXR72MCdvU9q1qTzIWDfhMf+DRvQJK4Q==",
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
+      "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
       "dev": true
     },
     "@nodelib/fs.walk": {
-      "version": "1.2.6",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.6.tgz",
-      "integrity": "sha512-8Broas6vTtW4GIXTAHDoE32hnN2M5ykgCpWGbuXHQ15vEMqr23pB76e/GZcYsZCHALv50ktd24qhEyKr6wBtow==",
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.7.tgz",
+      "integrity": "sha512-BTIhocbPBSrRmHxOAJFtR18oLhxTtAFDAvL8hY1S3iU8k+E60W/YFs4jrixGzQjMpF4qPXxIQHcjVD9dz1C2QA==",
       "dev": true,
       "requires": {
-        "@nodelib/fs.scandir": "2.1.4",
+        "@nodelib/fs.scandir": "2.1.5",
         "fastq": "^1.6.0"
       }
     },
@@ -2171,12 +2331,20 @@
         "mkdirp": "^1.0.4",
         "multimatch": "^5.0.0",
         "typescript": "~4.1.3"
+      },
+      "dependencies": {
+        "typescript": {
+          "version": "4.1.6",
+          "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.1.6.tgz",
+          "integrity": "sha512-pxnwLxeb/Z5SP80JDRzVjh58KsM6jZHRAOtTpS7sXLS4ogXNKC9ANxHHZqLLeVHZN35jCtI4JdmLLbLiC1kBow==",
+          "dev": true
+        }
       }
     },
     "@tsconfig/node14": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.0.tgz",
-      "integrity": "sha512-RKkL8eTdPv6t5EHgFKIVQgsDapugbuOptNd9OOunN/HAkzmmTnZELx1kNCK0rSdUYGmiFMM3rRQMAWiyp023LQ==",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.1.tgz",
+      "integrity": "sha512-509r2+yARFfHHE7T6Puu2jjkoycftovhXRqW328PDXTVGKihlb1P8Z9mMZH04ebyajfRY7dedfGynlrFHJUQCg==",
       "dev": true
     },
     "@types/command-line-args": {
@@ -2191,6 +2359,18 @@
       "integrity": "sha512-/xUgezxxYePeXhg5S04hUjxG9JZi+rJTs1+4NwpYPfSaS7BeDa6tVJkH6lN9Cb6rl8d24Fi2uX0s0Ngg2JT6gg==",
       "dev": true
     },
+    "@types/he": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@types/he/-/he-1.1.1.tgz",
+      "integrity": "sha512-jpzrsR1ns0n3kyWt92QfOUQhIuJGQ9+QGa7M62rO6toe98woQjnsnzjdMtsQXCdvjjmqjS2ZBCC7xKw0cdzU+Q==",
+      "dev": true
+    },
+    "@types/js-yaml": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.1.tgz",
+      "integrity": "sha512-xdOvNmXmrZqqPy3kuCQ+fz6wA0xU5pji9cd1nDrflWaAWtYLLGk5ykW0H6yg5TVyehHP1pfmuuSaZkhP+kspVA==",
+      "dev": true
+    },
     "@types/json-schema": {
       "version": "7.0.7",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.7.tgz",
@@ -2203,6 +2383,12 @@
       "integrity": "sha512-bpcvu/MKHHeYX+qeEN8GE7DIravODWdACVA1ctevD8CN24RhPZIKMn9ntfAsrvLfSX3cR5RrBKAbYm9bGs0A+Q==",
       "dev": true
     },
+    "@types/memory-cache": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/@types/memory-cache/-/memory-cache-0.2.1.tgz",
+      "integrity": "sha512-6rmz3bMqJNkj0HIN3AMhOfRN+JhnxxTULeBkwgilfnspjABtKG6ig8mbIzkOjtmiRw+WG1B56z+BO6htGz3IBw==",
+      "dev": true
+    },
     "@types/minimatch": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.4.tgz",
@@ -2222,9 +2408,44 @@
       "dev": true
     },
     "@types/node": {
-      "version": "15.12.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.0.tgz",
-      "integrity": "sha512-+aHJvoCsVhO2ZCuT4o5JtcPrCPyDE3+1nvbDprYes+pPkEsbjH7AGUCNtjMOXS0fqH14t+B7yLzaqSz92FPWyw==",
+      "version": "15.12.2",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.2.tgz",
+      "integrity": "sha512-zjQ69G564OCIWIOHSXyQEEDpdpGl+G348RAKY0XXy9Z5kU9Vzv1GMNnkar/ZJ8dzXB3COzD9Mo9NtRZ4xfgUww==",
+      "dev": true
+    },
+    "@types/node-fetch": {
+      "version": "2.5.10",
+      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.10.tgz",
+      "integrity": "sha512-IpkX0AasN44hgEad0gEF/V6EgR5n69VEqPEgnmoM8GsIGro3PowbWs4tR6IhxUTyPLpOn+fiGG6nrQhcmoCuIQ==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*",
+        "form-data": "^3.0.0"
+      },
+      "dependencies": {
+        "form-data": {
+          "version": "3.0.1",
+          "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
+          "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
+          "dev": true,
+          "requires": {
+            "asynckit": "^0.4.0",
+            "combined-stream": "^1.0.8",
+            "mime-types": "^2.1.12"
+          }
+        }
+      }
+    },
+    "@types/object-hash": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@types/object-hash/-/object-hash-2.1.0.tgz",
+      "integrity": "sha512-RW3VRiuQIMo5PJ4Q1IwBtdLHL/t8ACpzUY40norN9ejE6CUBwKetmSxJnITJ0NlzN/ymF1nvPvlpvegtns7yOg==",
+      "dev": true
+    },
+    "@types/pako": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@types/pako/-/pako-1.0.1.tgz",
+      "integrity": "sha512-GdZbRSJ3Cv5fiwT6I0SQ3ckeN2PWNqxd26W9Z2fCK1tGrrasGy4puvNFtnddqH9UJFMQYXxEuuB7B8UK+LLwSg==",
       "dev": true
     },
     "@types/strip-bom": {
@@ -2240,13 +2461,13 @@
       "dev": true
     },
     "ajv": {
-      "version": "6.12.6",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
-      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+      "version": "7.2.4",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-7.2.4.tgz",
+      "integrity": "sha512-nBeQgg/ZZA3u3SYxyaDvpvDtgZ/EZPF547ARgZBrG9Bhu1vKDwAIjtIf+sDtJUKa2zOcEbmRLBRSyMraS/Oy1A==",
       "requires": {
         "fast-deep-equal": "^3.1.1",
-        "fast-json-stable-stringify": "^2.0.0",
-        "json-schema-traverse": "^0.4.1",
+        "json-schema-traverse": "^1.0.0",
+        "require-from-string": "^2.0.2",
         "uri-js": "^4.2.2"
       }
     },
@@ -2271,6 +2492,11 @@
       "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==",
       "dev": true
     },
+    "argparse": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
+      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
+    },
     "array-back": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz",
@@ -2504,8 +2730,7 @@
     "commander": {
       "version": "7.2.0",
       "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz",
-      "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==",
-      "dev": true
+      "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw=="
     },
     "concat-map": {
       "version": "0.0.1",
@@ -2745,6 +2970,24 @@
       "requires": {
         "ajv": "^6.12.3",
         "har-schema": "^2.0.0"
+      },
+      "dependencies": {
+        "ajv": {
+          "version": "6.12.6",
+          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+          "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+          "requires": {
+            "fast-deep-equal": "^3.1.1",
+            "fast-json-stable-stringify": "^2.0.0",
+            "json-schema-traverse": "^0.4.1",
+            "uri-js": "^4.2.2"
+          }
+        },
+        "json-schema-traverse": {
+          "version": "0.4.1",
+          "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+          "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
+        }
       }
     },
     "harmony-reflect": {
@@ -2758,6 +3001,11 @@
       "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
       "dev": true
     },
+    "he": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
+      "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw=="
+    },
     "http-signature": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
@@ -2875,6 +3123,14 @@
       "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
       "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
     },
+    "js-yaml": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
+      "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+      "requires": {
+        "argparse": "^2.0.1"
+      }
+    },
     "jsbn": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
@@ -2886,9 +3142,9 @@
       "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
     },
     "json-schema-traverse": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
-      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
+      "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="
     },
     "json-stable-stringify": {
       "version": "1.0.1",
@@ -2904,6 +3160,14 @@
       "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
       "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
     },
+    "json5": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.0.tgz",
+      "integrity": "sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA==",
+      "requires": {
+        "minimist": "^1.2.5"
+      }
+    },
     "jsonify": {
       "version": "0.0.0",
       "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz",
@@ -2960,6 +3224,11 @@
       "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
       "dev": true
     },
+    "memory-cache": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/memory-cache/-/memory-cache-0.2.0.tgz",
+      "integrity": "sha1-eJCwHVLADI68nVM+H46xfjA0hxo="
+    },
     "merge2": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
@@ -3037,6 +3306,11 @@
       "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz",
       "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ=="
     },
+    "node-fetch": {
+      "version": "2.6.1",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
+      "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
+    },
     "oauth-sign": {
       "version": "0.9.0",
       "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
@@ -3085,6 +3359,11 @@
         "p-finally": "^1.0.0"
       }
     },
+    "pako": {
+      "version": "0.2.6",
+      "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.6.tgz",
+      "integrity": "sha1-PgxUg1O4WaucgAX6xwa91sevUF8="
+    },
     "path-is-absolute": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
@@ -3204,6 +3483,11 @@
       "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
       "dev": true
     },
+    "require-from-string": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
+      "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="
+    },
     "reusify": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
@@ -3437,14 +3721,6 @@
         "glob": "^7.1.7",
         "json-stable-stringify": "^1.0.1",
         "typescript": "~4.3.2"
-      },
-      "dependencies": {
-        "typescript": {
-          "version": "4.3.2",
-          "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.2.tgz",
-          "integrity": "sha512-zZ4hShnmnoVnAHpVHWpTcxdv7dWP60S2FsydQLV8V5PbS3FifjWFFRiHSWpDJahly88PRyV5teTSLoq4eG7mKw==",
-          "dev": true
-        }
       }
     },
     "ts-morph": {
@@ -3498,10 +3774,9 @@
       "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
     },
     "typescript": {
-      "version": "4.1.5",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.1.5.tgz",
-      "integrity": "sha512-6OSu9PTIzmn9TCDiovULTnET6BgXtDYL4Gg4szY+cGsc3JP1dQL8qvE8kShTRx1NIw4Q9IBHlwODjkjWEtMUyA==",
-      "dev": true
+      "version": "4.3.4",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz",
+      "integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew=="
     },
     "typescript-json-schema": {
       "version": "0.50.1",
@@ -3519,9 +3794,9 @@
       },
       "dependencies": {
         "@types/node": {
-          "version": "14.17.2",
-          "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.2.tgz",
-          "integrity": "sha512-sld7b/xmFum66AAKuz/rp/CUO8+98fMpyQ3SBfzzBNGMd/1iHBTAg9oyAvcYlAj46bpc74r91jSw2iFdnx29nw==",
+          "version": "14.17.3",
+          "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.3.tgz",
+          "integrity": "sha512-e6ZowgGJmTuXa3GyaPbTGxX17tnThl2aSSizrFthQ7m9uLGZBXiGhgE55cjRZTF5kjZvYn9EOPOMljdjwbflxw==",
           "dev": true
         },
         "typescript": {
@@ -3578,9 +3853,9 @@
       }
     },
     "winston": {
-      "version": "git+ssh://git@github.com/FoxxMD/winston.git#9639da027cd4f3b46b055b0193f240639ef53409",
-      "integrity": "sha512-StxHu2puJAl2Ky8mXitI2nQ7lDNT5PPS4cnTj+2FF4orQSKVRIGhulMkRpoAtk9Z40QuhkgNRfCmBGQXa30BZQ==",
-      "from": "winston@FoxxMD/winston#9639da027cd4f3b46b055b0193f240639ef53409",
+      "version": "git+ssh://git@github.com/FoxxMD/winston.git#fbab8de969ecee578981c77846156c7f43b5f01e",
+      "integrity": "sha512-OnunfctuocKBmG0uzkBkVYGSW13VYmzglQPwj4ZFOBEtV2e2ECqe65eCaYRYHSvKynGc8T5hiaEWAVR2hVj+Yg==",
+      "from": "winston@FoxxMD/winston#fbab8de969ecee578981c77846156c7f43b5f01e",
       "requires": {
         "@dabh/diagnostics": "^2.0.2",
         "async": "^3.1.0",
@@ -3753,6 +4028,11 @@
       "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",
       "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==",
       "dev": true
+    },
+    "zlib": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/zlib/-/zlib-1.0.5.tgz",
+      "integrity": "sha1-bnyXL8NxxkWmr7A6sUdp3vEU/MA="
     }
   }
 }

package.json

@@ -9,11 +9,13 @@
     "start": "node server.js",
     "guard": "ts-auto-guard src/JsonConfig.ts",
     "schema": "npm run -s schema-app & npm run -s schema-ruleset & npm run -s schema-rule & npm run -s schema-action",
-    "schema-app": "typescript-json-schema tsconfig.json JSONConfig --out src/Schema/App.json --required --tsNodeRegister --refs --propOrder",
-    "schema-ruleset": "typescript-json-schema tsconfig.json RuleSetJSONConfig --out src/Schema/RuleSet.json --required --tsNodeRegister --refs --propOrder",
-    "schema-rule": "typescript-json-schema tsconfig.json RuleJSONConfig --out src/Schema/Rule.json --required --tsNodeRegister --refs --propOrder",
-    "schema-action": "typescript-json-schema tsconfig.json ActionJSONConfig --out src/Schema/Action.json --required --tsNodeRegister --refs --propOrder",
-    "schemaNotWorking": "./node_modules/.bin/ts-json-schema-generator -f tsconfig.json -p src/JsonConfig.ts -t JSONConfig --out src/Schema/vegaSchema.json"
+    "schema-app": "typescript-json-schema tsconfig.json JSONConfig --out src/Schema/App.json --required --tsNodeRegister --refs",
+    "schema-ruleset": "typescript-json-schema tsconfig.json RuleSetJson --out src/Schema/RuleSet.json --required --tsNodeRegister --refs",
+    "schema-rule": "typescript-json-schema tsconfig.json RuleJson --out src/Schema/Rule.json --required --tsNodeRegister --refs",
+    "schema-action": "typescript-json-schema tsconfig.json ActionJson --out src/Schema/Action.json --required --tsNodeRegister --refs",
+    "schemaNotWorking": "./node_modules/.bin/ts-json-schema-generator -f tsconfig.json -p src/JsonConfig.ts -t JSONConfig --out src/Schema/vegaSchema.json",
+    "circular": "madge --circular --extensions ts src/index.ts",
+    "circular-graph": "madge --image graph.svg --circular --extensions ts src/index.ts"
   },
   "engines": {
     "node": ">=15"
@@ -22,24 +24,39 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "ajv": "^6.12.6",
+    "ajv": "^7.2.4",
+    "commander": "^7.2.0",
     "dayjs": "^1.10.5",
     "es6-error": "^4.1.1",
     "fast-deep-equal": "^3.1.3",
-    "minimist": "^1.2.5",
+    "he": "^1.2.0",
+    "js-yaml": "^4.1.0",
+    "json5": "^2.2.0",
+    "memory-cache": "^0.2.0",
     "mustache": "^4.2.0",
+    "node-fetch": "^2.6.1",
+    "object-hash": "^2.2.0",
     "p-event": "^4.2.0",
+    "pako": "^0.2.6",
     "safe-stable-stringify": "^1.1.1",
     "snoostorm": "^1.5.2",
     "snoowrap": "^1.23.0",
-    "winston": "FoxxMD/winston#9639da027cd4f3b46b055b0193f240639ef53409",
-    "winston-daily-rotate-file": "^4.5.5"
+    "typescript": "^4.3.4",
+    "winston": "FoxxMD/winston#fbab8de969ecee578981c77846156c7f43b5f01e",
+    "winston-daily-rotate-file": "^4.5.5",
+    "zlib": "^1.0.5"
   },
   "devDependencies": {
     "@tsconfig/node14": "^1.0.0",
+    "@types/he": "^1.1.1",
+    "@types/js-yaml": "^4.0.1",
+    "@types/memory-cache": "^0.2.1",
     "@types/minimist": "^1.2.1",
     "@types/mustache": "^4.1.1",
     "@types/node": "^15.6.1",
+    "@types/node-fetch": "^2.5.10",
+    "@types/object-hash": "^2.1.0",
+    "@types/pako": "^1.0.1",
     "ts-auto-guard": "*",
     "ts-json-schema-generator": "^0.93.0",
     "typescript-json-schema": "^0.50.1"

src/Action/ActionFactory.ts

@@ -1,23 +1,33 @@
-import {CommentAction, CommentActionJSONConfig} from "./CommentAction";
+import {CommentAction, CommentActionJson} from "./CommentAction";
 import LockAction from "./LockAction";
 import {RemoveAction} from "./RemoveAction";
-import {ReportAction, ReportActionJSONConfig} from "./ReportAction";
-import {FlairAction, FlairActionJSONConfig} from "./SubmissionAction/FlairAction";
-import Action, {ActionJSONConfig} from "./index";
+import {ReportAction, ReportActionJson} from "./ReportAction";
+import {FlairAction, FlairActionJson} from "./SubmissionAction/FlairAction";
+import Action, {ActionJson} from "./index";
+import {Logger} from "winston";
+import {UserNoteAction, UserNoteActionJson} from "./UserNoteAction";
+import ApproveAction, {ApproveActionConfig} from "./ApproveAction";
+import BanAction, {BanActionJson} from "./BanAction";
 
 export function actionFactory
-(config: ActionJSONConfig): Action {
+(config: ActionJson, logger: Logger, subredditName: string): Action {
     switch (config.kind) {
         case 'comment':
-            return new CommentAction(config as CommentActionJSONConfig);
+            return new CommentAction({...config as CommentActionJson, logger, subredditName});
         case 'lock':
-            return new LockAction();
+            return new LockAction({...config, logger, subredditName});
         case 'remove':
-            return new RemoveAction();
+            return new RemoveAction({...config, logger, subredditName});
         case 'report':
-            return new ReportAction(config as ReportActionJSONConfig);
+            return new ReportAction({...config as ReportActionJson, logger, subredditName});
         case 'flair':
-            return new FlairAction(config as FlairActionJSONConfig);
+            return new FlairAction({...config as FlairActionJson, logger, subredditName});
+        case 'approve':
+            return new ApproveAction({...config as ApproveActionConfig, logger, subredditName});
+        case 'usernote':
+            return new UserNoteAction({...config as UserNoteActionJson, logger, subredditName});
+        case 'ban':
+            return new BanAction({...config as BanActionJson, logger, subredditName});
         default:
             throw new Error('rule "kind" was not recognized.');
     }

src/Action/ApproveAction.ts

@@ -0,0 +1,35 @@
+import {ActionJson, ActionConfig} from "./index";
+import Action from "./index";
+import Snoowrap, {Comment, Submission} from "snoowrap";
+import {RuleResult} from "../Rule";
+
+export class ApproveAction extends Action {
+    getKind() {
+        return 'Approve';
+    }
+
+    async process(item: Comment | Submission, ruleResults: RuleResult[]): Promise<void> {
+        //snoowrap typing issue, thinks comments can't be locked
+        // @ts-ignore
+        if (item.approved) {
+            this.logger.warn('Item is already approved');
+        }
+        if (!this.dryRun) {
+            // @ts-ignore
+            await item.approve();
+        }
+    }
+}
+
+export interface ApproveActionConfig extends ActionConfig {
+
+}
+
+/**
+ * Ban the Author of the Activity this Check is run on
+ * */
+export interface ApproveActionJson extends ApproveActionConfig, ActionJson {
+    kind: 'approve'
+}
+
+export default ApproveAction;

src/Action/BanAction.ts

@@ -0,0 +1,107 @@
+import {ActionJson, ActionConfig, ActionOptions} from "./index";
+import Action from "./index";
+import Snoowrap, {Comment, Submission} from "snoowrap";
+import {RuleResult} from "../Rule";
+import {renderContent} from "../Utils/SnoowrapUtils";
+import {Footer} from "../Common/interfaces";
+
+export class BanAction extends Action {
+
+    message?: string;
+    reason?: string;
+    duration?: number;
+    note?: string;
+    footer?: false | string;
+
+    constructor(options: BanActionOptions) {
+        super(options);
+        const {
+            message,
+            reason,
+            duration,
+            note,
+            footer,
+        } = options;
+        this.footer = footer;
+        this.message = message;
+        this.reason = reason;
+        this.duration = duration;
+        this.note = note;
+    }
+
+    getKind() {
+        return 'Ban';
+    }
+
+    async process(item: Comment | Submission, ruleResults: RuleResult[]): Promise<void> {
+        const content = this.message === undefined ? undefined : await this.resources.getContent(this.message, item.subreddit);
+        const renderedBody = content === undefined ? undefined : await renderContent(content, item, ruleResults, this.resources.userNotes);
+        const renderedContent = renderedBody === undefined ? undefined : `${renderedBody}${await this.resources.generateFooter(item, this.footer)}`;
+
+        let banPieces = [];
+        banPieces.push(`Message: ${renderedContent === undefined ? 'None' : `${renderedContent.length > 100 ? `\r\n${renderedContent}` : renderedContent}`}`);
+        banPieces.push(`Reason:  ${this.reason || 'None'}`);
+        banPieces.push(`Note:    ${this.note || 'None'}`);
+        const durText = this.duration === undefined ? 'permanently' : `for ${this.duration} days`;
+        this.logger.info(`Banning ${item.author.name} ${durText}${this.reason !== undefined ? ` (${this.reason})` : ''}`);
+        this.logger.verbose(`\r\n${banPieces.join('\r\n')}`);
+        if (!this.dryRun) {
+            // @ts-ignore
+            await item.subreddit.banUser({
+                name: item.author.id,
+                banMessage: renderedContent === undefined ? undefined : renderedContent,
+                banReason: this.reason,
+                banNote: this.note,
+                duration: this.duration
+            });
+        }
+    }
+}
+
+export interface BanActionConfig extends ActionConfig, Footer {
+    /**
+     * The message that is sent in the ban notification. `message` is interpreted as reddit-flavored Markdown.
+     *
+     * If value starts with `wiki:` then the proceeding value will be used to get a wiki page
+     *
+     * EX `wiki:botconfig/mybot` tries to get `https://reddit.com/mySubredditExample/wiki/botconfig/mybot`
+     *
+     * EX `this is plain text` => "this is plain text"
+     *
+     * EX `this is **bold** markdown text` => "this is **bold** markdown text"
+     *
+     * @examples ["This is the content of a comment/report/usernote", "this is **bold** markdown text", "wiki:botconfig/acomment" ]
+     * */
+    message?: string
+    /**
+     * Reason for ban.
+     * @maxLength 100
+     * @examples ["repeat spam"]
+     * */
+    reason?: string
+    /**
+     * Number of days to ban the Author. If not specified Author will be banned permanently.
+     * @minimum 1
+     * @maximum 999
+     * @examples [90]
+     * */
+    duration?: number
+    /**
+     * A mod note for this ban
+     * @maxLength 100
+     * @examples ["Sock puppet for u/AnotherUser"]
+     * */
+    note?: string
+}
+
+export interface BanActionOptions extends BanActionConfig, ActionOptions {
+}
+
+/**
+ * Ban the Author of the Activity this Check is run on
+ * */
+export interface BanActionJson extends BanActionConfig, ActionJson {
+    kind: 'ban',
+}
+
+export default BanAction;

src/Action/CommentAction.ts

@@ -1,21 +1,16 @@
-import Action, {ActionJSONConfig, ActionConfig, ActionOptions} from "./index";
-import Snoowrap, {Comment} from "snoowrap";
+import Action, {ActionJson, ActionOptions} from "./index";
+import {Comment} from "snoowrap";
 import Submission from "snoowrap/dist/objects/Submission";
-import dayjs, {Dayjs} from "dayjs";
 import {renderContent} from "../Utils/SnoowrapUtils";
-import {RichContent} from "../Common/interfaces";
-
-export const WIKI_DESCRIM = 'wiki:';
+import {Footer, RequiredRichContent, RichContent} from "../Common/interfaces";
+import {RuleResult} from "../Rule";
 
 export class CommentAction extends Action {
     content: string;
-    hasWiki: boolean;
-    wiki?: string;
-    wikiFetched?: Dayjs;
     lock: boolean = false;
     sticky: boolean = false;
     distinguish: boolean = false;
-    name?: string = 'Comment';
+    footer?: false | string;
 
     constructor(options: CommentActionOptions) {
         super(options);
@@ -24,42 +19,52 @@ export class CommentAction extends Action {
             lock = false,
             sticky = false,
             distinguish = false,
+            footer,
         } = options;
-        this.hasWiki = content.trim().substring(0, WIKI_DESCRIM.length) === WIKI_DESCRIM;
+        this.footer = footer;
         this.content = content;
-        if (this.hasWiki) {
-            this.wiki = this.content.trim().substring(WIKI_DESCRIM.length);
-        }
         this.lock = lock;
         this.sticky = sticky;
         this.distinguish = distinguish;
     }
 
-    async handle(item: Comment | Submission, client: Snoowrap): Promise<void> {
-        if (this.hasWiki && (this.wikiFetched === undefined || Math.abs(dayjs().diff(this.wikiFetched, 'minute')) > 5)) {
-            try {
-                const wiki = item.subreddit.getWikiPage(this.wiki as string);
-                this.content = await wiki.content_md;
-                this.wikiFetched = dayjs();
-            } catch (err) {
-                this.logger.error(err);
-                throw new Error(`Could not read wiki page. Please ensure the page '${this.wiki}' exists and is readable`);
+    getKind() {
+        return 'Comment';
+    }
+
+    async process(item: Comment | Submission, ruleResults: RuleResult[]): Promise<void> {
+        const content = await this.resources.getContent(this.content, item.subreddit);
+        const body = await renderContent(content, item, ruleResults, this.resources.userNotes);
+
+        const footer = await this.resources.generateFooter(item, this.footer);
+
+        const renderedContent = `${body}${footer}`;
+        this.logger.verbose(`Contents:\r\n${renderedContent.length > 100 ? `\r\n${renderedContent}` : renderedContent}`);
+
+        if(item.archived) {
+            this.logger.warn('Cannot comment because Item is archived');
+            return;
+        }
+        let reply: Comment;
+        if(!this.dryRun) {
+            // @ts-ignore
+           reply = await item.reply(renderedContent);
+        }
+        if (this.lock) {
+            if (!this.dryRun) {
+                // snoopwrap typing issue, thinks comments can't be locked
+                // @ts-ignore
+                await item.lock();
             }
         }
-        // @ts-ignore
-        const reply: Comment = await item.reply(renderContent(this.content, item));
-        if (this.lock && item instanceof Submission) {
-            // @ts-ignore
-            await item.lock();
-        }
-        if (this.distinguish) {
+        if (this.distinguish && !this.dryRun) {
             // @ts-ignore
             await reply.distinguish({sticky: this.sticky});
         }
     }
 }
 
-export interface CommentActionConfig extends RichContent {
+export interface CommentActionConfig extends RequiredRichContent, Footer {
     /**
      * Lock the comment after creation?
      * */
@@ -74,12 +79,12 @@ export interface CommentActionConfig extends RichContent {
     distinguish?: boolean,
 }
 
-export interface CommentActionOptions extends CommentActionConfig,ActionOptions {
+export interface CommentActionOptions extends CommentActionConfig, ActionOptions {
 }
 
 /**
  * Reply to the Activity. For a submission the reply will be a top-level comment.
  * */
-export interface CommentActionJSONConfig extends CommentActionConfig, ActionJSONConfig {
-
+export interface CommentActionJson extends CommentActionConfig, ActionJson {
+kind: 'comment'
 }

src/Action/LockAction.ts

@@ -1,11 +1,21 @@
-import {ActionJSONConfig, ActionConfig} from "./index";
+import {ActionJson, ActionConfig} from "./index";
 import Action from "./index";
 import Snoowrap, {Comment, Submission} from "snoowrap";
+import {RuleResult} from "../Rule";
 
 export class LockAction extends Action {
-    name?: string = 'Lock';
-    async handle(item: Comment|Submission, client: Snoowrap): Promise<void> {
-        if (item instanceof Submission) {
+    getKind() {
+        return 'Lock';
+    }
+
+    async process(item: Comment | Submission, ruleResults: RuleResult[]): Promise<void> {
+        //snoowrap typing issue, thinks comments can't be locked
+        // @ts-ignore
+        if (item.locked) {
+            this.logger.warn('Item is already locked');
+        }
+        if (!this.dryRun) {
+            //snoowrap typing issue, thinks comments can't be locked
             // @ts-ignore
             await item.lock();
         }
@@ -19,8 +29,8 @@ export interface LockActionConfig extends ActionConfig {
 /**
  * Lock the Activity
  * */
-export interface LockActionJSONConfig extends LockActionConfig, ActionJSONConfig {
-
+export interface LockActionJson extends LockActionConfig, ActionJson {
+kind: 'lock'
 }
 
 export default LockAction;

src/Action/RemoveAction.ts

@@ -1,12 +1,25 @@
-import {ActionJSONConfig, ActionConfig} from "./index";
+import {ActionJson, ActionConfig} from "./index";
 import Action from "./index";
 import Snoowrap, {Comment, Submission} from "snoowrap";
+import {RuleResult} from "../Rule";
+import {activityIsRemoved} from "../Utils/SnoowrapUtils";
 
 export class RemoveAction extends Action {
-    name?: string = 'Remove';
-    async handle(item: Comment|Submission, client: Snoowrap): Promise<void> {
+    getKind() {
+        return 'Remove';
+    }
+
+    async process(item: Comment | Submission, ruleResults: RuleResult[]): Promise<void> {
+        // issue with snoowrap typings, doesn't think prop exists on Submission
         // @ts-ignore
-        await item.remove();
+        if (activityIsRemoved(item)) {
+            this.logger.warn('Item is already removed');
+            return;
+        }
+        if (!this.dryRun) {
+            // @ts-ignore
+            await item.remove();
+        }
     }
 }
 
@@ -17,6 +30,6 @@ export interface RemoveActionConfig extends ActionConfig {
 /**
  * Remove the Activity
  * */
-export interface RemoveActionJSONConfig extends RemoveActionConfig, ActionJSONConfig {
-
+export interface RemoveActionJson extends RemoveActionConfig, ActionJson {
+kind: 'remove'
 }

src/Action/ReportAction.ts

@@ -1,27 +1,41 @@
-import {ActionJSONConfig, ActionConfig, ActionOptions} from "./index";
+import {ActionJson, ActionConfig, ActionOptions} from "./index";
 import Action from "./index";
 import Snoowrap, {Comment, Submission} from "snoowrap";
+import {truncateStringToLength} from "../util";
+import {renderContent} from "../Utils/SnoowrapUtils";
+import {RuleResult} from "../Rule";
+import {RichContent} from "../Common/interfaces";
+
+// https://www.reddit.com/dev/api/oauth#POST_api_report
+// denotes 100 characters maximum
+const reportTrunc = truncateStringToLength(100);
+// actually only applies to VISIBLE text on OLD reddit... on old reddit rest of text is visible on hover. on new reddit the whole thing displays (up to at least 400 characters)
 
 export class ReportAction extends Action {
     content: string;
-    name?: string = 'Report';
 
     constructor(options: ReportActionOptions) {
         super(options);
-        this.content = options.content;
+        this.content = options.content || '';
     }
 
-    async handle(item: Comment | Submission, client: Snoowrap): Promise<void> {
-        // @ts-ignore
-        await item.report({reason: this.content});
+    getKind() {
+        return 'Report';
+    }
+
+    async process(item: Comment | Submission, ruleResults: RuleResult[]): Promise<void> {
+        const content = await this.resources.getContent(this.content, item.subreddit);
+        const renderedContent = await renderContent(content, item, ruleResults, this.resources.userNotes);
+        this.logger.verbose(`Contents:\r\n${renderedContent}`);
+        const truncatedContent = reportTrunc(renderedContent);
+        if(!this.dryRun) {
+            // @ts-ignore
+            await item.report({reason: truncatedContent});
+        }
     }
 }
 
-export interface ReportActionConfig {
-    /**
-     * The text of the report
-     * */
-    content: string,
+export interface ReportActionConfig extends RichContent {
 }
 
 export interface ReportActionOptions extends ReportActionConfig, ActionOptions {
@@ -30,6 +44,6 @@ export interface ReportActionOptions extends ReportActionConfig, ActionOptions {
 /**
  * Report the Activity
  * */
-export interface ReportActionJSONConfig extends ReportActionConfig, ActionJSONConfig {
-
+export interface ReportActionJson extends ReportActionConfig, ActionJson {
+    kind: 'report'
 }

src/Action/SubmissionAction/FlairAction.ts

@@ -1,11 +1,11 @@
 import {SubmissionActionConfig} from "./index";
-import Action, {ActionJSONConfig} from "../index";
+import Action, {ActionJson, ActionOptions} from "../index";
 import Snoowrap, {Comment, Submission} from "snoowrap";
+import {RuleResult} from "../../Rule";
 
 export class FlairAction extends Action {
     text: string;
     css: string;
-    name?: string = 'Flair';
 
     constructor(options: FlairActionOptions) {
         super(options);
@@ -16,10 +16,18 @@ export class FlairAction extends Action {
         this.css = options.css || '';
     }
 
-    async handle(item: Comment | Submission, client: Snoowrap): Promise<void> {
+    getKind() {
+        return 'Flair';
+    }
+
+    async process(item: Comment | Submission, ruleResults: RuleResult[]): Promise<void> {
         if (item instanceof Submission) {
-            // @ts-ignore
-            await item.assignFlair({text: this.text, cssClass: this.css})
+            if(!this.dryRun) {
+                // @ts-ignore
+                await item.assignFlair({text: this.text, cssClass: this.css})
+            }
+        } else {
+            this.logger.warn('Cannot flair Comment');
         }
     }
 }
@@ -28,7 +36,7 @@ export class FlairAction extends Action {
  * @minProperties 1
  * @additionalProperties false
  * */
-export interface FlairActionOptions extends SubmissionActionConfig {
+export interface FlairActionConfig extends SubmissionActionConfig {
     /**
      * The text of the flair to apply
     * */
@@ -39,9 +47,13 @@ export interface FlairActionOptions extends SubmissionActionConfig {
     css?: string,
 }
 
+export interface FlairActionOptions extends FlairActionConfig,ActionOptions {
+
+}
+
 /**
  * Flair the Submission
  * */
-export interface FlairActionJSONConfig extends FlairActionOptions, ActionJSONConfig {
-
+export interface FlairActionJson extends FlairActionConfig, ActionJson {
+kind: 'flair'
 }

src/Action/UserNoteAction.ts

@@ -0,0 +1,65 @@
+import {ActionJson, ActionConfig, ActionOptions} from "./index";
+import Action from "./index";
+import {Comment} from "snoowrap";
+import {renderContent} from "../Utils/SnoowrapUtils";
+import {RuleResult} from "../Rule";
+import {UserNote, UserNoteJson} from "../Subreddit/UserNotes";
+import Submission from "snoowrap/dist/objects/Submission";
+
+
+export class UserNoteAction extends Action {
+    content: string;
+    type: string;
+    allowDuplicate: boolean;
+
+    constructor(options: UserNoteActionOptions) {
+        super(options);
+        const {type, content = '', allowDuplicate = false} = options;
+        this.type = type;
+        this.content = content;
+        this.allowDuplicate = allowDuplicate;
+    }
+
+    getKind() {
+        return 'User Note';
+    }
+
+    async process(item: Comment | Submission, ruleResults: RuleResult[]): Promise<void> {
+        const content = await this.resources.getContent(this.content, item.subreddit);
+        const renderedContent = await renderContent(content, item, ruleResults, this.resources.userNotes);
+        this.logger.verbose(`Note:\r\n(${this.type}) ${renderedContent}`);
+
+        if (!this.allowDuplicate) {
+            const notes = await this.resources.userNotes.getUserNotes(item.author);
+            const existingNote = notes.find((x) => x.link.includes(item.id));
+            if (existingNote) {
+                this.logger.info(`Will not add note because one already exists for this Activity (${existingNote.time.local().format()}) and allowDuplicate=false`);
+                return;
+            }
+        }
+        if (!this.dryRun) {
+            await this.resources.userNotes.addUserNote(item, this.type, renderedContent);
+        } else if (!await this.resources.userNotes.warningExists(this.type)) {
+            this.logger.warn(`UserNote type '${this.type}' does not exist. If you meant to use this please add it through Toolbox first.`);
+        }
+    }
+}
+
+export interface UserNoteActionConfig extends ActionConfig,UserNoteJson {
+    /**
+     * Add Note even if a Note already exists for this Activity
+     * @examples [false]
+     * @default false
+     * */
+    allowDuplicate?: boolean,
+}
+
+export interface UserNoteActionOptions extends UserNoteActionConfig, ActionOptions {
+}
+
+/**
+ * Add a Toolbox User Note to the Author of this Activity
+ * */
+export interface UserNoteActionJson extends UserNoteActionConfig, ActionJson {
+    kind: 'usernote'
+}

src/Action/index.ts

@@ -1,50 +1,139 @@
-import Snoowrap, {Comment, Submission} from "snoowrap";
+import {Comment, Submission} from "snoowrap";
 import {Logger} from "winston";
-import {createLabelledLogger, loggerMetaShuffle} from "../util";
+import {RuleResult} from "../Rule";
+import ResourceManager, {SubredditResources} from "../Subreddit/SubredditResources";
+import {ChecksActivityState, TypedActivityStates} from "../Common/interfaces";
+import Author, {AuthorOptions} from "../Author/Author";
+import {isItem} from "../Utils/SnoowrapUtils";
 
 export abstract class Action {
     name?: string;
     logger: Logger;
+    resources: SubredditResources;
+    authorIs: AuthorOptions;
+    itemIs: TypedActivityStates;
+    dryRun: boolean;
 
-    constructor(options: ActionOptions = {}) {
+    constructor(options: ActionOptions) {
         const {
-            name,
-            loggerPrefix = '',
+            name = this.getKind(),
             logger,
+            subredditName,
+            dryRun = false,
+            authorIs: {
+                include = [],
+                exclude = [],
+            } = {},
+            itemIs = [],
         } = options;
-        if (name !== undefined) {
-            this.name = name;
-        }
-        if (logger === undefined) {
-            const prefix = `${loggerPrefix}|${this.name}`;
-            this.logger = createLabelledLogger(prefix, prefix);
-        } else {
-            this.logger = logger.child(loggerMetaShuffle(logger, name || 'Action', undefined, {truncateLength: 100}));
+
+        this.name = name;
+        this.dryRun = dryRun;
+        this.resources = ResourceManager.get(subredditName) as SubredditResources;
+        this.logger = logger.child({labels: [`Action ${this.getActionUniqueName()}`]});
+
+        this.authorIs = {
+            exclude: exclude.map(x => new Author(x)),
+            include: include.map(x => new Author(x)),
         }
+
+        this.itemIs = itemIs;
     }
 
-    abstract handle(item: Comment | Submission, client: Snoowrap): Promise<void>;
+    abstract getKind(): string;
+
+    getActionUniqueName() {
+        return this.name === this.getKind() ? this.getKind() : `${this.getKind()} - ${this.name}`;
+    }
+
+    async handle(item: Comment | Submission, ruleResults: RuleResult[]): Promise<void> {
+        let actionRun = false;
+        const [itemPass, crit] = isItem(item, this.itemIs, this.logger);
+        if (!itemPass) {
+            this.logger.verbose(`Activity did not pass 'itemIs' test, Action not run`);
+            return;
+        }
+        const authorRun = async () => {
+            if (this.authorIs.include !== undefined && this.authorIs.include.length > 0) {
+                for (const auth of this.authorIs.include) {
+                    if (await this.resources.testAuthorCriteria(item, auth)) {
+                        await this.process(item, ruleResults);
+                        return true;
+                    }
+                }
+                this.logger.verbose('Inclusive author criteria not matched, Action not run');
+                return false;
+            }
+            if (!actionRun && this.authorIs.exclude !== undefined && this.authorIs.exclude.length > 0) {
+                for (const auth of this.authorIs.exclude) {
+                    if (await this.resources.testAuthorCriteria(item, auth, false)) {
+                        await this.process(item, ruleResults);
+                        return true;
+                    }
+                }
+                this.logger.verbose('Exclusive author criteria not matched, Action not run');
+                return false;
+            }
+            return null;
+        }
+        const authorRunResults = await authorRun();
+        if (null === authorRunResults) {
+            await this.process(item, ruleResults);
+        } else if (!authorRunResults) {
+            return;
+        }
+        this.logger.verbose(`${this.dryRun ? 'DRYRUN - ' : ''}Done`);
+    }
+
+    abstract process(item: Comment | Submission, ruleResults: RuleResult[]): Promise<void>;
 }
 
-export interface ActionOptions {
-    name?: string;
-    logger?: Logger,
-    loggerPrefix?: string,
+export interface ActionOptions extends ActionConfig {
+    logger: Logger;
+    subredditName: string;
 }
 
-export interface ActionConfig {
+export interface ActionConfig extends ChecksActivityState {
     /**
-     * A friendly name for this Action
+     * An optional, but highly recommended, friendly name for this Action. If not present will default to `kind`.
+     *
+     * Can only contain letters, numbers, underscore, spaces, and dashes
+     *
+     * @pattern ^[a-zA-Z]([\w -]*[\w])?$
+     * @examples ["myDescriptiveAction"]
      * */
     name?: string;
+    /**
+     * If `true` the Action will not make the API request to Reddit to perform its action.
+     *
+     * @default false
+     * @examples [false, true]
+     * */
+    dryRun?: boolean;
+
+    /**
+     * If present then these Author criteria are checked before running the Action. If criteria fails then the Action is not run.
+     * */
+    authorIs?: AuthorOptions
+
+    /**
+     * A list of criteria to test the state of the `Activity` against before running the Action.
+     *
+     * If any set of criteria passes the Action will be run.
+     *
+     * */
+    itemIs?: TypedActivityStates
 }
 
-/** @see {isActionConfig} ts-auto-guard:type-guard */
-export interface ActionJSONConfig extends ActionConfig {
+export interface ActionJson extends ActionConfig {
     /**
      * The type of action that will be performed
      */
-    kind: 'comment' | 'lock' | 'remove' | 'report' | 'flair'
+    kind: 'comment' | 'lock' | 'remove' | 'report' | 'approve' | 'ban' | 'flair' | 'usernote'
+}
+
+export const isActionJson = (obj: object): obj is ActionJson => {
+    return (obj as ActionJson).kind !== undefined;
 }
 
 export default Action;

src/App.ts

@@ -0,0 +1,302 @@
+import Snoowrap, { Subreddit } from "snoowrap";
+import {Manager} from "./Subreddit/Manager";
+import winston, {Logger} from "winston";
+import {argParseInt, labelledFormat, parseBool, parseFromJsonOrYamlToObject, parseSubredditName, sleep} from "./util";
+import snoowrap from "snoowrap";
+import pEvent from "p-event";
+import EventEmitter from "events";
+import CacheManager from './Subreddit/SubredditResources';
+import dayjs, {Dayjs} from "dayjs";
+import LoggedError from "./Utils/LoggedError";
+import ConfigParseError from "./Utils/ConfigParseError";
+
+const {transports} = winston;
+
+const snooLogWrapper = (logger: Logger) => {
+    return {
+        warn: (...args: any[]) => logger.warn(args.slice(0, 2).join(' '), [args.slice(2)]),
+        debug: (...args: any[]) => logger.debug(args.slice(0, 2).join(' '), [args.slice(2)]),
+        info: (...args: any[]) => logger.info(args.slice(0, 2).join(' '), [args.slice(2)]),
+        trace: (...args: any[]) => logger.debug(args.slice(0, 2).join(' '), [args.slice(2)]),
+    }
+}
+
+export class App {
+
+    client: Snoowrap;
+    subreddits: string[];
+    subManagers: Manager[] = [];
+    logger: Logger;
+    wikiLocation: string;
+    dryRun?: true | undefined;
+    heartbeatInterval: number;
+    apiLimitWarning: number;
+    heartBeating: boolean = false;
+
+    constructor(options: any = {}) {
+        const {
+            subreddits = [],
+            clientId,
+            clientSecret,
+            accessToken,
+            refreshToken,
+            logDir = process.env.LOG_DIR || `${process.cwd()}/logs`,
+            logLevel = process.env.LOG_LEVEL || 'verbose',
+            wikiConfig = process.env.WIKI_CONFIG || 'botconfig/contextbot',
+            snooDebug = process.env.SNOO_DEBUG || false,
+            dryRun = process.env.DRYRUN || false,
+            heartbeat = process.env.HEARTBEAT || 300,
+            apiLimitWarning = process.env.API_REMAINING || 250,
+            version,
+            authorTTL = process.env.AUTHOR_TTL || 10000,
+            disableCache = process.env.DISABLE_CACHE || false,
+        } = options;
+
+        CacheManager.authorTTL = argParseInt(authorTTL);
+        CacheManager.enabled = !parseBool(disableCache);
+
+        this.dryRun = parseBool(dryRun) === true ? true : undefined;
+        this.heartbeatInterval = argParseInt(heartbeat);
+        this.apiLimitWarning = argParseInt(apiLimitWarning);
+        this.wikiLocation = wikiConfig;
+
+        const consoleTransport = new transports.Console();
+
+        const myTransports = [
+            consoleTransport,
+        ];
+        let errorTransports = [];
+
+        if (logDir !== false) {
+            let logPath = logDir;
+            if (logPath === true) {
+                logPath = `${process.cwd()}/logs`;
+            }
+            const rotateTransport = new winston.transports.DailyRotateFile({
+                dirname: logPath,
+                createSymlink: true,
+                symlinkName: 'contextBot-current.log',
+                filename: 'contextBot-%DATE%.log',
+                datePattern: 'YYYY-MM-DD',
+                maxSize: '5m'
+            });
+            // @ts-ignore
+            myTransports.push(rotateTransport);
+            errorTransports.push(rotateTransport);
+        }
+
+        const loggerOptions = {
+            level: logLevel || 'info',
+            format: labelledFormat(),
+            transports: myTransports,
+            levels: {
+                error: 0,
+                warn: 1,
+                info: 2,
+                http: 3,
+                verbose: 4,
+                debug: 5,
+                trace: 5,
+                silly: 6
+            },
+            exceptionHandlers: errorTransports,
+            rejectionHandlers: errorTransports,
+        };
+
+        winston.loggers.add('default', loggerOptions);
+
+        this.logger = winston.loggers.get('default');
+
+        if (this.dryRun) {
+            this.logger.info('Running in DRYRUN mode');
+        }
+
+        let subredditsArg = [];
+        if (subreddits !== undefined) {
+            if (Array.isArray(subreddits)) {
+                subredditsArg = subreddits;
+            } else {
+                subredditsArg = subreddits.split(',');
+            }
+        }
+        this.subreddits = subredditsArg.map(parseSubredditName);
+
+        const creds = {
+            userAgent: `web:contextBot:${version}`,
+            clientId,
+            clientSecret,
+            refreshToken,
+            accessToken,
+        };
+
+        this.client = new snoowrap(creds);
+        this.client.config({
+            warnings: true,
+            maxRetryAttempts: 5,
+            debug: parseBool(snooDebug),
+            logger: snooLogWrapper(this.logger.child({labels: ['Snoowrap']})),
+            continueAfterRatelimitError: true,
+        });
+    }
+
+    async buildManagers(subreddits: string[] = []) {
+        let availSubs = [];
+        const name = await this.client.getMe().name;
+        this.logger.info(`Reddit API Limit Remaining: ${this.client.ratelimitRemaining}`);
+        this.logger.info(`Authenticated Account: /u/${name}`);
+        for (const sub of await this.client.getModeratedSubreddits()) {
+            // TODO don't know a way to check permissions yet
+            availSubs.push(sub);
+        }
+        this.logger.info(`/u/${name} is a moderator of these subreddits: ${availSubs.map(x => x.display_name_prefixed).join(', ')}`);
+
+        let subsToRun: Subreddit[] = [];
+        const subsToUse = subreddits.length > 0 ? subreddits.map(parseSubredditName) : this.subreddits;
+        if (subsToUse.length > 0) {
+            this.logger.info(`User-defined subreddit constraints detected (CLI argument or environmental variable), will try to run on: ${subsToUse.join(', ')}`);
+            for (const sub of subsToUse) {
+                const asub = availSubs.find(x => x.display_name.toLowerCase() === sub.toLowerCase())
+                if (asub === undefined) {
+                    this.logger.warn(`Will not run on ${sub} because is not modded by, or does not have appropriate permissions to mod with, for this client.`);
+                } else {
+                    // @ts-ignore
+                    const fetchedSub = await asub.fetch();
+                    subsToRun.push(fetchedSub);
+                }
+            }
+        } else {
+            // otherwise assume all moddable subs from client should be run on
+            this.logger.info('No user-defined subreddit constraints detected, will try to run on all');
+            subsToRun = availSubs;
+        }
+
+        let subSchedule: Manager[] = [];
+        // get configs for subs we want to run on and build/validate them
+        for (const sub of subsToRun) {
+            let content = undefined;
+            let wiki;
+            try {
+                // @ts-ignore
+                wiki = await sub.getWikiPage(this.wikiLocation).fetch();
+                content = wiki.content_md;
+            } catch (err) {
+                this.logger.error(`[${sub.display_name_prefixed}] Could not read wiki configuration. Please ensure the page https://reddit.com${sub.url}wiki/${this.wikiLocation} exists and is readable -- error: ${err.message}`);
+                continue;
+            }
+
+            if(content === '') {
+                this.logger.error(`[${sub.display_name_prefixed}] Wiki page contents was empty`);
+                continue;
+            }
+
+            const [configObj, jsonErr, yamlErr] = parseFromJsonOrYamlToObject(content);
+
+            if (configObj === undefined) {
+                this.logger.error(`[${sub.display_name_prefixed}] Could not parse wiki page contents as JSON or YAML:`);
+                this.logger.error(jsonErr);
+                this.logger.error(yamlErr);
+                continue;
+            }
+
+            try {
+                const manager = new Manager(sub, this.client, this.logger, configObj, {dryRun: this.dryRun});
+                manager.lastWikiCheck = dayjs();
+                manager.lastWikiRevision = dayjs.unix(wiki.revision_date);
+                subSchedule.push(manager);
+            } catch (err) {
+                if(!(err instanceof LoggedError)) {
+                    this.logger.error(`[${sub.display_name_prefixed}] Config was not valid`, err);
+                }
+            }
+        }
+        this.subManagers = subSchedule;
+    }
+
+    async heartbeat() {
+        try {
+            this.heartBeating = true;
+            while (true) {
+                await sleep(this.heartbeatInterval * 1000);
+                const heartbeat = `HEARTBEAT -- Reddit API Rate Limit remaining: ${this.client.ratelimitRemaining}`
+                if (this.apiLimitWarning >= this.client.ratelimitRemaining) {
+                    this.logger.warn(heartbeat);
+                } else {
+                    this.logger.info(heartbeat);
+                }
+                for(const s of this.subManagers) {
+                    try {
+                        await s.parseConfiguration();
+                        if(!s.running) {
+                            s.handle();
+                        }
+                    } catch (err) {
+                        s.stop();
+                        this.logger.info('Will retry parsing config on next heartbeat...');
+                    }
+                }
+            }
+        } finally {
+            this.heartBeating = false;
+        }
+    }
+
+    async runManagers() {
+
+        // basic backoff delay if reddit is under load and not responding
+        let timeoutCount = 0;
+        let maxTimeoutCount = 4;
+        let otherRetryCount = 0;
+        // not sure should even allow so set to 0 for now
+        let maxOtherCount = 0;
+        let keepRunning = true;
+        let lastErrorAt: Dayjs | undefined;
+
+        while (keepRunning) {
+            try {
+                for (const manager of this.subManagers) {
+                    if (!manager.running) {
+                        manager.handle();
+                    }
+                }
+
+                if (this.heartbeatInterval !== 0 && !this.heartBeating) {
+                    this.heartbeat();
+                }
+
+                const emitter = new EventEmitter();
+                await pEvent(emitter, 'end');
+                keepRunning = false;
+            } catch (err) {
+                if (lastErrorAt !== undefined && dayjs().diff(lastErrorAt, 'minute') >= 5) {
+                    // if its been longer than 5 minutes since last error clear counters
+                    timeoutCount = 0;
+                    otherRetryCount = 0;
+                }
+
+                lastErrorAt = dayjs();
+
+                if (err.message.includes('ETIMEDOUT') || (err.code !== undefined && err.code.includes('ETIMEDOUT'))) {
+                    timeoutCount++;
+                    if (timeoutCount > maxTimeoutCount) {
+                        this.logger.error(`Timeouts (${timeoutCount}) exceeded max allowed (${maxTimeoutCount})`);
+                        throw err;
+                    }
+                    // exponential backoff
+                    const ms = (Math.pow(2, timeoutCount - 1) + (Math.random() - 0.3)) * 1000;
+                    this.logger.warn(`Reddit response timed out. Will wait ${ms / 1000} seconds before restarting managers`);
+                    await sleep(ms);
+
+                } else {
+                    // linear backoff
+                    otherRetryCount++;
+                    if (maxOtherCount > otherRetryCount) {
+                        throw err;
+                    }
+                    const ms = (3 * 1000) * otherRetryCount;
+                    this.logger.warn(`Non-timeout error occurred. Will wait ${ms / 1000} seconds before restarting managers`);
+                    await sleep(ms);
+                }
+            }
+        }
+    }
+}

src/Author/Author.ts

@@ -0,0 +1,129 @@
+import {DurationComparor, UserNoteCriteria} from "../Rule";
+import {CompareValue, CompareValueOrPercent} from "../Common/interfaces";
+
+/**
+ * If present then these Author criteria are checked before running the rule. If criteria fails then the rule is skipped.
+ * @examples [{"include": [{"flairText": ["Contributor","Veteran"]}, {"isMod": true}]}]
+ * */
+export interface AuthorOptions {
+    /**
+     * Will "pass" if any set of AuthorCriteria passes
+     * */
+    include?: AuthorCriteria[];
+    /**
+     * Only runs if `include` is not present. Will "pass" if any of set of the AuthorCriteria **does not** pass
+     * */
+    exclude?: AuthorCriteria[];
+}
+
+/**
+ * Criteria with which to test against the author of an Activity. The outcome of the test is based on:
+ *
+ * 1. All present properties passing and
+ * 2. If a property is a list then any value from the list matching
+ *
+ * @minProperties 1
+ * @additionalProperties false
+ * @examples [{"flairText": ["Contributor","Veteran"], "isMod": true, "name": ["FoxxMD", "AnotherUser"] }]
+ * */
+export interface AuthorCriteria {
+    /**
+     * A list of reddit usernames (case-insensitive) to match against. Do not include the "u/" prefix
+     *
+     *  EX to match against /u/FoxxMD and /u/AnotherUser use ["FoxxMD","AnotherUser"]
+     * @examples ["FoxxMD","AnotherUser"]
+     * */
+    name?: string[],
+    /**
+     * A list of (user) flair css class values from the subreddit to match against
+     * @examples ["red"]
+     * */
+    flairCssClass?: string[],
+    /**
+     * A list of (user) flair text values from the subreddit to match against
+     * @examples ["Approved"]
+     * */
+    flairText?: string[],
+    /**
+     * Is the author a moderator?
+     * */
+    isMod?: boolean,
+    /**
+     * A list of UserNote properties to check against the User Notes attached to this Author in this Subreddit (must have Toolbox enabled and used User Notes at least once)
+     * */
+    userNotes?: UserNoteCriteria[]
+
+    /**
+     * Test the age of the Author's account (when it was created) against this comparison
+     *
+     * The syntax is `(< OR > OR <= OR >=) <number> <unit>`
+     *
+     * * EX `> 100 days` => Passes if Author's account is older than 100 days
+     * * EX `<= 2 months` => Passes if Author's account is younger than or equal to 2 months
+     *
+     * Unit must be one of [DayJS Duration units](https://day.js.org/docs/en/durations/creating)
+     *
+     * [See] https://regexr.com/609n8 for example
+     *
+     * @pattern ^\s*(?<opStr>>|>=|<|<=)\s*(?<time>\d+)\s*(?<unit>days?|weeks?|months?|years?|hours?|minutes?|seconds?|milliseconds?)\s*$
+     * */
+    age?: DurationComparor
+
+    /**
+     * A string containing a comparison operator and a value to compare link karma against
+     *
+     * The syntax is `(< OR > OR <= OR >=) <number>[percent sign]`
+     *
+     * * EX `> 100`  => greater than 100 link karma
+     * * EX `<= 75%` => link karma is less than or equal to 75% of **all karma**
+     *
+     * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(%?)(.*)$
+     * */
+    linkKarma?: CompareValueOrPercent
+
+    /**
+     * A string containing a comparison operator and a value to compare karma against
+     *
+     * The syntax is `(< OR > OR <= OR >=) <number>[percent sign]`
+     *
+     * * EX `> 100`  => greater than 100 comment karma
+     * * EX `<= 75%` => comment karma is less than or equal to 75% of **all karma**
+     *
+     * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(%?)(.*)$
+     * */
+    commentKarma?: CompareValueOrPercent
+
+    totalKarma?: CompareValue
+
+    /**
+     * Does Author's account have a verified email?
+     * */
+    verified?: boolean
+}
+
+export class Author implements AuthorCriteria {
+    name?: string[];
+    flairCssClass?: string[];
+    flairText?: string[];
+    isMod?: boolean;
+    userNotes?: UserNoteCriteria[];
+    age?: string;
+    commentKarma?: string;
+    linkKarma?: string;
+    totalKarma?: string;
+    verified?: boolean;
+
+    constructor(options: AuthorCriteria) {
+        this.name = options.name;
+        this.flairCssClass = options.flairCssClass;
+        this.flairText = options.flairText;
+        this.isMod = options.isMod;
+        this.userNotes = options.userNotes;
+        this.age = options.age;
+        this.commentKarma = options.commentKarma;
+        this.linkKarma = options.linkKarma;
+        this.totalKarma = options.totalKarma;
+    }
+}
+
+export default Author;

src/Check/CommentCheck.js

@@ -1,8 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CommentCheck = void 0;
-const index_1 = require("./index");
-class CommentCheck extends index_1.Check {
-}
-exports.CommentCheck = CommentCheck;
-//# sourceMappingURL=CommentCheck.js.map

src/Check/CommentCheck.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"CommentCheck.js","sourceRoot":"","sources":["CommentCheck.ts"],"names":[],"mappings":";;;AAAA,mCAA8B;AAE9B,MAAa,YAAa,SAAQ,aAAK;CAEtC;AAFD,oCAEC"}

src/Check/CommentCheck.ts

@@ -1,5 +1,13 @@
-import {Check} from "./index";
+import {Check, CheckOptions} from "./index";
+import {CommentState} from "../Common/interfaces";
 
 export class CommentCheck extends Check {
+    itemIs: CommentState[];
 
+    constructor(options: CheckOptions) {
+        super(options);
+        const {itemIs = []} = options;
+        this.itemIs = itemIs;
+        this.logSummary('comment');
+    }
 }

src/Check/SubmissionCheck.js

@@ -1,8 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.SubmissionCheck = void 0;
-const index_1 = require("./index");
-class SubmissionCheck extends index_1.Check {
-}
-exports.SubmissionCheck = SubmissionCheck;
-//# sourceMappingURL=SubmissionCheck.js.map

src/Check/SubmissionCheck.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"SubmissionCheck.js","sourceRoot":"","sources":["SubmissionCheck.ts"],"names":[],"mappings":";;;AACA,mCAA8B;AAE9B,MAAa,eAAgB,SAAQ,aAAK;CAEzC;AAFD,0CAEC"}

src/Check/SubmissionCheck.ts

@@ -1,6 +1,14 @@
 
-import {Check} from "./index";
+import {Check, CheckOptions} from "./index";
+import {SubmissionState} from "../Common/interfaces";
 
 export class SubmissionCheck extends Check {
+    itemIs: SubmissionState[];
 
+    constructor(options: CheckOptions) {
+        super(options);
+        const {itemIs = []} = options;
+        this.itemIs = itemIs;
+        this.logSummary('submission');
+    }
 }

src/Check/index.ts

@@ -1,26 +1,36 @@
-import {RuleSet, IRuleSet, RuleSetJSONConfig} from "../Rule/RuleSet";
-import {IRule, Triggerable, Rule, RuleJSONConfig, RuleResult} from "../Rule";
-import Action, {ActionConfig, ActionJSONConfig} from "../Action";
+import {RuleSet, IRuleSet, RuleSetJson, RuleSetObjectJson} from "../Rule/RuleSet";
+import {IRule, isRuleSetResult, Rule, RuleJSONConfig, RuleResult, RuleSetResult} from "../Rule";
+import Action, {ActionConfig, ActionJson} from "../Action";
 import {Logger} from "winston";
-import Snoowrap, {Comment, Submission} from "snoowrap";
-import {RecentActivityRuleJSONConfig} from "../Rule/RecentActivityRule";
-import {RepeatSubmissionJSONConfig} from "../Rule/SubmissionRule/RepeatSubmissionRule";
-import {FlairActionJSONConfig} from "../Action/SubmissionAction/FlairAction";
-import {CommentActionJSONConfig} from "../Action/CommentAction";
+import {Comment, Submission} from "snoowrap";
 import {actionFactory} from "../Action/ActionFactory";
 import {ruleFactory} from "../Rule/RuleFactory";
-import {createLabelledLogger, determineNewResults, loggerMetaShuffle, mergeArr} from "../util";
-import {AuthorRuleJSONConfig} from "../Rule/AuthorRule";
-import {ReportActionJSONConfig} from "../Action/ReportAction";
-import {LockActionJSONConfig} from "../Action/LockAction";
-import {RemoveActionJSONConfig} from "../Action/RemoveAction";
-import {JoinCondition, JoinOperands} from "../Common/interfaces";
+import {
+    createAjvFactory,
+    FAIL,
+    mergeArr,
+    PASS,
+    resultsSummary,
+    ruleNamesFromResults,
+    truncateStringToLength
+} from "../util";
+import {
+    ChecksActivityState,
+    CommentState,
+    JoinCondition,
+    JoinOperands,
+    SubmissionState,
+    TypedActivityStates
+} from "../Common/interfaces";
 import * as RuleSchema from '../Schema/Rule.json';
 import * as RuleSetSchema from '../Schema/RuleSet.json';
 import * as ActionSchema from '../Schema/Action.json';
-import Ajv from 'ajv';
+import {ActionObjectJson, RuleJson, RuleObjectJson, ActionJson as ActionTypeJson} from "../Common/types";
+import {isItem} from "../Utils/SnoowrapUtils";
+import ResourceManager, {SubredditResources} from "../Subreddit/SubredditResources";
+import {Author, AuthorCriteria, AuthorOptions} from "../Author/Author";
 
-const ajv = new Ajv();
+const checkLogName = truncateStringToLength(25);
 
 export class Check implements ICheck {
     actions: Action[] = [];
@@ -29,6 +39,13 @@ export class Check implements ICheck {
     condition: JoinOperands;
     rules: Array<RuleSet | Rule> = [];
     logger: Logger;
+    itemIs: TypedActivityStates;
+    authorIs: {
+        include: AuthorCriteria[],
+        exclude: AuthorCriteria[]
+    };
+    dryRun?: boolean;
+    resources: SubredditResources;
 
     constructor(options: CheckOptions) {
         const {
@@ -37,18 +54,30 @@ export class Check implements ICheck {
             condition = 'AND',
             rules = [],
             actions = [],
+            subredditName,
+            itemIs = [],
+            authorIs: {
+                include = [],
+                exclude = [],
+            } = {},
+            dryRun,
         } = options;
 
-        if (options.logger !== undefined) {
-            // @ts-ignore
-            this.logger = options.logger.child(loggerMetaShuffle(options.logger, undefined, [`CHK ${name}`]), mergeArr);
-        } else {
-            this.logger = createLabelledLogger('Check');
-        }
+        this.logger = options.logger.child({labels: [`CHK ${checkLogName(name)}`]}, mergeArr);
+
+        const ajv = createAjvFactory(this.logger);
+
+        this.resources = ResourceManager.get(subredditName) as SubredditResources;
 
         this.name = name;
         this.description = description;
         this.condition = condition;
+        this.itemIs = itemIs;
+        this.authorIs = {
+            exclude: exclude.map(x => new Author(x)),
+            include: include.map(x => new Author(x)),
+        }
+        this.dryRun = dryRun;
         for (const r of rules) {
             if (r instanceof Rule || r instanceof RuleSet) {
                 this.rules.push(r);
@@ -57,16 +86,13 @@ export class Check implements ICheck {
                 let setErrors: any = [];
                 let ruleErrors: any = [];
                 if (valid) {
-                    // @ts-ignore
-                    r.logger = this.logger;
-                    this.rules.push(new RuleSet(r as RuleSetJSONConfig));
+                    const ruleConfig = r as RuleSetObjectJson;
+                    this.rules.push(new RuleSet({...ruleConfig, logger: this.logger, subredditName}));
                 } else {
                     setErrors = ajv.errors;
                     valid = ajv.validate(RuleSchema, r);
                     if (valid) {
-                        // @ts-ignore
-                        r.logger = this.logger;
-                        this.rules.push(ruleFactory(r as RuleJSONConfig));
+                        this.rules.push(ruleFactory(r as RuleJSONConfig, this.logger, subredditName));
                     } else {
                         ruleErrors = ajv.errors;
                         const leastErrorType = setErrors.length < ruleErrors ? 'RuleSet' : 'Rule';
@@ -86,7 +112,11 @@ export class Check implements ICheck {
             } else {
                 let valid = ajv.validate(ActionSchema, a);
                 if (valid) {
-                    this.actions.push(actionFactory(a as ActionJSONConfig));
+                    const aj = a as ActionJson;
+                    this.actions.push(actionFactory({
+                        ...aj,
+                        dryRun: this.dryRun || aj.dryRun
+                    }, this.logger, subredditName));
                     // @ts-ignore
                     a.logger = this.logger;
                 } else {
@@ -94,75 +124,232 @@ export class Check implements ICheck {
                 }
             }
         }
-
-
     }
 
-    async run(item: Submission | Comment, existingResults: RuleResult[] = []): Promise<[boolean, RuleResult[]]> {
-        this.logger.debug('Starting check');
-        let allResults: RuleResult[] = [];
-        let runOne = false;
+    logSummary(type: string) {
+        const runStats = [];
+        const ruleSetCount = this.rules.reduce((x, r) => r instanceof RuleSet ? x + 1 : x, 0);
+        const rulesInSetsCount = this.rules.reduce((x, r) => r instanceof RuleSet ? x + r.rules.length : x, 0);
+        if (ruleSetCount > 0) {
+            runStats.push(`${ruleSetCount} Rule Sets (${rulesInSetsCount} Rules)`);
+        }
+        const topRuleCount = this.rules.reduce((x, r) => r instanceof Rule ? x + 1 : x, 0);
+        if (topRuleCount > 0) {
+            runStats.push(`${topRuleCount} Top-Level Rules`);
+        }
+        runStats.push(`${this.actions.length} Actions`);
+        // not sure if this should be info or verbose
+        this.logger.info(`${type.toUpperCase()} (${this.condition}) => ${runStats.join(' | ')}${this.description !== undefined ? ` => ${this.description}` : ''}`);
+        if (this.rules.length === 0 && this.itemIs.length === 0 && this.authorIs.exclude.length === 0 && this.authorIs.include.length === 0) {
+            this.logger.warn('No rules, item tests, or author test found -- this check will ALWAYS PASS!');
+        }
+        let ruleSetIndex = 1;
         for (const r of this.rules) {
-            const combinedResults = [...existingResults, ...allResults];
-            const [passed, results] = await r.run(item, combinedResults);
-            //allResults = allResults.concat(determineNewResults(combinedResults, results));
-            allResults = allResults.concat(results);
-            if (passed === null) {
-                continue;
-            }
-            runOne = true;
-            if (passed) {
-                if (this.condition === 'OR') {
-                    return [true, allResults];
+            if (r instanceof RuleSet) {
+                for (const ru of r.rules) {
+                    this.logger.verbose(`(Rule Set ${ruleSetIndex} ${r.condition}) => ${ru.getRuleUniqueName()}`);
                 }
-            } else if (this.condition === 'AND') {
-                return [false, allResults];
+                ruleSetIndex++;
+            } else {
+                this.logger.verbose(`(Rule) => ${r.getRuleUniqueName()}`);
             }
         }
-        if (!runOne) {
-            return [false, allResults];
+        for (const a of this.actions) {
+            this.logger.verbose(`(Action) => ${a.getActionUniqueName()}`);
         }
-        return [true, allResults];
     }
 
-    async runActions(item: Submission | Comment, client: Snoowrap): Promise<void> {
-        for (const a of this.actions) {
-            await a.handle(item, client);
+    async runRules(item: Submission | Comment, existingResults: RuleResult[] = []): Promise<[boolean, RuleResult[]]> {
+        try {
+            let allRuleResults: RuleResult[] = [];
+            let allResults: (RuleResult | RuleSetResult)[] = [];
+            const [itemPass, crit] = isItem(item, this.itemIs, this.logger);
+            if (!itemPass) {
+                this.logger.verbose(`${FAIL} => Item did not pass 'itemIs' test`);
+                return [false, allRuleResults];
+            }
+            let authorPass = null;
+            if (this.authorIs.include !== undefined && this.authorIs.include.length > 0) {
+                for (const auth of this.authorIs.include) {
+                    if (await this.resources.testAuthorCriteria(item, auth)) {
+                        authorPass = true;
+                        break;
+                    }
+                }
+                if (!authorPass) {
+                    this.logger.verbose(`${FAIL} => Inclusive author criteria not matched`);
+                    return Promise.resolve([false, allRuleResults]);
+                }
+            }
+            if (authorPass === null && this.authorIs.exclude !== undefined && this.authorIs.exclude.length > 0) {
+                for (const auth of this.authorIs.exclude) {
+                    if (await this.resources.testAuthorCriteria(item, auth, false)) {
+                        authorPass = true;
+                        break;
+                    }
+                }
+                if (!authorPass) {
+                    this.logger.verbose(`${FAIL} =>  Exclusive author criteria not matched`);
+                    return Promise.resolve([false, allRuleResults]);
+                }
+            }
+
+            if (this.rules.length === 0) {
+                this.logger.info(`${PASS} => No rules to run, check auto-passes`);
+                return [true, allRuleResults];
+            }
+
+            let runOne = false;
+            for (const r of this.rules) {
+                //let results: RuleResult | RuleSetResult;
+                const combinedResults = [...existingResults, ...allRuleResults];
+                const [passed, results] = await r.run(item, combinedResults);
+                if (isRuleSetResult(results)) {
+                    allRuleResults = allRuleResults.concat(results.results);
+                } else {
+                    allRuleResults = allRuleResults.concat(results as RuleResult);
+                }
+                allResults.push(results);
+                if (passed === null) {
+                    continue;
+                }
+                runOne = true;
+                if (passed) {
+                    if (this.condition === 'OR') {
+                        this.logger.info(`${PASS} => Rules: ${resultsSummary(allResults, this.condition)}`);
+                        return [true, allRuleResults];
+                    }
+                } else if (this.condition === 'AND') {
+                    this.logger.verbose(`${FAIL} => Rules: ${resultsSummary(allResults, this.condition)}`);
+                    return [false, allRuleResults];
+                }
+            }
+            if (!runOne) {
+                this.logger.verbose(`${FAIL} => All Rules skipped because of Author checks or itemIs tests`);
+                return [false, allRuleResults];
+            } else if (this.condition === 'OR') {
+                // if OR and did not return already then none passed
+                this.logger.verbose(`${FAIL} => Rules: ${resultsSummary(allResults, this.condition)}`);
+                return [false, allRuleResults];
+            }
+            // otherwise AND and did not return already so all passed
+            this.logger.info(`${PASS} => Rules: ${resultsSummary(allResults, this.condition)}`);
+            return [true, allRuleResults];
+        } catch (e) {
+            e.logged = true;
+            this.logger.warn(`Running rules failed due to uncaught exception`, e);
+            throw e;
         }
     }
+
+    async runActions(item: Submission | Comment, ruleResults: RuleResult[]): Promise<Action[]> {
+        this.logger.debug(`${this.dryRun ? 'DRYRUN - ' : ''}Running Actions`);
+        const runActions: Action[] = [];
+        for (const a of this.actions) {
+            try {
+                await a.handle(item, ruleResults);
+                runActions.push(a);
+            } catch (err) {
+                this.logger.error(`Action ${a.getActionUniqueName()} encountered an error while running`, err);
+            }
+        }
+        this.logger.info(`${this.dryRun ? 'DRYRUN - ' : ''}Ran Actions: ${runActions.map(x => x.getActionUniqueName()).join(' | ')}`);
+        return runActions;
+    }
 }
 
-export interface ICheck extends JoinCondition {
+export interface ICheck extends JoinCondition, ChecksActivityState {
     /**
-     * A friendly name for this check (highly recommended) -- EX "repeatCrosspostReport"
+     * Friendly name for this Check EX "crosspostSpamCheck"
+     *
+     * Can only contain letters, numbers, underscore, spaces, and dashes
+     *
+     * @pattern ^[a-zA-Z]([\w -]*[\w])?$
+     * @examples ["myNewCheck"]
      * */
     name: string,
+    /**
+     * @examples ["A short description of what this check looks for and actions it performs"]
+     * */
     description?: string,
+
+    /**
+     * Use this option to override the `dryRun` setting for all of its `Actions`
+     * @examples [false, true]
+     * */
+    dryRun?: boolean;
+
+    /**
+     * A list of criteria to test the state of the `Activity` against before running the check.
+     *
+     * If any set of criteria passes the Check will be run. If the criteria fails then the Check will fail.
+     *
+     * * @examples [[{"over_18": true, "removed': false}]]
+     * */
+    itemIs?: TypedActivityStates
+
+    /**
+     * If present then these Author criteria are checked before running the Check. If criteria fails then the Check will fail.
+     * */
+    authorIs?: AuthorOptions
 }
 
 export interface CheckOptions extends ICheck {
     rules: Array<IRuleSet | IRule>
     actions: ActionConfig[]
-    logger?: Logger
+    logger: Logger
+    subredditName: string
 }
 
-/**
- * An object consisting of Rules (tests) and Actions to perform if Rules are triggered
- * @see {isCheckConfig} ts-auto-guard:type-guard
- * */
-export interface CheckJSONConfig extends ICheck {
+export interface CheckJson extends ICheck {
     /**
      * The type of event (new submission or new comment) this check should be run against
+     * @examples ["submission", "comment"]
      */
     kind: 'submission' | 'comment'
     /**
-     * Rules are run in the order found in configuration. Can be Rules or RuleSets
-     * @minItems 1
+     * A list of Rules to run.
+     *
+     * If `Rule` objects are triggered based on `condition` then `actions` will be performed.
+     *
+     * Can be `Rule`, `RuleSet`, or the `name` of any **named** `Rule` in your subreddit's configuration.
+     *
+     * **If `rules` is an empty array or not present then `actions` are performed immediately.**
      * */
-    rules: Array<RuleSetJSONConfig | RecentActivityRuleJSONConfig | RepeatSubmissionJSONConfig | AuthorRuleJSONConfig>
+    rules?: Array<RuleSetJson | RuleJson>
     /**
-     * The actions to run after the check is successfully triggered. ALL actions will run in the order they are listed
+     * The `Actions` to run after the check is successfully triggered. ALL `Actions` will run in the order they are listed
+     *
+     *  Can be `Action` or the `name` of any **named** `Action` in your subreddit's configuration
+     *
      * @minItems 1
+     * @examples [[{"kind": "comment", "content": "this is the content of the comment", "distinguish": true}, {"kind": "lock"}]]
      * */
-    actions: Array<FlairActionJSONConfig | CommentActionJSONConfig | ReportActionJSONConfig | LockActionJSONConfig | RemoveActionJSONConfig>
+    actions: Array<ActionTypeJson>
+}
+
+export interface SubmissionCheckJson extends CheckJson {
+    kind: 'submission'
+    itemIs?: SubmissionState[]
+}
+
+export interface CommentCheckJson extends CheckJson {
+    kind: 'comment'
+    itemIs?: CommentState[]
+}
+
+export type CheckStructuredJson = SubmissionCheckStructuredJson | CommentCheckStructuredJson;
+// export interface CheckStructuredJson extends CheckJson {
+//     rules: Array<RuleSetObjectJson | RuleObjectJson>
+//     actions: Array<ActionObjectJson>
+// }
+
+export interface SubmissionCheckStructuredJson extends SubmissionCheckJson {
+    rules: Array<RuleSetObjectJson | RuleObjectJson>
+    actions: Array<ActionObjectJson>
+}
+
+export interface CommentCheckStructuredJson extends CommentCheckJson {
+    rules: Array<RuleSetObjectJson | RuleObjectJson>
+    actions: Array<ActionObjectJson>
 }

src/Common/interfaces.ts

@@ -1,51 +1,189 @@
+import {Duration} from "dayjs/plugin/duration";
+
 /**
  * An ISO 8601 Duration
  * @pattern ^(-?)P(?=\d|T\d)(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)([DW]))?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?$
  * */
 export type ISO8601 = string;
-export type ActivityWindowType = Duration | number | ActivityWindowCriteria;
-export type Duration = ISO8601 | DurationObject;
 
 /**
- * If both properties are defined then the first criteria met will be used IE if # of activities = count before duration is reached then count will be used, or vice versa
+ * A shorthand value for a DayJS duration consisting of a number value and time unit
+ *
+ * * EX `9 days`
+ * * EX `3 months`
+ * @pattern ^\s*(?<time>\d+)\s*(?<unit>days?|weeks?|months?|years?|hours?|minutes?|seconds?|milliseconds?)\s*$
+ * */
+export type DayJSShorthand = string;
+export type DurationString = DayJSShorthand | ISO8601;
+
+/**
+ * A value to define the range of Activities to retrieve.
+ *
+ * Acceptable values:
+ *
+ * **`ActivityWindowCriteria` object**
+ *
+ * Allows specify multiple range properties and more specific behavior
+ *
+ * **A `number` of Activities to retrieve**
+ *
+ * * EX `100` => 100 Activities
+ *
+ * *****
+ *
+ * Any of the below values that specify the amount of time to subtract from `NOW` to create a time range IE `NOW <---> [duration] ago`
+ *
+ * Acceptable values:
+ *
+ * **A `string` consisting of a value and a [Day.js](https://day.js.org/docs/en/durations/creating#list-of-all-available-units) time UNIT**
+ *
+ * * EX `9 days` => Range is `NOW <---> 9 days ago`
+ *
+ * **A [Day.js](https://day.js.org/docs/en/durations/creating) `object`**
+ *
+ * * EX `{"days": 90, "minutes": 15}` => Range is `NOW <---> 90 days and 15 minutes ago`
+ *
+ * **An [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) `string`**
+ *
+ * * EX `PT15M` => 15 minutes => Range is `NOW <----> 15 minutes ago`
+ *
+ * @examples ["90 days"]
+ * */
+export type ActivityWindowType = ActivityWindowCriteria | DurationVal | number;
+export type DurationVal = DurationString | DurationObject;
+
+/**
+ * Multiple properties that may be used to define what range of Activity to retrieve.
+ *
+ * May specify one, or both properties along with the `satisfyOn` property, to affect the retrieval behavior.
+ *
+ * @examples [{"count": 100, "duration": {"days": 90}}]
  * @minProperties 1
  * @additionalProperties false
  * */
 export interface ActivityWindowCriteria {
     /**
      * The number of activities (submission/comments) to consider
+     * @examples [15]
      * */
     count?: number,
     /**
-     * An ISO 8601 duration or Day.js duration object.
+     * A value that specifies the amount of time to subtract from `NOW` to create a time range IE `NOW <---> [duration] ago`
      *
-     * The duration will be subtracted from the time when the rule is run to create a time range like this:
+     * Acceptable values:
      *
-     * endTime = NOW  <----> startTime = (NOW - duration)
+     * **A `string` consisting of a value and a [Day.js](https://day.js.org/docs/en/durations/creating) time unit**
      *
-     * EX endTime = 3:00PM <----> startTime = (NOW - 15 minutes) = 2:45PM -- so look for activities between 2:45PM and 3:00PM
-     * @examples ["PT1M", {"minutes": 15}]
+     * * EX `9 days` => Range is `NOW <---> 9 days ago`
+     *
+     * **A [Day.js](https://day.js.org/docs/en/durations/creating) `object`**
+     *
+     * * EX `{"days": 90, "minutes": 15}` => Range is `NOW <---> 90 days and 15 minutes ago`
+     *
+     * **An [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) `string`**
+     *
+     * * EX `PT15M` => 15 minutes => Range is `NOW <----> 15 minutes ago`
+     *
+     * @examples ["90 days", "PT15M", {"minutes": 15}]
      * */
-    duration?: Duration
+    duration?: DurationVal
+
+    /**
+     * Define the condition under which both criteria are considered met
+     *
+     * **If `any` then it will retrieve Activities until one of the criteria is met, whichever occurs first**
+     *
+     * EX `{"count": 100, duration: "90 days"}`:
+     * * If 90 days of activities = 40 activities => returns 40 activities
+     * * If 100 activities is only 20 days => 100 activities
+     *
+     * **If `all` then both criteria must be met.**
+     *
+     * Effectively, whichever criteria produces the most Activities...
+     *
+     * EX `{"count": 100, duration: "90 days"}`:
+     * * If at 90 days of activities => 40 activities, continue retrieving results until 100 => results in >90 days of activities
+     * * If at 100 activities => 20 days of activities, continue retrieving results until 90 days => results in >100 activities
+     *
+     * @examples ["any"]
+     * @default any
+     * */
+    satisfyOn?: 'any' | 'all';
+
+    /**
+     * Filter which subreddits (case-insensitive) Activities are retrieved from.
+     *
+     * **Note:** Filtering occurs **before** `duration/count` checks are performed.
+     * */
+    subreddits?: {
+        /**
+         * Include only results from these subreddits
+         *
+         * @examples [["mealtimevideos","askscience"]]
+         * */
+        include?: string[],
+        /**
+         * Exclude any results from these subreddits
+         *
+         * **Note:** `exclude` is ignored if `include` is present
+         *
+         * @examples [["mealtimevideos","askscience"]]
+         * */
+        exclude?: string[],
+    }
 }
 
 /**
- * A Day.js duration object
+ * A [Day.js duration object](https://day.js.org/docs/en/durations/creating)
  *
- * https://day.js.org/docs/en/durations/creating
+ * @examples [{"minutes": 30, "hours": 1}]
  * @minProperties 1
  * @additionalProperties false
  * */
 export interface DurationObject {
+    /**
+     * @examples [15]
+     * */
     seconds?: number
+    /**
+     * @examples [50]
+     * */
     minutes?: number
+    /**
+     * @examples [4]
+     * */
     hours?: number
+    /**
+     * @examples [7]
+     * */
     days?: number
+    /**
+     * @examples [2]
+     * */
     weeks?: number
+    /**
+     * @examples [3]
+     * */
     months?: number
+    /**
+     * @examples [0]
+     * */
     years?: number
 }
 
+export interface DurationComparison {
+    operator: StringOperator,
+    duration: Duration
+}
+
+export interface GenericComparison {
+    operator: StringOperator,
+    value: number,
+    isPercent: boolean,
+    extra?: string,
+    displayText: string,
+}
+
 
 export const windowExample: ActivityWindowType[] = [
     15,
@@ -68,15 +206,7 @@ export const windowExample: ActivityWindowType[] = [
 
 
 export interface ActivityWindow {
-    /**
-     * Criteria for defining what set of activities should be considered.
-     *
-     * The value of this property may be either count OR duration -- to use both write it as an ActivityWindowCriteria
-     *
-     * See ActivityWindowCriteria for descriptions of what count/duration do
-     * @examples require('./interfaces.ts').windowExample
-     * @default 15
-     */
+
     window?: ActivityWindowType,
 }
 
@@ -92,81 +222,363 @@ export interface RichContent {
     /**
      * The Content to submit for this Action. Content is interpreted as reddit-flavored Markdown.
      *
-     * If value starts with 'wiki:' then the proceeding value will be used to get a wiki page
+     * If value starts with `wiki:` then the proceeding value will be used to get a wiki page from the current subreddit
      *
-     * EX "wiki:botconfig/mybot" tries to get https://reddit.com/mySubredditExample/wiki/botconfig/mybot
+     *  * EX `wiki:botconfig/mybot` tries to get `https://reddit.com/r/currentSubreddit/wiki/botconfig/mybot`
      *
-     * EX "this is plain text"
+     * If the value starts with `wiki:` and ends with `|someValue` then `someValue` will be used as the base subreddit for the wiki page
      *
-     * EX "this is **bold** markdown text"
+     * * EX `wiki:replytemplates/test|ContextModBot` tries to get `https://reddit.com/r/ContextModBot/wiki/replytemplates/test`
      *
-     * @examples ["this is plain text", "this is **bold** markdown text", "wiki:botconfig/acomment" ]
+     * If the value starts with `url:` then the value is fetched as an external url and expects raw text returned
+     *
+     * * EX `url:https://pastebin.com/raw/38qfL7mL` tries to get the text response of `https://pastebin.com/raw/38qfL7mL`
+     *
+     * If none of the above is used the value is treated as the raw context
+     *
+     *  * EX `this is **bold** markdown text` => "this is **bold** markdown text"
+     *
+     * All Content is rendered using [mustache](https://github.com/janl/mustache.js/#templates) to enable [Action Templating](https://github.com/FoxxMD/reddit-context-bot#action-templating).
+     *
+     * The following properties are always available in the template (view individual Rules to see rule-specific template data):
+     * ```
+     * item.kind      => The type of Activity that was checked (comment/submission)
+     * item.author    => The name of the Author of the Activity EX FoxxMD
+     * item.permalink => A permalink URL to the Activity EX https://reddit.com/r/yourSub/comments/o1h0i0/title_name/1v3b7x
+     * item.url       => If the Activity is Link Sumbission then the external URL
+     * item.title     => If the Activity is a Submission then the title of that Submission
+     * rules          => An object containing RuleResults of all the rules run for this check. See Action Templating for more details on naming
+     * ```
+     *
+     * @examples ["This is the content of a comment/report/usernote", "this is **bold** markdown text", "wiki:botconfig/acomment" ]
      * */
-    content: string,
+    content?: string,
+}
+
+export interface RequiredRichContent extends RichContent {
+    content: string
 }
 
 /**
- * A list of subreddits (case-insensitive) to look for. Do not include "r/" prefix.
+ * A list of subreddits (case-insensitive) to look for.
  *
- * EX to match against /r/mealtimevideos and /r/askscience use ["mealtimevideos","askscience"]
+ * EX ["mealtimevideos","askscience"]
  * @examples ["mealtimevideos","askscience"]
  * @minItems 1
  * */
 export type SubredditList = string[];
 
 export interface SubredditCriteria {
-    subreddits: SubredditList
+    /**
+     * A list of Subreddits (by name, case-insensitive) to look for.
+     *
+     * EX ["mealtimevideos","askscience"]
+     * @examples [["mealtimevideos","askscience"]]
+     * @minItems 1
+     * */
+    subreddits: string[]
 }
 
 export type JoinOperands = 'OR' | 'AND';
 
 export interface JoinCondition {
     /**
-     * Under what condition should a set of rules be considered "successful"?
+     * Under what condition should a set of run `Rule` objects be considered "successful"?
      *
-     * If "OR" then ANY triggered rule results in success.
+     * If `OR` then **any** triggered `Rule` object results in success.
      *
-     * If "AND" then ALL rules must be triggered to result in success.
+     * If `AND` then **all** `Rule` objects must be triggered to result in success.
      *
      * @default "AND"
+     * @examples ["AND"]
      * */
     condition?: JoinOperands,
 }
 
+export type PollOn = 'unmoderated' | 'modqueue' | 'newSub' | 'newComm';
+
+export interface PollingOptionsStrong extends PollingOptions {
+    limit: number,
+    interval: number,
+}
+
 /**
- * You may specify polling options independently for submissions/comments
+ * A configuration for where, how, and when to poll Reddit for Activities to process
+ *
+ * @examples [{"pollOn": "unmoderated","limit": 25, "interval": 20000}]
  * */
 export interface PollingOptions {
+
     /**
-     * Polling options for submission events
+     * What source to get Activities from. The source you choose will modify how the bots behaves so choose carefully.
+     *
+     * ### unmoderated (default)
+     *
+     * Activities that have yet to be approved/removed by a mod. This includes all modqueue (reports/spam) **and new submissions**.
+     *
+     * Use this if you want the bot to act like a regular moderator and act on anything that can be seen from mod tools.
+     *
+     * **Note:** Does NOT include new comments, only comments that are reported/filtered by Automoderator. If you want to process all unmoderated AND all new comments then use some version of `polling: ["unmoderated","newComm"]`
+     *
+     * ### modqueue
+     *
+     * Activities requiring moderator review, such as reported things and items caught by the spam filter.
+     *
+     * Use this if you only want the Bot to process reported/filtered Activities.
+     *
+     * ### newSub
+     *
+     * Get only `Submissions` that show up in `/r/mySubreddit/new`
+     *
+     * Use this if you want the bot to process Submissions only when:
+     *
+     * * they are not initially filtered by Automoderator or
+     * * after they have been manually approved from modqueue
+     *
+     * ## newComm
+     *
+     * Get only new `Comments`
+     *
+     * Use this if you want the bot to process Comments only when:
+     *
+     * * they are not initially filtered by Automoderator or
+     * * after they have been manually approved from modqueue
+     *
      * */
-    submissions?: {
-        /**
-         * The number of submissions to pull from /r/subreddit/new on every request
-         * @default 10
-         * */
-        limit?: number,
-        /**
-         * Amount of time, in milliseconds, to wait between requests to /r/subreddit/new
-         *
-         * @default 10000
-         * */
-        interval?: number,
-    },
+    pollOn: 'unmoderated' | 'modqueue' | 'newSub' | 'newComm'
     /**
-     * Polling options for comment events
+     * The maximum number of Activities to get on every request
+     * @default 25
+     * @examples [25]
      * */
-    comments?: {
-        /**
-         * The number of new comments to pull on every request
-         * @default 10
-         * */
-        limit?: number,
-        /**
-         * Amount of time, in milliseconds, to wait between requests for new comments
-         *
-         * @default 10000
-         * */
-        interval?: number,
-    }
+    limit?: number
+
+    /**
+     * Amount of time, in milliseconds, to wait between requests
+     *
+     * @default 20000
+     * @examples [20000]
+     * */
+    interval?: number,
+}
+
+export interface SubredditCacheConfig {
+    /**
+     * Amount of time, in milliseconds, author activities (Comments/Submission) should be cached
+     * @examples [10000]
+     * @default 10000
+     * */
+    authorTTL?: number;
+    /**
+     * Amount of time, in milliseconds, wiki content pages should be cached
+     * @examples [300000]
+     * @default 300000
+     * */
+    wikiTTL?: number;
+
+    /**
+     * Amount of time, in milliseconds, [Toolbox User Notes](https://www.reddit.com/r/toolbox/wiki/docs/usernotes) should be cached
+     * @examples [60000]
+     * @default 60000
+     * */
+    userNotesTTL?: number;
+}
+
+export interface Footer {
+    /**
+     * Customize the footer for Actions that send replies (Comment/Ban)
+     *
+     * If `false` no footer is appended
+     *
+     * If `string` the value is rendered as markdown or will use `wiki:` parser the same way `content` properties on Actions are rendered with [templating](https://github.com/FoxxMD/reddit-context-bot#action-templating).
+     *
+     * If footer is `undefined` (not set) the default footer will be used:
+     *
+     * > *****
+     * > This action was performed by [a bot.] Mention a moderator or [send a modmail] if you any ideas, questions, or concerns about this action.
+     *
+     * *****
+     *
+     * The following properties are available for [templating](https://github.com/FoxxMD/reddit-context-bot#action-templating):
+     * ```
+     * subName    => name of subreddit Action was performed in (EX 'mealtimevideos')
+     * permaLink  => The permalink for the Activity the Action was performed on EX https://reddit.com/r/yourSub/comments/o1h0i0/title_name/1v3b7x
+     * modmaiLink => An encoded URL that will open a new message to your subreddit with the Action permalink appended to the body
+     * botLink    => A permalink to the FAQ for this bot.
+     * ```
+     * If you use your own footer or no footer **please link back to the bot FAQ** using the `{{botLink}}` property in your content :)
+     *
+     * */
+    footer?: false | string
+}
+
+export interface ManagerOptions {
+    /**
+     * An array of sources to process Activities from
+     *
+     * Values in the array may be either:
+     *
+     * **A `string` representing the `pollOn` value to use**
+     *
+     * One of:
+     *
+     * * `unmoderated`
+     * * `modqueue`
+     * * `newSub`
+     * * `newComm`
+     *
+     * with the rest of the `PollingOptions` properties as defaults
+     *
+     * **A `PollingOptions` object**
+     *
+     * If you want to specify non-default preoperties
+     *
+     * ****
+     * If not specified the default is `["unmoderated"]`
+     *
+     * @default [["unmoderated"]]
+     * @example [["unmoderated","newComm"]]
+    * */
+    polling?: (string|PollingOptions)[]
+
+    /**
+     * Per-subreddit config for caching TTL values. If set to `false` caching is disabled.
+     * */
+    caching?: false | SubredditCacheConfig
+
+    /**
+     * Use this option to override the `dryRun` setting for all `Checks`
+     *
+     * @default undefined
+     * @examples [false,true]
+     * */
+    dryRun?: boolean;
+
+    /**
+     * Customize the footer for Actions that send replies (Comment/Ban). **This sets the default value for all Actions without `footer` specified in their configuration.**
+     *
+     * If `false` no footer is appended
+     *
+     * If `string` the value is rendered as markdown or will use `wiki:` parser the same way `content` properties on Actions are rendered with [templating](https://github.com/FoxxMD/reddit-context-bot#action-templating).
+     *
+     * If footer is `undefined` (not set) the default footer will be used:
+     *
+     * > *****
+     * > This action was performed by [a bot.] Mention a moderator or [send a modmail] if you any ideas, questions, or concerns about this action.
+     *
+     * *****
+     *
+     * The following properties are available for [templating](https://github.com/FoxxMD/reddit-context-bot#action-templating):
+     * ```
+     * subName    => name of subreddit Action was performed in (EX 'mealtimevideos')
+     * permaLink  => The permalink for the Activity the Action was performed on EX https://reddit.com/r/yourSub/comments/o1h0i0/title_name/1v3b7x
+     * modmaiLink => An encoded URL that will open a new message to your subreddit with the Action permalink appended to the body
+     * botLink    => A permalink to the FAQ for this bot.
+     * ```
+     * If you use your own footer or no footer **please link back to the bot FAQ** using the `{{botLink}}` property in your content :)
+     *
+     * @default undefined
+     * */
+    footer?: false | string
+
+    /*
+    * An alternate identifier to use in logs to identify your subreddit
+    *
+    * If your subreddit has a very long name it can make logging unwieldy. Specify a shorter name here to make log statements more readable (and shorter)
+    * @example ["shortName"]
+    * */
+    nickname?: string
+}
+
+/**
+ * A string containing a comparison operator and a value to compare against
+ *
+ * The syntax is `(< OR > OR <= OR >=) <number>`
+ *
+ * * EX `> 100`  => greater than 100
+ *
+ * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(%?)(.*)$
+ * */
+export type CompareValue = string;
+
+/**
+ * A string containing a comparison operator and a value to compare against
+ *
+ * The syntax is `(< OR > OR <= OR >=) <number>[percent sign]`
+ *
+ * * EX `> 100`  => greater than 100
+ * * EX `<= 75%` => less than or equal to 75%
+ *
+ * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(%?)(.*)$
+ * */
+export type CompareValueOrPercent = string;
+
+export type StringOperator = '>' | '>=' | '<' | '<=';
+
+export interface ThresholdCriteria {
+    /**
+     * The number or percentage to trigger this criteria at
+     *
+     * * If `threshold` is a `number` then it is the absolute number of items to trigger at
+     * * If `threshold` is a `string` with percentage (EX `40%`) then it is the percentage of the total this item must reach to trigger
+     *
+     * @default 10%
+     * @examples ["10%", 15]
+     * */
+    threshold: number | string
+
+    /**
+     * @examples [">",">=","<","<="]
+     * */
+    condition: StringOperator
+}
+
+export interface ChecksActivityState {
+    itemIs?: TypedActivityStates
+}
+
+export interface ActivityState {
+    removed?: boolean
+    filtered?: boolean
+    deleted?: boolean
+    locked?: boolean
+    spam?: boolean
+    stickied?: boolean
+    distinguished?: boolean
+    approved?: boolean
+}
+
+/**
+ * Different attributes a `Submission` can be in. Only include a property if you want to check it.
+ * @examples [{"over_18": true, "removed": false}]
+ * */
+export interface SubmissionState extends ActivityState {
+    pinned?: boolean
+    spoiler?: boolean
+    /**
+     * NSFW
+     * */
+    over_18?: boolean
+    is_self?: boolean
+}
+
+/**
+ * Different attributes a `Comment` can be in. Only include a property if you want to check it.
+ * @examples [{"op": true, "removed": false}]
+ * */
+export interface CommentState extends ActivityState {
+    /*
+    * Is this Comment Author also the Author of the Submission this comment is in?
+    * */
+    op?: boolean
+}
+
+export type TypedActivityStates = SubmissionState[] | CommentState[];
+
+export interface DomainInfo {
+    display: string,
+    domain: string,
+    aliases: string[],
+    provider?: string,
+    mediaType?: string
 }

src/Common/types.ts

@@ -0,0 +1,19 @@
+import {RecentActivityRuleJSONConfig} from "../Rule/RecentActivityRule";
+import {RepeatActivityJSONConfig} from "../Rule/SubmissionRule/RepeatActivityRule";
+import {AuthorRuleJSONConfig} from "../Rule/AuthorRule";
+import {AttributionJSONConfig} from "../Rule/AttributionRule";
+import {FlairActionJson} from "../Action/SubmissionAction/FlairAction";
+import {CommentActionJson} from "../Action/CommentAction";
+import {ReportActionJson} from "../Action/ReportAction";
+import {LockActionJson} from "../Action/LockAction";
+import {RemoveActionJson} from "../Action/RemoveAction";
+import {HistoryJSONConfig} from "../Rule/HistoryRule";
+import {UserNoteActionJson} from "../Action/UserNoteAction";
+import {ApproveActionJson} from "../Action/ApproveAction";
+import {BanActionJson} from "../Action/BanAction";
+
+export type RuleJson = RecentActivityRuleJSONConfig | RepeatActivityJSONConfig | AuthorRuleJSONConfig | AttributionJSONConfig | HistoryJSONConfig | string;
+export type RuleObjectJson = Exclude<RuleJson, string>
+
+export type ActionJson = CommentActionJson | FlairActionJson | ReportActionJson | LockActionJson | RemoveActionJson | ApproveActionJson | BanActionJson | UserNoteActionJson | string;
+export type ActionObjectJson = Exclude<ActionJson, string>;

src/ConfigBuilder.ts

@@ -1,5 +1,5 @@
 import {Logger} from "winston";
-import {createLabelledLogger, loggerMetaShuffle, mergeArr} from "./util";
+import {createAjvFactory, mergeArr, normalizeName} from "./util";
 import {CommentCheck} from "./Check/CommentCheck";
 import {SubmissionCheck} from "./Check/SubmissionCheck";
 
@@ -7,44 +7,211 @@ import Ajv from 'ajv';
 import * as schema from './Schema/App.json';
 import {JSONConfig} from "./JsonConfig";
 import LoggedError from "./Utils/LoggedError";
-
-const ajv = new Ajv();
+import {CheckStructuredJson} from "./Check";
+import {PollingOptions, PollingOptionsStrong, PollOn} from "./Common/interfaces";
+import {isRuleSetJSON, RuleSetJson, RuleSetObjectJson} from "./Rule/RuleSet";
+import deepEqual from "fast-deep-equal";
+import {ActionJson, ActionObjectJson, RuleJson, RuleObjectJson} from "./Common/types";
+import {isActionJson} from "./Action";
 
 export interface ConfigBuilderOptions {
-    logger?: Logger,
+    logger: Logger,
 }
 
 export class ConfigBuilder {
+    configLogger: Logger;
     logger: Logger;
 
     constructor(options: ConfigBuilderOptions) {
 
-        if (options.logger !== undefined) {
-            this.logger = options.logger.child(loggerMetaShuffle(options.logger, 'Config'), mergeArr);
-        } else {
-            this.logger = createLabelledLogger(`Config`, `Config`);
-        }
+        this.configLogger = options.logger.child({leaf: 'Config'}, mergeArr);
+        this.logger = options.logger;
     }
 
-    buildFromJson(config: object): (Array<SubmissionCheck> | Array<CommentCheck>)[] {
-        const commentChecks: Array<CommentCheck> = [];
-        const subChecks: Array<SubmissionCheck> = [];
+    validateJson(config: object): JSONConfig {
+        const ajv = createAjvFactory(this.logger);
         const valid = ajv.validate(schema, config);
-        if(valid) {
-            const validConfig = config as JSONConfig;
-            for (const jCheck of validConfig.checks) {
-                if (jCheck.kind === 'comment') {
-                    commentChecks.push(new CommentCheck({...jCheck, logger: this.logger}));
-                } else if (jCheck.kind === 'submission') {
-                    subChecks.push(new SubmissionCheck({...jCheck, logger: this.logger}));
+        if (valid) {
+            return config as JSONConfig;
+        } else {
+            this.configLogger.error('Json config was not valid. Please use schema to check validity.');
+            if (Array.isArray(ajv.errors)) {
+                for (const err of ajv.errors) {
+                    let parts = [
+                        `At: ${err.dataPath}`,
+                    ];
+                    let data;
+                    if (typeof err.data === 'string') {
+                        data = err.data;
+                    } else if (err.data !== null && typeof err.data === 'object' && (err.data as any).name !== undefined) {
+                        data = `Object named '${(err.data as any).name}'`;
+                    }
+                    if (data !== undefined) {
+                        parts.push(`Data: ${data}`);
+                    }
+                    let suffix = '';
+                    // @ts-ignore
+                    if (err.params.allowedValues !== undefined) {
+                        // @ts-ignore
+                        suffix = err.params.allowedValues.join(', ');
+                        suffix = ` [${suffix}]`;
+                    }
+                    parts.push(`${err.keyword}: ${err.schemaPath} => ${err.message}${suffix}`);
+
+                    // if we have a reference in the description parse it out so we can log it here for context
+                    if(err.parentSchema !== undefined && err.parentSchema.description !== undefined) {
+                        const desc = err.parentSchema.description as string;
+                        const seeIndex = desc.indexOf('[See]');
+                        if(seeIndex !== -1) {
+                            let newLineIndex: number | undefined = desc.indexOf('\n', seeIndex);
+                            if(newLineIndex === -1) {
+                                newLineIndex = undefined;
+                            }
+                            const seeFragment = desc.slice(seeIndex + 5, newLineIndex);
+                            parts.push(`See:${seeFragment}`);
+                        }
+                    }
+
+                    this.configLogger.error(`Schema Error:\r\n${parts.join('\r\n')}`);
                 }
             }
-        } else {
-            this.logger.error('Json config was not valid. Please use schema to check validity.', ajv.errors);
-            this.logger.error(ajv.errors);
-            throw new LoggedError();
+            throw new LoggedError('Config schema validity failure');
+        }
+    }
+
+    parseToStructured(config: JSONConfig): CheckStructuredJson[] {
+        let namedRules: Map<string, RuleObjectJson> = new Map();
+        let namedActions: Map<string, ActionObjectJson> = new Map();
+        const {checks = []} = config;
+        for (const c of checks) {
+            const {rules = []} = c;
+            namedRules = extractNamedRules(rules, namedRules);
+            namedActions = extractNamedActions(c.actions, namedActions);
         }
 
-        return [subChecks, commentChecks];
+        const structuredChecks: CheckStructuredJson[] = [];
+        for (const c of checks) {
+            const {rules = []} = c;
+            const strongRules = insertNamedRules(rules, namedRules);
+            const strongActions = insertNamedActions(c.actions, namedActions);
+            const strongCheck = {...c, rules: strongRules, actions: strongActions} as CheckStructuredJson;
+            structuredChecks.push(strongCheck);
+        }
+
+        return structuredChecks;
     }
 }
+
+export const buildPollingOptions = (values: (string | PollingOptions)[]): PollingOptionsStrong[] => {
+    let opts: PollingOptionsStrong[] = [];
+    for (const v of values) {
+        if (typeof v === 'string') {
+            opts.push({pollOn: v as PollOn, interval: 10000, limit: 25});
+        } else {
+            const {
+                pollOn: p,
+                interval = 20000,
+                limit = 25
+            } = v;
+            opts.push({pollOn: p as PollOn, interval, limit});
+        }
+    }
+    return opts;
+}
+
+export const extractNamedRules = (rules: Array<RuleSetJson | RuleJson>, namedRules: Map<string, RuleObjectJson> = new Map()): Map<string, RuleObjectJson> => {
+    //const namedRules = new Map();
+    for (const r of rules) {
+        let rulesToAdd: RuleObjectJson[] = [];
+        if ((typeof r === 'object')) {
+            if ((r as RuleObjectJson).kind !== undefined) {
+                // itsa rule
+                const rule = r as RuleObjectJson;
+                if (rule.name !== undefined) {
+                    rulesToAdd.push(rule);
+                }
+            } else {
+                const ruleSet = r as RuleSetJson;
+                const nestedNamed = extractNamedRules(ruleSet.rules);
+                rulesToAdd = [...nestedNamed.values()];
+            }
+            for (const rule of rulesToAdd) {
+                const name = rule.name as string;
+                const normalName = normalizeName(name);
+                const {name: n, ...rest} = rule;
+                const ruleNoName = {...rest};
+
+                if (namedRules.has(normalName)) {
+                    const {name: nn, ...ruleRest} = namedRules.get(normalName) as RuleObjectJson;
+                    if (!deepEqual(ruleRest, ruleNoName)) {
+                        throw new Error(`Rule names must be unique (case-insensitive). Conflicting name: ${name}`);
+                    }
+                } else {
+                    namedRules.set(normalName, rule);
+                }
+            }
+        }
+    }
+    return namedRules;
+}
+
+export const insertNamedRules = (rules: Array<RuleSetJson | RuleJson>, namedRules: Map<string, RuleObjectJson> = new Map()): Array<RuleSetObjectJson | RuleObjectJson> => {
+    const strongRules: Array<RuleSetObjectJson | RuleObjectJson> = [];
+    for (const r of rules) {
+        if (typeof r === 'string') {
+            const foundRule = namedRules.get(r.toLowerCase());
+            if (foundRule === undefined) {
+                throw new Error(`No named Rule with the name ${r} was found`);
+            }
+            strongRules.push(foundRule);
+        } else if (isRuleSetJSON(r)) {
+            const {rules: sr, ...rest} = r;
+            const setRules = insertNamedRules(sr, namedRules);
+            const strongSet = {rules: setRules, ...rest} as RuleSetObjectJson;
+            strongRules.push(strongSet);
+        } else {
+            strongRules.push(r);
+        }
+    }
+
+    return strongRules;
+}
+
+export const extractNamedActions = (actions: Array<ActionJson>, namedActions: Map<string, ActionObjectJson> = new Map()): Map<string, ActionObjectJson> => {
+    for (const a of actions) {
+        if (!(typeof a === 'string')) {
+            if (isActionJson(a) && a.name !== undefined) {
+                const normalName = a.name.toLowerCase();
+                const {name: n, ...rest} = a;
+                const actionNoName = {...rest};
+                if (namedActions.has(normalName)) {
+                    // @ts-ignore
+                    const {name: nn, ...aRest} = namedActions.get(normalName) as ActionObjectJson;
+                    if (!deepEqual(aRest, actionNoName)) {
+                        throw new Error(`Actions names must be unique (case-insensitive). Conflicting name: ${a.name}`);
+                    }
+                } else {
+                    namedActions.set(normalName, a);
+                }
+            }
+        }
+    }
+    return namedActions;
+}
+
+export const insertNamedActions = (actions: Array<ActionJson>, namedActions: Map<string, ActionObjectJson> = new Map()): Array<ActionObjectJson> => {
+    const strongActions: Array<ActionObjectJson> = [];
+    for (const a of actions) {
+        if (typeof a === 'string') {
+            const foundAction = namedActions.get(a.toLowerCase());
+            if (foundAction === undefined) {
+                throw new Error(`No named Action with the name ${a} was found`);
+            }
+            strongActions.push(foundAction);
+        } else {
+            strongActions.push(a);
+        }
+    }
+
+    return strongActions;
+}

src/JsonConfig.ts

@@ -1,7 +1,7 @@
-import {CheckJSONConfig} from "./Check";
-import {PollingOptions} from "./Common/interfaces";
+import {CheckJson, CommentCheckJson, SubmissionCheckJson} from "./Check";
+import {ManagerOptions} from "./Common/interfaces";
 
-export interface JSONConfig {
+export interface JSONConfig extends ManagerOptions {
     /**
      * A list of all the checks that should be run for a subreddit.
      *
@@ -12,6 +12,5 @@ export interface JSONConfig {
      * When a check "passes", and actions are performed, then all subsequent checks are skipped.
      * @minItems 1
      * */
-    checks: CheckJSONConfig[]
-    polling?: PollingOptions
+    checks: Array<SubmissionCheckJson|CommentCheckJson>
 }

src/Rule/AttributionRule.ts

@@ -0,0 +1,440 @@
+import {SubmissionRule, SubmissionRuleJSONConfig} from "./SubmissionRule";
+import {ActivityWindowType, DomainInfo, ReferenceSubmission} from "../Common/interfaces";
+import {Rule, RuleOptions, RuleResult} from "./index";
+import Submission from "snoowrap/dist/objects/Submission";
+import {getAttributionIdentifier} from "../Utils/SnoowrapUtils";
+import dayjs from "dayjs";
+import {
+    comparisonTextOp,
+    FAIL,
+    formatNumber,
+    parseGenericValueOrPercentComparison,
+    parseSubredditName,
+    PASS
+} from "../util";
+import { Comment } from "snoowrap/dist/objects";
+import SimpleError from "../Utils/SimpleError";
+
+
+export interface AttributionCriteria {
+    /**
+     * A string containing a comparison operator and a value to compare comments against
+     *
+     * The syntax is `(< OR > OR <= OR >=) <number>[percent sign]`
+     *
+     * * EX `> 12`  => greater than 12 activities originate from same attribution
+     * * EX `<= 10%` => less than 10% of all Activities have the same attribution
+     *
+     * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(%?)(.*)$
+     * @default "> 10%"
+     * */
+    threshold: string
+    window: ActivityWindowType
+    /**
+     * What activities to use for total count when determining what percentage an attribution comprises
+     *
+     * EX:
+     *
+     * Author has 100 activities, 40 are submissions and 60 are comments
+     *
+     * * If `submission` then if 10 submission are for Youtube Channel A then percentage => 10/40 = 25%
+     * * If `all` then if 10 submission are for Youtube Channel A then percentage => 10/100 = 10%
+     *
+     * @default all
+     **/
+    thresholdOn?: 'submissions' | 'all'
+    /**
+     * The minimum number of activities that must exist for this criteria to run
+     * @default 5
+     * */
+    minActivityCount?: number
+
+    /**
+     * A list of domains whose Activities will be tested against `threshold`.
+     *
+     * If this is present then `aggregateOn` is ignored.
+     *
+     * The values are tested as partial strings so you do not need to include full URLs, just the part that matters.
+     *
+     * EX `["youtube"]` will match submissions with the domain `https://youtube.com/c/aChannel`
+     * EX `["youtube.com/c/bChannel"]` will NOT match submissions with the domain `https://youtube.com/c/aChannel`
+     *
+     * If you wish to aggregate on self-posts for a subreddit use the syntax `self.[subreddit]` EX `self.AskReddit`
+     *
+     * **If this Rule is part of a Check for a Submission and you wish to aggregate on the domain of the Submission use the special string `AGG:SELF`**
+     *
+     * If nothing is specified or list is empty (default) aggregate using `aggregateOn`
+     *
+     * @default [[]]
+     * */
+    domains?: string[],
+    /**
+     * Set to `true` if you wish to combine all of the Activities from `domains` to test against `threshold` instead of testing each `domain` individually
+     *
+     * @default false
+     * @examples [false]
+     * */
+    domainsCombined?: boolean,
+
+    /**
+     * Only include Activities from this list of Subreddits (by name, case-insensitive)
+     *
+     *
+     * EX `["mealtimevideos","askscience"]`
+     * @examples ["mealtimevideos","askscience"]
+     * @minItems 1
+     * */
+    include?: string[],
+    /**
+     * Do not include Activities from this list of Subreddits (by name, case-insensitive)
+     *
+     * Will be ignored if `include` is present.
+     *
+     * EX `["mealtimevideos","askscience"]`
+     * @examples ["mealtimevideos","askscience"]
+     * @minItems 1
+     * */
+    exclude?: string[],
+
+    /**
+     * If `domains` is not specified this list determines which categories of domains should be aggregated on. All aggregated domains will be tested against `threshold`
+     *
+     * * If `media` is included then aggregate author's submission history which reddit recognizes as media (youtube, vimeo, etc.)
+     * * If `self` is included then aggregate on author's submission history which are self-post (`self.[subreddit]`) or reddit image/video (i.redd.it / v.redd.it)
+     * * If `link` is included then aggregate author's submission history which is external links but not media
+     *
+     * If nothing is specified or list is empty (default) all domains are aggregated
+     *
+     *  @default undefined
+     *  @examples [[]]
+     * */
+    aggregateOn?: ('media' | 'self' | 'link')[],
+
+    /**
+     * Should the criteria consolidate recognized media domains into the parent domain?
+     *
+     * Submissions to major media domains (youtube, vimeo) can be identified by individual Channel/Author...
+     *
+     * * If `false` then domains will be aggregated at the channel level IE Youtube Channel A (2 counts), Youtube Channel B  (3 counts)
+     * * If `true` then then media domains will be consolidated at domain level and then aggregated IE youtube.com (5 counts)
+     *
+     *  @default false
+     *  @examples [false]
+     * */
+    consolidateMediaDomains?: boolean
+
+    name?: string
+}
+
+const SUBMISSION_DOMAIN = 'AGG:SELF';
+
+const defaultCriteria = [{threshold: '10%', window: 100}];
+
+interface DomainAgg {
+    info: DomainInfo,
+    count: number
+}
+
+export class AttributionRule extends Rule {
+    criteria: AttributionCriteria[];
+    criteriaJoin: 'AND' | 'OR';
+
+    constructor(options: AttributionOptions) {
+        super(options);
+        const {
+            criteria = defaultCriteria,
+            criteriaJoin = 'OR',
+        } = options || {};
+
+        this.criteria = criteria;
+        this.criteriaJoin = criteriaJoin;
+        if (this.criteria.length === 0) {
+            throw new Error('Must provide at least one AttributionCriteria');
+        }
+    }
+
+    getKind(): string {
+        return "Attr";
+    }
+
+    protected getSpecificPremise(): object {
+        return {
+            criteria: this.criteria,
+            criteriaJoin: this.criteriaJoin,
+        }
+    }
+
+    protected async process(item: Comment | Submission): Promise<[boolean, RuleResult]> {
+        let criteriaResults = [];
+
+        for (const criteria of this.criteria) {
+
+            const {
+                threshold = '> 10%',
+                window,
+                thresholdOn = 'all',
+                minActivityCount = 10,
+                aggregateOn = [],
+                consolidateMediaDomains = false,
+                domains = [],
+                domainsCombined = false,
+                include: includeRaw = [],
+                exclude: excludeRaw = [],
+            } = criteria;
+
+            const include = includeRaw.map(x => parseSubredditName(x).toLowerCase());
+            const exclude = excludeRaw.map(x => parseSubredditName(x).toLowerCase());
+
+            const {operator, value, isPercent, extra = ''} = parseGenericValueOrPercentComparison(threshold);
+
+            let activities = thresholdOn === 'submissions' ? await this.resources.getAuthorSubmissions(item.author, {window: window}) : await this.resources.getAuthorActivities(item.author, {window: window});
+            activities = activities.filter(act => {
+                if (include.length > 0) {
+                    return include.some(x => x === act.subreddit.display_name.toLowerCase());
+                } else if (exclude.length > 0) {
+                    return !exclude.some(x => x === act.subreddit.display_name.toLowerCase())
+                }
+                return true;
+            });
+
+            let activityTotal = 0;
+            let firstActivity, lastActivity;
+
+            if(activities.length === 0) {
+                this.logger.debug(`No activities retrieved for criteria`);
+                continue;
+            }
+
+            activityTotal = activities.length;
+            firstActivity = activities[0];
+            lastActivity = activities[activities.length - 1];
+
+            const activityTotalWindow = dayjs.duration(dayjs(firstActivity.created_utc * 1000).diff(dayjs(lastActivity.created_utc * 1000)));
+
+            if (activities.length < minActivityCount) {
+                criteriaResults.push({criteria, activityTotal, activityTotalWindow, triggered: false, aggDomains: [], minCountMet: false});
+                this.logger.debug(`${activities.length } activities retrieved was less than min activities required to run criteria (${minActivityCount})`);
+                continue;
+            }
+
+            const realDomains: DomainInfo[] = domains.map(x => {
+                if(x === SUBMISSION_DOMAIN) {
+                    if(!(item instanceof Submission)) {
+                        throw new SimpleError('Cannot run Attribution Rule with the domain SELF:AGG on a Comment');
+                    }
+                    return getAttributionIdentifier(item, consolidateMediaDomains);
+                }
+                return {display: x, domain: x, aliases: [x]};
+            });
+            const realDomainIdents = realDomains.map(x => x.aliases).flat(1).map(x => x.toLowerCase());
+
+            const submissions: Submission[] = thresholdOn === 'submissions' ? activities as Submission[] : activities.filter(x => x instanceof Submission) as Submission[];
+            const aggregatedSubmissions = submissions.reduce((acc: Map<string, DomainAgg>, sub) => {
+                const domainInfo = getAttributionIdentifier(sub, consolidateMediaDomains)
+
+                let domainType = 'link';
+                if(sub.secure_media !== undefined && sub.secure_media !== null) {
+                    domainType = 'media';
+                } else if((sub.is_self || sub.is_video || sub.domain === 'i.redd.it')) {
+                    domainType = 'self';
+                }
+
+                if(realDomains.length === 0 && aggregateOn.length !== 0) {
+                    if(domainType === 'media' && !aggregateOn.includes('media')) {
+                        return acc;
+                    }
+                    if(domainType === 'self' && !aggregateOn.includes('self')) {
+                        return acc;
+                    }
+                    if(domainType === 'link' && !aggregateOn.includes('link')) {
+                        return acc;
+                    }
+                }
+
+                if(realDomains.length > 0) {
+                    if(domainInfo.aliases.map(x => x.toLowerCase()).some(x => realDomainIdents.includes(x))) {
+                        const domainAgg = acc.get(domainInfo.display) || {info: domainInfo, count: 0};
+                        acc.set(domainInfo.display, {...domainAgg, count: domainAgg.count + 1});
+                    }
+                } else {
+                    const domainAgg = acc.get(domainInfo.display) || {info: domainInfo, count: 0};
+                    acc.set(domainInfo.display, {...domainAgg, count: domainAgg.count + 1});
+                }
+
+                return acc;
+            }, new Map());
+
+            let aggDomains = [];
+
+            if(domainsCombined) {
+                let combinedCount = 0;
+                let domains = [];
+                let triggered = false;
+                for (const [domain, dAgg] of aggregatedSubmissions) {
+                    domains.push(domain);
+                    combinedCount += dAgg.count;
+                }
+                if(isPercent) {
+                    triggered = comparisonTextOp(combinedCount / activityTotal, operator, (value/100));
+                }
+                else {
+                    triggered = comparisonTextOp(combinedCount, operator, value);
+                }
+                const combinedDomain = Array.from(aggregatedSubmissions.values()).map(x => x.info.domain).join(' and ');
+                const combinedDisplay = Array.from(aggregatedSubmissions.values()).map(x => `${x.info.display}${x.info.provider !== undefined ? ` (${x.info.provider})` : ''}`).join(' and ');
+                aggDomains.push({
+                    domain: {display: combinedDisplay, domain: combinedDomain, aliases: [combinedDomain]},
+                    count: combinedCount,
+                    percent: Math.round((combinedCount / activityTotal) * 100),
+                    triggered,
+                });
+
+            } else {
+                for (const [domain, dAgg] of aggregatedSubmissions) {
+                    let triggered = false;
+                    if(isPercent) {
+                        triggered = comparisonTextOp(dAgg.count / activityTotal, operator, (value/100));
+                    }
+                    else {
+                        triggered = comparisonTextOp(dAgg.count, operator, value);
+                    }
+
+                    aggDomains.push({
+                        domain: dAgg.info,
+                        count: dAgg.count,
+                        percent: Math.round((dAgg.count / activityTotal) * 100),
+                        triggered,
+                    });
+                }
+            }
+
+            criteriaResults.push({criteria, activityTotal, activityTotalWindow, aggDomains, minCountMet: true});
+        }
+
+        let criteriaMeta = false;
+        if (this.criteriaJoin === 'OR') {
+            criteriaMeta = criteriaResults.some(x => x.aggDomains.length > 0 && x.aggDomains.some(y => y.triggered === true));
+        } else {
+            criteriaMeta = criteriaResults.every(x => x.aggDomains.length > 0 && x.aggDomains.some(y => y.triggered === true));
+        }
+
+        let usableCriteria = criteriaResults.filter(x => x.aggDomains.length > 0 && x.aggDomains.some(y => y.triggered === true));
+        if (usableCriteria.length === 0) {
+            usableCriteria = criteriaResults.filter(x => x.aggDomains.length > 0)
+        }
+        // probably none hit min count then
+        if(criteriaResults.every(x => x.minCountMet === false)) {
+            const result = `${FAIL} No criteria had their min activity count met`;
+            this.logger.verbose(result);
+            return Promise.resolve([false, this.getResult(false, {result})]);
+        }
+
+        let result;
+        const refCriteriaResults = usableCriteria.find(x => x !== undefined);
+        if(refCriteriaResults === undefined) {
+            result = `${FAIL} No criteria results found??`;
+            return Promise.resolve([false, this.getResult(false, {result})])
+        }
+
+        const {
+            aggDomains = [],
+            activityTotal,
+            activityTotalWindow,
+            criteria: {threshold, window}
+        } = refCriteriaResults;
+
+        const largestCount = aggDomains.reduce((acc, curr) => Math.max(acc, curr.count), 0);
+        const largestPercent = aggDomains.reduce((acc, curr) => Math.max(acc, curr.percent), 0);
+        const smallestCount = aggDomains.reduce((acc, curr) => Math.min(acc, curr.count), aggDomains[0].count);
+        const smallestPercent = aggDomains.reduce((acc, curr) => Math.min(acc, curr.percent), aggDomains[0].percent);
+        const windowText = typeof window === 'number' ? `${activityTotal} Items` : activityTotalWindow.humanize();
+        const countRange = smallestCount === largestCount ? largestCount : `${smallestCount} - ${largestCount}`
+        const percentRange = formatNumber(smallestPercent, {toFixed: 0}) === formatNumber(largestPercent, {toFixed: 0}) ? `${largestPercent}%` : `${smallestPercent}% - ${largestPercent}%`
+
+        let data: any = {};
+        const resultAgnostic = `met the threshold of ${threshold}, with ${countRange} (${percentRange}) of ${activityTotal} Total -- window: ${windowText}`;
+
+        if(criteriaMeta) {
+            result = `${PASS} ${aggDomains.length} Attribution(s) ${resultAgnostic}`;
+            data = {
+                triggeredDomainCount: aggDomains.length,
+                activityTotal,
+                largestCount,
+                largestPercent: `${largestPercent}%`,
+                smallestCount,
+                smallestPercent: `${smallestPercent}%`,
+                countRange,
+                percentRange,
+                domains: aggDomains.map(x => x.domain.domain),
+                domainsDelim: aggDomains.map(x => x.domain.domain).join(', '),
+                titles: aggDomains.map(x => `${x.domain.display}${x.domain.provider !== undefined ? ` (${x.domain.provider})` :''}`),
+                titlesDelim: aggDomains.map(x => `${x.domain.display}${x.domain.provider !== undefined ? ` (${x.domain.provider})` :''}`).join(', '),
+                threshold: threshold,
+                window: windowText
+            };
+        } else {
+            result = `${FAIL} No Attributions ${resultAgnostic}`;
+        }
+
+        this.logger.verbose(result);
+        return Promise.resolve([criteriaMeta, this.getResult(criteriaMeta, {
+            result,
+            data,
+        })]);
+    }
+
+}
+
+interface AttributionConfig extends ReferenceSubmission {
+
+    /**
+     * A list threshold-window values to test attribution against
+     *
+     * If none is provided the default set used is:
+     *
+     * ```
+     * threshold: 10%
+     * window: 100
+     * ```
+     *
+     * @minItems 1
+     * */
+    criteria?: AttributionCriteria[]
+
+    /**
+     * * If `OR` then any set of AttributionCriteria that produce an Attribution over the threshold will trigger the rule.
+     * * If `AND` then all AttributionCriteria sets must product an Attribution over the threshold to trigger the rule.
+     * */
+    criteriaJoin?: 'AND' | 'OR'
+}
+
+export interface AttributionOptions extends AttributionConfig, RuleOptions {
+
+}
+
+/**
+ * Aggregates all of the domain/media accounts attributed to an author's Submission history. If any domain is over the threshold the rule is triggered
+ *
+ * Available data for [Action templating](https://github.com/FoxxMD/reddit-context-bot#action-templating):
+ *
+ * ```
+ * triggeredDomainCount => Number of domains that met the threshold
+ * activityTotal        => Number of Activities considered from window
+ * window               => The date range of the Activities considered
+ * largestCount         => The count from the largest aggregated domain
+ * largestPercentage    => The percentage of Activities the largest aggregated domain comprises
+ * smallestCount        => The count from the smallest aggregated domain
+ * smallestPercentage   => The percentage of Activities the smallest aggregated domain comprises
+ * countRange           => A convenience string displaying "smallestCount - largestCount" or just one number if both are the same
+ * percentRange         => A convenience string displaying "smallestPercentage - largestPercentage" or just one percentage if both are the same
+ * domains              => An array of all the domain URLs that met the threshold
+ * domainsDelim         => A comma-delimited string of all the domain URLs that met the threshold
+ * titles               => The friendly-name of the domain if one is present, otherwise the URL (IE youtube.com/c/34ldfa343 => "My Youtube Channel Title")
+ * titlesDelim          => A comma-delimited string of all the domain friendly-names
+ * threshold            => The threshold you configured for this Rule to trigger
+ * url                  => Url of the submission that triggered the rule
+ * ```
+ * */
+export interface AttributionJSONConfig extends AttributionConfig, SubmissionRuleJSONConfig {
+    kind: 'attribution'
+}

src/Rule/AuthorRule.ts

@@ -1,7 +1,7 @@
-import {Author, AuthorOptions, AuthorCriteria, Rule, RuleJSONConfig, RuleOptions, RuleResult} from "./index";
+import {Rule, RuleJSONConfig, RuleOptions, RuleResult} from "./index";
 import {Comment} from "snoowrap";
 import Submission from "snoowrap/dist/objects/Submission";
-import {testAuthorCriteria} from "../Utils/SnoowrapUtils";
+import {Author, AuthorCriteria} from "../Author/Author";
 
 /**
  * Checks the author of the Activity against AuthorCriteria. This differs from a Rule's AuthorOptions as this is a full Rule and will only pass/fail, not skip.
@@ -53,21 +53,21 @@ export class AuthorRule extends Rule {
         };
     }
 
-    protected async process(item: Comment | Submission): Promise<[boolean, RuleResult[]]> {
+    protected async process(item: Comment | Submission): Promise<[boolean, RuleResult]> {
         if (this.include.length > 0) {
             for (const auth of this.include) {
-                if (await testAuthorCriteria(item, auth)) {
-                    return Promise.resolve([true, [this.getResult(true)]]);
+                if (await this.resources.testAuthorCriteria(item, auth)) {
+                    return Promise.resolve([true, this.getResult(true)]);
                 }
             }
-            return Promise.resolve([false, [this.getResult(false)]]);
+            return Promise.resolve([false, this.getResult(false)]);
         }
         for (const auth of this.exclude) {
-            if (await testAuthorCriteria(item, auth, false)) {
-                return Promise.resolve([true, [this.getResult(true)]]);
+            if (await this.resources.testAuthorCriteria(item, auth, false)) {
+                return Promise.resolve([true, this.getResult(true)]);
             }
         }
-        return Promise.resolve([false, [this.getResult(false)]]);
+        return Promise.resolve([false, this.getResult(false)]);
     }
 }
 

src/Rule/HistoryRule.ts

@@ -0,0 +1,340 @@
+
+import {ActivityWindowType, CompareValueOrPercent, ThresholdCriteria} from "../Common/interfaces";
+import {Rule, RuleJSONConfig, RuleOptions, RuleResult} from "./index";
+import Submission from "snoowrap/dist/objects/Submission";
+import {getAuthorActivities} from "../Utils/SnoowrapUtils";
+import dayjs from "dayjs";
+import {
+    comparisonTextOp,
+    FAIL,
+    formatNumber,
+    parseGenericValueOrPercentComparison, parseSubredditName,
+    PASS,
+    percentFromString
+} from "../util";
+
+export interface CommentThresholdCriteria extends ThresholdCriteria {
+    /**
+     * If `true` then when threshold...
+     *
+     * * is `number` it will be number of comments where author is OP
+     * * is `percent` it will be **percent of total comments where author is OP**
+     * */
+    asOp?: boolean
+}
+/**
+ * If both `submission` and `comment` are defined then criteria will only trigger if BOTH thresholds are met
+ * */
+export interface HistoryCriteria {
+
+    /**
+     * A string containing a comparison operator and a value to compare submissions against
+     *
+     * The syntax is `(< OR > OR <= OR >=) <number>[percent sign]`
+     *
+     * * EX `> 100`  => greater than 100 submissions
+     * * EX `<= 75%` => submissions are equal to or less than 75% of all Activities
+     *
+     * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(%?)(.*)$
+     * */
+    submission?: CompareValueOrPercent
+    /**
+     * A string containing a comparison operator and a value to compare comments against
+     *
+     * The syntax is `(< OR > OR <= OR >=) <number>[percent sign] [OP]`
+     *
+     * * EX `> 100`  => greater than 100 comments
+     * * EX `<= 75%` => comments are equal to or less than 75% of all Activities
+     *
+     * If your string also contains the text `OP` somewhere **after** `<number>[percent sign]`...:
+     *
+     * * EX `> 100 OP`  => greater than 100 comments as OP
+     * * EX `<= 25% as OP` => Comments as OP were less then or equal to 25% of **all Comments**
+     *
+     * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(%?)(.*)$
+     * */
+    comment?: CompareValueOrPercent
+
+    window: ActivityWindowType
+
+    /**
+     * The minimum number of activities that must exist from the `window` results for this criteria to run
+     * @default 5
+     * */
+    minActivityCount?: number
+    name?: string
+}
+
+export class HistoryRule extends Rule {
+    criteria: HistoryCriteria[];
+    condition: 'AND' | 'OR';
+    include: string[];
+    exclude: string[];
+
+    constructor(options: HistoryOptions) {
+        super(options);
+        const {
+            criteria,
+            condition = 'OR',
+            include = [],
+            exclude = [],
+        } = options || {};
+
+        this.criteria = criteria;
+        this.condition = condition;
+        if (this.criteria.length === 0) {
+            throw new Error('Must provide at least one HistoryCriteria');
+        }
+        this.include = include.map(x => parseSubredditName(x).toLowerCase());
+        this.exclude = exclude.map(x => parseSubredditName(x).toLowerCase());
+    }
+
+    getKind(): string {
+        return "History";
+    }
+
+    protected getSpecificPremise(): object {
+        return {
+            criteria: this.criteria,
+            include: this.include,
+            exclude: this.exclude,
+        }
+    }
+
+    protected async process(item: Submission): Promise<[boolean, RuleResult]> {
+        // TODO reuse activities between ActivityCriteria to reduce api calls
+
+        let criteriaResults = [];
+
+        for (const criteria of this.criteria) {
+
+            const {comment, window, submission, minActivityCount = 5} = criteria;
+
+            let activities = await this.resources.getAuthorActivities(item.author, {window: window});
+            activities = activities.filter(act => {
+                if (this.include.length > 0) {
+                    return this.include.some(x => x === act.subreddit.display_name.toLowerCase());
+                } else if (this.exclude.length > 0) {
+                    return !this.exclude.some(x => x === act.subreddit.display_name.toLowerCase())
+                }
+                return true;
+            });
+
+            if (activities.length < minActivityCount) {
+                continue;
+            }
+
+            const activityTotal = activities.length;
+            const {submissionTotal, commentTotal, opTotal} = activities.reduce((acc, act) => {
+                if(act instanceof Submission) {
+                    return {...acc, submissionTotal: acc.submissionTotal + 1};
+                }
+                let a = {...acc, commentTotal: acc.commentTotal + 1};
+                if(act.is_submitter) {
+                    a.opTotal = a.opTotal + 1;
+                }
+                return a;
+            },{submissionTotal: 0, commentTotal: 0, opTotal: 0});
+
+            let commentTrigger = undefined;
+            if(comment !== undefined) {
+                const {operator, value, isPercent, extra = ''} = parseGenericValueOrPercentComparison(comment);
+                const asOp = extra.toLowerCase().includes('op');
+                if(isPercent) {
+                    const per = value / 100;
+                    if(asOp) {
+                        commentTrigger = comparisonTextOp(opTotal / commentTotal, operator, per);
+                    } else {
+                        commentTrigger = comparisonTextOp(commentTotal / activityTotal, operator, per);
+                    }
+                } else {
+                    if(asOp) {
+                        commentTrigger = comparisonTextOp(opTotal, operator, value);
+                    } else {
+                        commentTrigger = comparisonTextOp(commentTotal, operator, value);
+                    }
+                }
+            }
+
+            let submissionTrigger = undefined;
+            if(submission !== undefined) {
+                const {operator, value, isPercent} = parseGenericValueOrPercentComparison(submission);
+                if(isPercent) {
+                    const per = value / 100;
+                    submissionTrigger = comparisonTextOp(submissionTotal / activityTotal, operator, per);
+                } else {
+                    submissionTrigger = comparisonTextOp(submissionTotal, operator, value);
+                }
+            }
+
+            const firstActivity = activities[0];
+            const lastActivity = activities[activities.length - 1];
+
+            const activityTotalWindow = dayjs.duration(dayjs(firstActivity.created_utc * 1000).diff(dayjs(lastActivity.created_utc * 1000)));
+
+            criteriaResults.push({
+                criteria,
+                activityTotal,
+                activityTotalWindow,
+                submissionTotal,
+                commentTotal,
+                opTotal,
+                submissionTrigger,
+                commentTrigger,
+                triggered: (submissionTrigger === undefined || submissionTrigger === true) && (commentTrigger === undefined || commentTrigger === true)
+            });
+        }
+
+        let criteriaMet = false;
+        let failCriteriaResult: string = '';
+        if (this.condition === 'OR') {
+            criteriaMet = criteriaResults.some(x => x.triggered);
+            if(!criteriaMet) {
+                failCriteriaResult = `${FAIL} No criteria was met`;
+            }
+        } else {
+            criteriaMet = criteriaResults.every(x => x.triggered);
+            if(!criteriaMet) {
+                if(criteriaResults.some(x => x.triggered)) {
+                    const met = criteriaResults.filter(x => x.triggered);
+                    failCriteriaResult = `${FAIL} ${met.length} out of ${criteriaResults.length} criteria met but Rule required all be met. Set log level to debug to see individual results`;
+                    const results = criteriaResults.map(x => this.generateResultDataFromCriteria(x, true));
+                    this.logger.debug(`\r\n ${results.map(x => x.result).join('\r\n')}`);
+                } else {
+                    failCriteriaResult = `${FAIL} No criteria was met`;
+                }
+            }
+        }
+
+        if(criteriaMet) {
+            // use first triggered criteria found
+            const refCriteriaResults = criteriaResults.find(x => x.triggered);
+            const resultData = this.generateResultDataFromCriteria(refCriteriaResults);
+
+            this.logger.verbose(`${PASS} ${resultData.result}`);
+            return Promise.resolve([true, this.getResult(true, resultData)]);
+        }
+
+        return Promise.resolve([false, this.getResult(false, {result: failCriteriaResult})]);
+    }
+
+    protected generateResultDataFromCriteria(results: any, includePassFailSymbols = false) {
+        const {
+            activityTotal,
+            activityTotalWindow,
+            submissionTotal,
+            commentTotal,
+            opTotal,
+            criteria: {
+                comment,
+                submission,
+                window,
+            },
+            criteria,
+            triggered,
+            submissionTrigger,
+            commentTrigger,
+        } = results;
+
+        const data: any = {
+            activityTotal,
+            submissionTotal,
+            commentTotal,
+            opTotal,
+            commentPercent: formatNumber((commentTotal/activityTotal)*100),
+            submissionPercent: formatNumber((submissionTotal/activityTotal)*100),
+            opPercent: formatNumber((opTotal/commentTotal)*100),
+            criteria,
+            window: typeof window === 'number' ? `${activityTotal} Items` : activityTotalWindow.humanize(true),
+            triggered,
+            submissionTrigger,
+            commentTrigger,
+        };
+
+        let thresholdSummary = [];
+        let submissionSummary;
+        let commentSummary;
+        if(submission !== undefined) {
+            const {operator, value, isPercent, displayText} = parseGenericValueOrPercentComparison(submission);
+            const suffix = !isPercent ? 'Items' : `(${formatNumber((submissionTotal/activityTotal)*100)}%) of ${activityTotal} Total`;
+            submissionSummary = `${includePassFailSymbols ? `${submissionTrigger ? PASS : FAIL} ` : ''}Submissions (${submissionTotal}) were${submissionTrigger ? '' : ' not'} ${displayText} ${suffix}`;
+            data.submissionSummary = submissionSummary;
+            thresholdSummary.push(submissionSummary);
+        }
+        if(comment !== undefined) {
+            const {operator, value, isPercent, displayText, extra = ''} = parseGenericValueOrPercentComparison(comment);
+            const asOp = extra.toLowerCase().includes('op');
+            const totalType = asOp ? 'Comments' : 'Activities'
+            const countType = asOp ? 'Comments as OP' : 'Comments';
+            const suffix = !isPercent ? 'Items' : `(${asOp ? formatNumber((opTotal/commentTotal)*100) : formatNumber((commentTotal/activityTotal)*100)}%) of ${activityTotal} Total ${totalType}`;
+            commentSummary = `${includePassFailSymbols ? `${commentTrigger ? PASS : FAIL} ` : ''}${countType} (${asOp ? opTotal : commentTotal}) were${commentTrigger ? '' : ' not'} ${displayText} ${suffix}`;
+            data.commentSummary = commentSummary;
+            thresholdSummary.push(commentSummary);
+        }
+
+        data.thresholdSummary = thresholdSummary.join(' and ');
+
+        const result = `${thresholdSummary} (${data.window})`;
+
+        return {result, data};
+    }
+
+}
+
+export default HistoryRule;
+
+interface HistoryConfig  {
+
+    /**
+     * A list threshold-window values to test activities against.
+     *
+     * @minItems 1
+     * */
+    criteria: HistoryCriteria[]
+
+    /**
+     * * If `OR` then any set of Criteria that pass will trigger the Rule
+     * * If `AND` then all Criteria sets must pass to trigger the Rule
+     * */
+    condition?: 'AND' | 'OR'
+
+    /**
+     * Only include Submissions from this list of Subreddits (by name, case-insensitive)
+     *
+     * EX `["mealtimevideos","askscience"]`
+     * @examples ["mealtimevideos","askscience"]
+     * @minItems 1
+     * */
+    include?: string[],
+    /**
+     * Do not include Submissions from this list of Subreddits (by name, case-insensitive)
+     *
+     * EX `["mealtimevideos","askscience"]`
+     * @examples ["mealtimevideos","askscience"]
+     * @minItems 1
+     * */
+    exclude?: string[],
+}
+
+export interface HistoryOptions extends HistoryConfig, RuleOptions {
+
+}
+
+/**
+ * Aggregates an Author's submission and comment history. Rule can be triggered on count/percent of total (for either or both comment/sub totals) as well as comment OP total.
+ *
+ * Available data for [Action templating](https://github.com/FoxxMD/reddit-context-bot#action-templating):
+ *
+ * ```
+ * activityTotal    => Total number of activities
+ * submissionTotal  => Total number of submissions
+ * commentTotal     => Total number of comments
+ * opTotal          => Total number of comments as OP
+ * thresholdSummary => A text summary of the first Criteria triggered with totals/percentages
+ * criteria         => The ThresholdCriteria object
+ * window           => A text summary of the range of Activities considered (# of Items if number, time range if Duration)
+ * ```
+ * */
+export interface HistoryJSONConfig extends HistoryConfig, RuleJSONConfig {
+    kind: 'history'
+}

src/Rule/RecentActivityRule.ts

@@ -1,8 +1,13 @@
 import {Rule, RuleJSONConfig, RuleOptions, RulePremise, RuleResult} from "./index";
 import {Comment, VoteableContent} from "snoowrap";
 import Submission from "snoowrap/dist/objects/Submission";
-import {getAuthorActivities, getAuthorComments, getAuthorSubmissions} from "../Utils/SnoowrapUtils";
-import {parseUsableLinkIdentifier} from "../util";
+import {
+    activityWindowText,
+    comparisonTextOp, FAIL, formatNumber,
+    parseGenericValueOrPercentComparison, parseSubredditName,
+    parseUsableLinkIdentifier,
+    PASS
+} from "../util";
 import {
     ActivityWindow,
     ActivityWindowCriteria,
@@ -33,7 +38,7 @@ export class RecentActivityRule extends Rule {
     }
 
     getKind(): string {
-        return 'Recent Activity';
+        return 'Recent';
     }
 
     getSpecificPremise(): object {
@@ -45,28 +50,27 @@ export class RecentActivityRule extends Rule {
         }
     }
 
-    async process(item: Submission | Comment): Promise<[boolean, RuleResult[]]> {
+    async process(item: Submission | Comment): Promise<[boolean, RuleResult]> {
         let activities;
 
         switch (this.lookAt) {
             case 'comments':
-                activities = await getAuthorComments(item.author, {window: this.window});
+                activities = await this.resources.getAuthorComments(item.author, {window: this.window});
                 break;
             case 'submissions':
-                activities = await getAuthorSubmissions(item.author, {window: this.window});
+                activities = await this.resources.getAuthorSubmissions(item.author, {window: this.window});
                 break;
             default:
-                activities = await getAuthorActivities(item.author, {window: this.window});
+                activities = await this.resources.getAuthorActivities(item.author, {window: this.window});
                 break;
         }
 
-
         let viableActivity = activities;
         if (this.useSubmissionAsReference) {
             if (!(item instanceof Submission)) {
-                this.logger.debug('Cannot use post as reference because triggered item is not a Submission');
+                this.logger.warn('Cannot use post as reference because triggered item is not a Submission');
             } else if (item.is_self) {
-                this.logger.debug('Cannot use post as reference because triggered Submission is not a link type');
+                this.logger.warn('Cannot use post as reference because triggered Submission is not a link type');
             } else {
                 const usableUrl = parseLink(await item.url);
                 viableActivity = viableActivity.filter((x) => {
@@ -85,40 +89,114 @@ export class RecentActivityRule extends Rule {
             grouped[s] = (grouped[s] || []).concat(activity);
             return grouped;
         }, {} as Record<string, (Submission | Comment)[]>);
-        const triggeredOn = [];
+
+
+        const summaries = [];
+        let totalTriggeredOn;
         for (const triggerSet of this.thresholds) {
-            const {count: threshold = 1, subreddits = []} = triggerSet;
-            for (const sub of subreddits) {
+            let currCount = 0;
+            const presentSubs = [];
+            const {threshold = '>= 1', subreddits = []} = triggerSet;
+            for (const sub of subreddits.map(x => parseSubredditName(x))) {
                 const isub = sub.toLowerCase();
                 const {[isub]: tSub = []} = groupedActivity;
-                if (tSub.length >= threshold) {
-                    triggeredOn.push({subreddit: sub, count: tSub.length});
+                if (tSub.length > 0) {
+                    currCount += tSub.length;
+                    presentSubs.push(sub);
                 }
             }
+            const {operator, value, isPercent} = parseGenericValueOrPercentComparison(threshold);
+            let sum = {subsWithActivity: presentSubs, subreddits, count: currCount, threshold, triggered: false, testValue: currCount.toString()};
+            if (isPercent) {
+                sum.testValue = `${formatNumber((currCount / viableActivity.length) * 100)}%`;
+                if (comparisonTextOp(currCount / viableActivity.length, operator, value / 100)) {
+                    sum.triggered = true;
+                    totalTriggeredOn = sum;
+                }
+            } else if (comparisonTextOp(currCount, operator, value)) {
+                sum.triggered = true;
+                totalTriggeredOn = sum;
+            }
+            summaries.push(sum);
+            // if either trigger condition is hit end the iteration early
+            if (totalTriggeredOn !== undefined) {
+                break;
+            }
         }
-        if (triggeredOn.length > 0) {
-            const friendlyText = triggeredOn.map(x => `${x.subreddit}(${x.count})`).join(' | ');
-            const friendly = `Triggered by: ${friendlyText}`;
-            this.logger.debug(friendly);
-            return Promise.resolve([true, [this.getResult(true, {result: friendly, data: triggeredOn})]]);
+        let result = '';
+        if (totalTriggeredOn !== undefined) {
+            const resultData = this.generateResultData(totalTriggeredOn, viableActivity);
+            result = `${PASS} ${resultData.result}`;
+            this.logger.verbose(result);
+            return Promise.resolve([true, this.getResult(true, resultData)]);
+        } else if(summaries.length === 1) {
+            // can display result if its only one summary otherwise need to log to debug
+            const res = this.generateResultData(summaries[0], viableActivity);
+            result = `${FAIL} ${res.result}`;
+        } else {
+            result = `${FAIL} No criteria was met. Use 'debug' to see individual results`;
+            this.logger.debug(`\r\n ${summaries.map(x => this.generateResultData(x, viableActivity).result).join('\r\n')}`);
         }
 
-        return Promise.resolve([false, [this.getResult(false)]]);
+        this.logger.verbose(result);
+
+        return Promise.resolve([false, this.getResult(false, {result})]);
+    }
+    
+    generateResultData(summary: any, activities: (Submission | Comment)[] = []) {
+        const {
+            count,
+            testValue,
+            subreddits = [],
+            subsWithActivity = [],
+            threshold,
+            triggered
+        } = summary;
+        const relevantSubs = subsWithActivity.length === 0 ? subreddits : subsWithActivity;
+        const totalSummary = `${testValue} activities over ${relevantSubs.length} subreddits ${triggered ? 'met' : 'did not meet'} threshold of ${threshold}`;
+        return {
+            result: totalSummary,
+            data: {
+                window: typeof this.window === 'number' ? `${activities.length} Items` : activityWindowText(activities),
+                summary: totalSummary,
+                subSummary: relevantSubs.join(', '),
+                subCount: relevantSubs.length,
+                totalCount: count,
+                threshold,
+                testValue
+            }
+        };
     }
 }
 
+/**
+ * At least one count property must be present. If both are present then either can trigger the rule
+ *
+ * @minProperties 1
+ * @additionalProperties false
+ * */
 export interface SubThreshold extends SubredditCriteria {
     /**
-     * The number of activities in each subreddit from the list that will trigger this rule
-     * @default 1
-     * @minimum 1
+     * A string containing a comparison operator and a value to compare recent activities against
+     *
+     * The syntax is `(< OR > OR <= OR >=) <number>[percent sign]`
+     *
+     * * EX `> 3`  => greater than 3 activities found in the listed subreddits
+     * * EX `<= 75%` => number of Activities in the subreddits listed are equal to or less than 75% of all Activities
+     *
+     * **Note:** If you use percentage comparison here as well as `useSubmissionAsReference` then "all Activities" is only pertains to Activities that had the Link of the Submission, rather than all Activities from this window.
+     *
+     * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(%?)(.*)$
+     * @default ">= 1"
+     * @examples [">= 1"]
      * */
-    count?: number,
+    threshold?: string
 }
 
 interface RecentActivityConfig extends ActivityWindow, ReferenceSubmission {
     /**
      * If present restricts the activities that are considered for count from SubThreshold
+     * @examples ["submissions","comments"]
      * */
     lookAt?: 'comments' | 'submissions',
     /**
@@ -133,8 +211,19 @@ export interface RecentActivityRuleOptions extends RecentActivityConfig, RuleOpt
 
 /**
  * Checks a user's history for any Activity (Submission/Comment) in the subreddits specified in thresholds
+ *
+ * Available data for [Action templating](https://github.com/FoxxMD/reddit-context-bot#action-templating):
+ *
+ * ```
+ * summary    => comma-deliminated list of subreddits that hit the threshold and their count EX subredditA(1), subredditB(4),...
+ * subCount   => Total number of subreddits that hit the threshold
+ * totalCount => Total number of all activity occurrences in subreddits
+ * ```
  * */
 export interface RecentActivityRuleJSONConfig extends RecentActivityConfig, RuleJSONConfig {
+    /**
+     * @examples ["recentActivity"]
+     * */
     kind: 'recentActivity'
 }
 

src/Rule/RuleFactory.ts

@@ -1,17 +1,30 @@
 import {RecentActivityRule, RecentActivityRuleJSONConfig} from "./RecentActivityRule";
-import RepeatSubmissionRule, {RepeatSubmissionJSONConfig} from "./SubmissionRule/RepeatSubmissionRule";
+import RepeatActivityRule, {RepeatActivityJSONConfig} from "./SubmissionRule/RepeatActivityRule";
 import {Rule, RuleJSONConfig} from "./index";
 import AuthorRule, {AuthorRuleJSONConfig} from "./AuthorRule";
+import {AttributionJSONConfig, AttributionRule} from "./AttributionRule";
+import {Logger} from "winston";
+import HistoryRule, {HistoryJSONConfig} from "./HistoryRule";
 
 export function ruleFactory
-(config: RuleJSONConfig): Rule {
+(config: RuleJSONConfig, logger: Logger, subredditName: string): Rule {
+    let cfg;
     switch (config.kind) {
         case 'recentActivity':
-            return new RecentActivityRule(config as RecentActivityRuleJSONConfig);
-        case 'repeatSubmission':
-            return new RepeatSubmissionRule(config as RepeatSubmissionJSONConfig);
+            cfg = config as RecentActivityRuleJSONConfig;
+            return new RecentActivityRule({...cfg, logger, subredditName});
+        case 'repeatActivity':
+            cfg = config as RepeatActivityJSONConfig;
+            return new RepeatActivityRule({...cfg, logger, subredditName});
         case 'author':
-            return new AuthorRule(config as AuthorRuleJSONConfig);
+            cfg = config as AuthorRuleJSONConfig;
+            return new AuthorRule({...cfg, logger, subredditName});
+        case 'attribution':
+            cfg = config as AttributionJSONConfig;
+            return new AttributionRule({...cfg, logger, subredditName});
+        case 'history':
+            cfg = config as HistoryJSONConfig;
+            return new HistoryRule({...cfg, logger, subredditName});
         default:
             throw new Error('rule "kind" was not recognized.');
     }

src/Rule/RuleSet.ts

@@ -1,39 +1,30 @@
-import {IRule, Triggerable, Rule, RuleJSONConfig, RuleResult} from "./index";
+import {IRule, Triggerable, Rule, RuleJSONConfig, RuleResult, RuleSetResult} from "./index";
 import {Comment, Submission} from "snoowrap";
 import {ruleFactory} from "./RuleFactory";
-import {RecentActivityRuleJSONConfig} from "./RecentActivityRule";
-import {RepeatSubmissionJSONConfig} from "./SubmissionRule/RepeatSubmissionRule";
-import {createLabelledLogger, determineNewResults, findResultByPremise, loggerMetaShuffle} from "../util";
+import {createAjvFactory, mergeArr} from "../util";
 import {Logger} from "winston";
-import {AuthorRuleJSONConfig} from "./AuthorRule";
 import {JoinCondition, JoinOperands} from "../Common/interfaces";
 import * as RuleSchema from '../Schema/Rule.json';
 import Ajv from 'ajv';
+import {RuleJson, RuleObjectJson} from "../Common/types";
 
-const ajv = new Ajv();
-
-export class RuleSet implements IRuleSet, Triggerable {
+export class RuleSet implements IRuleSet {
     rules: Rule[] = [];
     condition: JoinOperands;
     logger: Logger;
 
     constructor(options: RuleSetOptions) {
         const {logger, condition = 'AND', rules = []} = options;
-        if (logger !== undefined) {
-            this.logger = logger.child(loggerMetaShuffle(logger, 'Rule Set'));
-        } else {
-            this.logger = createLabelledLogger('Rule Set');
-        }
+        this.logger = logger.child({leaf: 'Rule Set'}, mergeArr);
         this.condition = condition;
+        const ajv = createAjvFactory(this.logger);
         for (const r of rules) {
             if (r instanceof Rule) {
                 this.rules.push(r);
             } else {
                 const valid = ajv.validate(RuleSchema, r);
                 if (valid) {
-                    // @ts-ignore
-                    r.logger = this.logger;
-                    this.rules.push(ruleFactory(r as RuleJSONConfig));
+                    this.rules.push(ruleFactory(r as RuleJSONConfig, logger, options.subredditName));
                 } else {
                     this.logger.warn('Could not build rule because of JSON errors', {}, {errors: ajv.errors, obj: r});
                 }
@@ -41,12 +32,12 @@ export class RuleSet implements IRuleSet, Triggerable {
         }
     }
 
-    async run(item: Comment | Submission, existingResults: RuleResult[] = []): Promise<[boolean, RuleResult[]]> {
+    async run(item: Comment | Submission, existingResults: RuleResult[] = []): Promise<[boolean, RuleSetResult]> {
         let results: RuleResult[] = [];
         let runOne = false;
         for (const r of this.rules) {
             const combinedResults = [...existingResults, ...results];
-            const [passed, [result]] = await r.run(item, combinedResults);
+            const [passed, result] = await r.run(item, combinedResults);
             //results = results.concat(determineNewResults(combinedResults, result));
             results.push(result);
             // skip rule if author check failed
@@ -56,17 +47,30 @@ export class RuleSet implements IRuleSet, Triggerable {
             runOne = true;
             if (passed) {
                 if (this.condition === 'OR') {
-                    return [true, results];
+                    return [true, this.generateResultSet(true, results)];
                 }
             } else if (this.condition === 'AND') {
-                return [false, results];
+                return [false, this.generateResultSet(false, results)];
             }
         }
         // if no rules were run it's the same as if nothing was triggered
         if (!runOne) {
-            return [false, results];
+            return [false, this.generateResultSet(false, results)];
         }
-        return [true, results];
+        if(this.condition === 'OR') {
+            // if OR and did not return already then none passed
+            return [false, this.generateResultSet(false, results)];
+        }
+        // otherwise AND and did not return already so all passed
+        return [true, this.generateResultSet(true, results)];
+    }
+
+    generateResultSet(triggered: boolean, results: RuleResult[]): RuleSetResult {
+        return {
+            results,
+            triggered,
+            condition: this.condition
+        };
     }
 }
 
@@ -79,16 +83,25 @@ export interface IRuleSet extends JoinCondition {
 
 export interface RuleSetOptions extends IRuleSet {
     rules: Array<IRule | RuleJSONConfig>,
-    logger?: Logger
+    logger: Logger
+    subredditName: string
 }
 
 /**
- * A RuleSet is a "nested" set of Rules that can be used to create more complex AND/OR behavior. Think of the outcome of a RuleSet as the result of all of it's Rules (based on condition)
- * @see {isRuleSetConfig} ts-auto-guard:type-guard
+ * A RuleSet is a "nested" set of `Rule` objects that can be used to create more complex AND/OR behavior. Think of the outcome of a `RuleSet` as the result of all of its run `Rule` objects (based on `condition`)
  * */
-export interface RuleSetJSONConfig extends IRuleSet {
+export interface RuleSetJson extends JoinCondition {
     /**
+     * Can be `Rule` or the `name` of any **named** `Rule` in your subreddit's configuration
      * @minItems 1
      * */
-    rules: Array<RecentActivityRuleJSONConfig | RepeatSubmissionJSONConfig | AuthorRuleJSONConfig>
+    rules: Array<RuleJson>
+}
+
+export interface RuleSetObjectJson extends RuleSetJson {
+    rules: Array<RuleObjectJson>
+}
+
+export const isRuleSetJSON = (obj: object): obj is RuleSetJson => {
+    return (obj as RuleSetJson).rules !== undefined;
 }

src/Rule/SubmissionRule/RepeatActivityRule.ts

@@ -0,0 +1,304 @@
+import {SubmissionRule, SubmissionRuleJSONConfig} from "./index";
+import {RuleOptions, RuleResult} from "../index";
+import {Comment} from "snoowrap";
+import {
+    activityWindowText,
+    comparisonTextOp, FAIL,
+    parseGenericValueComparison, parseSubredditName,
+    parseUsableLinkIdentifier as linkParser, PASS
+} from "../../util";
+import {ActivityWindow, ActivityWindowType, ReferenceSubmission} from "../../Common/interfaces";
+import Submission from "snoowrap/dist/objects/Submission";
+import dayjs from "dayjs";
+
+const parseUsableLinkIdentifier = linkParser();
+
+interface RepeatActivityData {
+    identifier: string,
+    sets: (Submission | Comment)[]
+}
+
+interface RepeatActivityReducer {
+    openSets: RepeatActivityData[]
+    allSets: RepeatActivityData[]
+}
+
+const getActivityIdentifier = (activity: (Submission | Comment), length = 200) => {
+    let identifier: string;
+    if (activity instanceof Submission) {
+        if (activity.is_self) {
+            identifier = activity.selftext.slice(0, length);
+        } else {
+            identifier = parseUsableLinkIdentifier(activity.url) as string;
+        }
+    } else {
+        identifier = activity.body.slice(0, length);
+    }
+    return identifier;
+}
+
+export class RepeatActivityRule extends SubmissionRule {
+    threshold: string;
+    window: ActivityWindowType;
+    gapAllowance?: number;
+    useSubmissionAsReference: boolean;
+    lookAt: 'submissions' | 'all';
+    include: string[];
+    exclude: string[];
+    keepRemoved: boolean;
+
+    constructor(options: RepeatActivityOptions) {
+        super(options);
+        const {
+            threshold = '> 5',
+            window = 100,
+            gapAllowance,
+            useSubmissionAsReference = true,
+            lookAt = 'all',
+            include = [],
+            exclude = [],
+            keepRemoved = false,
+        } = options;
+        this.keepRemoved = keepRemoved;
+        this.threshold = threshold;
+        this.window = window;
+        this.gapAllowance = gapAllowance;
+        this.useSubmissionAsReference = useSubmissionAsReference;
+        this.include = include.map(x => parseSubredditName(x).toLowerCase());
+        this.exclude = exclude.map(x => parseSubredditName(x).toLowerCase());
+        this.lookAt = lookAt;
+    }
+
+    getKind(): string {
+        return 'Repeat';
+    }
+
+    getSpecificPremise(): object {
+        return {
+            threshold: this.threshold,
+            window: this.window,
+            gapAllowance: this.gapAllowance,
+            useSubmissionAsReference: this.useSubmissionAsReference,
+            include: this.include,
+            exclude: this.exclude,
+        }
+    }
+
+    async process(item: Submission): Promise<[boolean, RuleResult]> {
+        const referenceUrl = await item.url;
+        if (referenceUrl === undefined && this.useSubmissionAsReference) {
+            this.logger.warn(`Rule not triggered because useSubmissionAsReference=true but submission is not a link`);
+            return Promise.resolve([false, this.getResult(false)]);
+        }
+
+        let filterFunc = (x: any) => true;
+        if(this.include.length > 0) {
+            filterFunc = (x: Submission|Comment) => this.include.includes(x.subreddit.display_name.toLowerCase());
+        } else if(this.exclude.length > 0) {
+            filterFunc = (x: Submission|Comment) => !this.exclude.includes(x.subreddit.display_name.toLowerCase());
+        }
+
+        let activities: (Submission | Comment)[] = [];
+        switch (this.lookAt) {
+            case 'submissions':
+                activities = await this.resources.getAuthorSubmissions(item.author, {window: this.window, keepRemoved: this.keepRemoved});
+                break;
+            default:
+                activities = await this.resources.getAuthorActivities(item.author, {window: this.window, keepRemoved: this.keepRemoved});
+                break;
+        }
+
+        const condensedActivities = activities.reduce((acc: RepeatActivityReducer, activity: (Submission | Comment), index: number) => {
+            const {openSets = [], allSets = []} = acc;
+
+            let identifier = getActivityIdentifier(activity);
+            const validSub = filterFunc(activity);
+
+            let updatedAllSets = [...allSets];
+            let updatedOpenSets: RepeatActivityData[] = [];
+
+            let currIdentifierInOpen = false;
+            const bufferedActivities = this.gapAllowance === undefined || this.gapAllowance === 0 ? [] : activities.slice(Math.max(0, index - this.gapAllowance), Math.max(0, index));
+            for (const o of openSets) {
+                if (o.identifier === identifier && validSub) {
+                    updatedOpenSets.push({...o, sets: [...o.sets, activity]});
+                    currIdentifierInOpen = true;
+                } else if (bufferedActivities.some(x => getActivityIdentifier(x) === identifier) && validSub) {
+                    updatedOpenSets.push(o);
+                } else {
+                    updatedAllSets.push(o);
+                }
+            }
+
+            if (!currIdentifierInOpen) {
+                updatedOpenSets.push({identifier, sets: [activity]})
+            }
+
+            return {openSets: updatedOpenSets, allSets: updatedAllSets};
+
+        }, {openSets: [], allSets: []});
+
+        const allRepeatSets = [...condensedActivities.allSets, ...condensedActivities.openSets];
+
+        const identifierGroupedActivities = allRepeatSets.reduce((acc, repeatActivityData) => {
+            let existingSets = [];
+            if (acc.has(repeatActivityData.identifier)) {
+                existingSets = acc.get(repeatActivityData.identifier);
+            }
+            acc.set(repeatActivityData.identifier, [...existingSets, repeatActivityData.sets].sort((a, b) => b.length < a.length ? 1 : -1));
+            return acc;
+        }, new Map());
+
+        let applicableGroupedActivities = identifierGroupedActivities;
+        if (this.useSubmissionAsReference) {
+            applicableGroupedActivities = new Map();
+            const referenceSubmissions = identifierGroupedActivities.get(getActivityIdentifier(item));
+            applicableGroupedActivities.set(getActivityIdentifier(item), referenceSubmissions || [])
+        }
+
+        const {operator, value: thresholdValue} = parseGenericValueComparison(this.threshold);
+        const greaterThan = operator.includes('>');
+        let allLessThan = true;
+
+        const identifiersSummary: SummaryData[] = [];
+        for (let [key, value] of applicableGroupedActivities) {
+            const summaryData: SummaryData = {
+                identifier: key,
+                totalSets: value.length,
+                totalTriggeringSets: 0,
+                largestTrigger: 0,
+                sets: [],
+                setsMarkdown: [],
+                triggeringSets: [],
+                triggeringSetsMarkdown: [],
+            };
+            for (let set of value) {
+                const test = comparisonTextOp(set.length, operator, thresholdValue);
+                const md = set.map((x: (Comment | Submission)) => `[${x instanceof Submission ? x.title : getActivityIdentifier(x, 50)}](https://reddit.com${x.permalink}) in ${x.subreddit_name_prefixed} on ${dayjs(x.created_utc * 1000).utc().format()}`);
+
+                summaryData.sets.push(set);
+                summaryData.largestTrigger = Math.max(summaryData.largestTrigger, set.length);
+                summaryData.setsMarkdown.push(md);
+                if (test) {
+                    summaryData.triggeringSets.push(set);
+                    summaryData.totalTriggeringSets++;
+                    summaryData.triggeringSetsMarkdown.push(md);
+                    // }
+                } else if (!greaterThan) {
+                    allLessThan = false;
+                }
+            }
+            identifiersSummary.push(summaryData);
+        }
+
+        const criteriaMet = identifiersSummary.filter(x => x.totalTriggeringSets > 0).length > 0 && (greaterThan || (!greaterThan && allLessThan));
+
+        const largestRepeat = identifiersSummary.reduce((acc, summ) => Math.max(summ.largestTrigger, acc), 0);
+        let result: string;
+        if (criteriaMet || greaterThan) {
+            result = `${criteriaMet ? PASS : FAIL} ${identifiersSummary.filter(x => x.totalTriggeringSets > 0).length} of ${identifiersSummary.length} unique items repeated ${this.threshold} times, largest repeat: ${largestRepeat}`;
+        } else {
+            result = `${FAIL} Not all of ${identifiersSummary.length} unique items repeated ${this.threshold} times, largest repeat: ${largestRepeat}`
+        }
+
+        this.logger.verbose(result);
+
+        if (criteriaMet) {
+            const triggeringSummaries = identifiersSummary.filter(x => x.totalTriggeringSets > 0);
+            return Promise.resolve([true, this.getResult(true, {
+                result,
+                data: {
+                    window: typeof this.window === 'number' ? `${activities.length} Items` : activityWindowText(activities),
+                    totalTriggeringSets: triggeringSummaries.length,
+                    largestRepeat,
+                    threshold: this.threshold,
+                    gapAllowance: this.gapAllowance,
+                    url: referenceUrl,
+                    triggeringSummaries,
+                }
+            })])
+        }
+
+        return Promise.resolve([false, this.getResult(false, {result})]);
+    }
+}
+
+interface SummaryData {
+    identifier: string,
+    totalSets: number,
+    totalTriggeringSets: number,
+    largestTrigger: number,
+    sets: (Comment | Submission)[],
+    setsMarkdown: string[],
+    triggeringSets: (Comment | Submission)[],
+    triggeringSetsMarkdown: string[]
+}
+
+interface RepeatActivityConfig extends ActivityWindow, ReferenceSubmission {
+    /**
+     * The number of repeat submissions that will trigger the rule
+     * @default ">= 5"
+     * */
+    threshold?: string,
+    /**
+     * The number of allowed non-identical Submissions between identical Submissions that can be ignored when checking against the threshold value
+     * */
+    gapAllowance?: number,
+    /**
+     * Only include Submissions from this list of Subreddits (by name, case-insensitive)
+     *
+     * EX `["mealtimevideos","askscience"]`
+     * @examples ["mealtimevideos","askscience"]
+     * @minItems 1
+     * */
+    include?: string[],
+    /**
+     * Do not include Submissions from this list of Subreddits (by name, case-insensitive)
+     *
+     * EX `["mealtimevideos","askscience"]`
+     * @examples ["mealtimevideos","askscience"]
+     * @minItems 1
+     * */
+    exclude?: string[],
+
+    /**
+     * If present determines which activities to consider for gapAllowance.
+     *
+     * * If `submissions` then only the author's submission history is considered IE gapAllowance = 2  ===> can have gap of two submissions between repeats
+     * * If `all` then the author's entire history (submissions/comments) is considered IE gapAllowance = 2  ===> can only have gap of two activities (submissions or comments) between repeats
+     *
+     *  @default all
+     * */
+    lookAt?: 'submissions' | 'all',
+    /**
+     * Count submissions/comments that have previously been removed.
+     *
+     * By default all `Submissions/Commments` that are in a `removed` state will be filtered from `window` (only applies to subreddits you mod).
+     *
+     * Setting to `true` could be useful if you also want to also detected removed repeat posts by a user like for example if automoderator removes multiple, consecutive submissions for not following title format correctly.
+     *
+     * @default false
+     * */
+    keepRemoved?: boolean
+}
+
+export interface RepeatActivityOptions extends RepeatActivityConfig, RuleOptions {
+
+}
+
+/**
+ * Checks a user's history for Submissions with identical content
+ *
+ * Available data for [Action templating](https://github.com/FoxxMD/reddit-context-bot#action-templating):
+ *
+ * ```
+ * count      => Total number of repeat Submissions
+ * threshold  => The threshold you configured for this Rule to trigger
+ * url        => Url of the submission that triggered the rule
+ * ```
+ * */
+export interface RepeatActivityJSONConfig extends RepeatActivityConfig, SubmissionRuleJSONConfig {
+    kind: 'repeatActivity'
+}
+
+export default RepeatActivityRule;

src/Rule/SubmissionRule/RepeatSubmissionRule.ts

@@ -1,162 +0,0 @@
-import {SubmissionRule, SubmissionRuleJSONConfig} from "./index";
-import {Rule, RuleOptions, RulePremise, RuleResult} from "../index";
-import {Submission} from "snoowrap";
-import {getAuthorSubmissions} from "../../Utils/SnoowrapUtils";
-import {groupBy, parseUsableLinkIdentifier as linkParser} from "../../util";
-import {ActivityWindow, ActivityWindowType, ReferenceSubmission} from "../../Common/interfaces";
-
-const groupByUrl = groupBy(['urlIdentifier']);
-const parseUsableLinkIdentifier = linkParser()
-
-export class RepeatSubmissionRule extends SubmissionRule {
-    threshold: number;
-    window: ActivityWindowType;
-    gapAllowance?: number;
-    useSubmissionAsReference: boolean;
-    include: string[];
-    exclude: string[];
-
-    constructor(options: RepeatSubmissionOptions) {
-        super(options);
-        const {
-            threshold = 5,
-            window = 15,
-            gapAllowance,
-            useSubmissionAsReference = true,
-            include = [],
-            exclude = []
-        } = options;
-        this.threshold = threshold;
-        this.window = window;
-        this.gapAllowance = gapAllowance;
-        this.useSubmissionAsReference = useSubmissionAsReference;
-        this.include = include;
-        this.exclude = exclude;
-    }
-
-    getKind(): string {
-        return 'Repeat Submission';
-    }
-
-    getSpecificPremise(): object {
-        return {
-            threshold: this.threshold,
-            window: this.window,
-            gapAllowance: this.gapAllowance,
-            useSubmissionAsReference: this.useSubmissionAsReference,
-            include: this.include,
-            exclude: this.exclude,
-        }
-    }
-
-    async process(item: Submission): Promise<[boolean, RuleResult[]]> {
-        const referenceUrl = await item.url;
-        if (referenceUrl === undefined && this.useSubmissionAsReference) {
-            throw new Error(`Cannot run Rule ${this.name} because submission is not a link`);
-        }
-        const submissions = await getAuthorSubmissions(item.author, {window: this.window});
-
-        // we need to check in order
-        if (this.gapAllowance !== undefined) {
-            let consecutivePosts = referenceUrl !== undefined ? 1 : 0;
-            let gap = 0;
-            let lastUrl = parseUsableLinkIdentifier(referenceUrl);
-            // start with second post since first is the one we triggered on (prob)
-            for (const sub of submissions.slice(1)) {
-                if (sub.url !== undefined) {
-                    const regUrl = parseUsableLinkIdentifier(sub.url);
-                    if (lastUrl === undefined || lastUrl === regUrl) {
-                        consecutivePosts++;
-                        gap = 0;
-                    } else {
-                        gap++;
-                        if (gap > this.gapAllowance) {
-                            gap = 0;
-                            consecutivePosts = 1;
-                        }
-                    }
-                    lastUrl = regUrl;
-                } else {
-                    gap++;
-                    if (gap > this.gapAllowance) {
-                        gap = 0;
-                        consecutivePosts = 0;
-                    }
-                }
-                if (consecutivePosts >= this.threshold) {
-                    const result = `Threshold of ${this.threshold} repeats triggered for submission with url ${sub.url}`;
-                    this.logger.debug(result);
-                    return Promise.resolve([true, [this.getResult(true, {result})]]);
-                }
-            }
-            return Promise.resolve([false, [this.getResult(false)]]);
-        }
-
-        // otherwise we can just group all occurrences together
-        const groupedPosts = groupByUrl(submissions.map(x => ({
-            ...x,
-            urlIdentifier: parseUsableLinkIdentifier(x.url)
-        })));
-        let groupsToCheck = [];
-        if (this.useSubmissionAsReference) {
-            const identifier = parseUsableLinkIdentifier(referenceUrl);
-            const {[identifier as string]: refGroup = []} = groupedPosts;
-            groupsToCheck.push(refGroup);
-        } else {
-            groupsToCheck = Object.values(groupedPosts)
-        }
-        for (const group of groupsToCheck) {
-            if (group.length >= this.threshold) {
-                // @ts-ignore
-                const result = `Threshold of ${this.threshold} repeats triggered for submission with url ${group[0].url}`;
-                this.logger.debug(result);
-                return Promise.resolve([true, [this.getResult(true, {result})]]);
-            }
-        }
-        return Promise.resolve([false, [this.getResult(false)]]);
-    }
-}
-
-interface RepeatSubmissionConfig extends ActivityWindow, ReferenceSubmission {
-    /**
-     * The number of repeat submissions that will trigger the rule
-     * @default 5
-     * */
-    threshold?: number,
-    /**
-     * The number of allowed non-identical Submissions between identical Submissions that can be ignored when checking against the threshold value
-     * */
-    gapAllowance?: number,
-    /**
-     * Only include Submissions from this list of Subreddits.
-     *
-     * A list of subreddits (case-insensitive) to look for. Do not include "r/" prefix.
-     *
-     * EX to match against /r/mealtimevideos and /r/askscience use ["mealtimevideos","askscience"]
-     * @examples ["mealtimevideos","askscience"]
-     * @minItems 1
-     * */
-    include?: string[],
-    /**
-     * Do not include Submissions from this list of Subreddits.
-     *
-     * A list of subreddits (case-insensitive) to look for. Do not include "r/" prefix.
-     *
-     * EX to match against /r/mealtimevideos and /r/askscience use ["mealtimevideos","askscience"]
-     * @examples ["mealtimevideos","askscience"]
-     * @minItems 1
-     * */
-    exclude?: string[],
-}
-
-export interface RepeatSubmissionOptions extends RepeatSubmissionConfig, RuleOptions {
-
-}
-/**
- * Checks a user's history for Submissions with identical content
- * */
-export interface RepeatSubmissionJSONConfig extends RepeatSubmissionConfig, SubmissionRuleJSONConfig {
-    kind: 'repeatSubmission'
-}
-
-export default RepeatSubmissionRule;

src/Rule/index.ts

@@ -1,14 +1,18 @@
-import {Comment, RedditUser} from "snoowrap";
+import {Comment} from "snoowrap";
 import Submission from "snoowrap/dist/objects/Submission";
 import {Logger} from "winston";
-import {createLabelledLogger, findResultByPremise, loggerMetaShuffle, mergeArr} from "../util";
-import {testAuthorCriteria} from "../Utils/SnoowrapUtils";
+import {findResultByPremise, mergeArr} from "../util";
+import ResourceManager, {SubredditResources} from "../Subreddit/SubredditResources";
+import {ChecksActivityState, TypedActivityStates} from "../Common/interfaces";
+import {isItem} from "../Utils/SnoowrapUtils";
+import Author, {AuthorOptions} from "../Author/Author";
 
 export interface RuleOptions {
     name?: string;
-    authors?: AuthorOptions;
-    logger?: Logger
-    loggerPrefix?: string
+    authorIs?: AuthorOptions;
+    itemIs?: TypedActivityStates;
+    logger: Logger
+    subredditName: string;
 }
 
 export interface RulePremise {
@@ -23,77 +27,95 @@ interface ResultContext {
 
 export interface RuleResult extends ResultContext {
     premise: RulePremise
-    name?: string
+    name: string
     triggered: (boolean | null)
 }
 
+export interface RuleSetResult {
+    results: RuleResult[],
+    condition: 'OR' | 'AND',
+    triggered: boolean
+}
+
+export const isRuleSetResult = (obj: any): obj is RuleSetResult => {
+    return typeof obj === 'object' && Array.isArray(obj.results) && obj.condition !== undefined && obj.triggered !== undefined;
+}
+
 export interface Triggerable {
-    run(item: Comment | Submission, existingResults: RuleResult[]): Promise<[(boolean | null), RuleResult[]]>;
+    run(item: Comment | Submission, existingResults: RuleResult[]): Promise<[(boolean | null), RuleResult?]>;
 }
 
 export abstract class Rule implements IRule, Triggerable {
-    name?: string;
+    name: string;
     logger: Logger
-    authors: AuthorOptions;
+    authorIs: AuthorOptions;
+    itemIs: TypedActivityStates;
+    resources: SubredditResources;
 
     constructor(options: RuleOptions) {
         const {
-            name,
-            loggerPrefix = '',
+            name = this.getKind(),
             logger,
-            authors: {
+            authorIs: {
                 include = [],
                 exclude = [],
             } = {},
+            itemIs = [],
+            subredditName,
         } = options;
         this.name = name;
+        this.resources = ResourceManager.get(subredditName) as SubredditResources;
 
-        this.authors = {
+        this.authorIs = {
             exclude: exclude.map(x => new Author(x)),
             include: include.map(x => new Author(x)),
         }
 
-        const ruleUniqueName = this.name === undefined ? this.getKind() : `${this.getKind()} - ${this.name}`;
-        if (logger === undefined) {
-            const prefix = `${loggerPrefix}|${ruleUniqueName}`;
-            this.logger = createLabelledLogger(prefix, prefix);
-        } else {
-            this.logger = logger.child(loggerMetaShuffle(logger, undefined, [ruleUniqueName], {truncateLength: 100}));
-        }
+        this.itemIs = itemIs;
+
+        this.logger = logger.child({labels: [`Rule ${this.getRuleUniqueName()}`]}, mergeArr);
     }
 
-    async run(item: Comment | Submission, existingResults: RuleResult[] = []): Promise<[(boolean | null), RuleResult[]]> {
-        this.logger = this.logger.child(loggerMetaShuffle(this.logger, `${item instanceof Submission ? 'SUB' : 'COMM'} ${item.id}`), mergeArr);
-        this.logger.debug('Starting rule run');
+    async run(item: Comment | Submission, existingResults: RuleResult[] = []): Promise<[(boolean | null), RuleResult]> {
         const existingResult = findResultByPremise(this.getPremise(), existingResults);
         if (existingResult) {
-            return Promise.resolve([existingResult.triggered, [existingResult]]);
+            this.logger.debug(`Returning existing result of ${existingResult.triggered ? '✔️' : '❌'}`);
+            return Promise.resolve([existingResult.triggered, {...existingResult, name: this.name}]);
         }
-        if (this.authors.include !== undefined && this.authors.include.length > 0) {
-            for (const auth of this.authors.include) {
-                if (await testAuthorCriteria(item, auth)) {
+        const [itemPass, crit] = isItem(item, this.itemIs, this.logger);
+        if(!itemPass) {
+            this.logger.verbose(`(Skipped) Item did not pass 'itemIs' test`);
+            return Promise.resolve([null, this.getResult(null, {result: `Item did not pass 'itemIs' test`})]);
+        }
+        if (this.authorIs.include !== undefined && this.authorIs.include.length > 0) {
+            for (const auth of this.authorIs.include) {
+                if (await this.resources.testAuthorCriteria(item, auth)) {
                     return this.process(item);
                 }
             }
-            this.logger.debug('Inclusive author criteria not matched, rule running skipped');
-            return Promise.resolve([false, [this.getResult(null, {result: 'Inclusive author criteria not matched, rule running skipped'})]]);
+            this.logger.verbose('(Skipped) Inclusive author criteria not matched');
+            return Promise.resolve([null, this.getResult(null, {result: 'Inclusive author criteria not matched'})]);
         }
-        if (this.authors.exclude !== undefined && this.authors.exclude.length > 0) {
-            for (const auth of this.authors.exclude) {
-                if (await testAuthorCriteria(item, auth, false)) {
+        if (this.authorIs.exclude !== undefined && this.authorIs.exclude.length > 0) {
+            for (const auth of this.authorIs.exclude) {
+                if (await this.resources.testAuthorCriteria(item, auth, false)) {
                     return this.process(item);
                 }
             }
-            this.logger.debug('Exclusive author criteria not matched, rule running skipped');
-            return Promise.resolve([false, [this.getResult(null, {result: 'Exclusive author criteria not matched, rule running skipped'})]]);
+            this.logger.verbose('(Skipped) Exclusive author criteria not matched');
+            return Promise.resolve([null, this.getResult(null, {result: 'Exclusive author criteria not matched'})]);
         }
         return this.process(item);
     }
 
-    protected abstract process(item: Comment | Submission): Promise<[boolean, RuleResult[]]>;
+    protected abstract process(item: Comment | Submission): Promise<[boolean, RuleResult]>;
 
     abstract getKind(): string;
 
+    getRuleUniqueName() {
+        return this.name === undefined ? this.getKind() : `${this.getKind()} - ${this.name}`;
+    }
+
     protected abstract getSpecificPremise(): object;
 
     getPremise(): RulePremise {
@@ -101,7 +123,8 @@ export abstract class Rule implements IRule, Triggerable {
         return {
             kind: this.getKind(),
             config: {
-                authors: this.authors,
+                authorIs: this.authorIs,
+                itemIs: this.itemIs,
                 ...config,
             },
         };
@@ -117,83 +140,100 @@ export abstract class Rule implements IRule, Triggerable {
     }
 }
 
-export class Author implements AuthorCriteria {
-    name?: string[];
-    flairCssClass?: string[];
-    flairText?: string[];
-    isMod?: boolean;
-
-    constructor(options: AuthorCriteria) {
-        this.name = options.name;
-        this.flairCssClass = options.flairCssClass;
-        this.flairText = options.flairText;
-        this.isMod = options.isMod;
-    }
-}
-
-/**
- * If present then these Author criteria are checked before running the rule. If criteria fails then the rule is skipped.
- * @minProperties 1
- * @additionalProperties false
- * */
-export interface AuthorOptions {
+export interface UserNoteCriteria {
     /**
-     * Will "pass" if any set of AuthorCriteria passes
+     * User Note type key to search for
+     * @examples ["spamwarn"]
      * */
-    include?: AuthorCriteria[];
+    type: string;
     /**
-     * Only runs if include is not present. Will "pass" if any of set of the AuthorCriteria does not pass
-     * */
-    exclude?: AuthorCriteria[];
-}
-
-/**
- * Criteria with which to test against the author of an Activity. The outcome of the test is based on:
- *
- * 1. All present properties passing and
- * 2. If a property is a list then any value from the list matching
- *
- * @minProperties 1
- * @additionalProperties false
- * */
-export interface AuthorCriteria {
-    /**
-     * A list of reddit usernames (case-insensitive) to match against. Do not include the "u/" prefix
+     * Number of occurrences of this type. Ignored if `search` is `current`
      *
-     *  EX to match against /u/FoxxMD and /u/AnotherUser use ["FoxxMD","AnotherUser"]
-     * @examples ["FoxxMD","AnotherUser"]
+     * A string containing a comparison operator and/or a value to compare number of occurrences against
+     *
+     * The syntax is `(< OR > OR <= OR >=) <number>[percent sign] [ascending|descending]`
+     *
+     * @examples [">= 1"]
+     * @default ">= 1"
+     * @pattern ^\s*(?<opStr>>|>=|<|<=)\s*(?<value>\d+)\s*(?<percent>%?)\s*(?<extra>asc.*|desc.*)*$
      * */
-    name?: string[],
+    count?: string;
+
     /**
-     * A list of (user) flair css class values from the subreddit to match against
+     * How to test the notes for this Author:
+     *
+     * ### current
+     *
+     * Only the most recent note is checked for `type`
+     *
+     * ### total
+     *
+     * The `count` comparison of `type` must be found within all notes
+     *
+     * * EX `count: > 3`   => Must have more than 3 notes of `type`, total
+     * * EX `count: <= 25%` => Must have 25% or less of notes of `type`, total
+     *
+     * ### consecutive
+     *
+     * The `count` **number** of `type` notes must be found in a row.
+     *
+     * You may also specify the time-based order in which to search the notes by specifying `ascending (asc)` or `descending (desc)` in the `count` value. Default is `descending`
+     *
+     * * EX `count: >= 3` => Must have 3 or more notes of `type` consecutively, in descending order
+     * * EX `count: < 2`  => Must have less than 2 notes of `type` consecutively, in descending order
+     * * EX `count: > 4 asc` => Must have greater than 4 notes of `type` consecutively, in ascending order
+     *
+     * @examples ["current"]
+     * @default current
      * */
-    flairCssClass?: string[],
-    /**
-     * A list of (user) flair text values from the subreddit to match against
-     * */
-    flairText?: string[],
-    /**
-     * Is the author a moderator?
-     * */
-    isMod?: boolean,
+    search?: 'current' | 'consecutive' | 'total'
 }
 
-export interface IRule {
+/**
+ * A duration and how to compare it against a value
+ *
+ * The syntax is `(< OR > OR <= OR >=) <number> <unit>` EX `> 100 days`, `<= 2 months`
+ *
+ * * EX `> 100 days` => Passes if the date being compared is before 100 days ago
+ * * EX `<= 2 months` => Passes if the date being compared is after or equal to 2 months
+ *
+ * Unit must be one of [DayJS Duration units](https://day.js.org/docs/en/durations/creating)
+ *
+ * [See] https://regexr.com/609n8 for example
+ *
+ * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(days|weeks|months|years|hours|minutes|seconds|milliseconds)\s*$
+ * */
+export type DurationComparor = string;
+
+export interface IRule extends ChecksActivityState {
     /**
-     * A friendly, descriptive name for this rule. Highly recommended to make it easier to track logs EX "repeatCrosspostRule"
+     * An optional, but highly recommended, friendly name for this rule. If not present will default to `kind`.
+     *
+     * Can only contain letters, numbers, underscore, spaces, and dashes
+     *
+     * name is used to reference Rule result data during Action content templating. See CommentAction or ReportAction for more details.
+     * @pattern ^[a-zA-Z]([\w -]*[\w])?$
+     * @examples ["myNewRule"]
      * */
     name?: string
     /**
      * If present then these Author criteria are checked before running the rule. If criteria fails then the rule is skipped.
      * */
-    authors?: AuthorOptions
+    authorIs?: AuthorOptions
+    /**
+     * A list of criteria to test the state of the `Activity` against before running the Rule.
+     *
+     * If any set of criteria passes the Rule will be run. If the criteria fails then the Rule is skipped.
+     *
+     * */
+    itemIs?: TypedActivityStates
 }
 
-/** @see {isRuleConfig} ts-auto-guard:type-guard */
 export interface RuleJSONConfig extends IRule {
     /**
      * The kind of rule to run
+     * @examples ["recentActivity", "repeatActivity", "author", "attribution", "history"]
      */
-    kind: 'recentActivity' | 'repeatSubmission' | 'author'
+    kind: 'recentActivity' | 'repeatActivity' | 'author' | 'attribution' | 'history'
 }
 

src/Schema/Action.json

@@ -1,26 +1,323 @@
 {
     "$schema": "http://json-schema.org/draft-07/schema#",
+    "definitions": {
+        "AuthorCriteria": {
+            "additionalProperties": false,
+            "description": "Criteria with which to test against the author of an Activity. The outcome of the test is based on:\n\n1. All present properties passing and\n2. If a property is a list then any value from the list matching",
+            "examples": [
+                {
+                    "flairText": [
+                        "Contributor",
+                        "Veteran"
+                    ],
+                    "isMod": true,
+                    "name": [
+                        "FoxxMD",
+                        "AnotherUser"
+                    ]
+                }
+            ],
+            "minProperties": 1,
+            "properties": {
+                "age": {
+                    "description": "Test the age of the Author's account (when it was created) against this comparison\n\nThe syntax is `(< OR > OR <= OR >=) <number> <unit>`\n\n* EX `> 100 days` => Passes if Author's account is older than 100 days\n* EX `<= 2 months` => Passes if Author's account is younger than or equal to 2 months\n\nUnit must be one of [DayJS Duration units](https://day.js.org/docs/en/durations/creating)\n\n[See] https://regexr.com/609n8 for example",
+                    "pattern": "^\\s*(?<opStr>>|>=|<|<=)\\s*(?<time>\\d+)\\s*(?<unit>days?|weeks?|months?|years?|hours?|minutes?|seconds?|milliseconds?)\\s*$",
+                    "type": "string"
+                },
+                "commentKarma": {
+                    "description": "A string containing a comparison operator and a value to compare karma against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 100`  => greater than 100 comment karma\n* EX `<= 75%` => comment karma is less than or equal to 75% of **all karma**",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "flairCssClass": {
+                    "description": "A list of (user) flair css class values from the subreddit to match against",
+                    "examples": [
+                        "red"
+                    ],
+                    "items": {
+                        "type": "string"
+                    },
+                    "type": "array"
+                },
+                "flairText": {
+                    "description": "A list of (user) flair text values from the subreddit to match against",
+                    "examples": [
+                        "Approved"
+                    ],
+                    "items": {
+                        "type": "string"
+                    },
+                    "type": "array"
+                },
+                "isMod": {
+                    "description": "Is the author a moderator?",
+                    "type": "boolean"
+                },
+                "linkKarma": {
+                    "description": "A string containing a comparison operator and a value to compare link karma against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 100`  => greater than 100 link karma\n* EX `<= 75%` => link karma is less than or equal to 75% of **all karma**",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "name": {
+                    "description": "A list of reddit usernames (case-insensitive) to match against. Do not include the \"u/\" prefix\n\n EX to match against /u/FoxxMD and /u/AnotherUser use [\"FoxxMD\",\"AnotherUser\"]",
+                    "examples": [
+                        "FoxxMD",
+                        "AnotherUser"
+                    ],
+                    "items": {
+                        "type": "string"
+                    },
+                    "type": "array"
+                },
+                "totalKarma": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "userNotes": {
+                    "description": "A list of UserNote properties to check against the User Notes attached to this Author in this Subreddit (must have Toolbox enabled and used User Notes at least once)",
+                    "items": {
+                        "$ref": "#/definitions/UserNoteCriteria"
+                    },
+                    "type": "array"
+                },
+                "verified": {
+                    "description": "Does Author's account have a verified email?",
+                    "type": "boolean"
+                }
+            },
+            "type": "object"
+        },
+        "AuthorOptions": {
+            "description": "If present then these Author criteria are checked before running the rule. If criteria fails then the rule is skipped.",
+            "examples": [
+                {
+                    "include": [
+                        {
+                            "flairText": [
+                                "Contributor",
+                                "Veteran"
+                            ]
+                        },
+                        {
+                            "isMod": true
+                        }
+                    ]
+                }
+            ],
+            "properties": {
+                "exclude": {
+                    "description": "Only runs if `include` is not present. Will \"pass\" if any of set of the AuthorCriteria **does not** pass",
+                    "items": {
+                        "$ref": "#/definitions/AuthorCriteria"
+                    },
+                    "type": "array"
+                },
+                "include": {
+                    "description": "Will \"pass\" if any set of AuthorCriteria passes",
+                    "items": {
+                        "$ref": "#/definitions/AuthorCriteria"
+                    },
+                    "type": "array"
+                }
+            },
+            "type": "object"
+        },
+        "CommentState": {
+            "description": "Different attributes a `Comment` can be in. Only include a property if you want to check it.",
+            "examples": [
+                {
+                    "op": true,
+                    "removed": false
+                }
+            ],
+            "properties": {
+                "approved": {
+                    "type": "boolean"
+                },
+                "deleted": {
+                    "type": "boolean"
+                },
+                "distinguished": {
+                    "type": "boolean"
+                },
+                "filtered": {
+                    "type": "boolean"
+                },
+                "locked": {
+                    "type": "boolean"
+                },
+                "op": {
+                    "type": "boolean"
+                },
+                "removed": {
+                    "type": "boolean"
+                },
+                "spam": {
+                    "type": "boolean"
+                },
+                "stickied": {
+                    "type": "boolean"
+                }
+            },
+            "type": "object"
+        },
+        "SubmissionState": {
+            "description": "Different attributes a `Submission` can be in. Only include a property if you want to check it.",
+            "examples": [
+                {
+                    "over_18": true,
+                    "removed": false
+                }
+            ],
+            "properties": {
+                "approved": {
+                    "type": "boolean"
+                },
+                "deleted": {
+                    "type": "boolean"
+                },
+                "distinguished": {
+                    "type": "boolean"
+                },
+                "filtered": {
+                    "type": "boolean"
+                },
+                "is_self": {
+                    "type": "boolean"
+                },
+                "locked": {
+                    "type": "boolean"
+                },
+                "over_18": {
+                    "description": "NSFW",
+                    "type": "boolean"
+                },
+                "pinned": {
+                    "type": "boolean"
+                },
+                "removed": {
+                    "type": "boolean"
+                },
+                "spam": {
+                    "type": "boolean"
+                },
+                "spoiler": {
+                    "type": "boolean"
+                },
+                "stickied": {
+                    "type": "boolean"
+                }
+            },
+            "type": "object"
+        },
+        "UserNoteCriteria": {
+            "properties": {
+                "count": {
+                    "default": ">= 1",
+                    "description": "Number of occurrences of this type. Ignored if `search` is `current`\n\nA string containing a comparison operator and/or a value to compare number of occurrences against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign] [ascending|descending]`",
+                    "examples": [
+                        ">= 1"
+                    ],
+                    "pattern": "^\\s*(?<opStr>>|>=|<|<=)\\s*(?<value>\\d+)\\s*(?<percent>%?)\\s*(?<extra>asc.*|desc.*)*$",
+                    "type": "string"
+                },
+                "search": {
+                    "default": "current",
+                    "description": "How to test the notes for this Author:\n\n### current\n\nOnly the most recent note is checked for `type`\n\n### total\n\nThe `count` comparison of `type` must be found within all notes\n\n* EX `count: > 3`   => Must have more than 3 notes of `type`, total\n* EX `count: <= 25%` => Must have 25% or less of notes of `type`, total\n\n### consecutive\n\nThe `count` **number** of `type` notes must be found in a row.\n\nYou may also specify the time-based order in which to search the notes by specifying `ascending (asc)` or `descending (desc)` in the `count` value. Default is `descending`\n\n* EX `count: >= 3` => Must have 3 or more notes of `type` consecutively, in descending order\n* EX `count: < 2`  => Must have less than 2 notes of `type` consecutively, in descending order\n* EX `count: > 4 asc` => Must have greater than 4 notes of `type` consecutively, in ascending order",
+                    "enum": [
+                        "consecutive",
+                        "current",
+                        "total"
+                    ],
+                    "examples": [
+                        "current"
+                    ],
+                    "type": "string"
+                },
+                "type": {
+                    "description": "User Note type key to search for",
+                    "examples": [
+                        "spamwarn"
+                    ],
+                    "type": "string"
+                }
+            },
+            "required": [
+                "type"
+            ],
+            "type": "object"
+        }
+    },
     "properties": {
+        "authorIs": {
+            "$ref": "#/definitions/AuthorOptions",
+            "description": "If present then these Author criteria are checked before running the Action. If criteria fails then the Action is not run.",
+            "examples": [
+                {
+                    "include": [
+                        {
+                            "flairText": [
+                                "Contributor",
+                                "Veteran"
+                            ]
+                        },
+                        {
+                            "isMod": true
+                        }
+                    ]
+                }
+            ]
+        },
+        "dryRun": {
+            "default": false,
+            "description": "If `true` the Action will not make the API request to Reddit to perform its action.",
+            "examples": [
+                false,
+                true
+            ],
+            "type": "boolean"
+        },
+        "itemIs": {
+            "anyOf": [
+                {
+                    "items": {
+                        "$ref": "#/definitions/SubmissionState"
+                    },
+                    "type": "array"
+                },
+                {
+                    "items": {
+                        "$ref": "#/definitions/CommentState"
+                    },
+                    "type": "array"
+                }
+            ],
+            "description": "A list of criteria to test the state of the `Activity` against before running the Action.\n\nIf any set of criteria passes the Action will be run."
+        },
         "kind": {
             "description": "The type of action that will be performed",
             "enum": [
+                "approve",
+                "ban",
                 "comment",
                 "flair",
                 "lock",
                 "remove",
-                "report"
+                "report",
+                "usernote"
             ],
             "type": "string"
         },
         "name": {
-            "description": "A friendly name for this Action",
+            "description": "An optional, but highly recommended, friendly name for this Action. If not present will default to `kind`.\n\nCan only contain letters, numbers, underscore, spaces, and dashes",
+            "examples": [
+                "myDescriptiveAction"
+            ],
+            "pattern": "^[a-zA-Z]([\\w -]*[\\w])?$",
             "type": "string"
         }
     },
-    "propertyOrder": [
-        "kind",
-        "name"
-    ],
     "required": [
         "kind"
     ],

src/Subreddit/Manager.ts

@@ -1,39 +1,135 @@
-import Snoowrap, {Comment, Submission, Subreddit} from "snoowrap";
+import Snoowrap, {Comment, Subreddit} from "snoowrap";
 import {Logger} from "winston";
 import {SubmissionCheck} from "../Check/SubmissionCheck";
 import {CommentCheck} from "../Check/CommentCheck";
-import {createLabelledLogger, determineNewResults, loggerMetaShuffle, mergeArr, sleep} from "../util";
-import {CommentStream, SubmissionStream} from "snoostorm";
+import {
+    determineNewResults,
+    mergeArr, parseFromJsonOrYamlToObject, sleep,
+} from "../util";
+import {CommentStream, SubmissionStream, Poll, ModQueueStream} from "snoostorm";
 import pEvent from "p-event";
 import {RuleResult} from "../Rule";
-import {ConfigBuilder} from "../ConfigBuilder";
-import {PollingOptions} from "../Common/interfaces";
-
-export interface ManagerOptions {
-    polling?: PollingOptions
-}
+import {ConfigBuilder, buildPollingOptions} from "../ConfigBuilder";
+import {ManagerOptions, PollingOptionsStrong} from "../Common/interfaces";
+import Submission from "snoowrap/dist/objects/Submission";
+import {itemContentPeek} from "../Utils/SnoowrapUtils";
+import LoggedError from "../Utils/LoggedError";
+import ResourceManager, {
+    SubredditResourceOptions,
+    SubredditResources,
+    SubredditResourceSetOptions
+} from "./SubredditResources";
+import {UnmoderatedStream} from "./Streams";
+import EventEmitter from "events";
+import ConfigParseError from "../Utils/ConfigParseError";
+import dayjs, { Dayjs as DayjsObj } from "dayjs";
 
 export class Manager {
     subreddit: Subreddit;
     client: Snoowrap;
     logger: Logger;
-    pollOptions: PollingOptions;
-    submissionChecks: SubmissionCheck[];
-    commentChecks: CommentCheck[];
+    pollOptions!: PollingOptionsStrong[];
+    submissionChecks!: SubmissionCheck[];
+    commentChecks!: CommentCheck[];
+    resources!: SubredditResources;
+    wikiLocation: string = 'botconfig/contextbot';
+    lastWikiRevision?: DayjsObj
+    lastWikiCheck: DayjsObj = dayjs();
+    wikiUpdateRunning: boolean = false;
 
-    subListedOnce = false;
-    streamSub?: SubmissionStream;
-    commentsListedOnce = false;
-    streamComments?: CommentStream;
+    streamListedOnce: string[] = [];
+    streams: Poll<Snoowrap.Submission | Snoowrap.Comment>[] = [];
+    dryRun?: boolean;
+    globalDryRun?: boolean;
+    emitter: EventEmitter = new EventEmitter();
+
+    displayLabel: string;
+    currentLabels?: string[];
+
+    running: boolean = false;
+
+    getCurrentLabels = () => {
+        return this.currentLabels;
+    }
 
     constructor(sub: Subreddit, client: Snoowrap, logger: Logger, sourceData: object, opts: ManagerOptions = {}) {
-        this.logger = logger.child(loggerMetaShuffle(logger, undefined, [`r/${sub.display_name}`], {truncateLength: 40}), mergeArr);
-
-        const configBuilder = new ConfigBuilder({logger: this.logger});
-        const [subChecks, commentChecks] = configBuilder.buildFromJson(sourceData);
-        this.pollOptions = opts.polling || {};
+        const displayLabel = `${sub.display_name_prefixed}`;
+        this.displayLabel = displayLabel;
+        this.currentLabels = [displayLabel];
+        const getLabels = this.getCurrentLabels;
+        // dynamic default meta for winston feasible using function getters
+        // https://github.com/winstonjs/winston/issues/1626#issuecomment-531142958
+        this.logger = logger.child({
+            get labels() {
+                return getLabels()
+            }
+        }, mergeArr);
         this.subreddit = sub;
         this.client = client;
+        this.parseConfigurationFromObject(sourceData);
+    }
+
+    protected parseConfigurationFromObject(configObj: object) {
+        const configBuilder = new ConfigBuilder({logger: this.logger});
+        const validJson = configBuilder.validateJson(configObj);
+        const {checks, ...configManagerOpts} = validJson;
+        const {
+            polling = [{pollOn: 'unmoderated', limit: 25, interval: 20000}],
+            caching,
+            dryRun,
+            footer,
+            nickname
+        } = configManagerOpts || {};
+        this.pollOptions = buildPollingOptions(polling);
+        this.dryRun = this.globalDryRun || dryRun;
+
+        if(nickname !== undefined) {
+            this.displayLabel = nickname;
+            this.currentLabels = [this.displayLabel];
+        }
+
+        if(footer !== undefined) {
+            this.resources.footer = footer;
+        }
+
+        let resourceConfig: SubredditResourceSetOptions = {
+            footer,
+            enabled: true
+        };
+        if(caching === false) {
+            resourceConfig.enabled = false;
+        } else {
+            resourceConfig = {...resourceConfig, ...caching};
+        }
+        if(this.resources === undefined) {
+            this.resources = ResourceManager.set(this.subreddit.display_name, {
+                ...resourceConfig,
+                logger: this.logger,
+                subreddit: this.subreddit
+            });
+        }
+        this.resources.setOptions(resourceConfig);
+
+        this.logger.info('Subreddit-specific options updated');
+        this.logger.info('Building Checks...');
+
+        const commentChecks: Array<CommentCheck> = [];
+        const subChecks: Array<SubmissionCheck> = [];
+        const structuredChecks = configBuilder.parseToStructured(validJson);
+        for (const jCheck of structuredChecks) {
+            const checkConfig = {
+                ...jCheck,
+                dryRun: this.dryRun || jCheck.dryRun,
+                logger: this.logger,
+                subredditName: this.subreddit.display_name
+            };
+            if (jCheck.kind === 'comment') {
+                commentChecks.push(new CommentCheck(checkConfig));
+            } else if (jCheck.kind === 'submission') {
+                subChecks.push(new SubmissionCheck(checkConfig));
+            }
+        }
+
         this.submissionChecks = subChecks;
         this.commentChecks = commentChecks;
         const checkSummary = `Found Checks -- Submission: ${this.submissionChecks.length} | Comment: ${this.commentChecks.length}`;
@@ -44,93 +140,220 @@ export class Manager {
         }
     }
 
-    async runChecks(checkType: ('Comment' | 'Submission'), item: (Submission | Comment)): Promise<void> {
+    async parseConfiguration(force: boolean = false) {
+        this.wikiUpdateRunning = true;
+        this.lastWikiCheck = dayjs();
+
+        let sourceData: string;
+        try {
+            // @ts-ignore
+            const wiki = await this.subreddit.getWikiPage(this.wikiLocation).fetch();
+            const revisionDate = dayjs.unix(wiki.revision_date);
+            if (!force && (this.lastWikiRevision !== undefined && this.lastWikiRevision.isSame(revisionDate))) {
+                // nothing to do, we already have this revision
+                this.wikiUpdateRunning = false;
+                this.logger.verbose('Config is up to date');
+                return;
+            }
+            if (this.lastWikiRevision !== undefined) {
+                this.logger.info(`Updating config due to stale wiki page (${dayjs.duration(dayjs().diff(revisionDate)).humanize()} old)`)
+            }
+            this.lastWikiRevision = revisionDate;
+            sourceData = await wiki.content_md;
+        } catch (err) {
+            const msg = `Could not read wiki configuration. Please ensure the page https://reddit.com${this.subreddit.url}wiki/${this.wikiLocation} exists and is readable -- error: ${err.message}`;
+            this.logger.error(msg);
+            this.wikiUpdateRunning = false;
+            throw new ConfigParseError(msg);
+        }
+
+        if (sourceData === '') {
+            this.logger.error(`Wiki page contents was empty`);
+            this.wikiUpdateRunning = false;
+            throw new ConfigParseError('Wiki page contents was empty');
+        }
+
+        const [configObj, jsonErr, yamlErr] = parseFromJsonOrYamlToObject(sourceData);
+
+        if (configObj === undefined) {
+            this.logger.error(`Could not parse wiki page contents as JSON or YAML:`);
+            this.logger.error(jsonErr);
+            this.logger.error(yamlErr);
+            this.wikiUpdateRunning = false;
+            throw new ConfigParseError('Could not parse wiki page contents as JSON or YAML:')
+        }
+
+        this.wikiUpdateRunning = false;
+        this.parseConfigurationFromObject(configObj);
+        this.logger.info('Checks updated');
+    }
+
+    async runChecks(checkType: ('Comment' | 'Submission'), item: (Submission | Comment), checkNames: string[] = []): Promise<void> {
         const checks = checkType === 'Comment' ? this.commentChecks : this.submissionChecks;
         const itemId = await item.id;
         let allRuleResults: RuleResult[] = [];
+        const itemIdentifier = `${checkType === 'Submission' ? 'SUB' : 'COM'} ${itemId}`;
+        this.currentLabels = [this.displayLabel, itemIdentifier];
+        const [peek, _] = await itemContentPeek(item);
+        this.logger.info(`<EVENT> ${peek}`);
 
-        for (const check of checks) {
-            this.logger.debug(`Running Check ${check.name} on ${checkType} (ID ${itemId})`);
+        while(this.wikiUpdateRunning) {
+            // sleep for a few seconds while we get new config zzzz
+            this.logger.verbose('A wiki config update is running, delaying checks by 3 seconds');
+            await sleep(3000);
+        }
+        if(this.lastWikiCheck.diff(dayjs(), 's') > 60) {
+            // last checked more than 60 seconds ago for config, try and update
+            await this.parseConfiguration();
+        }
+
+        const startingApiLimit = this.client.ratelimitRemaining;
+
+        if(item instanceof Submission) {
+            if(await item.removed_by_category === 'deleted') {
+                this.logger.warn('Submission was deleted, cannot process.');
+                return;
+            }
+        } else if(item.author.name === '[deleted]') {
+            this.logger.warn('Comment was deleted, cannot process.');
+            return;
+        }
+
+        let checksRun = 0;
+        let actionsRun = 0;
+        let totalRulesRun = 0;
+
+        try {
             let triggered = false;
-            try {
-                const [checkTriggered, checkResults] = await check.run(item, allRuleResults);
-                allRuleResults = allRuleResults.concat(determineNewResults(allRuleResults, checkResults));
-                triggered = checkTriggered;
-                const invokedRules = checkResults.map(x => x.name || x.premise.kind).join(' | ');
-                if (checkTriggered) {
-                    this.logger.debug(`Check ${check.name} was triggered with invoked Rules: ${invokedRules}`);
-                } else {
-                    this.logger.debug(`Check ${check.name} was not triggered using invoked Rule(s): ${invokedRules}`);
+            for (const check of checks) {
+                if (checkNames.length > 0 && !checkNames.map(x => x.toLowerCase()).some(x => x === check.name.toLowerCase())) {
+                    this.logger.warn(`Check ${check} not in array of requested checks to run, skipping`);
+                    continue;
+                }
+                checksRun++;
+                triggered = false;
+                let currentResults: RuleResult[] = [];
+                try {
+                    const [checkTriggered, checkResults] = await check.runRules(item, allRuleResults);
+                    currentResults = checkResults;
+                    totalRulesRun += checkResults.length;
+                    allRuleResults = allRuleResults.concat(determineNewResults(allRuleResults, checkResults));
+                    triggered = checkTriggered;
+                } catch (e) {
+                    if(e.logged !== true) {
+                        this.logger.warn(`Running rules for Check ${check.name} failed due to uncaught exception`, e);
+                    }
                 }
 
-            } catch (e) {
-                this.logger.warn(`Check ${check.name} on Submission (ID ${itemId}) failed with error: ${e.message}`, e);
+                if (triggered) {
+                    const runActions = await check.runActions(item, currentResults.filter(x => x.triggered));
+                    actionsRun = runActions.length;
+                    break;
+                }
             }
 
-            if (triggered) {
-                // TODO give actions a name
-                await check.runActions(item, this.client);
-                this.logger.debug(`Ran actions for Check ${check.name}`);
-                break;
+            if(!triggered) {
+                this.logger.info('No checks triggered');
             }
+
+        } catch (err) {
+            if (!(err instanceof LoggedError) && err.logged !== true) {
+                this.logger.error('An unhandled error occurred while running checks', err);
+            }
+        } finally {
+            this.logger.verbose(`Run Stats:        Checks ${checksRun} | Rules => Total: ${totalRulesRun} Unique: ${allRuleResults.length} Cached: ${totalRulesRun - allRuleResults.length} | Actions ${actionsRun}`);
+            this.logger.verbose(`Reddit API Stats: Initial Limit ${startingApiLimit} | Current Limit ${this.client.ratelimitRemaining} | Calls Made ${startingApiLimit - this.client.ratelimitRemaining}`);
+            this.currentLabels = [this.displayLabel];
         }
     }
 
     async handle(): Promise<void> {
-        if (this.submissionChecks.length > 0) {
-            const {
-                submissions: {
-                    limit = 10,
-                    interval = 10000,
-                } = {}
-            } = this.pollOptions
-            this.streamSub = new SubmissionStream(this.client, {
-                subreddit: this.subreddit.display_name,
-                limit,
-                pollTime: interval,
-            });
-
-
-            this.streamSub.once('listing', async (listing) => {
-                this.subListedOnce = true;
-                // for debugging
-                // await this.runChecks('Submission', listing[0]);
-            });
-            this.streamSub.on('item', async (item) => {
-                if (!this.subListedOnce) {
-                    return;
-                }
-                await this.runChecks('Submission', item)
-            });
+        if(this.submissionChecks.length === 0 && this.commentChecks.length === 0) {
+            this.logger.warn('No submission or comment checks to run! Bot will not run.');
+            return;
         }
 
-        if (this.commentChecks.length > 0) {
-            const {
-                comments: {
-                    limit = 10,
-                    interval = 10000,
-                } = {}
-            } = this.pollOptions
-            this.streamComments = new CommentStream(this.client, {
-                subreddit: this.subreddit.display_name,
-                limit,
-                pollTime: interval,
-            });
-            this.streamComments.once('listing', () => this.commentsListedOnce = true);
-            this.streamComments.on('item', async (item) => {
-                if (!this.commentsListedOnce) {
-                    return;
-                }
-                await this.runChecks('Comment', item)
-            });
-        }
+        try {
 
-        if (this.streamSub !== undefined) {
-            await pEvent(this.streamSub, 'end');
-        } else if (this.streamComments !== undefined) {
-            await pEvent(this.streamComments, 'end');
-        } else {
-            this.logger.warn('No submission or comment checks to run!');
+            for(const pollOpt of this.pollOptions) {
+                let stream: Poll<Snoowrap.Submission | Snoowrap.Comment>;
+
+                switch(pollOpt.pollOn) {
+                    case 'unmoderated':
+                        stream = new UnmoderatedStream(this.client, {
+                            subreddit: this.subreddit.display_name,
+                            limit: pollOpt.limit,
+                            pollTime: pollOpt.interval,
+                        });
+                        break;
+                    case 'modqueue':
+                        stream = new ModQueueStream(this.client, {
+                            subreddit: this.subreddit.display_name,
+                            limit: pollOpt.limit,
+                            pollTime: pollOpt.interval,
+                        });
+                        break;
+                    case 'newSub':
+                        stream = new SubmissionStream(this.client, {
+                            subreddit: this.subreddit.display_name,
+                            limit: pollOpt.limit,
+                            pollTime: pollOpt.interval,
+                        });
+                        break;
+                    case 'newComm':
+                        stream = new CommentStream(this.client, {
+                            subreddit: this.subreddit.display_name,
+                            limit: pollOpt.limit,
+                            pollTime: pollOpt.interval,
+                        });
+                        break;
+                }
+
+                stream.once('listing', async (listing) => {
+                    // warning if poll event could potentially miss activities
+                    if(this.commentChecks.length === 0 && ['unmoderated','modqueue','newComm'].some(x => x === pollOpt.pollOn)) {
+                        this.logger.warn(`Polling '${pollOpt.pollOn}' may return Comments but no comments checks were configured.`);
+                    }
+                    if(this.submissionChecks.length === 0 && ['unmoderated','modqueue','newSub'].some(x => x === pollOpt.pollOn)) {
+                        this.logger.warn(`Polling '${pollOpt.pollOn}' may return Submissions but no submission checks were configured.`);
+                    }
+                    this.streamListedOnce.push(pollOpt.pollOn);
+                });
+                stream.on('item', async (item) => {
+                    if (!this.streamListedOnce.includes(pollOpt.pollOn)) {
+                        return;
+                    }
+                    if(item instanceof Submission) {
+                        if(this.submissionChecks.length > 0) {
+                            await this.runChecks('Submission', item);
+                        }
+                    } else if(this.commentChecks.length > 0) {
+                        await this.runChecks('Comment', item)
+                    }
+                });
+                this.streams.push(stream);
+            }
+
+            this.running = true;
+            this.logger.info('Bot Running');
+
+            await pEvent(this.emitter, 'end');
+        } catch (err) {
+            this.logger.error('Encountered unhandled error, manager is bailing out');
+            this.logger.error(err);
+        } finally {
+            this.stop();
+        }
+    }
+
+    stop() {
+        if(this.running) {
+            for(const s of this.streams) {
+                s.end();
+            }
+            this.emitter.emit('end');
+            this.running = false;
+            this.logger.info('Bot Stopped');
         }
     }
 }

src/Subreddit/Streams.ts

@@ -0,0 +1,16 @@
+import { Poll, SnooStormOptions } from "snoostorm"
+import Snoowrap from "snoowrap";
+
+export class UnmoderatedStream extends Poll<
+    Snoowrap.Submission | Snoowrap.Comment
+    > {
+    constructor(
+        client: Snoowrap,
+        options: SnooStormOptions & { subreddit: string }) {
+        super({
+            frequency: options.pollTime || 20000,
+            get: async () => client.getSubreddit(options.subreddit).getUnmoderated(options),
+            identifier: "id",
+        });
+    }
+}

src/Subreddit/SubredditResources.ts

@@ -0,0 +1,246 @@
+import Snoowrap, {RedditUser, Comment, Submission} from "snoowrap";
+import cache from 'memory-cache';
+import objectHash from 'object-hash';
+import {
+    AuthorActivitiesOptions,
+    AuthorTypedActivitiesOptions, BOT_LINK,
+    getAuthorActivities,
+    testAuthorCriteria
+} from "../Utils/SnoowrapUtils";
+import Subreddit from 'snoowrap/dist/objects/Subreddit';
+import winston, {Logger} from "winston";
+import fetch from 'node-fetch';
+import {mergeArr, parseExternalUrl, parseWikiContext} from "../util";
+import LoggedError from "../Utils/LoggedError";
+import {Footer, SubredditCacheConfig} from "../Common/interfaces";
+import UserNotes from "./UserNotes";
+import Mustache from "mustache";
+import he from "he";
+import {AuthorCriteria} from "../Author/Author";
+
+export const DEFAULT_FOOTER = '\r\n*****\r\nThis action was performed by [a bot.]({{botLink}}) Mention a moderator or [send a modmail]({{modmailLink}}) if you any ideas, questions, or concerns about this action.';
+
+export interface SubredditResourceOptions extends SubredditCacheConfig, Footer {
+    enabled: boolean;
+    subreddit: Subreddit,
+    logger: Logger;
+}
+
+export interface SubredditResourceSetOptions extends SubredditCacheConfig, Footer {
+    enabled: boolean;
+}
+
+export class SubredditResources {
+    enabled!: boolean;
+    protected authorTTL!: number;
+    protected useSubredditAuthorCache!: boolean;
+    protected wikiTTL!: number;
+    name: string;
+    protected logger: Logger;
+    userNotes: UserNotes;
+    footer!: false | string;
+    subreddit: Subreddit
+
+    constructor(name: string, options: SubredditResourceOptions) {
+        const {
+            subreddit,
+            logger,
+            enabled = true,
+            userNotesTTL = 60000,
+        } = options || {};
+
+        this.subreddit = subreddit;
+        this.name = name;
+        if (logger === undefined) {
+            const alogger = winston.loggers.get('default')
+            this.logger = alogger.child({labels: [this.name, 'Resource Cache']}, mergeArr);
+        } else {
+            this.logger = logger.child({labels: ['Resource Cache']}, mergeArr);
+        }
+
+        this.userNotes = new UserNotes(enabled ? userNotesTTL : 0, this.subreddit, this.logger)
+        this.setOptions(options);
+    }
+
+    setOptions (options: SubredditResourceSetOptions) {
+        const {
+            enabled = true,
+            authorTTL,
+            userNotesTTL = 60000,
+            wikiTTL = 300000, // 5 minutes
+            footer = DEFAULT_FOOTER
+        } = options || {};
+
+        this.footer = footer;
+        this.enabled = manager.enabled ? enabled : false;
+        if (authorTTL === undefined) {
+            this.useSubredditAuthorCache = false;
+            this.authorTTL = manager.authorTTL;
+        } else {
+            this.useSubredditAuthorCache = true;
+            this.authorTTL = authorTTL;
+        }
+        this.wikiTTL = wikiTTL;
+        this.userNotes.notesTTL = enabled ? userNotesTTL : 0;
+    }
+
+    async getAuthorActivities(user: RedditUser, options: AuthorTypedActivitiesOptions): Promise<Array<Submission | Comment>> {
+        const useCache = this.enabled && this.authorTTL > 0;
+        let hash;
+        if (useCache) {
+            const userName = user.name;
+            const hashObj: any = {...options, userName};
+            if (this.useSubredditAuthorCache) {
+                hashObj.subreddit = this.name;
+            }
+            hash = objectHash.sha1({...options, userName});
+
+            const cacheVal = cache.get(hash);
+            if (null !== cacheVal) {
+                this.logger.debug(`Cache Hit: ${userName} (${options.type || 'overview'})`);
+                return cacheVal as Array<Submission | Comment>;
+            }
+        }
+
+
+        const items = await getAuthorActivities(user, options);
+
+        if (useCache) {
+            cache.put(hash, items, this.authorTTL);
+        }
+        return Promise.resolve(items);
+    }
+
+    async getAuthorComments(user: RedditUser, options: AuthorActivitiesOptions): Promise<Comment[]> {
+        return await this.getAuthorActivities(user, {...options, type: 'comment'}) as unknown as Promise<Comment[]>;
+    }
+
+    async getAuthorSubmissions(user: RedditUser, options: AuthorActivitiesOptions): Promise<Submission[]> {
+        return await this.getAuthorActivities(user, {
+            ...options,
+            type: 'submission'
+        }) as unknown as Promise<Submission[]>;
+    }
+
+    async getContent(val: string, subredditArg?: Subreddit): Promise<string> {
+        const subreddit = subredditArg || this.subreddit;
+        let cacheKey;
+        const wikiContext = parseWikiContext(val);
+        if (wikiContext !== undefined) {
+            cacheKey = `${wikiContext.wiki}${wikiContext.subreddit !== undefined ? `|${wikiContext.subreddit}` : ''}`;
+        }
+        const extUrl = wikiContext === undefined ? parseExternalUrl(val) : undefined;
+        if (extUrl !== undefined) {
+            cacheKey = extUrl;
+        }
+
+        if (cacheKey === undefined) {
+            return val;
+        }
+
+        const useCache = this.enabled && this.wikiTTL > 0;
+        // try to get cached value first
+        let hash = `${subreddit.display_name}-${cacheKey}`;
+        if (useCache) {
+            const cachedContent = cache.get(hash);
+            if (cachedContent !== null) {
+                this.logger.debug(`Cache Hit: ${cacheKey}`);
+                return cachedContent;
+            }
+        }
+
+        let wikiContent: string;
+
+        // no cache hit, get from source
+        if (wikiContext !== undefined) {
+            let sub;
+            if (wikiContext.subreddit === undefined || wikiContext.subreddit.toLowerCase() === subreddit.display_name) {
+                sub = subreddit;
+            } else {
+                // @ts-ignore
+                const client = subreddit._r as Snoowrap;
+                sub = client.getSubreddit(wikiContext.subreddit);
+            }
+            try {
+                const wikiPage = sub.getWikiPage(wikiContext.wiki);
+                wikiContent = await wikiPage.content_md;
+            } catch (err) {
+                const msg = `Could not read wiki page. Please ensure the page 'https://reddit.com${sub.display_name_prefixed}wiki/${wikiContext}' exists and is readable`;
+                this.logger.error(msg, err);
+                throw new LoggedError(msg);
+            }
+        } else {
+            try {
+                const response = await fetch(extUrl as string);
+                wikiContent = await response.text();
+            } catch (err) {
+                const msg = `Error occurred while trying to fetch the url ${extUrl}`;
+                this.logger.error(msg, err);
+                throw new LoggedError(msg);
+            }
+        }
+
+        if (useCache) {
+            cache.put(hash, wikiContent, this.wikiTTL);
+        }
+
+        return wikiContent;
+    }
+
+    async testAuthorCriteria(item: (Comment | Submission), authorOpts: AuthorCriteria, include = true) {
+        const useCache = this.enabled && this.authorTTL > 0;
+        let hash;
+        if (useCache) {
+            const hashObj = {itemId: item.id, ...authorOpts, include};
+            hash = `authorCrit-${objectHash.sha1(hashObj)}`;
+            const cachedAuthorTest = cache.get(hash);
+            if (null !== cachedAuthorTest) {
+                this.logger.debug(`Cache Hit: Author Check on ${item.id}`);
+                return cachedAuthorTest;
+            }
+        }
+
+        const result = await testAuthorCriteria(item, authorOpts, include, this.userNotes);
+        if (useCache) {
+            cache.put(hash, result, this.authorTTL);
+        }
+        return result;
+    }
+
+    async generateFooter(item: Submission | Comment, actionFooter?: false | string)
+    {
+        let footer = actionFooter !== undefined ? actionFooter : this.footer;
+        if(footer === false) {
+            return '';
+        }
+        const subName = await item.subreddit.display_name;
+        const permaLink = `https://reddit.com${await item.permalink}`
+        const modmailLink = `https://www.reddit.com/message/compose?to=%2Fr%2F${subName}&message=${encodeURIComponent(permaLink)}`
+
+        const footerRawContent = await this.getContent(footer, item.subreddit);
+        return he.decode(Mustache.render(footerRawContent, {subName, permaLink, modmailLink, botLink: BOT_LINK}));
+    }
+}
+
+class SubredditResourcesManager {
+    resources: Map<string, SubredditResources> = new Map();
+    authorTTL: number = 10000;
+    enabled: boolean = true;
+
+    get(subName: string): SubredditResources | undefined {
+        if (this.resources.has(subName)) {
+            return this.resources.get(subName) as SubredditResources;
+        }
+        return undefined;
+    }
+
+    set(subName: string, initOptions: SubredditResourceOptions): SubredditResources {
+        const resource = new SubredditResources(subName, initOptions);
+        this.resources.set(subName, resource);
+        return resource;
+    }
+}
+
+const manager = new SubredditResourcesManager();
+
+export default manager;

src/Subreddit/UserNotes.ts

@@ -0,0 +1,264 @@
+import dayjs, {Dayjs} from "dayjs";
+import {Comment, RedditUser, WikiPage} from "snoowrap";
+import cache from 'memory-cache';
+import {COMMENT_URL_ID, deflateUserNotes, inflateUserNotes, parseLinkIdentifier, SUBMISSION_URL_ID} from "../util";
+import Subreddit from "snoowrap/dist/objects/Subreddit";
+import {Logger} from "winston";
+import LoggedError from "../Utils/LoggedError";
+import Submission from "snoowrap/dist/objects/Submission";
+import {RichContent} from "../Common/interfaces";
+
+interface RawUserNotesPayload {
+    ver: number,
+    constants: UserNotesConstants,
+    blob: RawBlobPayload
+}
+
+interface RawBlobPayload {
+    [username: string]: RawUserNoteRoot
+}
+
+interface RawUserNoteRoot {
+    ns: RawNote[]
+}
+
+export interface RawNote {
+    /**
+     * Note Text
+     * */
+    n: string;
+    /**
+     * Unix epoch in seconds
+     * */
+    t: number;
+    /**
+     * Moderator index from constants.users
+     * */
+    m: number;
+    /**
+     * Link shorthand
+     * */
+    l: string;
+    /**
+     * type/color index from constants.warnings
+     * */
+    w: number;
+}
+
+export type UserNotesConstants = Pick<any, "users" | "warnings">;
+
+export class UserNotes {
+    notesTTL: number;
+    subreddit: Subreddit;
+    wiki: WikiPage;
+    moderators?: RedditUser[];
+    logger: Logger;
+    identifier: string;
+
+    users: Map<string, UserNote[]> = new Map();
+
+    constructor(ttl: number, subreddit: Subreddit, logger: Logger) {
+        this.notesTTL = ttl;
+        this.subreddit = subreddit;
+        this.logger = logger;
+        this.wiki = subreddit.getWikiPage('usernotes');
+        this.identifier = `${this.subreddit.display_name}-usernotes`;
+    }
+
+    async getUserNotes(user: RedditUser): Promise<UserNote[]> {
+        let notes: UserNote[] | undefined = [];
+
+        if (this.users !== undefined) {
+            notes = this.users.get(user.name);
+            if (notes !== undefined) {
+                this.logger.debug('Returned cached notes');
+                return notes;
+            }
+        }
+
+        const payload = await this.retrieveData();
+        const rawNotes = payload.blob[user.name];
+        if (rawNotes !== undefined) {
+            if (this.moderators === undefined) {
+                this.moderators = await this.subreddit.getModerators();
+            }
+            const notes = rawNotes.ns.map(x => UserNote.fromRaw(x, payload.constants, this.moderators as RedditUser[]));
+            // sort in ascending order by time
+            notes.sort((a, b) => a.time.isBefore(b.time) ? -1 : 1);
+            if (this.notesTTL > 0) {
+                this.users.set(user.name, notes);
+            }
+            return notes;
+        } else {
+            return [];
+        }
+    }
+
+    async addUserNote(item: (Submission|Comment), type: string | number, text: string = ''): Promise<UserNote>
+    {
+        const payload = await this.retrieveData();
+
+        // idgaf
+        // @ts-ignore
+        const mod = await this.subreddit._r.getMe();
+        if(!payload.constants.users.includes(mod.name)) {
+            this.logger.info(`Mod ${mod.name} does not exist in UserNote constants, adding them`);
+            payload.constants.users.push(mod.name);
+        }
+        if(!payload.constants.warnings.find((x: string) => x === type)) {
+            this.logger.warn(`UserNote type '${type}' does not exist, adding it but make sure spelling and letter case is correct`);
+            payload.constants.warnings.push(type);
+            //throw new LoggedError(`UserNote type '${type}' does not exist. If you meant to use this please add it through Toolbox first.`);
+        }
+        const newNote = new UserNote(dayjs(), text, mod, type, `https://reddit.com${item.permalink}`);
+
+        if(payload.blob[item.author.name] === undefined) {
+            payload.blob[item.author.name] = {ns: []};
+        }
+        payload.blob[item.author.name].ns.push(newNote.toRaw(payload.constants));
+
+        await this.saveData(payload);
+        if(this.notesTTL > 0) {
+            const currNotes = this.users.get(item.author.name) || [];
+            currNotes.push(newNote);
+            this.users.set(item.author.name, currNotes);
+        }
+        return newNote;
+    }
+
+    async warningExists(type: string): Promise<boolean>
+    {
+        const payload = await this.retrieveData();
+        return payload.constants.warnings.some((x: string) => x === type);
+    }
+
+    async retrieveData(): Promise<RawUserNotesPayload> {
+        if (this.notesTTL > 0) {
+            const cachedPayload = cache.get(this.identifier);
+            if (cachedPayload !== null) {
+                return cachedPayload as RawUserNotesPayload;
+            }
+        }
+
+        try {
+            // @ts-ignore
+            this.wiki = await this.subreddit.getWikiPage('usernotes').fetch();
+            const wikiContent = this.wiki.content_md;
+            // TODO don't handle for versions lower than 6
+            const userNotes = JSON.parse(wikiContent);
+
+            userNotes.blob = inflateUserNotes(userNotes.blob);
+
+            if (this.notesTTL > 0) {
+                cache.put(`${this.subreddit.display_name}-usernotes`, userNotes, this.notesTTL, () => {
+                    this.users = new Map();
+                });
+            }
+
+            return userNotes as RawUserNotesPayload;
+        } catch (err) {
+            const msg = `Could not read usernotes. Make sure at least one moderator has used toolbox and usernotes before.`;
+            this.logger.error(msg, err);
+            throw new LoggedError(msg);
+        }
+    }
+
+    async saveData(payload: RawUserNotesPayload): Promise<RawUserNotesPayload> {
+
+        const blob = deflateUserNotes(payload.blob);
+        const wikiPayload = {...payload, blob};
+
+        try {
+            // @ts-ignore
+            //this.wiki = await this.wiki.refresh();
+            // @ts-ignore
+            this.wiki = await this.subreddit.getWikiPage('usernotes').edit({text: JSON.stringify(wikiPayload), reason: 'ContextBot edited usernotes'});
+            if (this.notesTTL > 0) {
+                cache.put(this.identifier, payload, this.notesTTL, () => {
+                    this.users = new Map();
+                });
+            }
+
+            return payload as RawUserNotesPayload;
+        } catch (err) {
+            const msg = `Could not edit usernotes. Make sure at least one moderator has used toolbox and usernotes before and that this account has editing permissions`;
+            this.logger.error(msg, err);
+            throw new LoggedError(msg);
+        }
+    }
+}
+
+export interface UserNoteJson extends RichContent {
+    /**
+     * User Note type key
+     * @examples ["spamwarn"]
+     * */
+    type: string,
+}
+
+export class UserNote {
+    //time: Dayjs;
+    // text?: string;
+    // moderator: RedditUser;
+    // noteTypeIndex: number;
+    // noteType: string | null;
+    // link: string;
+
+    constructor(public time: Dayjs, public text: string, public moderator: RedditUser, public noteType: string | number, public link: string) {
+
+    }
+
+    public toRaw(constants: UserNotesConstants): RawNote {
+        return {
+            t: this.time.unix(),
+            n: this.text,
+            m: constants.users.findIndex((x: string) => x === this.moderator.name),
+            w: typeof this.noteType === 'number' ? this.noteType : constants.warnings.findIndex((x: string) => x === this.noteType),
+            l: usernoteLinkShorthand(this.link)
+        }
+    }
+
+    public static fromRaw(obj: RawNote, constants: UserNotesConstants, mods: RedditUser[]) {
+        const mod = mods.find(x => x.name === constants.users[obj.m]);
+        if (mod === undefined) {
+            throw new Error('Could not find moderator for Usernote');
+        }
+        return new UserNote(dayjs.unix(obj.t), obj.n, mod, constants.warnings[obj.w] === null ? obj.w : constants.warnings[obj.w], usernoteLinkExpand(obj.l))
+    }
+}
+
+// https://github.com/toolbox-team/reddit-moderator-toolbox/wiki/Subreddit-Wikis%3A-usernotes#link-string-formats
+export const usernoteLinkExpand = (link: string) => {
+    if (link.charAt(0) === 'l') {
+        const pieces = link.split(',');
+        if (pieces.length === 3) {
+            // it's a comment
+            return `https://www.reddit.com/comments/${pieces[1]}/_/${pieces[2]}`;
+        }
+        // its a submission
+        return `https://redd.it/${pieces[1]}`;
+    } else {
+        // its an old modmail thread
+        return `https://www.reddit.com/message/messages/${link.split(',')[1]}`;
+    }
+}
+export const usernoteLinkShorthand = (link: string) => {
+
+    const commentReg = parseLinkIdentifier([COMMENT_URL_ID]);
+    const submissionReg = parseLinkIdentifier([SUBMISSION_URL_ID]);
+
+    let commentId = commentReg(link);
+    let submissionId = submissionReg(link);
+
+    if (commentId !== undefined) {
+        commentId = commentReg(link);
+        return `l,${submissionId},${commentId}`;
+    } else if (submissionId !== undefined) {
+        return `l,${submissionId}`;
+    }
+
+    // aren't dealing with messages at this point so just store whole thing if we didn't get a shorthand
+    return link;
+}
+
+export default UserNotes;

src/Utils/CommandConfig.ts

@@ -0,0 +1,86 @@
+import commander, {InvalidOptionArgumentError} from "commander";
+import {argParseInt, parseBool} from "../util";
+
+export const clientId = new commander.Option('-c, --clientId <id>', 'Client ID for your Reddit application (default: process.env.CLIENT_ID)')
+    .default(process.env.CLIENT_ID);
+clientId.required = true;
+
+export const clientSecret = new commander.Option('-e, --clientSecret <secret>', 'Client Secret for your Reddit application (default: process.env.CLIENT_SECRET)')
+    .default(process.env.CLIENT_SECRET);
+clientSecret.required = true;
+
+export const accessToken = new commander.Option('-a, --accessToken <token>', 'Access token retrieved from authenticating an account with your Reddit Application (default: process.env.ACCESS_TOKEN)')
+    .default(process.env.ACCESS_TOKEN);
+accessToken.required = true;
+
+export const refreshToken = new commander.Option('-r, --refreshToken <token>', 'Refresh token retrieved from authenticating an account with your Reddit Application (default: process.env.REFRESH_TOKEN)')
+    .default(process.env.REFRESH_TOKEN);
+refreshToken.required = true;
+
+export const subreddits = new commander.Option('-s, --subreddits <list...>', 'List of subreddits to run on. Bot will run on all subs it has access to if not defined')
+    .default(process.env.SUBREDDITS || [], 'process.env.SUBREDDITS (comma-seperated)');
+
+export const logDir = new commander.Option('-d, --logDir <dir>', 'Absolute path to directory to store rotated logs in')
+    .default(process.env.LOG_DIR || `${process.cwd()}/logs`, 'process.env.LOG_DIR || process.cwd()/logs');
+
+export const logLevel = new commander.Option('-l, --logLevel <level>', 'Log level')
+    .default(process.env.LOG_LEVEL || 'verbose', 'process.env.LOG_LEVEL || verbose');
+
+export const wikiConfig = new commander.Option('-w, --wikiConfig <path>', 'Relative url to contextbot wiki page EX https://reddit.com/r/subreddit/wiki/<path>')
+    .default(process.env.WIKI_CONFIG || 'botconfig/contextbot', "process.env.WIKI_CONFIG || 'botconfig/contextbot'");
+
+export const snooDebug = new commander.Option('--snooDebug', 'Set Snoowrap to debug')
+    .argParser(parseBool)
+    .default(process.env.SNOO_DEBUG || false, 'process.env.SNOO_DEBUG || false');
+
+export const authorTTL = new commander.Option('--authorTTL <ms>', 'Set the TTL (ms) for the Author Activities shared cache')
+    .argParser(argParseInt)
+    .default(process.env.AUTHOR_TTL || 10000, 'process.env.AUTHOR_TTL || 10000');
+
+export const heartbeat = new commander.Option('--heartbeat <s>', 'Interval, in seconds, between heartbeat logs. Set to 0 to disable')
+    .argParser(argParseInt)
+    //heartbeat.defaultValueDescription = 'process.env.HEARTBEAT || 300';
+    .default(process.env.HEARTBEAT || 300, 'process.env.HEARTBEAT || 300');
+
+export const apiRemaining = new commander.Option('--apiLimitWarning <remaining>', 'When API limit remaining (600/10min) is lower than this value log statements for limit will be raised to WARN level')
+    .argParser(argParseInt)
+    .default(process.env.API_REMAINING || 250, 'process.env.API_REMAINING || 250');
+
+export const dryRun = new commander.Option('--dryRun', 'Set dryRun=true for all checks/actions on all subreddits (overrides any existing)')
+    .argParser(parseBool)
+    .default(process.env.DRYRUN || false, 'process.env.DRYRUN || false');
+
+export const disableCache = new commander.Option('--disableCache', 'Disable caching for all subreddits')
+    .argParser(parseBool)
+    .default(process.env.DISABLE_CACHE || false, 'process.env.DISABLE_CACHE || false');
+
+export const checks = new commander.Option('-h, --checks <checkNames...>', 'An optional list of Checks, by name, that should be run. If none are specified all Checks for the Subreddit the Activity is in will be run');
+
+export const limit = new commander.Option('--limit <limit>', 'Limit the number of unmoderated activities pulled for each subreddit')
+    .argParser(parseInt);
+
+export const getUniversalOptions = (): commander.Option[] => {
+    let options = [];
+
+    options.push(dryRun);
+
+    options = [
+        clientId,
+        clientSecret,
+        accessToken,
+        refreshToken,
+        subreddits,
+        logDir,
+        logLevel,
+        wikiConfig,
+        snooDebug,
+        authorTTL,
+        heartbeat,
+        apiRemaining,
+        dryRun,
+        disableCache
+    ]
+
+
+    return options;
+}

src/Utils/ConfigParseError.ts

@@ -0,0 +1,7 @@
+import LoggedError from "./LoggedError";
+
+class ConfigParseError extends LoggedError {
+
+}
+
+export default ConfigParseError

src/Utils/InvalidRegexError.ts

@@ -0,0 +1,22 @@
+import ExtendableError from "es6-error";
+
+class InvalidRegexError extends ExtendableError {
+    constructor(regex: RegExp | RegExp[], val?: string, url?: string) {
+        const msgParts = [
+            'Regex(es) did not match the value given.',
+        ];
+        let regArr = Array.isArray(regex) ? regex : [regex];
+        for(const r of regArr) {
+            msgParts.push(`Regex: ${r}`)
+        }
+        if (val !== undefined) {
+            msgParts.push(`Value: ${val}`);
+        }
+        if (url !== undefined) {
+            msgParts.push(`Sample regex: ${url}`);
+        }
+        super(msgParts.join('\r\n'));
+    }
+}
+
+export default InvalidRegexError;

src/Utils/SimpleError.ts

@@ -0,0 +1,7 @@
+import ExtendableError from "es6-error";
+
+class SimpleError extends ExtendableError {
+
+}
+
+export default SimpleError;

src/Utils/SnoowrapUtils.ts

@@ -1,10 +1,32 @@
-import {Comment, RedditUser} from "snoowrap";
+import Snoowrap, {RedditUser} from "snoowrap";
 import Submission from "snoowrap/dist/objects/Submission";
+import Comment from "snoowrap/dist/objects/Comment";
 import {Duration, DurationUnitsObjectType} from "dayjs/plugin/duration";
 import dayjs, {Dayjs} from "dayjs";
 import Mustache from "mustache";
-import {AuthorOptions, AuthorCriteria} from "../Rule";
-import {ActivityWindowCriteria, ActivityWindowType} from "../Common/interfaces";
+import he from "he";
+import {RuleResult, UserNoteCriteria} from "../Rule";
+import {
+    ActivityWindowType, CommentState, DomainInfo,
+    DurationVal,
+    SubmissionState,
+    TypedActivityStates
+} from "../Common/interfaces";
+import {
+    compareDurationValue, comparisonTextOp,
+    isActivityWindowCriteria,
+    normalizeName, parseDuration,
+    parseDurationComparison, parseGenericValueComparison, parseGenericValueOrPercentComparison, parseSubredditName,
+    truncateStringToLength
+} from "../util";
+import UserNotes from "../Subreddit/UserNotes";
+import {Logger} from "winston";
+import InvalidRegexError from "./InvalidRegexError";
+import SimpleError from "./SimpleError";
+import {AuthorCriteria} from "../Author/Author";
+import { URL } from "url";
+
+export const BOT_LINK = 'https://www.reddit.com/r/ContextModBot/comments/o1dugk/introduction_to_contextmodbot_and_rcb';
 
 export interface AuthorTypedActivitiesOptions extends AuthorActivitiesOptions {
     type?: 'comment' | 'submission',
@@ -12,33 +34,90 @@ export interface AuthorTypedActivitiesOptions extends AuthorActivitiesOptions {
 
 export interface AuthorActivitiesOptions {
     window: ActivityWindowType | Duration
+    chunkSize?: number,
+    // TODO maybe move this into window
+    keepRemoved?: boolean,
 }
 
 export async function getAuthorActivities(user: RedditUser, options: AuthorTypedActivitiesOptions): Promise<Array<Submission | Comment>> {
 
-    let window: number | Dayjs,
-        chunkSize = 30;
-    if (typeof options.window !== 'number') {
-        const endTime = dayjs();
-        let d;
-        if (dayjs.isDuration(options.window)) {
-            d = options.window;
-        } else {
-            // @ts-ignore
-            d = dayjs.duration(options.window);
+    const {
+        chunkSize: cs = 100,
+        window: optWindow,
+        keepRemoved = true,
+    } = options;
+
+    let satisfiedCount: number | undefined,
+        satisfiedEndtime: Dayjs | undefined,
+        chunkSize = Math.min(cs, 100),
+        satisfy = 'any';
+
+    let durVal: DurationVal | undefined;
+    let duration: Duration | undefined;
+
+    let includes: string[] = [];
+    let excludes: string[] = [];
+
+    if(isActivityWindowCriteria(optWindow)) {
+        const {
+            satisfyOn = 'any',
+            count,
+            duration,
+            subreddits: {
+                include = [],
+                exclude = [],
+            } = {},
+        } = optWindow;
+
+        includes = include.map(x => parseSubredditName(x).toLowerCase());
+        excludes = exclude.map(x => parseSubredditName(x).toLowerCase());
+
+        if(includes.length > 0 && excludes.length > 0) {
+            // TODO add logger so this can be logged...
+            // this.logger.warn('include and exclude both specified, exclude will be ignored');
         }
-        if (!dayjs.isDuration(d)) {
-            // TODO print object
-            throw new Error('window given was not a number, a valid ISO8601 duration, a Day.js duration, or well-formed Duration options');
-        }
-        window = endTime.subtract(d.asMilliseconds(), 'milliseconds');
+        satisfiedCount = count;
+        durVal = duration;
+        satisfy = satisfyOn
+    } else if(typeof optWindow === 'number') {
+        satisfiedCount = optWindow;
     } else {
-        window = options.window;
-        // use whichever is smaller so we only do one api request if window is smaller than default chunk size
-        chunkSize = Math.min(chunkSize, window);
+        durVal = optWindow as DurationVal;
     }
+
+    // if count is less than max limit (100) go ahead and just get that many. may result in faster response time for low numbers
+    if(satisfiedCount !== undefined) {
+        chunkSize = Math.min(chunkSize, satisfiedCount);
+    }
+
+    if(durVal !== undefined) {
+        const endTime = dayjs();
+        if (typeof durVal === 'object') {
+            duration = dayjs.duration(durVal);
+            if (!dayjs.isDuration(duration)) {
+                throw new Error('window value given was not a well-formed Duration object');
+            }
+        } else {
+            try {
+                duration = parseDuration(durVal);
+            } catch (e) {
+                if (e instanceof InvalidRegexError) {
+                    throw new Error(`window value of '${durVal}' could not be parsed as a valid ISO8601 duration or DayJS duration shorthand (see Schema)`);
+                }
+                throw e;
+            }
+        }
+        satisfiedEndtime = endTime.subtract(duration.asMilliseconds(), 'milliseconds');
+    }
+
+    if(satisfiedCount === undefined && satisfiedEndtime === undefined) {
+        throw new Error('window value was not valid');
+    } else if(satisfy === 'all' && !(satisfiedCount !== undefined && satisfiedEndtime !== undefined)) {
+        // even though 'all' was requested we don't have two criteria so its really 'any' logic
+        satisfy = 'any';
+    }
+
     let items: Array<Submission | Comment> = [];
-    let lastItemDate;
     //let count = 1;
     let listing;
     switch (options.type) {
@@ -53,23 +132,78 @@ export async function getAuthorActivities(user: RedditUser, options: AuthorTyped
             break;
     }
     let hitEnd = false;
+    let offset = chunkSize;
     while (!hitEnd) {
-        items = items.concat(listing);
-        if (typeof window === 'number') {
-            hitEnd = items.length >= window
-        } else {
-            const lastItem = listing[listing.length - 1];
-            const lastUtc = await lastItem.created_utc
-            lastItemDate = dayjs(lastUtc);
-            if (lastItemDate.isBefore(window)) {
-                hitEnd = true;
+
+        let countOk = false,
+            timeOk = false;
+
+        let listSlice = listing.slice(offset - chunkSize)
+        // TODO partition list by filtered so we can log a debug statement with count of filtered out activities
+        if (includes.length > 0) {
+            listSlice = listSlice.filter(x => {
+                const actSub = x.subreddit.display_name.toLowerCase();
+                return includes.includes(actSub);
+            });
+        } else if (excludes.length > 0) {
+            listSlice = listSlice.filter(x => {
+                const actSub = x.subreddit.display_name.toLowerCase();
+                return !excludes.includes(actSub);
+            });
+        }
+
+        if(!keepRemoved) {
+            // snoowrap typings think 'removed' property does not exist on submission
+            // @ts-ignore
+            listSlice = listSlice.filter(x => !activityIsRemoved(x));
+        }
+
+        if (satisfiedCount !== undefined && items.length + listSlice.length >= satisfiedCount) {
+            // satisfied count
+            if(satisfy === 'any') {
+                items = items.concat(listSlice).slice(0, satisfiedCount);
+                break;
+            }
+            countOk = true;
+        }
+
+        let truncatedItems: Array<Submission | Comment> = [];
+        if(satisfiedEndtime !== undefined) {
+            truncatedItems = listSlice.filter((x) => {
+                const utc = x.created_utc * 1000;
+                const itemDate = dayjs(utc);
+                // @ts-ignore
+                return satisfiedEndtime.isBefore(itemDate);
+            });
+
+            if (truncatedItems.length !== listSlice.length) {
+                if(satisfy === 'any') {
+                    // satisfied duration
+                    items = items.concat(truncatedItems);
+                    break;
+                }
+                timeOk = true;
             }
         }
-        if (!hitEnd) {
-            hitEnd = listing.isFinished;
+
+        // if we've satisfied everything take whichever is bigger
+        if(satisfy === 'all' && countOk && timeOk) {
+            if(satisfiedCount as number > items.length + truncatedItems.length) {
+                items = items.concat(listSlice).slice(0, satisfiedCount);
+            } else {
+                items = items.concat(truncatedItems);
+            }
+            break;
         }
+
+        // if we got this far neither count nor time was satisfied (or both) so just add all items from listing and fetch more if possible
+        items = items.concat(listSlice);
+
+        hitEnd = listing.isFinished;
+
         if (!hitEnd) {
-            listing.fetchMore({amount: chunkSize});
+            offset += chunkSize;
+            listing = await listing.fetchMore({amount: chunkSize});
         }
     }
     return Promise.resolve(items);
@@ -83,77 +217,500 @@ export const getAuthorSubmissions = async (user: RedditUser, options: AuthorActi
     return await getAuthorActivities(user, {...options, type: 'submission'}) as unknown as Promise<Submission[]>;
 }
 
-export const renderContent = async (content: string, data: (Submission | Comment), additionalData = {}) => {
+export const renderContent = async (template: string, data: (Submission | Comment), ruleResults: RuleResult[] = [], usernotes: UserNotes) => {
     const templateData: any = {
         kind: data instanceof Submission ? 'submission' : 'comment',
         author: await data.author.name,
+        // make this a getter so that if we don't load notes (and api call) if we don't need to
+        // didn't work either for some reason
+        // tried to get too fancy :(
+        // get notes() {
+        //     return usernotes.getUserNotes(data.author).then((notesData) => {
+        //         // return usable notes data with some stats
+        //         const current = notesData.length > 0 ? notesData[notesData.length -1] : undefined;
+        //         // group by type
+        //         const grouped = notesData.reduce((acc: any, x) => {
+        //             const {[x.noteType]: nt = []} = acc;
+        //             return Object.assign(acc, {[x.noteType]: nt.concat(x)});
+        //         }, {});
+        //         return {
+        //             data: notesData,
+        //             current,
+        //             ...grouped,
+        //         };
+        //     });
+        // },
+        // when i was trying to use mustache-async (didn't work)
+        // notes: async () => {
+        //     const notesData = await usernotes.getUserNotes(data.author);
+        //     // return usable notes data with some stats
+        //     const current = notesData.length > 0 ? notesData[notesData.length -1] : undefined;
+        //     // group by type
+        //     const grouped = notesData.reduce((acc: any, x) => {
+        //         const {[x.noteType]: nt = []} = acc;
+        //         return Object.assign(acc, {[x.noteType]: nt.concat(x)});
+        //     }, {});
+        //     return {
+        //         data: notesData,
+        //         current,
+        //         ...grouped,
+        //     };
+        // },
         permalink: data.permalink,
+        botLink: BOT_LINK,
+    }
+    if(template.includes('{{item.notes')) {
+        // we need to get notes
+        const notesData = await usernotes.getUserNotes(data.author);
+        // return usable notes data with some stats
+        const current = notesData.length > 0 ? notesData[notesData.length -1] : undefined;
+        // group by type
+        const grouped = notesData.reduce((acc: any, x) => {
+            const {[x.noteType]: nt = []} = acc;
+            return Object.assign(acc, {[x.noteType]: nt.concat(x)});
+        }, {});
+        templateData.notes = {
+            data: notesData,
+            current,
+            ...grouped,
+        };
     }
     if (data instanceof Submission) {
         templateData.url = data.url;
         templateData.title = data.title;
     }
+    // normalize rule names and map context data
+    // NOTE: we are relying on users to use unique names for rules. If they don't only the last rule run of kind X will have its results here
+    const normalizedRuleResults = ruleResults.reduce((acc: object, ruleResult) => {
+        const {
+            name, triggered,
+            data = {},
+            result,
+            premise: {
+                kind
+            }
+        } = ruleResult;
+        // remove all non-alphanumeric characters (spaces, dashes, underscore) and set to lowercase
+        // we will set this as the rule property name to make it easy to access results from mustache template
+        const normalName = normalizeName(name);
+        return {
+            ...acc, [normalName]: {
+                kind,
+                triggered,
+                result,
+                ...data,
+            }
+        };
+    }, {});
 
-    return Mustache.render(content, {...templateData, ...additionalData});
+    const view = {item: templateData, rules: normalizedRuleResults};
+    const rendered = Mustache.render(template, view) as string;
+    return he.decode(rendered);
 }
 
-export const testAuthorCriteria = async (item: (Comment|Submission), authorOpts: AuthorCriteria, include = true) => {
+export const testAuthorCriteria = async (item: (Comment | Submission), authorOpts: AuthorCriteria, include = true, userNotes: UserNotes) => {
     // @ts-ignore
     const author: RedditUser = await item.author;
-    for(const k of Object.keys(authorOpts)) {
-        switch(k) {
-            case 'name':
-               const authPass = () => {
-                   // @ts-ignore
-                   for (const n of authorOpts[k]) {
-                       if (n.toLowerCase() === author.name.toLowerCase()) {
-                          return true;
-                       }
-                   }
-                   return false;
-               }
-               if((include && !authPass) || (!include && authPass)) {
-                   return false;
-               }
-               break;
-            case 'flairCssClass':
-                const css = await item.author_flair_css_class;
-                const cssPass = () => {
-                    // @ts-ignore
-                    for(const c of authorOpts[k]) {
-                        if(c === css) {
-                            return;
+    for (const k of Object.keys(authorOpts)) {
+        // @ts-ignore
+        if (authorOpts[k] !== undefined) {
+            switch (k) {
+                case 'name':
+                    const authPass = () => {
+                        // @ts-ignore
+                        for (const n of authorOpts[k]) {
+                            if (n.toLowerCase() === author.name.toLowerCase()) {
+                                return true;
+                            }
                         }
+                        return false;
                     }
-                    return false;
-                }
-                if((include && !cssPass) || (!include && cssPass)) {
-                    return false;
-                }
-                break;
-            case 'flairText':
-                const text = await item.author_flair_text;
-                const textPass = () => {
-                    // @ts-ignore
-                    for(const c of authorOpts[k]) {
-                        if(c === text) {
-                            return
+                    const authResult = authPass();
+                    if ((include && !authResult) || (!include && authResult)) {
+                        return false;
+                    }
+                    break;
+                case 'flairCssClass':
+                    const css = await item.author_flair_css_class;
+                    const cssPass = () => {
+                        // @ts-ignore
+                        for (const c of authorOpts[k]) {
+                            if (c === css) {
+                                return;
+                            }
                         }
+                        return false;
                     }
-                    return false;
-                }
-                if((include && !textPass) || (!include && textPass)) {
-                    return false;
-                }
-                break;
-            case 'isMod':
-                const mods: RedditUser[] = await item.subreddit.getModerators();
-                const isModerator = mods.some(x => x.name === item.author.name);
-                const modMatch = authorOpts.isMod === isModerator;
-                if((include && !modMatch) || (!include && !modMatch)) {
-                    return false;
-                }
+                    const cssResult = cssPass();
+                    if ((include && !cssResult) || (!include && cssResult)) {
+                        return false;
+                    }
+                    break;
+                case 'flairText':
+                    const text = await item.author_flair_text;
+                    const textPass = () => {
+                        // @ts-ignore
+                        for (const c of authorOpts[k]) {
+                            if (c === text) {
+                                return
+                            }
+                        }
+                        return false;
+                    };
+                    const textResult = textPass();
+                    if ((include && !textResult) || (!include && textResult)) {
+                        return false;
+                    }
+                    break;
+                case 'isMod':
+                    const mods: RedditUser[] = await item.subreddit.getModerators();
+                    const isModerator = mods.some(x => x.name === item.author.name);
+                    const modMatch = authorOpts.isMod === isModerator;
+                    if ((include && !modMatch) || (!include && modMatch)) {
+                        return false;
+                    }
+                    break;
+                case 'age':
+                    const ageTest = compareDurationValue(parseDurationComparison(await authorOpts.age as string), dayjs.unix(await item.author.created));
+                    if ((include && !ageTest) || (!include && ageTest)) {
+                        return false;
+                    }
+                    break;
+                case 'linkKarma':
+                    const lkCompare = parseGenericValueOrPercentComparison(await authorOpts.linkKarma as string);
+                    let lkMatch;
+                    if(lkCompare.isPercent) {
+                        // @ts-ignore
+                        const tk = author.total_karma as number;
+                        lkMatch = comparisonTextOp(author.link_karma / tk, lkCompare.operator, lkCompare.value/100);
+                    } else {
+                        lkMatch = comparisonTextOp(author.link_karma, lkCompare.operator, lkCompare.value);
+                    }
+                    if ((include && !lkMatch) || (!include && lkMatch)) {
+                        return false;
+                    }
+                    break;
+                case 'commentKarma':
+                    const ckCompare = parseGenericValueOrPercentComparison(await authorOpts.commentKarma as string);
+                    let ckMatch;
+                    if(ckCompare.isPercent) {
+                        // @ts-ignore
+                        const ck = author.total_karma as number;
+                        ckMatch = comparisonTextOp(author.comment_karma / ck, ckCompare.operator, ckCompare.value/100);
+                    } else {
+                        ckMatch = comparisonTextOp(author.comment_karma, ckCompare.operator, ckCompare.value);
+                    }
+                    if ((include && !ckMatch) || (!include && ckMatch)) {
+                        return false;
+                    }
+                    break;
+                case 'totalKarma':
+                    const tkCompare = parseGenericValueComparison(await authorOpts.totalKarma as string);
+                    if(tkCompare.isPercent) {
+                        throw new SimpleError(`'totalKarma' value on AuthorCriteria cannot be a percentage`);
+                    }
+                    // @ts-ignore
+                    const totalKarma = author.total_karma as number;
+                    const tkMatch = comparisonTextOp(totalKarma, tkCompare.operator, tkCompare.value);
+                    if ((include && !tkMatch) || (!include && tkMatch)) {
+                        return false;
+                    }
+                    break;
+                case 'verified':
+                    const vMatch = await author.has_verified_mail === authorOpts.verified as boolean;
+                    if ((include && !vMatch) || (!include && vMatch)) {
+                        return false;
+                    }
+                    break;
+                case 'userNotes':
+                    const notes = await userNotes.getUserNotes(item.author);
+                    const notePass = () => {
+                        for (const noteCriteria of authorOpts[k] as UserNoteCriteria[]) {
+                            const {count = '>= 1', search = 'current', type} = noteCriteria;
+                            const {value, operator, isPercent, extra = ''} = parseGenericValueOrPercentComparison(count);
+                            const order = extra.includes('asc') ? 'ascending' : 'descending';
+                            switch (search) {
+                                case 'current':
+                                    if (notes.length > 0 && notes[notes.length - 1].noteType === type) {
+                                        return true;
+                                    }
+                                    break;
+                                case 'consecutive':
+                                    let orderedNotes = notes;
+                                    if (order === 'descending') {
+                                        orderedNotes = [...notes];
+                                        orderedNotes.reverse();
+                                    }
+                                    let currCount = 0;
+                                    for (const note of orderedNotes) {
+                                        if (note.noteType === type) {
+                                            currCount++;
+                                        } else {
+                                            currCount = 0;
+                                        }
+                                        if(isPercent) {
+                                            throw new SimpleError(`When comparing UserNotes with 'consecutive' search 'count' cannot be a percentage. Given: ${count}`);
+                                        }
+                                        if (comparisonTextOp(currCount, operator, value)) {
+                                            return true;
+                                        }
+                                    }
+                                    break;
+                                case 'total':
+                                    if(isPercent) {
+                                        if(comparisonTextOp(notes.filter(x => x.noteType === type).length / notes.length, operator, value/100)) {
+                                            return true;
+                                        }
+                                    } else if(comparisonTextOp(notes.filter(x => x.noteType === type).length, operator, value)) {
+                                        return true;
+                                    }
+                            }
+                        }
+                        return false;
+                    }
+                    const noteResult = notePass();
+                    if ((include && !noteResult) || (!include && noteResult)) {
+                        return false;
+                    }
+                    break;
+            }
         }
     }
     return true;
 }
+
+export interface ItemContent {
+    submissionTitle: string,
+    content: string,
+    author: string,
+    permalink: string,
+}
+
+export const itemContentPeek = async (item: (Comment | Submission), peekLength = 200): Promise<[string, ItemContent]> => {
+    const truncatePeek = truncateStringToLength(peekLength);
+    let content = '';
+    let submissionTitle = '';
+    let peek = '';
+    const author = item.author.name;
+    if (item instanceof Submission) {
+        submissionTitle = item.title;
+        peek = `${truncatePeek(item.title)} by ${author} https://reddit.com${item.permalink}`;
+
+    } else if (item instanceof Comment) {
+        content = truncatePeek(item.body)
+        try {
+            // @ts-ignore
+            const client = item._r as Snoowrap; // protected? idgaf
+            // @ts-ignore
+            const commentSub = await client.getSubmission(item.link_id);
+            const [p, {submissionTitle: subTitle}] = await itemContentPeek(commentSub);
+            submissionTitle = subTitle;
+            peek = `${truncatePeek(content)} in ${subTitle} by ${author} https://reddit.com${item.permalink}`;
+        } catch (err) {
+            // possible comment is not on a submission, just swallow
+        }
+    }
+
+    return [peek, {submissionTitle, content, author, permalink: item.permalink}];
+}
+
+// @ts-ignore
+export const getSubmissionFromComment = async (item: Comment): Promise<Submission> => {
+    try {
+        // @ts-ignore
+        const client = item._r as Snoowrap; // protected? idgaf
+        // @ts-ignore
+        return client.getSubmission(item.link_id);
+    } catch (err) {
+        // possible comment is not on a submission, just swallow
+    }
+}
+
+const SPOTIFY_PODCAST_AUTHOR_REGEX: RegExp = /this episode from (?<author>.*?) on Spotify./;
+const SPOTIFY_PODCAST_AUTHOR_REGEX_URL = 'https://regexr.com/61c2f';
+
+const SPOTIFY_MUSIC_AUTHOR_REGEX: RegExp = /Listen to .*? on Spotify.\s(?<author>.+?)\s·\s(?<mediaType>.+?)\s/;
+const SPOTIFY_MUSIC_AUTHOR_REGEX_URL = 'https://regexr.com/61c2r';
+
+const ANCHOR_AUTHOR_REGEX: RegExp = /by (?<author>.+?)$/;
+const ANCHOR_AUTHOR_REGEX_URL = 'https://regexr.com/61c31';
+
+export const getAttributionIdentifier = (sub: Submission, useParentMediaDomain = false): DomainInfo => {
+    let domain: string = '';
+    let displayDomain: string = '';
+    let domainIdents: string[] = useParentMediaDomain ? [sub.domain] : [];
+    let provider: string | undefined;
+    let mediaType: string | undefined;
+    if (!useParentMediaDomain && sub.secure_media?.oembed !== undefined) {
+        const {
+            author_url,
+            author_name,
+            description,
+            provider_name,
+        } = sub.secure_media?.oembed;
+        switch(provider_name) {
+            case 'Spotify':
+                if(description !== undefined) {
+                    let match = description.match(SPOTIFY_PODCAST_AUTHOR_REGEX);
+                    if(match !== null) {
+                        const {author} = match.groups as any;
+                        displayDomain = author;
+                        domainIdents.push(author);
+                        mediaType = 'Podcast';
+                    } else {
+                        match = description.match(SPOTIFY_MUSIC_AUTHOR_REGEX);
+                        if(match !== null) {
+                            const {author, mediaType: mt} = match.groups as any;
+                            displayDomain = author;
+                            domainIdents.push(author);
+                            mediaType = mt.toLowerCase();
+                        }
+                    }
+                }
+            break;
+            case 'Anchor FM Inc.':
+                if(author_name !== undefined) {
+                    let match = author_name.match(ANCHOR_AUTHOR_REGEX);
+                    if(match !== null) {
+                        const {author} = match.groups as any;
+                        displayDomain = author;
+                        domainIdents.push(author);
+                        mediaType = 'podcast';
+                    }
+                }
+            break;
+            case 'YouTube':
+                mediaType = 'Video/Audio';
+                break;
+            default:
+                // nah
+        }
+        // handles yt, vimeo, twitter fine
+        if(displayDomain === '') {
+            if (author_name !== undefined) {
+                domainIdents.push(author_name);
+                if (displayDomain === '') {
+                    displayDomain = author_name;
+                }
+            }
+            if (author_url !== undefined) {
+                domainIdents.push(author_url);
+                domain = author_url;
+                if (displayDomain === '') {
+                    displayDomain = author_url;
+                }
+            }
+        }
+        if(displayDomain === '') {
+            // we have media but could not parse stuff for some reason just use url
+            const u = new URL(sub.url);
+            displayDomain = u.pathname;
+            domainIdents.push(u.pathname);
+        }
+        provider = provider_name;
+    } else if(sub.secure_media?.type !== undefined) {
+        domainIdents.push(sub.secure_media?.type);
+        domain = sub.secure_media?.type;
+    } else {
+        domain = sub.domain;
+    }
+
+    if(domain === '') {
+        domain = sub.domain;
+    }
+    if (displayDomain === '') {
+        displayDomain = domain;
+    }
+
+    return {display: displayDomain, domain, aliases: domainIdents, provider, mediaType};
+}
+
+export const isItem = (item: Submission | Comment, stateCriteria: TypedActivityStates, logger: Logger): [boolean, SubmissionState|CommentState|undefined] => {
+    if (stateCriteria.length === 0) {
+        return [true, undefined];
+    }
+
+    const log = logger.child({leaf: 'Item Check'});
+
+    for (const crit of stateCriteria) {
+        const [pass, passCrit] = (() => {
+            for (const k of Object.keys(crit)) {
+                // @ts-ignore
+                if (crit[k] !== undefined) {
+                    switch(k) {
+                        case 'removed':
+                            const removed = activityIsRemoved(item);
+                            if (removed !== crit['removed']) {
+                                // @ts-ignore
+                                log.debug(`Failed: Expected => ${k}:${crit[k]} | Found => ${k}:${removed}`)
+                                return [false, crit];
+                            }
+                            break;
+                        case 'deleted':
+                            const deleted = activityIsDeleted(item);
+                            if (deleted !== crit['deleted']) {
+                                // @ts-ignore
+                                log.debug(`Failed: Expected => ${k}:${crit[k]} | Found => ${k}:${deleted}`)
+                                return [false, crit];
+                            }
+                            break;
+                        case 'filtered':
+                            const filtered = activityIsFiltered(item);
+                            if (filtered !== crit['filtered']) {
+                                // @ts-ignore
+                                log.debug(`Failed: Expected => ${k}:${crit[k]} | Found => ${k}:${filtered}`)
+                                return [false, crit];
+                            }
+                            break;
+                        default:
+                            // @ts-ignore
+                            if (item[k] !== undefined) {
+                                // @ts-ignore
+                                if (item[k] !== crit[k]) {
+                                    // @ts-ignore
+                                    log.debug(`Failed: Expected => ${k}:${crit[k]} | Found => ${k}:${item[k]}`)
+                                    return [false, crit];
+                                }
+                            } else {
+                                log.warn(`Tried to test for Item property '${k}' but it did not exist`);
+                            }
+                            break;
+                    }
+                }
+            }
+            log.debug(`Passed: ${JSON.stringify(crit)}`);
+            return [true, crit];
+        })() as [boolean, SubmissionState|CommentState|undefined];
+        if (pass) {
+            return [true, passCrit];
+        }
+    }
+    return [false, undefined];
+}
+
+export const activityIsRemoved = (item: Submission|Comment): boolean => {
+    if(item instanceof Submission) {
+        // when automod filters a post it gets this category
+        return item.banned_at_utc !== null && item.removed_by_category !== 'automod_filtered';
+    }
+    // when automod filters a comment item.removed === false
+    // so if we want to processing filtered comments we need to check for this
+    return item.banned_at_utc !== null && item.removed;
+}
+
+export const activityIsFiltered = (item: Submission|Comment): boolean => {
+    if(item instanceof Submission) {
+        // when automod filters a post it gets this category
+        return item.banned_at_utc !== null && item.removed_by_category === 'automod_filtered';
+    }
+    // when automod filters a comment item.removed === false
+    // so if we want to processing filtered comments we need to check for this
+    return item.banned_at_utc !== null && !item.removed;
+}
+
+export const activityIsDeleted = (item: Submission|Comment): boolean => {
+    if(item instanceof Submission) {
+        return item.removed_by_category === 'deleted';
+    }
+    return item.author.name === '[deleted]'
+}

src/index.ts

@@ -1,158 +1,136 @@
-import snoowrap from "snoowrap";
-import minimist from 'minimist';
 import winston from 'winston';
 import 'winston-daily-rotate-file';
 import dayjs from 'dayjs';
 import utc from 'dayjs/plugin/utc.js';
 import dduration from 'dayjs/plugin/duration.js';
-import {labelledFormat} from "./util";
-import EventEmitter from "events";
+import relTime from 'dayjs/plugin/relativeTime.js';
+import sameafter from 'dayjs/plugin/isSameOrAfter.js';
+import samebefore from 'dayjs/plugin/isSameOrBefore.js';
 import {Manager} from "./Subreddit/Manager";
-import pEvent from "p-event";
+import {Command} from 'commander';
+import {checks, getUniversalOptions, limit} from "./Utils/CommandConfig";
+import {App} from "./App";
+import Submission from "snoowrap/dist/objects/Submission";
+import {COMMENT_URL_ID, parseLinkIdentifier, SUBMISSION_URL_ID} from "./util";
 
 dayjs.extend(utc);
 dayjs.extend(dduration);
+dayjs.extend(relTime);
+dayjs.extend(sameafter);
+dayjs.extend(samebefore);
 
-const {transports} = winston;
+const commentReg = parseLinkIdentifier([COMMENT_URL_ID]);
+const submissionReg = parseLinkIdentifier([SUBMISSION_URL_ID]);
 
-const argv = minimist(process.argv.slice(2));
-const {
-    _: subredditsArgs = [],
-    clientId = process.env.CLIENT_ID,
-    clientSecret = process.env.CLIENT_SECRET,
-    accessToken = process.env.ACCESS_TOKEN,
-    refreshToken = process.env.REFRESH_TOKEN,
-    logDir = process.env.LOG_DIR,
-    logLevel = process.env.LOG_LEVEL,
-} = argv;
-
-const logPath = logDir ?? `${process.cwd()}/logs`;
-
-// @ts-ignore
-const rotateTransport = new winston.transports.DailyRotateFile({
-    dirname: logPath,
-    createSymlink: true,
-    symlinkName: 'contextBot-current.log',
-    filename: 'contextBot-%DATE%.log',
-    datePattern: 'YYYY-MM-DD',
-    maxSize: '5m'
-});
-
-const consoleTransport = new transports.Console();
-
-const myTransports = [
-    consoleTransport,
-];
-
-if (typeof logPath === 'string') {
-    // @ts-ignore
-    myTransports.push(rotateTransport);
+const program = new Command();
+for (const o of getUniversalOptions()) {
+    program.addOption(o);
 }
 
-const loggerOptions = {
-    level: logLevel || 'info',
-    format: labelledFormat(),
-    transports: myTransports,
-};
-
-winston.loggers.add('default', loggerOptions);
-
-const logger = winston.loggers.get('default');
-
-const version = process.env.VERSION || 'dev';
-
-let subredditsArg = subredditsArgs;
-if (subredditsArg.length === 0) {
-    // try to get from comma delim env variable
-    const subenv = process.env.SUBREDDITS;
-    if (typeof subenv === 'string') {
-        subredditsArg = subenv.split(',');
-    }
-}
 (async function () {
-    const creds = {
-        userAgent: `web:contextBot:${version}`,
-        clientId,
-        clientSecret,
-        refreshToken,
-        accessToken,
-    };
     try {
-        const client = new snoowrap(creds);
-        client.config({warnings: true, retryErrorCodes: [500], maxRetryAttempts: 2, debug: logLevel === 'debug'});
 
-        //const me = await client.getMe().name;
+        program
+            .command('run')
+            .description('Runs bot normally')
+            .action(async (run, command) => {
+                const app = new App(program.opts());
+                await app.buildManagers();
+                await app.runManagers();
+            });
 
-// determine which subreddits this account has appropriate access to
-        let availSubs = [];
-        for (const sub of await client.getModeratedSubreddits()) {
-            // TODO don't know a way to check permissions yet
-            availSubs.push(sub);
-            // if(sub.user_is_moderator) {
-            //     const modUser = sub.getModerators().find(x => x.name === myName);
-            //     const canMod = modUser.features
-            // }
-        }
+        program
+            .command('check <activityIdentifier> [type]')
+            .description('Run check(s) on a specific activity', {
+                activityIdentifier: 'Either a permalink URL or the ID of the Comment or Submission',
+                type: `If activityIdentifier is not a permalink URL then the type of activity ('comment' or 'submission'). May also specify 'submission' type when using a permalink to a comment to get the Submission`,
+            })
+            .addOption(checks)
+            .action(async (activityIdentifier, type, commandOptions = {}) => {
+                const {checks = []} = commandOptions;
+                const app = new App(program.opts());
 
-        let subsToRun = [];
-// if user specified subs to run on check they are all subs client can mod
-        if (subredditsArgs.length > 0) {
-            for (const sub of subredditsArg) {
-                const asub = availSubs.find(x => x.name.toLowerCase() === sub.trim().toLowerCase())
-                if (asub === undefined) {
-                    logger.error(`Will not run on ${sub} because is not modded by, or does not have appropriate permissions to mod with, for this client.`);
-                } else {
-                    subsToRun.push(asub);
+                let a;
+                const commentId = commentReg(activityIdentifier);
+                if (commentId !== undefined) {
+                    if (type !== 'submission') {
+                        // @ts-ignore
+                        a = await app.client.getComment(commentId);
+                    } else {
+                        // @ts-ignore
+                        a = await app.client.getSubmission(submissionReg(activityIdentifier) as string);
+                    }
+                }
+                if (a === undefined) {
+                    const submissionId = submissionReg(activityIdentifier);
+                    if (submissionId !== undefined) {
+                        if (type === 'comment') {
+                            throw new Error(`Detected URL was for a submission but type was 'comment', cannot get activity`);
+                        } else {
+                            // @ts-ignore
+                            a = await app.client.getSubmission(submissionId);
+                        }
+                    }
                 }
-            }
-        } else {
-            // otherwise assume all moddable subs from client should be run on
-            subsToRun = availSubs;
-        }
 
-        let subSchedule: Manager[] = [];
+                if (a === undefined) {
+                    // if we get this far then probably not a URL
+                    if (type === undefined) {
+                        throw new Error(`activityIdentifier was not a valid Reddit URL and type was not specified`);
+                    }
+                    if (type === 'comment') {
+                        // @ts-ignore
+                        a = await app.client.getComment(activityIdentifier);
+                    } else {
+                        // @ts-ignore
+                        a = await app.client.getSubmission(activityIdentifier);
+                    }
+                }
 
-        // get configs for subs we want to run on and build/validate them
-        for (const sub of subsToRun) {
-            let content = undefined;
-            let json = undefined;
-            try {
-                const wiki = sub.getWikiPage('contextbot');
-                content = await wiki.content_md;
-            } catch (err) {
-                logger.error(`Could not read wiki configuration for ${sub.display_name}. Please ensure the page 'contextbot' exists and is readable -- error: ${err.message}`);
-                continue;
-            }
-            try {
-                json = JSON.parse(content);
+                // @ts-ignore
+                const activity = await a.fetch();
+                const sub = await activity.subreddit.display_name;
+                await app.buildManagers([sub]);
+                if (app.subManagers.length > 0) {
+                    const manager = app.subManagers.find(x => x.subreddit.display_name === sub) as Manager;
+                    await manager.runChecks(type === 'comment' ? 'Comment' : 'Submission', activity, checks);
+                }
+            });
 
-            } catch (err) {
-                logger.error(`Wiki page contents for ${sub.display_name} was not valid -- error: ${err.message}`);
-                continue;
-            }
-            try {
-                subSchedule.push(new Manager(sub, client, logger, json));
-            } catch (err) {
-                logger.error(`Config for ${sub.display_name} was not valid, will not run for this subreddit`);
-            }
-        }
+        program.command('unmoderated <subreddits...>')
+            .description('Run checks on all unmoderated activity in the modqueue', {
+                subreddits: 'The list of subreddits to run on. If not specified will run on all subreddits the account has moderation access to.'
+            })
+            .addOption(checks)
+            .addOption(limit)
+            .action(async (subreddits = [], commandOptions = {}) => {
+                const {checks = [], limit = 100} = commandOptions;
+                const app = new App(program.opts());
 
-        const emitter = new EventEmitter();
+                await app.buildManagers(subreddits);
 
-        for (const manager of subSchedule) {
-            manager.handle();
-        }
+                for(const manager of app.subManagers) {
+                    const activities = await manager.subreddit.getUnmoderated({limit});
+                    for(const a of activities.reverse()) {
+                        await manager.runChecks(a instanceof Submission ? 'Submission' : 'Comment', a, checks);
+                    }
+                }
+            });
+
+
+        await program.parseAsync();
 
-        // never hits so we can run indefinitely
-        await pEvent(emitter, 'end');
     } catch (err) {
-        if(err.name === 'StatusCodeError' && err.response !== undefined) {
+        const logger = winston.loggers.get('default');
+        if (err.name === 'StatusCodeError' && err.response !== undefined) {
             const authHeader = err.response.headers['www-authenticate'];
-            if(authHeader !== undefined && authHeader.includes('insufficient_scope')) {
+            if (authHeader !== undefined && authHeader.includes('insufficient_scope')) {
                 logger.error('Reddit responded with a 403 insufficient_scope, did you choose the correct scopes?');
             }
         }
-        debugger;
         console.log(err);
     }
 }());
+export {Author} from "./Author/Author";
+export {AuthorCriteria} from "./Author/Author";
+export {AuthorOptions} from "./Author/Author";

src/util.ts

@@ -1,62 +1,81 @@
 import winston, {Logger} from "winston";
 import jsonStringify from 'safe-stable-stringify';
-import dayjs from 'dayjs';
-import {RulePremise, RuleResult} from "./Rule";
+import dayjs, {Dayjs, OpUnitType} from 'dayjs';
+import {isRuleSetResult, RulePremise, RuleResult, RuleSetResult} from "./Rule";
 import deepEqual from "fast-deep-equal";
-import utc from 'dayjs/plugin/utc.js';
-import dduration from 'dayjs/plugin/duration.js';
-
-dayjs.extend(utc);
-dayjs.extend(dduration);
+import {Duration} from 'dayjs/plugin/duration.js';
+import Ajv from "ajv";
+import {InvalidOptionArgumentError} from "commander";
+import Submission from "snoowrap/dist/objects/Submission";
+import {Comment} from "snoowrap";
+import {inflateSync, deflateSync} from "zlib";
+import {ActivityWindowCriteria, DurationComparison, GenericComparison, StringOperator} from "./Common/interfaces";
+import JSON5 from "json5";
+import yaml, {JSON_SCHEMA} from "js-yaml";
+import SimpleError from "./Utils/SimpleError";
+import InvalidRegexError from "./Utils/InvalidRegexError";
 
 const {format} = winston;
 const {combine, printf, timestamp, label, splat, errors} = format;
 
 const s = splat();
 const SPLAT = Symbol.for('splat')
-const errorsFormat = errors({stack: true});
+//const errorsFormat = errors({stack: true});
 const CWD = process.cwd();
 
-export const truncateStringToLength = (length: number, truncStr = '...') => (str: string) => str.length > length ? `${str.slice(0, length)}${truncStr}` : str;
-
-export const loggerMetaShuffle = (logger: Logger, newLeaf: (string | undefined | null) = null, extraLabels: string[] = [], {truncateLength = 15} = {}) => {
-    const labelTrunc = truncateStringToLength(truncateLength);
-    const {labels = [], leaf} = logger.defaultMeta || {};
-    return {
-        labels: labels.concat(extraLabels.map(x => labelTrunc(x))),
-        leaf: newLeaf
-    };
+// const errorAwareFormat = (info: any) => {
+//     if(info instanceof SimpleError) {
+//         return errors()(info);
+//     }
+// }
+const errorAwareFormat = {
+    transform: (info: any, opts: any) => {
+        // don't need to log stack trace if we know the error is just a simple message (we handled it)
+        const stack = !(info instanceof SimpleError) && !(info.message instanceof SimpleError);
+        return errors().transform(info, { stack });
+    }
 }
 
-let longestLabel = 3;
-// @ts-ignore
-export const defaultFormat = printf(({level, message, label = 'App', labels = [], leaf, itemId, timestamp, [SPLAT]: splatObj, stack, ...rest}) => {
+export const PASS = '✔';
+export const FAIL = '✘';
+
+export const truncateStringToLength = (length: number, truncStr = '...') => (str: string) => str.length > length ? `${str.slice(0, length - truncStr.length - 1)}${truncStr}` : str;
+
+export const defaultFormat = printf(({
+                                         level,
+                                         message,
+                                         labels = ['App'],
+                                         leaf,
+                                         itemId,
+                                         timestamp,
+                                        // @ts-ignore
+                                         [SPLAT]: splatObj,
+                                         stack,
+                                         ...rest
+                                     }) => {
     let stringifyValue = splatObj !== undefined ? jsonStringify(splatObj) : '';
-    if (label.length > longestLabel) {
-        longestLabel = label.length;
-    }
     let msg = message;
     let stackMsg = '';
     if (stack !== undefined) {
         const stackArr = stack.split('\n');
-        msg = stackArr[0];
+        const stackTop = stackArr[0];
         const cleanedStack = stackArr
             .slice(1) // don't need actual error message since we are showing it as msg
             .map((x: string) => x.replace(CWD, 'CWD')) // replace file location up to cwd for user privacy
             .join('\n'); // rejoin with newline to preserve formatting
         stackMsg = `\n${cleanedStack}`;
+        if(msg === undefined || msg === null || typeof message === 'object') {
+            msg = stackTop;
+        } else {
+            stackMsg = `\n${stackTop}${stackMsg}`
+        }
     }
 
-    let labelContent = `[${label.padEnd(longestLabel)}]`;
-    if(labels.length > 0 || leaf !== null) {
-        let nodes = labels;
-        if(leaf !== null) {
-            nodes.push(leaf);
-        }
-        //labelContent = `${labels.slice(0, labels.length).map((x: string) => `[${x}]`).join(' ')}`
-        labelContent = `${nodes.map((x: string) => `[${x}]`).join(' ')}`;
+    let nodes = labels;
+    if (leaf !== null && leaf !== undefined) {
+        nodes.push(leaf);
     }
-    //let leafContent = leaf !== undefined ? ` (${leaf})` : '';
+    const labelContent = `${nodes.map((x: string) => `[${x}]`).join(' ')}`;
 
     return `${timestamp} ${level.padEnd(7)}: ${labelContent} ${msg}${stringifyValue !== '' ? ` ${stringifyValue}` : ''}${stackMsg}`;
 });
@@ -72,24 +91,12 @@ export const labelledFormat = (labelName = 'App') => {
         ),
         l,
         s,
-        errorsFormat,
+        errorAwareFormat,
+        //errorsFormat,
         defaultFormat,
     );
 }
 
-export const createLabelledLogger = (name = 'default', label = 'App') => {
-    if (winston.loggers.has(name)) {
-        return winston.loggers.get(name);
-    }
-    const def = winston.loggers.get('default');
-    winston.loggers.add(name, {
-        transports: def.transports,
-        level: def.level,
-        format: labelledFormat(label)
-    });
-    return winston.loggers.get(name);
-}
-
 export interface groupByOptions {
     lowercase?: boolean
 }
@@ -114,6 +121,10 @@ export const groupBy = <T>(keys: (keyof T)[], opts: groupByOptions = {}) => (arr
     }, {} as Record<string, T[]>)
 };
 
+// match /mealtimesvideos/ /comments/ etc... (?:\/.*\/)
+// matches https://old.reddit.com/r  (?:^.+?)(?:reddit.com\/r)
+// (?:^.+?)(?:reddit.com\/r\/.+\/.\/)
+// (?:.*\/)([\d\w]+?)(?:\/*)
 
 /**
  * @see https://stackoverflow.com/a/61033353/1469797
@@ -129,12 +140,29 @@ export const parseUsableLinkIdentifier = (regexes: RegExp[] = [REGEX_YOUTUBE]) =
         if (matches.length > 0) {
             // use first capture group
             // TODO make this configurable at some point?
-            return matches[0][matches[0].length - 1];
+            const captureGroup = matches[0][matches[0].length - 1];
+            if(captureGroup !== '') {
+                return captureGroup;
+            }
         }
     }
     return val;
 }
 
+export const parseLinkIdentifier = (regexes: RegExp[]) => {
+    const u = parseUsableLinkIdentifier(regexes);
+    return (val: string): (string | undefined) => {
+        const id = u(val);
+        if (id === val) {
+            return undefined;
+        }
+        return id;
+    }
+}
+
+export const SUBMISSION_URL_ID: RegExp = /(?:^.+?)(?:reddit.com\/r)(?:\/[\w\d]+){2}(?:\/)([\w\d]*)/g;
+export const COMMENT_URL_ID: RegExp = /(?:^.+?)(?:reddit.com\/r)(?:\/[\w\d]+){4}(?:\/)([\w\d]*)/g;
+
 export function sleep(ms: number) {
     return new Promise(resolve => setTimeout(resolve, ms));
 }
@@ -173,8 +201,355 @@ export const determineNewResults = (existing: RuleResult[], val: RuleResult | Ru
     return newResults;
 }
 
-export const mergeArr = (objValue: [], srcValue: []): (any[]|undefined) => {
+export const mergeArr = (objValue: [], srcValue: []): (any[] | undefined) => {
     if (Array.isArray(objValue)) {
         return objValue.concat(srcValue);
     }
 }
+
+export const ruleNamesFromResults = (results: RuleResult[]) => {
+    return results.map(x => x.name || x.premise.kind).join(' | ')
+}
+
+export const triggeredIndicator = (val: boolean | null): string => {
+    if(val === null) {
+        return '-';
+    }
+    return val ? PASS : FAIL;
+}
+
+export const resultsSummary = (results: (RuleResult|RuleSetResult)[], topLevelCondition: 'OR' | 'AND'): string => {
+    const parts: string[] = results.map((x) => {
+        if(isRuleSetResult(x)) {
+            return `${triggeredIndicator(x.triggered)} (${resultsSummary(x.results, x.condition)}${x.results.length === 1 ? ` [${x.condition}]` : ''})`;
+        }
+        const res = x as RuleResult;
+        return `${triggeredIndicator(x.triggered)} ${res.name}`;
+    });
+    return parts.join(` ${topLevelCondition} `)
+    //return results.map(x => x.name || x.premise.kind).join(' | ')
+}
+
+export const createAjvFactory = (logger: Logger) => {
+    return  new Ajv({logger: logger, verbose: true, strict: "log", allowUnionTypes: true});
+}
+
+export const percentFromString = (str: string): number => {
+   const n = Number.parseInt(str.replace('%', ''));
+   if(Number.isNaN(n)) {
+       throw new Error(`${str} could not be parsed to a number`);
+   }
+   return n / 100;
+}
+
+export interface numberFormatOptions {
+    toFixed: number,
+    defaultVal?: any,
+    prefix?: string,
+    suffix?: string,
+    round?: {
+        type?: string,
+        enable: boolean,
+        indicate?: boolean,
+    }
+}
+
+export const formatNumber = (val: number | string, options?: numberFormatOptions) => {
+    const {
+        toFixed = 2,
+        defaultVal = null,
+        prefix = '',
+        suffix = '',
+        round,
+    } = options || {};
+    let parsedVal = typeof val === 'number' ? val : Number.parseFloat(val);
+    if (Number.isNaN(parsedVal)) {
+        return defaultVal;
+    }
+    let prefixStr = prefix;
+    const {enable = false, indicate = true, type = 'round'} = round || {};
+    if (enable && !Number.isInteger(parsedVal)) {
+        switch (type) {
+            case 'round':
+                parsedVal = Math.round(parsedVal);
+                break;
+            case 'ceil':
+                parsedVal = Math.ceil(parsedVal);
+                break;
+            case 'floor':
+                parsedVal = Math.floor(parsedVal);
+        }
+        if (indicate) {
+            prefixStr = `~${prefix}`;
+        }
+    }
+    const localeString = parsedVal.toLocaleString(undefined, {
+        minimumFractionDigits: toFixed,
+        maximumFractionDigits: toFixed,
+    });
+    return `${prefixStr}${localeString}${suffix}`;
+};
+
+export function argParseInt(value: any, prev: any = undefined): number {
+    let usedVal = value;
+    if (value === undefined || value === '') {
+        usedVal = prev;
+    }
+    if(usedVal === undefined || usedVal === '') {
+        return usedVal;
+    }
+
+    if (typeof usedVal === 'string') {
+        const parsedValue = parseInt(usedVal, 10);
+        if (isNaN(parsedValue)) {
+            throw new InvalidOptionArgumentError('Not a number.');
+        }
+        return parsedValue;
+    } else if (typeof usedVal === 'number') {
+        return usedVal;
+    }
+    throw new InvalidOptionArgumentError('Not a number.');
+}
+
+export function parseBool(value: any, prev: any = false): boolean {
+    let usedVal = value;
+    if (value === undefined || value === '') {
+        usedVal = prev;
+    }
+    if(usedVal === undefined || usedVal === '') {
+        return false;
+    }
+    if (typeof usedVal === 'string') {
+        return usedVal === 'true';
+    } else if (typeof usedVal === 'boolean') {
+        return usedVal;
+    }
+    throw new InvalidOptionArgumentError('Not a boolean value.');
+}
+
+export function activityWindowText(activities: (Submission | Comment)[], suffix = false): (string | undefined) {
+    if (activities.length === 0) {
+        return undefined;
+    }
+    if (activities.length === 1) {
+        return `1 Item`;
+    }
+
+    return dayjs.duration(dayjs(activities[0].created_utc * 1000).diff(dayjs(activities[activities.length - 1].created_utc * 1000))).humanize(suffix);
+}
+
+export function normalizeName(val: string) {
+    return val.trim().replace(/\W+/g, '').toLowerCase()
+}
+
+// https://github.com/toolbox-team/reddit-moderator-toolbox/wiki/Subreddit-Wikis%3A-usernotes#working-with-the-blob
+export const inflateUserNotes = (blob: string) => {
+    //const binaryData = Buffer.from(blob, 'base64').toString('binary');
+    //const str = pako.inflate(binaryData, {to: 'string'});
+
+    const buffer = Buffer.from(blob, 'base64');
+    const str = inflateSync(buffer).toString('utf-8');
+
+    // @ts-ignore
+    return JSON.parse(str);
+}
+// https://github.com/toolbox-team/reddit-moderator-toolbox/wiki/Subreddit-Wikis%3A-usernotes#working-with-the-blob
+export const deflateUserNotes = (usersObject: object) => {
+    const jsonString = JSON.stringify(usersObject);
+
+    // Deflate/compress the string
+    //const binaryData = pako.deflate(jsonString);
+    const binaryData = deflateSync(jsonString);
+
+    // Convert binary data to a base64 string with a Buffer
+    const blob = Buffer.from(binaryData).toString('base64');
+    return blob;
+}
+
+export const isActivityWindowCriteria = (val: any): val is ActivityWindowCriteria => {
+    if (val !== null && typeof val === 'object') {
+        return (val.count !== undefined && typeof val.count === 'number') ||
+            // close enough
+            val.duration !== undefined;
+    }
+    return false;
+}
+
+export const parseFromJsonOrYamlToObject = (content: string): [object?, Error?, Error?] => {
+    let obj;
+    let jsonErr,
+        yamlErr;
+
+    try {
+        obj = JSON5.parse(content);
+        const oType = obj === null ? 'null' : typeof obj;
+        if (oType !== 'object') {
+            jsonErr = new SimpleError(`Parsing as json produced data of type '${oType}' (expected 'object')`);
+            obj = undefined;
+        }
+    } catch (err) {
+        jsonErr = err;
+    }
+    if (obj === undefined) {
+        try {
+            obj = yaml.load(content, {schema: JSON_SCHEMA, json: true});
+            const oType = obj === null ? 'null' : typeof obj;
+            if (oType !== 'object') {
+                yamlErr = new SimpleError(`Parsing as yaml produced data of type '${oType}' (expected 'object')`);
+                obj = undefined;
+            }
+        } catch (err) {
+            yamlErr = err;
+        }
+    }
+    return [obj, jsonErr, yamlErr];
+}
+
+export const comparisonTextOp = (val1: number, strOp: string, val2: number): boolean => {
+    switch (strOp) {
+        case '>':
+            return val1 > val2;
+        case '>=':
+            return val1 >= val2;
+        case '<':
+            return val1 < val2;
+        case '<=':
+            return val1 <= val2;
+        default:
+            throw new Error(`${strOp} was not a recognized operator`);
+    }
+}
+
+const GENERIC_VALUE_COMPARISON = /^\s*(?<opStr>>|>=|<|<=)\s*(?<value>\d+)(?<extra>\s+.*)*$/
+const GENERIC_VALUE_COMPARISON_URL = 'https://regexr.com/60dq4';
+export const parseGenericValueComparison = (val: string): GenericComparison => {
+    const matches = val.match(GENERIC_VALUE_COMPARISON);
+    if (matches === null) {
+        throw new InvalidRegexError(GENERIC_VALUE_COMPARISON, val, GENERIC_VALUE_COMPARISON_URL)
+    }
+    const groups = matches.groups as any;
+
+    return {
+        operator: groups.opStr as StringOperator,
+        value: Number.parseFloat(groups.value),
+        isPercent: false,
+        extra: groups.extra,
+        displayText: `${groups.opStr} ${groups.value}`
+    }
+}
+
+const GENERIC_VALUE_PERCENT_COMPARISON = /^\s*(?<opStr>>|>=|<|<=)\s*(?<value>\d+)\s*(?<percent>%?)(?<extra>.*)$/
+const GENERIC_VALUE_PERCENT_COMPARISON_URL = 'https://regexr.com/60a16';
+export const parseGenericValueOrPercentComparison = (val: string): GenericComparison => {
+    const matches = val.match(GENERIC_VALUE_PERCENT_COMPARISON);
+    if (matches === null) {
+        throw new InvalidRegexError(GENERIC_VALUE_PERCENT_COMPARISON, val, GENERIC_VALUE_PERCENT_COMPARISON_URL)
+    }
+    const groups = matches.groups as any;
+
+    return {
+        operator: groups.opStr as StringOperator,
+        value: Number.parseFloat(groups.value),
+        isPercent: groups.percent !== '',
+        extra: groups.extra,
+        displayText: `${groups.opStr} ${groups.value}${groups.percent === undefined ? '': '%'}`
+    }
+}
+
+export const dateComparisonTextOp = (val1: Dayjs, strOp: StringOperator, val2: Dayjs, granularity?: OpUnitType): boolean => {
+    switch (strOp) {
+        case '>':
+            return val1.isBefore(val2, granularity);
+        case '>=':
+            return val1.isSameOrBefore(val2, granularity);
+        case '<':
+            return val1.isAfter(val2, granularity);
+        case '<=':
+            return val1.isSameOrAfter(val2, granularity);
+        default:
+            throw new Error(`${strOp} was not a recognized operator`);
+    }
+}
+
+const ISO8601_REGEX: RegExp = /^(-?)P(?=\d|T\d)(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)([DW]))?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?$/;
+const DURATION_REGEX: RegExp = /^\s*(?<time>\d+)\s*(?<unit>days?|weeks?|months?|years?|hours?|minutes?|seconds?|milliseconds?)\s*$/;
+export const parseDuration = (val: string): Duration => {
+    let matches = val.match(DURATION_REGEX);
+    if (matches !== null) {
+        const groups = matches.groups as any;
+        const dur: Duration = dayjs.duration(groups.time, groups.unit);
+        if (!dayjs.isDuration(dur)) {
+            throw new SimpleError(`Parsed value '${val}' did not result in a valid Dayjs Duration`);
+        }
+        return dur;
+    }
+    matches = val.match(ISO8601_REGEX);
+    if (matches !== null) {
+        const dur: Duration = dayjs.duration(val);
+        if (!dayjs.isDuration(dur)) {
+            throw new SimpleError(`Parsed value '${val}' did not result in a valid Dayjs Duration`);
+        }
+        return dur;
+    }
+    throw new InvalidRegexError([DURATION_REGEX, ISO8601_REGEX], val)
+}
+
+/**
+ * Named groups: operator, time, unit
+ * */
+const DURATION_COMPARISON_REGEX: RegExp = /^\s*(?<opStr>>|>=|<|<=)\s*(?<time>\d+)\s*(?<unit>days?|weeks?|months?|years?|hours?|minutes?|seconds?|milliseconds?)\s*$/;
+const DURATION_COMPARISON_REGEX_URL = 'https://regexr.com/609n8';
+export const parseDurationComparison = (val: string): DurationComparison => {
+    const matches = val.match(DURATION_COMPARISON_REGEX);
+    if (matches === null) {
+        throw new InvalidRegexError(DURATION_COMPARISON_REGEX, val, DURATION_COMPARISON_REGEX_URL)
+    }
+    const groups = matches.groups as any;
+    const dur: Duration = dayjs.duration(groups.time, groups.unit);
+    if (!dayjs.isDuration(dur)) {
+        throw new SimpleError(`Parsed value '${val}' did not result in a valid Dayjs Duration`);
+    }
+    return {
+        operator: groups.opStr as StringOperator,
+        duration: dur
+    }
+}
+export const compareDurationValue = (comp: DurationComparison, date: Dayjs) => {
+    const dateToCompare = dayjs().subtract(comp.duration.asSeconds(), 'seconds');
+    return dateComparisonTextOp(date, comp.operator, dateToCompare);
+}
+
+const SUBREDDIT_NAME_REGEX: RegExp = /^\s*(?:\/r\/|r\/)*(\w+)*\s*$/;
+const SUBREDDIT_NAME_REGEX_URL = 'https://regexr.com/61a1d';
+export const parseSubredditName = (val:string): string => {
+    const matches = val.match(SUBREDDIT_NAME_REGEX);
+    if (matches === null) {
+        throw new InvalidRegexError(SUBREDDIT_NAME_REGEX, val, SUBREDDIT_NAME_REGEX_URL)
+    }
+    return matches[1] as string;
+}
+
+const WIKI_REGEX: RegExp = /^\s*wiki:(?<url>[^|]+)\|*(?<subreddit>[^\s]*)\s*$/;
+const WIKI_REGEX_URL = 'https://regexr.com/61bq1';
+const URL_REGEX: RegExp = /^\s*url:(?<url>[^\s]+)\s*$/;
+const URL_REGEX_URL = 'https://regexr.com/61bqd';
+
+export const parseWikiContext = (val: string) => {
+    const matches = val.match(WIKI_REGEX);
+    if (matches === null) {
+        return undefined;
+    }
+    const sub = (matches.groups as any).subreddit as string;
+    return {
+        wiki: (matches.groups as any).url as string,
+        subreddit: sub === '' ? undefined : parseSubredditName(sub)
+    };
+}
+
+export const parseExternalUrl = (val: string) => {
+    const matches = val.match(URL_REGEX);
+    if (matches === null) {
+        return undefined;
+    }
+    return (matches.groups as any).url as string;
+}