Merge branch 'edge'

* On hard limit stop shared streams
* On nanny mode turned off restart any stopped shared streams

.gitignore

@@ -383,3 +383,12 @@ dist
 **/src/**/*.js
 !src/Web/assets/public/yaml/*
 **/src/**/*.map
+/**/*.sqlite
+/**/*.bak
+*.yaml
+*.json5
+
+!src/Schema/*.json
+!docs/**/*.json5
+!docs/**/*.yaml
+!docs/**/*.json

README.md

@@ -1,6 +1,7 @@
-[![Latest Release](https://img.shields.io/github/v/release/foxxmd/context-mod)](https://github.com/FoxxMD/context-mod/releases)
-[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
-[![Docker Pulls](https://img.shields.io/docker/pulls/foxxmd/context-mod)](https://hub.docker.com/r/foxxmd/context-mod)
+# ContextMod [![Latest Release](https://img.shields.io/github/v/release/foxxmd/context-mod)](https://github.com/FoxxMD/context-mod/releases) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Docker Pulls](https://img.shields.io/docker/pulls/foxxmd/context-mod)](https://hub.docker.com/r/foxxmd/context-mod)
+
+<img src="/docs/logo.png" align="right"
+alt="ContextMod logo" width="180" height="176">
 
 **Context Mod** (CM) is an event-based, [reddit](https://reddit.com) moderation bot built on top of [snoowrap](https://github.com/not-an-aardvark/snoowrap) and written in [typescript](https://www.typescriptlang.org/).
 
@@ -19,8 +20,8 @@ Some feature highlights:
   * Default/no configuration runs "All In One" behavior
   * Additional configuration allows web interface to connect to multiple servers
   * Each server instance can run multiple reddit accounts as bots
-* **Per-subreddit configuration** is handled by JSON stored in the subreddit wiki
-* Any text-based actions (comment, submission, message, usernotes, ban, etc...) can be configured via a wiki page or raw text in JSON and support [mustache](https://mustache.github.io) [templating](/docs/actionTemplating.md)
+* **Per-subreddit configuration** is handled by YAML (**like automoderator!**) or JSON stored in the subreddit wiki
+* Any text-based actions (comment, submission, message, usernotes, ban, etc...) can be configured via a wiki page or raw text and supports [mustache](https://mustache.github.io) [templating](/docs/actionTemplating.md)
 * History-based rules support multiple "valid window" types -- [ISO 8601 Durations](https://en.wikipedia.org/wiki/ISO_8601#Durations), [Day.js Durations](https://day.js.org/docs/en/durations/creating), and submission/comment count limits.
 * Support Activity skipping based on:
   * Author criteria (name, css flair/text, age, karma, moderator status, and [Toolbox User Notes](https://www.reddit.com/r/toolbox/wiki/docs/usernotes))
@@ -85,7 +86,7 @@ See the [Moderator's Getting Started Guide](/docs/gettingStartedMod.md)
 
 ## Configuration and Documentation
 
-Context Bot's configuration can be written in JSON, [JSON5](https://json5.org/) or YAML. Its schema conforms to [JSON Schema Draft 7](https://json-schema.org/). Additionally, many **operator** settings can be passed via command line or environmental variables.
+Context Bot's configuration can be written in YAML (like automoderator) or [JSON5](https://json5.org/). Its schema conforms to [JSON Schema Draft 7](https://json-schema.org/). Additionally, many **operator** settings can be passed via command line or environmental variables.
 
 * For **operators** (running the bot instance) see the [Operator Configuration](/docs/operatorConfiguration.md) guide
 * For **moderators** consult the [app schema and examples folder](/docs/#configuration-and-usage)

docs/README.md

@@ -120,6 +120,15 @@ It consists of:
 * **rules** -- The **Rules** for the Rule Set.
 
 Example
+
+YAML
+```yaml
+condition: AND
+# rules are an array
+rules:
+ - aRule
+```
+JSON
 ```json5
 {
   "condition": "AND",

docs/activitiesWindow.md

@@ -17,7 +17,28 @@ Examples of all of the above
 
 <details>
 
+```yaml
+# count, last 100 activities
+window: 100
+
+# duration, last 10 days
+window: 10 days
+
+# duration object, last 2 months and 5 days
+window:
+  months: 2
+  days: 5
+
+# iso 8601 string, last 15 minutes
+window: PT15M
+
+# ActivityWindowCriteria, last 100 activities or 6 weeks of activities (whichever is found first)
+window:
+  count: 100
+  duration: 6 weeks
 ```
+
+```json5
 // count, last 100 activities
 {
   "window": 100
@@ -49,6 +70,7 @@ Examples of all of the above
   }
 }
 ```
+
 </details>
 
 ## Types of Ranges
@@ -95,6 +117,7 @@ If you need to specify multiple units of time for your duration you can instead
 
 Example
 
+JSON
 ```json
 {
   "days": 4,
@@ -102,6 +125,13 @@ Example
   "minutes": 20
 }
 ```
+YAML
+```yaml
+window:
+  days: 4
+  hours: 6
+  minutes: 20
+```
 
 ##### An ISO 8601 duration string
 
@@ -119,6 +149,7 @@ This is an object that lets you specify more granular conditions for your range.
 
 The full object looks like this:
 
+JSON
 ```json
 {
   "count": 100,
@@ -130,6 +161,19 @@ The full object looks like this:
   }
 }
 ```
+YAML
+```yaml
+window:
+  count: 100
+  duration: 10 days
+  satisfyOn: any
+  subreddits:
+    include:
+      - mealtimevideos
+      - pooptimevideos
+    exclude:
+      - videos
+```
 
 ### Specifying Range
 
@@ -142,7 +186,9 @@ If both range properties are specified then the value `satisfyOn` determines how
 
 If **any** then Activities will be retrieved until one of the range properties is met, **whichever occurs first.**
 
-Example 
+Example
+
+JSON
 ```json
 {
   "count": 80,
@@ -150,6 +196,13 @@ Example
   "satisfyOn": "any"
 }
 ```
+YAML
+```yaml
+window:
+  count: 80
+  duration: 90 days
+  satisfyOn: any
+```
 Activities are retrieved in chunks of 100 (or `count`, whichever is smaller)
 
 * If 90 days of activities returns only 40 activities => returns 40 activities
@@ -160,6 +213,8 @@ Activities are retrieved in chunks of 100 (or `count`, whichever is smaller)
 If **all** then both ranges must be satisfied. Effectively, whichever range produces the most Activities will be the one that is used.
 
 Example
+
+JSON
 ```json
 {
   "count": 100,
@@ -167,6 +222,13 @@ Example
   "satisfyOn": "all"
 }
 ```
+YAML
+```yaml
+window:
+  count: 100
+  duration: 90 days
+  satisfyOn: all
+```
 Activities are retrieved in chunks of 100 (or `count`, whichever is smaller)
 
 * If at 90 days of activities => 40 activities retrieved
@@ -187,6 +249,8 @@ You may filter retrieved Activities using an array of subreddits.
 Use **include** to specify which subreddits should be included from results
 
 Example where only activities from /r/mealtimevideos and /r/modsupport will be returned
+
+JSON
 ```json
 {
   "count": 100,
@@ -196,7 +260,17 @@ Example where only activities from /r/mealtimevideos and /r/modsupport will be r
     "include": ["mealtimevideos","modsupport"]
   }
 }
-
+```
+YAML
+```yaml
+window:
+  count: 100
+  duruation: 90 days
+  satisfyOn: any
+  subreddits:
+    include:
+      - mealtimevideos
+      - modsupport
 ```
 
 #### Exclude
@@ -204,6 +278,8 @@ Example where only activities from /r/mealtimevideos and /r/modsupport will be r
 Use **exclude** to specify which subreddits should NOT be in the results
 
 Example where activities from /r/mealtimevideos and /r/modsupport will not be returned in results
+
+JSON
 ```json
 {
   "count": 100,
@@ -214,4 +290,15 @@ Example where activities from /r/mealtimevideos and /r/modsupport will not be re
   }
 }
 ```
+YAML
+```yaml
+window:
+  count: 100
+  duruation: 90 days
+  satisfyOn: any
+  subreddits:
+    exclude:
+      - mealtimevideos
+      - modsupport
+```
 **Note:** `exclude` will be ignored if `include` is also present.

docs/examples/README.md

@@ -18,6 +18,7 @@ This directory contains example of valid, ready-to-go configurations for Context
   * [Author](/docs/examples/author)
   * [Regex](/docs/examples/regex)
   * [Repost](/docs/examples/repost)
+  * [Author and post flairs](/docs/examples/onlyfansFlair)
 * [Toolbox User Notes](/docs/examples/userNotes)
 * [Advanced Concepts](/docs/examples/advancedConcepts)
   * [Rule Sets](/docs/examples/advancedConcepts/ruleSets.json5)

docs/examples/advancedConcepts/README.md

@@ -1,6 +1,6 @@
 ### Named Rules
 
-See [ruleNameReuse.json5](/docs/examples/advancedConcepts/ruleNameReuse.json5)
+See **Rule Name Reuse Examples [YAML](/docs/examples/advancedConcepts/ruleNameReuse.yaml) | [JSON](/docs/examples/advancedConcepts/ruleNameReuse.json5)**
 
 ### Check Order
 
@@ -23,7 +23,7 @@ The `rules` array on a `Checks` can contain both `Rule` objects and `RuleSet` ob
 
 A **Rule Set** is a "nested" set of `Rule` objects with a passing condition specified. These allow you to create more complex trigger behavior by combining multiple rules. 
 
-See **[ruleSets.json5](/docs/examples/advancedConcepts/ruleSets.json5)** for a complete example as well as consulting the [schema](https://json-schema.app/view/%23%2Fdefinitions%2FRuleSetJson?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Fcontext-mod%2Fmaster%2Fsrc%2FSchema%2FApp.json).
+See **ruleSets [YAML](/docs/examples/advancedConcepts/ruleSets.yaml) | [JSON](/docs/examples/advancedConcepts/ruleSets.json5)** for a complete example as well as consulting the [schema](https://json-schema.app/view/%23%2Fdefinitions%2FRuleSetJson?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Fcontext-mod%2Fmaster%2Fsrc%2FSchema%2FApp.json).
 
 ### Rule Order
 

docs/examples/advancedConcepts/ruleNameReuse.yaml

@@ -0,0 +1,52 @@
+checks:
+  - name: Auto Remove SP Karma
+    description: >-
+      Remove submission because author has self-promo >10% and posted in karma
+      subs recently
+    kind: submission
+    rules:
+      # named rules can be referenced at any point in the configuration (where they occur does not matter)
+      # and can be used in any Check
+      # Note: rules do not transfer between subreddit configurations
+      - freekarmasub
+      - name: attr10all
+        kind: attribution
+        criteria:
+          - threshold: '> 10%'
+            window: 90 days
+          - threshold: '> 10%'
+            window: 100
+    actions:
+      - kind: remove
+      - kind: comment
+        content: >-
+          Your submission was removed because you are over reddit's threshold
+          for self-promotion and recently posted this content in a karma sub
+  - name: Free Karma On Submission Alert
+    description: Check if author has posted this submission in 'freekarma' subreddits
+    kind: submission
+    rules:
+      # rules can be re-used throughout a configuration by referencing them by name
+      #
+      # The rule name itself can only contain spaces, hyphens and underscores
+      # The value used to reference it will have all of these removed, and lower-cased
+      #
+      # so to reference this rule use the value 'freekarmasub'
+      - name: Free_Karma-SUB
+        kind: recentActivity
+        lookAt: submissions
+        useSubmissionAsReference: true
+        thresholds:
+          - threshold: '>= 1'
+            subreddits:
+              - DeFreeKarma
+              - FreeKarma4U
+              - FreeKarma4You
+              - upvote
+        window: 7 days
+    actions:
+      - kind: report
+        content: >-
+          Submission posted {{rules.freekarmasub.totalCount}} times in karma
+          {{rules.freekarmasub.subCount}} subs over
+          {{rules.freekarmasub.window}}: {{rules.freekarmasub.subSummary}}

docs/examples/advancedConcepts/ruleSets.yaml

@@ -0,0 +1,53 @@
+checks:
+  - name: Self Promo All or low comment
+    description: >-
+      SP >10% of all activities or >10% of submissions with low comment
+      engagement
+    kind: submission
+    rules:
+      # this attribution rule is looking at all activities
+      #
+      # we want want this one rule to trigger the check because >10% of all activity (submission AND comments) is a good requirement
+      - name: attr10all
+        kind: attribution
+        criteria:
+          - threshold: '> 10%'
+            window: 90 days
+          - threshold: '> 10%'
+            window: 100
+        # this is a RULE SET
+        #
+        # it is made up of "nested" rules with a pass condition (AND/OR)
+        # if the nested rules pass the condition then the Rule Set triggers the Check
+        #
+        # AND = all nested rules must be triggered to make the Rule Set trigger
+        # AND = any of the nested Rules will be the Rule Set trigger
+      - condition: AND
+        # in this check we use an Attribution >10% on ONLY submissions, which is a lower requirement then the above attribution rule
+        # and combine it with a History rule looking for low comment engagement
+        # to make a "higher" requirement Rule Set our of two low requirement Rules
+        rules:
+          - name: attr20sub
+            kind: attribution
+            criteria:
+              - threshold: '> 10%'
+                thresholdOn: submissions
+                window: 90 days
+              - threshold: '> 10%'
+                thresholdOn: submissions
+                window: 100
+            lookAt: media
+          - name: lowOrOpComm
+            kind: history
+            criteriaJoin: OR
+            criteria:
+              - window: 90 days
+                comment: < 50%
+              - window: 90 days
+                comment: '> 40% OP'
+    actions:
+      - kind: remove
+      - kind: comment
+        content: >-
+          Your submission was removed because you are over reddit's threshold
+          for self-promotion or exhibit low comment engagement

docs/examples/attribution/README.md

@@ -10,5 +10,5 @@ Consult the [schema](https://json-schema.app/view/%23/%23%2Fdefinitions%2FCheckJ
 
 ### Examples
 
-* [Self Promotion as percentage of all Activities](/docs/examples/attribution/redditSelfPromoAll.json5) - Check if Author is submitting much more than they comment.
-* [Self Promotion as percentage of Submissions](/docs/examplesm/attribution/redditSelfPromoSubmissionsOnly.json5) - Check if any of Author's aggregated submission origins are >10% of their submissions
+* Self Promotion as percentage of all Activities [YAML](/docs/examples/attribution/redditSelfPromoAll.yaml) | [JSON](/docs/examples/attribution/redditSelfPromoAll.json5) - Check if Author is submitting much more than they comment.
+* Self Promotion as percentage of Submissions [YAML](/docs/examples/attribution/redditSelfPromoSubmissionsOnly.yaml) | [JSON](/docs/examplesm/attribution/redditSelfPromoSubmissionsOnly.json5) - Check if any of Author's aggregated submission origins are >10% of their submissions

docs/examples/attribution/redditSelfPromoAll.yaml

@@ -0,0 +1,27 @@
+checks:
+  - name: Self Promo Activities
+    description: >-
+      Check if any of Author's aggregated submission origins are >10% of entire
+      history
+    # check will run on a new submission in your subreddit and look at the Author of that submission
+    kind: submission
+    rules:
+      - name: attr10all
+        kind: attribution
+        # criteria defaults to OR -- so either of these criteria will trigger the rule
+        criteria:
+          - threshold: '> 10%' # threshold can be a percent or an absolute number
+            # The default is "all" -- calculate percentage of entire history (submissions & comments)
+            #thresholdOn: all
+            #
+            # look at last 90 days of Author's activities (comments and submissions)
+            window: 90 days
+          - threshold: '> 10%'
+            # look at Author's last 100 activities (comments and submissions)
+            window: 100
+    actions:
+      - kind: report
+        content: >-
+          {{rules.attr10all.largestPercent}}% of
+          {{rules.attr10all.activityTotal}} items over
+          {{rules.attr10all.window}}

docs/examples/attribution/redditSelfPromoSubmissionOnly.yaml

@@ -0,0 +1,24 @@
+checks:
+  - name: Self Promo Submissions
+    description: >-
+      Check if any of Author's aggregated submission origins are >10% of their
+      submissions
+    # check will run on a new submission in your subreddit and look at the Author of that submission
+    kind: submission
+    rules:
+      - name: attr10sub
+        kind: attribution
+        # criteria defaults to OR -- so either of these criteria will trigger the rule
+        criteria:
+          - threshold: '> 10%' # threshold can be a percent or an absolute number
+            thresholdOn: submissions # calculate percentage of submissions, rather than entire history (submissions & comments)
+            window: 90 days # look at last 90 days of Author's activities (comments and submissions)
+          - threshold: '> 10%'
+            thresholdOn: submissions
+            window: 100 # look at Author's last 100 activities (comments and submissions)
+    actions:
+      - kind: report
+        content: >-
+          {{rules.attr10sub.largestPercent}}% of
+          {{rules.attr10sub.activityTotal}} items over
+          {{rules.attr10sub.window}}

docs/examples/author/README.md

@@ -18,10 +18,10 @@ Consult the [schema](https://json-schema.app/view/%23%2Fdefinitions%2FAuthorRule
 ### Examples
 
 * Basic examples
-    * [Flair new user Submission](/docs/examples/author/flairNewUserSubmission.json5) - If the Author does not have the `vet` flair then flair the Submission with `New User`
-    * [Flair vetted user Submission](/docs/examples/author/flairNewUserSubmission.json5) - If the Author does have the `vet` flair then flair the Submission with `Vetted`
+    * Flair new user Submission [YAML](/docs/examples/author/flairNewUserSubmission.yaml) | [JSON](/docs/examples/author/flairNewUserSubmission.json5) - If the Author does not have the `vet` flair then flair the Submission with `New User`
+    * Flair vetted user Submission [YAML](/docs/examples/author/flairNewUserSubmission.yaml) | [JSON](/docs/examples/author/flairNewUserSubmission.json5) - If the Author does have the `vet` flair then flair the Submission with `Vetted`
 * Used with other Rules
-    * [Ignore vetted user](/docs/examples/author/flairNewUserSubmission.json5) - Short-circuit the Check if the Author has the `vet` flair
+    * Ignore vetted user [YAML](/docs/examples/author/flairNewUserSubmission.yaml) | [JSON](/docs/examples/author/flairNewUserSubmission.json5) - Short-circuit the Check if the Author has the `vet` flair
     
 ## Filter
 
@@ -35,4 +35,4 @@ All **Rules** and **Checks** have an optional `authorIs` property that takes an
 
 ### Examples
 
-* [Skip recent activity check based on author](/docs/examples/author/authorFilter.json5) - Skip a Recent Activity check for a set of subreddits if the Author of the Submission has any set of flairs.
+* Skip recent activity check based on author [YAML](/docs/examples/author/authorFilter.yaml) | [JSON](/docs/examples/author/authorFilter.json5) - Skip a Recent Activity check for a set of subreddits if the Author of the Submission has any set of flairs.

docs/examples/author/authorFilter.yaml

@@ -0,0 +1,48 @@
+checks:
+  - name: Karma/Meme Sub Activity
+    description: Report on karma sub activity or meme sub activity if user isn't a memelord
+    # check will run on a new submission in your subreddit and look at the Author of that submission
+    kind: submission
+    rules:
+      - name: freekarma
+        kind: recentActivity
+        lookAt: submissions
+        thresholds:
+          - threshold: '>= 1'
+            subreddits:
+              - DeFreeKarma
+              - FreeKarma4U
+        window: 7 days
+      - name: noobmemer
+        kind: recentActivity
+        # authors filter will be checked before a rule is run. If anything passes then the Rule is skipped -- it is not failed or triggered.
+        # if *all* Rules for a Check are skipped due to authors filter then the Check will fail
+        authorIs:
+          # each property (include/exclude) can contain multiple AuthorCriteria
+          # if any AuthorCriteria passes its test the Rule is skipped
+          #
+          # for an AuthorCriteria to pass all properties present on it must pass
+          #
+          # if include is present it will always run and exclude will be skipped
+          #-include:
+          exclude:
+              # for this to pass the Author of the Submission must not have the flair "Supreme Memer" and have the name "user1" or "user2"
+            - flairText:
+                - Supreme Memer
+              names:
+                - user1
+                - user2
+              # for this to pass the Author of the Submission must not have the flair "Decent Memer"
+            - flairText:
+                - Decent Memer
+        lookAt: submissions
+        thresholds:
+          - threshold: '>= 1'
+            subreddits:
+              - dankmemes
+        window: 7 days
+    actions:
+      - kind: report
+        content: >-
+          Author has posted in free karma sub, or in /r/dankmemes and does not
+          have meme flair in this subreddit

docs/examples/author/flairNewUserSubmission.yaml

@@ -0,0 +1,16 @@
+checks:
+  - name: Flair New User Sub
+    description: Flair submission as sketchy if user does not have vet flair
+    # check will run on a new submission in your subreddit and look at the Author of that submission
+    kind: submission
+    rules:
+      - name: newflair
+        kind: author
+        # rule will trigger if Author does not have "vet" flair text
+        exclude:
+          - flairText:
+              - vet
+    actions:
+      - kind: flair
+        text: New User
+        css: orange

docs/examples/author/flairVettedUserSubmission.yaml

@@ -0,0 +1,16 @@
+checks:
+  - name: Flair Vetted User Submission
+    description: Flair submission as Approved if user has vet flair
+    # check will run on a new submission in your subreddit and look at the Author of that submission
+    kind: submission
+    rules:
+      - name: newflair
+        kind: author
+        # rule will trigger if Author has "vet" flair text
+        include:
+          - flairText:
+              - vet
+    actions:
+      - kind: flair
+        text: Vetted
+        css: green

docs/examples/author/ignoreVettedUser.yaml

@@ -0,0 +1,45 @@
+checks:
+  - name: non-vetted karma/meme activity
+    description: >-
+      Report if Author has SP and has recent karma/meme sub activity and isn't
+      vetted
+    # check will run on a new submission in your subreddit and look at the Author of that submission
+    kind: submission
+    rules:
+      # The Author Rule is best used in conjunction with other Rules --
+      # instead of having to write an AuthorFilter for every Rule where you want to skip it based on Author criteria
+      # you can write one Author Rule and make it fail on the required criteria
+      # so that the check fails and Actions don't run
+      - name: nonvet
+        kind: author
+        exclude:
+          - flairText:
+              - vet
+      - name: attr10
+        kind: attribution
+        criteria:
+          - threshold: '> 10%'
+            window: 90 days
+          - threshold: '> 10%'
+            window: 100
+      - name: freekarma
+        kind: recentActivity
+        lookAt: submissions
+        thresholds:
+          - threshold: '>= 1'
+            subreddits:
+              - DeFreeKarma
+              - FreeKarma4U
+        window: 7 days
+      - name: memes
+        kind: recentActivity
+        lookAt: submissions
+        thresholds:
+          - threshold: '>= 3'
+            subreddits:
+              - dankmemes
+        window: 7 days
+    # will NOT run if the Author for this Submission has the flair "vet"
+    actions:
+      - kind: report
+        content: Author has posted in free karma or meme subs recently

docs/examples/history/README.md

@@ -9,5 +9,5 @@ Consult the [schema](https://json-schema.app/view/%23%2Fdefinitions%2FHistoryJSO
 
 ### Examples
 
-* [Low Comment Engagement](/docs/examples/history/lowEngagement.json5) - Check if Author is submitting much more than they comment.
-* [OP Comment Engagement](/docs/examples/history/opOnlyEngagement.json5) - Check if Author is mostly engaging only in their own content
+* Low Comment Engagement [YAML](/docs/examples/history/lowEngagement.yaml) | [JSON](/docs/examples/history/lowEngagement.json5) - Check if Author is submitting much more than they comment.
+* OP Comment Engagement [YAML](/docs/examples/history/opOnlyEngagement.yaml) | [JSON](/docs/examples/history/opOnlyEngagement.json5) - Check if Author is mostly engaging only in their own content

docs/examples/history/lowEngagement.yaml

@@ -0,0 +1,21 @@
+checks:
+  - name: Low Comment Engagement
+    description: Check if Author is submitting much more than they comment
+    # check will run on a new submission in your subreddit and look at the Author of that submission
+    kind: submission
+    rules:
+      - name: lowComm
+        kind: history
+        criteria:
+          - comment: '< 30%'
+            window:
+             # get author's last 90 days of activities or 100 activities, whichever is less
+             duration: 90 days
+             count: 100
+            # trigger if less than 30% of their activities in this time period are comments
+
+    actions:
+      - kind: report
+        content: >-
+          Low engagement: comments were {{rules.lowcomm.commentPercent}} of
+          {{rules.lowcomm.activityTotal}} over {{rules.lowcomm.window}}

docs/examples/history/opOnlyEngagement.yaml

@@ -0,0 +1,22 @@
+checks:
+  - name: Engaging Own Content Only
+    description: Check if Author is mostly engaging in their own content only
+    # check will run on a new submission in your subreddit and look at the Author of that submission
+    kind: submission
+    rules:
+      - name: opOnly
+        kind: history
+        criteria:
+            # trigger if more than 60% of their activities in this time period are comments as OP
+          - comment: '> 60% OP'
+            window:
+              # get author's last 90 days of activities or 100 activities, whichever is less
+              duration: 90 days
+              count: 100
+
+    actions:
+      - kind: report
+        content: >-
+          Selfish OP: {{rules.oponly.opPercent}} of
+          {{rules.oponly.commentTotal}} comments over {{rules.oponly.window}}
+          are as OP

docs/examples/onlyfansFlair/README.md

@@ -0,0 +1,9 @@
+# Flair users and submissions
+
+Flair users and submissions based on certain keywords from submitter's profile.
+
+Consult [User Flair schema](https://json-schema.app/view/%23%2Fdefinitions%2FUserFlairActionJson?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Fcontext-mod%2Fmaster%2Fsrc%2FSchema%2FApp.json) and [Submission Flair schema](https://json-schema.app/view/%23%2Fdefinitions%2FFlairActionJson?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Fcontext-mod%2Fmaster%2Fsrc%2FSchema%2FApp.json) for a complete reference of the rule's properties.
+
+### Examples
+
+* OnlyFans submissions [YAML](/docs/examples/onlyFansFlair/onlyFansFlair.yaml) | [JSON](/docs/examples/onlyfansFlair/onlyfansFlair.json5) - Check whether submitter has typical OF keywords in their profile and flair both author + submission accordingly.

docs/examples/onlyfansFlair/onlyfansFlair.json5

@@ -0,0 +1,68 @@
+{
+  "checks": [
+    {
+      "name": "Flair OF submitters",
+      "description": "Flair submission as OF if user does not have Verified flair and has certain keywords in their profile",
+      "kind": "submission",
+      "authorIs": {
+        "exclude": [
+          {
+            "flairCssClass": ["verified"]
+          }
+        ]
+      },
+      "rules": [
+        {
+          "name": "OnlyFans strings in description",
+          "kind": "author",
+          "include": [
+            {
+              "description": [
+                "/(cashapp|allmylinks|linktr|onlyfans\\.com)/i",
+                "/(see|check|my|view) (out|of|onlyfans|kik|skype|insta|ig|profile|links)/i",
+                "my links",
+                "$"
+              ]
+            }
+          ]
+        }
+      ],
+      "actions": [
+        {
+          "name": "Set OnlyFans user flair",
+          "kind": "userflair",
+          "flair_template_id": "put-your-onlyfans-user-flair-id-here"
+        },
+        {
+          "name":"Set OF Creator SUBMISSION flair",
+          "kind": "flair",
+          "flair_template_id": "put-your-onlyfans-post-flair-id-here"
+        }
+      ]
+    },
+
+    {
+      "name": "Flair posts of OF submitters",
+      "description": "Flair submission as OnlyFans if submitter has OnlyFans userflair (override post flair set by submitter)",
+      "kind": "submission",
+      "rules": [
+        {
+          "name": "Include OF submitters",
+          "kind": "author",
+          "include": [
+            {
+              "flairCssClass": ["onlyfans"]
+            }
+          ]
+        }
+      ],
+      "actions": [
+        {
+          "name":"Set OF Creator SUBMISSION flair",
+          "kind": "flair",
+          "flair_template_id": "put-your-onlyfans-post-flair-id-here"
+        }
+      ]
+    }
+  ]
+}

docs/examples/onlyfansFlair/onlyfansFlair.yaml

@@ -0,0 +1,38 @@
+checks:
+  - name: Flair OF submitters
+    description: Flair submission as OF if user does not have Verified flair and has
+      certain keywords in their profile
+    kind: submission
+    authorIs:
+      exclude:
+        - flairCssClass:
+            - verified
+    rules:
+      - name: OnlyFans strings in description
+        kind: author
+        include:
+          - description:
+              - '/(cashapp|allmylinks|linktr|onlyfans\.com)/i'
+              - '/(see|check|my|view) (out|of|onlyfans|kik|skype|insta|ig|profile|links)/i'
+              - my links
+              - "$"
+    actions:
+      - name: Set OnlyFans user flair
+        kind: userflair
+        flair_template_id: put-your-onlyfans-user-flair-id-here
+      - name: Set OF Creator SUBMISSION flair
+        kind: flair
+        flair_template_id: put-your-onlyfans-post-flair-id-here
+  - name: Flair posts of OF submitters
+    description: Flair submission as OnlyFans if submitter has OnlyFans userflair (override post flair set by submitter)
+    kind: submission
+    rules:
+      - name: Include OF submitters
+        kind: author
+        include:
+          - flairCssClass:
+              - onlyfans
+    actions:
+      - name: Set OF Creator SUBMISSION flair
+        kind: flair
+        flair_template_id: put-your-onlyfans-post-flair-id-here

docs/examples/recentActivity/README.md

@@ -6,5 +6,5 @@ Consult the [schema](https://json-schema.app/view/%23%2Fdefinitions%2FRecentActi
 
 ### Examples
 
-* [Free Karma Subreddits](/docs/examples/recentActivity/freeKarma.json5) - Check if the Author has recently posted in any "free karma" subreddits
-* [Submission in Free Karma Subreddits](/docs/examples/recentActivity/freeKarmaOnSubmission.json5) - Check if the Author has posted the Submission this check is running on in any "free karma" subreddits recently
+* Free Karma Subreddits [YAML](/docs/examples/recentActivity/freeKarma.yaml) | [JSON](/docs/examples/recentActivity/freeKarma.json5) - Check if the Author has recently posted in any "free karma" subreddits
+* Submission in Free Karma Subreddits [YAML](/docs/examples/recentActivity/freeKarmaOnSubmission.yaml) | [JSON](/docs/examples/recentActivity/freeKarmaOnSubmission.json5) - Check if the Author has posted the Submission this check is running on in any "free karma" subreddits recently

docs/examples/recentActivity/freeKarma.yaml

@@ -0,0 +1,27 @@
+checks:
+  - name: Free Karma Alert
+    description: Check if author has posted in 'freekarma' subreddits
+    # check will run on a new submission in your subreddit and look at the Author of that submission
+    kind: submission
+    rules:
+      - name: freekarma
+        kind: recentActivity
+        # // when lookAt is not present this rule will look for submissions and comments
+        #lookAt: comments
+        useSubmissionAsReference: false
+        thresholds:
+            # if the number of activities (sub/comment) found CUMULATIVELY in the subreddits listed is
+            # equal to or greater than 1 then the rule is triggered
+          - threshold: '>= 1'
+            subreddits:
+              - DeFreeKarma
+              - FreeKarma4U
+              - FreeKarma4You
+              - upvote
+        window: 7 days
+    actions:
+      - kind: report
+        content: >-
+          {{rules.freekarma.totalCount}} activities in karma
+          {{rules.freekarma.subCount}} subs over {{rules.freekarma.window}}:
+          {{rules.freekarma.subSummary}}

docs/examples/recentActivity/freeKarmaOnSubmission.yaml

@@ -0,0 +1,26 @@
+checks:
+  - name: Free Karma On Submission Alert
+    description: Check if author has posted this submission in 'freekarma' subreddits
+    kind: submission
+    rules:
+      - name: freekarmasub
+        kind: recentActivity
+        # rule will only look at Author's submissions in these subreddits
+        lookAt: submissions
+        # rule will only look at Author's submissions in these subreddits that have the same content (link) as the submission this event was made on
+        # In simpler terms -- rule will only check to see if the same link the author just posted is also posted in these subreddits
+        useSubmissionAsReference: true
+        thresholds:
+          - threshold: '>= 1'
+            subreddits:
+              - DeFreeKarma
+              - FreeKarma4U
+              - FreeKarma4You
+              - upvote
+        window: 7 days
+    actions:
+      - kind: report
+        content: >-
+          Submission posted {{rules.freekarmasub.totalCount}} times in karma
+          {{rules.freekarmasub.subCount}} subs over
+          {{rules.freekarmasub.window}}: {{rules.freekarmasub.subSummary}}

docs/examples/regex/README.md

@@ -11,12 +11,12 @@ Which can then be used in conjunction with a [`window`](https://github.com/FoxxM
 
 ### Examples
 
-* [Trigger if regex matches against the current activity](/docs/examples/regex/matchAnyCurrentActivity.json5)
-* [Trigger if regex matches 5 times against the current activity](/docs/examples/regex/matchThresholdCurrentActivity.json5)
-* [Trigger if regex matches against any part of a Submission](/docs/examples/regex/matchSubmissionParts.json5)
-* [Trigger if regex matches any of Author's last 10 activities](/docs/examples/regex/matchHistoryActivity.json5)
-* [Trigger if regex matches at least 3 of Author's last 10 activities](/docs/examples/regex/matchActivityThresholdHistory.json5)
-* [Trigger if there are 5 regex matches in the Author's last 10 activities](/docs/examples/regex/matchTotalHistoryActivity.json5)
-* [Trigger if there are 5 regex matches in the Author's last 10 comments](/docs/examples/regex/matchSubsetHistoryActivity.json5)
-* [Remove comments that are spamming discord links](/docs/examples/regex/removeDiscordSpam.json5)
+* Trigger if regex matches against the current activity - [YAML](/docs/examples/regex/matchAnyCurrentActivity.yaml) | [JSON](/docs/examples/regex/matchAnyCurrentActivity.json5)
+* Trigger if regex matches 5 times against the current activity - [YAML](/docs/examples/regex/matchThresholdCurrentActivity.yaml) | [JSON](/docs/examples/regex/matchThresholdCurrentActivity.json5)
+* Trigger if regex matches against any part of a Submission - [YAML](/docs/examples/regex/matchSubmissionParts.yaml) | [JSON](/docs/examples/regex/matchSubmissionParts.json5)
+* Trigger if regex matches any of Author's last 10 activities - [YAML](/docs/examples/regex/matchHistoryActivity.yaml) | [JSON](/docs/examples/regex/matchHistoryActivity.json5)
+* Trigger if regex matches at least 3 of Author's last 10 activities - [YAML](/docs/examples/regex/matchActivityThresholdHistory.json5) | [JSON](/docs/examples/regex/matchActivityThresholdHistory.json5)
+* Trigger if there are 5 regex matches in the Author's last 10 activities - [YAML](/docs/examples/regex/matchTotalHistoryActivity.yaml) | [JSON](/docs/examples/regex/matchTotalHistoryActivity.json5) 
+* Trigger if there are 5 regex matches in the Author's last 10 comments - [YAML](/docs/examples/regex/matchSubsetHistoryActivity.yaml) | [JSON](/docs/examples/regex/matchSubsetHistoryActivity.json5)
+* Remove comments that are spamming discord links - [YAML](/docs/examples/regex/removeDiscordSpam.yaml) | [JSON](/docs/examples/regex/removeDiscordSpam.json5)
   * Differs from just using automod because this config can allow one-off/organic links from users who DO NOT spam discord links but will still remove the comment if the user is spamming them

docs/examples/regex/matchActivityThresholdHistory.yaml

@@ -0,0 +1,13 @@
+name: swear
+kind: regex
+criteria:
+    # triggers if more than 3 activities in the last 10 match the regex
+  - regex: '/fuck|shit|damn/'
+    # this differs from "totalMatchThreshold"
+    #
+    # activityMatchThreshold => # of activities from window must match regex
+    # totalMatchThreshold => # of matches across all activities from window must match regex
+    activityMatchThreshold: '> 3'
+    # if `window` is specified it tells the rule to check the current activity as well as the activities returned from `window`
+    # learn more about `window` here https://github.com/FoxxMD/context-mod/blob/master/docs/activitiesWindow.md
+    window: 10

docs/examples/regex/matchAnyCurrentActivity.yaml

@@ -0,0 +1,6 @@
+name: swear
+kind: regex
+criteria:
+  - regex: '/fuck|shit|damn/'
+    # if "matchThreshold" is not specified it defaults to this -- default behavior is to trigger if there are any matches
+    #matchThreshold: "> 0"

docs/examples/regex/matchHistoryActivity.yaml

@@ -0,0 +1,8 @@
+name: swear
+kind: regex
+criteria:
+    # triggers if any activity in the last 10 (including current activity) match the regex
+  - regex: '/fuck|shit|damn/'
+    # if `window` is specified it tells the rule to check the current activity as well as the activities returned from `window`
+    # learn more about `window` here https://github.com/FoxxMD/context-mod/blob/master/docs/activitiesWindow.md
+    window: 10

docs/examples/regex/matchSubmissionParts.yaml

@@ -0,0 +1,11 @@
+name: swear
+kind: regex
+criteria:
+  - regex: '/fuck|shit|damn/'
+    # triggers if the current activity has more than 0 matches
+    # if the activity is a submission then matches against title, body, and url
+    # if "testOn" is not provided then `title, body` are the defaults
+    testOn:
+      - title
+      - body
+      - url

docs/examples/regex/matchSubsetHistoryActivity.yaml

@@ -0,0 +1,16 @@
+name: swear
+kind: regex
+criteria:
+    # triggers if there are more than 5 regex matches in the last 10 activities (comments only)
+  - regex: '/fuck|shit|damn/'
+    # this differs from "activityMatchThreshold"
+    #
+    # activityMatchThreshold => # of activities from window must match regex
+    # totalMatchThreshold => # of matches across all activities from window must match regex
+    totalMatchThreshold: '> 5'
+    # if `window` is specified it tells the rule to check the current activity as well as the activities returned from `window`
+    # learn more about `window` here https://github.com/FoxxMD/context-mod/blob/master/docs/activitiesWindow.md
+    window: 10
+    # determines which activities from window to consider
+    # defaults to "all" (submissions and comments)
+    lookAt: comments

docs/examples/regex/matchThresholdCurrentActivity.yaml

@@ -0,0 +1,6 @@
+name: swear
+kind: regex
+criteria:
+  - regex: '/fuck|shit|damn/'
+    # triggers if current activity has greater than 5 matches
+    matchThreshold: '> 5'

docs/examples/regex/matchTotalHistoryActivity.yaml

@@ -0,0 +1,13 @@
+name: swear
+kind: regex
+criteria:
+    # triggers if there are more than 5 regex matches in the last 10 activities (comments or submission)
+  - regex: '/fuck|shit|damn/'
+    # this differs from "activityMatchThreshold"
+    #
+    # activityMatchThreshold => # of activities from window must match regex
+    # totalMatchThreshold => # of matches across all activities from window must match regex
+    totalMatchThreshold: '> 5'
+    # if `window` is specified it tells the rule to check the current activity as well as the activities returned from `window`
+    # learn more about `window` here https://github.com/FoxxMD/context-mod/blob/master/docs/activitiesWindow.md
+    window: 10

docs/examples/regex/removeDiscordSpam.yaml

@@ -0,0 +1,36 @@
+checks:
+  - name: remove discord spam
+    notifyOnTrigger: true
+    description: remove comments from users who are spamming discord links
+    kind: comment
+    authorIs:
+      exclude:
+        - isMod: true
+    itemIs:
+      - removed: false
+        approved: false
+    condition: OR
+    rules:
+      - enable: true
+        name: linkOnlySpam
+        kind: regex
+        criteria:
+          - name: only link
+            regex: '/^.*(discord\.gg\/[\w\d]+)$/i'
+      - condition: AND
+        rules:
+          - name: linkAnywhereSpam
+            kind: regex
+            criteria:
+              - name: contains link anywhere
+                regex: '/^.*(discord\.gg\/[\w\d]+).*$/i'
+          - name: linkAnywhereHistoricalSpam
+            kind: regex
+            criteria:
+              - name: contains links anywhere historically
+                regex: '/^.*(discord\.gg\/[\w\d]+).*$/i'
+                totalMatchThreshold: '>= 3'
+                lookAt: comments
+                window: 10
+    actions:
+      - kind: remove

docs/examples/repeatActivity/README.md

@@ -45,5 +45,5 @@ With only `gapAllowance: 2` this rule **would trigger** because the the 1 and 2
 
 ## Examples
 
-* [Crosspost Spamming](/docs/examples/repeatActivity/crosspostSpamming.json5) - Check if an Author is spamming their Submissions across multiple subreddits
-* [Burst-posting](/docs/examples/repeatActivity/burstPosting.json5) - Check if Author is crossposting their Submissions in short bursts
+* Crosspost Spamming [JSON](/docs/examples/repeatActivity/crosspostSpamming.json5) | [YAML](/docs/examples/repeatActivity/crosspostSpamming.yaml) - Check if an Author is spamming their Submissions across multiple subreddits
+* Burst-posting [JSON](/docs/examples/repeatActivity/burstPosting.json5) | [YAML](/docs/examples/repeatActivity/burstPosting.yaml) - Check if Author is crossposting their Submissions in short bursts

docs/examples/repeatActivity/burstPosting.yaml

@@ -0,0 +1,23 @@
+checks:
+  - name: Burstpost Spam
+    description: Check if Author is crossposting in short bursts
+    # check will run on a new submission in your subreddit and look at the Author of that submission
+    kind: submission
+    rules:
+      - name: burstpost
+        kind: repeatActivity
+        # will only look at Submissions in Author's history that contain the same content (link) as the Submission this check was initiated by
+        useSubmissionAsReference: true
+        # the number of non-repeat activities (submissions or comments) to ignore between repeat submissions
+        gapAllowance: 3
+        # if the Author has posted this Submission 6 times, ignoring 3 non-repeat activities between each repeat, then this rule will trigger
+        threshold: '>= 6'
+        # look at all of the Author's submissions in the last 7 days or 100 submissions
+        window:
+          duration: 7 days
+          count: 100
+    actions:
+      - kind: report
+        content: >-
+          Author has burst-posted this link {{rules.burstpost.largestRepeat}}
+          times over {{rules.burstpost.window}}

docs/examples/repeatActivity/crosspostSpamming.yaml

@@ -0,0 +1,19 @@
+checks:
+  - name: Crosspost Spam
+    description: Check if Author is spamming Submissions across subreddits
+    # check will run on a new submission in your subreddit and look at the Author of that submission
+    kind: submission
+    rules:
+      - name: xpostspam
+        kind: repeatActivity
+        # will only look at Submissions in Author's history that contain the same content (link) as the Submission this check was initiated by
+        useSubmissionAsReference: true
+        # if the Author has posted this Submission 5 times consecutively then this rule will trigger
+        threshold: '>= 5'
+        # look at all of the Author's submissions in the last 7 days
+        window: 7 days
+    actions:
+      - kind: report
+        content: >-
+          Author has posted this link {{rules.xpostspam.largestRepeat}} times
+          over {{rules.xpostspam.window}}

docs/examples/repost/README.md

@@ -31,6 +31,17 @@ Specify one or more types of facets as a string to use their default configurati
 
 <details>
 
+YAML
+```yaml
+kind: repost
+criteria:
+  - searchOn:
+      - title
+      - url
+      - crossposts
+```
+
+JSON
 ```json5
 {
   "kind": "repost",
@@ -53,6 +64,16 @@ Specify one or more types of facets as a string to use their default configurati
 
 <details>
 
+```yaml
+kind: repost
+criteria:
+  - searchOn:
+      - title
+      - kind: url
+        matchScore: 90
+      - external
+```
+
 ```json5
 {
   "kind": "repost",
@@ -129,6 +150,22 @@ Define a set of criteria to test against the **number of reposts**, **time repos
 
 <details>
 
+```yaml
+kind: repost
+criteria:
+  - searchOn:
+      - title
+      - url
+      - crossposts
+    occurrences:
+      criteria:
+        - count:
+            condition: AND
+            test:
+              - '> 3'
+              - <= 5
+```
+
 ```json5
 {
   "kind": "repost",
@@ -163,6 +200,22 @@ Define a test or array of tests to run against **when reposts were created**
 
 <details>
 
+```yaml
+kind: repost
+criteria:
+  - searchOn:
+      - title
+      - url
+      - crossposts
+    occurrences:
+      criteria:
+        - time:
+            condition: AND
+            test:
+              - testOn: all
+                condition: '> 3 months'
+```
+
 ```json5
 {
   "kind": "repost",
@@ -233,6 +286,21 @@ This is the same behavior described in the [TLDR](#TLDR) section above -- find a
 
 <details>
 
+```yaml
+polling:
+  - unmoderated
+checks:
+  - name: subRepost
+    description: Check if submission has been reposted
+    kind: submission
+    condition: AND
+    rules:
+      - kind: repost
+    actions:
+      - kind: report
+        content: This submission was reposted
+```
+
 ```json5
 {
   "polling": [
@@ -276,6 +344,24 @@ Find any submissions with:
 
 <details>
 
+```yaml
+polling:
+  - unmoderated
+checks:
+  - name: subRepost
+    description: Check if submission has been reposted
+    kind: submission
+    condition: AND
+    rules:
+      - kind: repost
+        criteria:
+          - searchOn:
+              - title
+    actions:
+      - kind: report
+        content: This submission was reposted
+```
+
 ```json5
 {
   "polling": [
@@ -325,6 +411,25 @@ Find any submissions with:
 
 <details>
 
+```yaml
+polling:
+  - unmoderated
+checks:
+  - name: subRepost
+    description: Check if submission has been reposted
+    kind: submission
+    condition: AND
+    rules:
+      - kind: repost
+        criteria:
+          - searchOn:
+              - kind: title
+                matchScore: '95'
+    actions:
+      - kind: report
+        content: This submission was reposted
+```
+
 ```json5
 {
   "polling": [
@@ -376,6 +481,26 @@ Find any submissions with:
 
 <details>
 
+```yaml
+polling:
+  - unmoderated
+checks:
+  - name: subRepost
+    description: Check if submission has been reposted
+    kind: submission
+    condition: AND
+    rules:
+      - kind: repost
+        criteria:
+          - searchOn:
+              - duplicates
+              - kind: title
+                matchScore: '95'
+    actions:
+      - kind: report
+        content: This submission was reposted
+```
+
 ```json5
 {
   "polling": [
@@ -420,7 +545,6 @@ Find any submissions with:
     }
   ]
 }
-
 ```
 
 </details>
@@ -429,6 +553,33 @@ Find any submissions with:
 
 <details>
 
+```yaml
+polling:
+  - unmoderated
+checks:
+  - name: subRepost
+    description: Check there are no reposts with same title in the last month
+    kind: submission
+    condition: AND
+    rules:
+      - kind: repost
+        criteria:
+          - searchOn:
+              - title
+            occurrences:
+              condition: OR
+              criteria:
+                - count:
+                    test:
+                      - < 1
+                - time:
+                    test:
+                      - testOn: newest
+                        condition: '> 1 month'
+    actions:
+      - kind: approve
+```
+
 ```json5
 {
   "polling": [
@@ -522,6 +673,21 @@ FINALLY
 
 <details>
 
+```yaml
+polling:
+  - newComm
+checks:
+  - name: commRepost
+    description: Check if comment has been reposted
+    kind: common
+    condition: AND
+    rules:
+      - kind: repost
+    actions:
+      - kind: report
+        content: This comment was reposted
+```
+
 ```json5
 {
   "polling": [
@@ -561,6 +727,24 @@ FINALLY
 
 <details>
 
+```yaml
+polling:
+  - newComm
+checks:
+  - name: commRepost
+    description: Check if comment has been reposted from youtube
+    kind: comment
+    condition: AND
+    rules:
+      - kind: repost
+        criteria:
+          - searchOn:
+              - external
+    actions:
+      - kind: report
+        content: This comment was reposted from youtube
+```
+
 ```json5
 {
   "polling": [
@@ -608,6 +792,25 @@ FINALLY
 
 <details>
 
+```yaml
+polling:
+  - newComm
+checks:
+  - name: commRepost
+    description: Check if comment has been reposted from youtube
+    kind: comment
+    condition: AND
+    rules:
+      - kind: repost
+        criteria:
+          - searchOn:
+              - external
+            matchScore: 95
+    actions:
+      - kind: report
+        content: This comment was reposted from youtube
+```
+
 ```json5
 {
   "polling": [
@@ -656,6 +859,28 @@ FINALLY
 
 <details>
 
+```yaml
+polling:
+  - newComm
+checks:
+  - name: commRepost
+    description: Check if comment has been reposted
+    kind: comment
+    condition: AND
+    rules:
+      - kind: repost
+        criteria:
+          - searchOn:
+              - external
+              - url
+            matchScore: 95
+    actions:
+      - kind: report
+        content: >-
+          This comment was reposted from youtube or from submission with the
+          same URL
+```
+
 ```json5
 {
   "polling": [
@@ -697,7 +922,6 @@ FINALLY
     }
   ]
 }
-
 ```
 
 </details>

docs/examples/subredditReady/README.md

@@ -11,21 +11,31 @@ All actions for these configurations are non-destructive in that:
 
 **You will have to remove the `report` action and `dryRun` settings yourself.** This is to ensure that you understand the behavior the bot will be performing. If you are unsure of this you should leave them in place until you are certain the behavior the bot is performing is acceptable.
 
+**YAML** is the same format as **automoderator**
+
 ## Submission-based Behavior
 
-### [Remove submissions from users who have used 'freekarma' subs to bypass karma checks](/docs/examples/subredditReady/freekarma.json5)
+### Remove submissions from users who have used 'freekarma' subs to bypass karma checks
+
+[YAML](/docs/examples/subredditReady/freekarma.yaml) | [JSON](/docs/examples/subredditReady/freekarma.json5)
 
 If the user has any activity (comment/submission) in known freekarma subreddits in the past (50 activities or 6 months) then remove the submission.
 
-### [Remove submissions from users who have crossposted the same submission 4 or more times](/docs/examples/subredditReady/crosspostSpam.json5)
+### Remove submissions from users who have crossposted the same submission 4 or more times
+
+[YAML](/docs/examples/subredditReady/crosspostSpam.yaml) | [JSON](/docs/examples/subredditReady/crosspostSpam.yaml)
 
 If the user has crossposted the same submission in the past (50 activities or 6 months) 4 or more times in a row then remove the submission.
 
-### [Remove submissions from users who have crossposted or used 'freekarma' subs](/docs/examples/subredditReady/freeKarmaOrCrosspostSpam.json5)
+### Remove submissions from users who have crossposted or used 'freekarma' subs
+
+[YAML](/docs/examples/subredditReady/freeKarmaOrCrosspostSpam.yaml) | [JSON](/docs/examples/subredditReady/freeKarmaOrCrosspostSpam.json5)
 
 Will remove submission if either of the above two behaviors is detected
 
-### [Remove link submissions where the user's history is comprised of 10% or more of the same link](/docs/examples/subredditReady/selfPromo.json5)
+### Remove link submissions where the user's history is comprised of 10% or more of the same link
+
+[YAML](/docs/examples/subredditReady/selfPromo.yaml) | [JSON](/docs/examples/subredditReady/selfPromo.json5)
 
 If the link origin (youtube author, twitter author, etc. or regular domain for non-media links)
 
@@ -36,11 +46,15 @@ then remove the submission
 
 ## Comment-based behavior
 
-### [Remove comment if the user has posted the same comment 4 or more times in a row](/docs/examples/subredditReady/commentSpam.json5)
+### Remove comment if the user has posted the same comment 4 or more times in a row
+
+[YAML](/docs/examples/subredditReady/commentSpam.yaml) | [JSON](/docs/examples/subredditReady/commentSpam.json5)
 
 If the user made the same comment (with some fuzzy matching) 4 or more times in a row in the past (50 activities or 6 months) then remove the comment.
 
-### [Remove comment if it is discord invite link spam](/docs/examples/subredditReady/discordSpam.json5)
+### Remove comment if it is discord invite link spam
+
+[YAML](/docs/examples/subredditReady/discordSpam.yaml) | [JSON](/docs/examples/subredditReady/discordSpam.json5)
 
 This rule goes a step further than automod can by being more discretionary about how it handles this type of spam. 
 

docs/examples/subredditReady/commentSpam.yaml

@@ -0,0 +1,25 @@
+polling:
+  - newComm
+checks:
+    # Stop users who spam the same comment many times
+  - name: low xp comment spam
+    description: X-posted comment >=4x
+    kind: comment
+    condition: AND
+    rules:
+      - name: xPostLow
+        kind: repeatActivity
+        # number of "non-repeat" comments allowed between "repeat comments"
+        gapAllowance: 2
+        # greater or more than 4 repeat comments triggers this rule
+        threshold: '>= 4'
+        # retrieve either last 50 comments or 6 months' of history, whichever is less
+        window:
+          count: 50
+          duration: 6 months
+    actions:
+      - kind: report
+        enable: true
+        content: 'Remove => Posted same comment {{rules.xpostlow.largestRepeat}}x times'
+      - kind: remove
+        enable: true

docs/examples/subredditReady/crosspostSpam.yaml

@@ -0,0 +1,48 @@
+polling:
+  - unmoderated
+checks:
+#   stop users who post low-effort, crossposted spam submissions
+#
+#   Remove a SUBMISSION if the user has crossposted it at least 4 times in recent history AND
+#   less than 50% of their activity is comments OR more than 40% of those comments are as OP (in the own submissions)
+  - name: low xp spam and engagement
+    description: X-posted 4x and low comment engagement
+    kind: submission
+    itemIs:
+      - removed: false
+    condition: AND
+    rules:
+      - name: xPostLow
+        kind: repeatActivity
+        gapAllowance: 2
+        threshold: '>= 4'
+        window:
+          count: 50
+          duration: 6 months
+      - name: lowOrOpComm
+        kind: history
+        criteriaJoin: OR
+        criteria:
+          - window:
+              count: 100
+              duration: 6 months
+            comment: < 50%
+          - window:
+              count: 100
+              duration: 6 months
+            comment: '> 40% OP'
+    actions:
+      - kind: report
+        enable: true
+        content: >-
+          Remove=>{{rules.xpostlow.largestRepeat}} X-P =>
+          {{rules.loworopcomm.thresholdSummary}}
+      - kind: remove
+        enable: true
+      - kind: comment
+        enable: true
+        content: >-
+          Your submission has been removed because you cross-posted it
+          {{rules.xpostlow.largestRepeat}} times and you have very low
+          engagement outside of making submissions
+        distinguish: true

docs/examples/subredditReady/discordSpam.yaml

@@ -0,0 +1,46 @@
+polling:
+    - newComm
+checks:
+    - name: ban discord only spammer
+      description: ban a user who spams only a discord link many times historically
+      kind: comment
+      condition: AND
+      rules:
+          - linkOnlySpam
+          - linkAnywhereHistoricalSpam
+      actions:
+          - kind: remove
+          - kind: ban
+            content: spamming discord links
+    - name: remove discord spam
+      description: >-
+          remove comments from users who only link to discord or mention discord
+          link many times historically
+      kind: comment
+      condition: OR
+      rules:
+          - name: linkOnlySpam
+            kind: regex
+            criteria:
+                - name: only link
+                  # single quotes are required to escape special characters
+                  regex: '/^.*(discord\.gg\/[\w\d]+)$/i'
+          - condition: AND
+            rules:
+                - name: linkAnywhereSpam
+                  kind: regex
+                  criteria:
+                      - name: contains link anywhere
+                        # single quotes are required to escape special characters
+                        regex: '/^.*(discord\.gg\/[\w\d]+).*$/i'
+                - name: linkAnywhereHistoricalSpam
+                  kind: regex
+                  criteria:
+                      - name: contains links anywhere historically
+                        # single quotes are required to escape special characters
+                        regex: '/^.*(discord\.gg\/[\w\d]+).*$/i'
+                        totalMatchThreshold: '>= 3'
+                        lookAt: comments
+                        window: 10
+      actions:
+          - kind: remove

docs/examples/subredditReady/freeKarmaOrCrosspostSpam.yaml

@@ -0,0 +1,84 @@
+polling:
+  - unmoderated
+checks:
+    #   stop users who post low-effort, crossposted spam submissions
+    #
+    #   Remove a SUBMISSION if the user has crossposted it at least 4 times in recent history AND
+    #   less than 50% of their activity is comments OR more than 40% of those comments are as OP (in the own submissions)
+  - name: remove on low xp spam and engagement
+    description: X-posted 4x and low comment engagement
+    kind: submission
+    itemIs:
+      - removed: false
+    condition: AND
+    rules:
+      - name: xPostLow
+        kind: repeatActivity
+        gapAllowance: 2
+        threshold: '>= 4'
+        window:
+          count: 50
+          duration: 6 months
+      - name: lowOrOpComm
+        kind: history
+        criteriaJoin: OR
+        criteria:
+          - window:
+              count: 100
+              duration: 6 months
+            comment: < 50%
+          - window:
+              count: 100
+              duration: 6 months
+            comment: '> 40% OP'
+    actions:
+      - kind: report
+        enable: true
+        content: >-
+          Remove=>{{rules.xpostlow.largestRepeat}} X-P =>
+          {{rules.loworopcomm.thresholdSummary}}
+      - kind: remove
+        enable: false
+      - kind: comment
+        enable: true
+        content: >-
+          Your submission has been removed because you cross-posted it
+          {{rules.xpostlow.largestRepeat}} times and you have very low
+          engagement outside of making submissions
+        distinguish: true
+        dryRun: true
+    # Remove submissions from users who have recent activity in freekarma subs within the last 50 activities or 6 months (whichever is less)
+  - name: freekarma removal
+    description: Remove submission if user has used freekarma sub recently
+    kind: submission
+    itemIs:
+      - removed: false
+    condition: AND
+    rules:
+      - name: freekarma
+        kind: recentActivity
+        window:
+          count: 50
+          duration: 6 months
+        useSubmissionAsReference: false
+        thresholds:
+          - subreddits:
+              - FreeKarma4U
+              - FreeKarma4You
+              - KarmaStore
+              - promote
+              - shamelessplug
+              - upvote
+    actions:
+      - kind: report
+        enable: true
+        content: 'Remove=> {{rules.newtube.totalCount}} activities in freekarma subs'
+      - kind: remove
+        enable: false
+      - kind: comment
+        enable: true
+        content: >-
+          Your submission has been removed because you have recent activity in
+          'freekarma' subs
+        distinguish: true
+        dryRun: true

docs/examples/subredditReady/freekarma.yaml

@@ -0,0 +1,35 @@
+polling:
+  - unmoderated
+checks:
+    # Remove submissions from users who have recent activity in freekarma subs within the last 50 activities or 6 months (whichever is less)
+  - name: freekarma removal
+    description: Remove submission if user has used freekarma sub recently
+    kind: submission
+    itemIs:
+      - removed: false
+    condition: AND
+    rules:
+      - name: freekarma
+        kind: recentActivity
+        window:
+          count: 50
+          duration: 6 months
+        useSubmissionAsReference: false
+        thresholds:
+          - subreddits:
+              - FreeKarma4U
+              - FreeKarma4You
+              - KarmaStore
+              - upvote
+    actions:
+      - kind: report
+        enable: true
+        content: 'Remove=> {{rules.newtube.totalCount}} activities in freekarma subs'
+      - kind: remove
+        enable: true
+      - kind: comment
+        enable: false
+        content: >-
+          Your submission has been removed because you have recent activity in
+          'freekarma' subs
+        distinguish: true

docs/examples/subredditReady/selfPromo.yaml

@@ -0,0 +1,71 @@
+polling:
+  - unmoderated
+checks:
+    #
+    # Stop users who make link submissions with a self-promotional agenda (with reddit's suggested 10% rule)
+    # https://www.reddit.com/wiki/selfpromotion#wiki_guidelines_for_self-promotion_on_reddit
+    #
+    # Remove a SUBMISSION if the link comprises more than or equal to 10% of users history (100 activities or 6 months) OR
+    #
+    # if link comprises 10% of submission history (100 activities or 6 months)
+    # AND less than 50% of their activity is comments OR more than 40% of those comments are as OP (in the own submissions)
+    #
+  - name: Self-promo all AND low engagement
+    description: Self-promo is >10% for all or just sub and low comment engagement
+    kind: submission
+    condition: OR
+    rules:
+      - name: attr
+        kind: attribution
+        criteria:
+          - threshold: '>= 10%'
+            window:
+              count: 100
+              duration: 6 months
+            domains:
+              - 'AGG:SELF'
+      - condition: AND
+        rules:
+          - name: attrsub
+            kind: attribution
+            criteria:
+              - threshold: '>= 10%'
+                thresholdOn: submissions
+                window:
+                  count: 100
+                  duration: 6 months
+                domains:
+                  - 'AGG:SELF'
+          - name: lowOrOpComm
+            kind: history
+            criteriaJoin: OR
+            criteria:
+              - window:
+                  count: 100
+                  duration: 6 months
+                comment: < 50%
+              - window:
+                  count: 100
+                  duration: 6 months
+                comment: '> 40% OP'
+    actions:
+      - kind: report
+        enable: true
+        content: >-
+          {{rules.attr.largestPercent}}{{rules.attrsub.largestPercent}} of
+          {{rules.attr.activityTotal}}{{rules.attrsub.activityTotal}} items
+          ({{rules.attr.window}}{{rules.attrsub.window}}){{#rules.loworopcomm.thresholdSummary}}
+          =>
+          {{rules.loworopcomm.thresholdSummary}}{{/rules.loworopcomm.thresholdSummary}}
+      - kind: remove
+        enable: false
+      - kind: comment
+        enable: true
+        content: >-
+          Your submission has been removed it comprises 10% or more of your
+          recent history
+          ({{rules.attr.largestPercent}}{{rules.attrsub.largestPercent}}). This
+          is against [reddit's self promotional
+          guidelines.](https://www.reddit.com/wiki/selfpromotion#wiki_guidelines_for_self-promotion_on_reddit)
+        distinguish: true
+        dryRun: true

docs/examples/userNotes/README.md

@@ -14,7 +14,7 @@ Consult the [schema](https://json-schema.app/view/%23%2Fdefinitions%2FUserNoteCr
 
 ### Examples
 
-* [Do not tag user with Good User note](/docs/examples/userNotes/usernoteFilter.json5)
+* Do not tag user with Good User note [JSON](/docs/examples/userNotes/usernoteFilter.json5) | [YAML](/docs/examples/userNotes/usernoteFilter.yaml)
 
 ## Action
 
@@ -23,4 +23,4 @@ A User Note can also be added to the Author of a Submission or Comment with the
 
 ### Examples
 
-* [Add note on user doing self promotion](/docs/examples/userNotes/usernoteSP.json5)
+* Add note on user doing self promotion [JSON](/docs/examples/userNotes/usernoteSP.json5) | [YAML](/docs/examples/userNotes/usernoteSP.yaml)

docs/examples/userNotes/usernoteFilter.yaml

@@ -0,0 +1,27 @@
+checks:
+  - name: Self Promo Activities
+    description: Tag SP only if user does not have good contributor user note
+    # check will run on a new submission in your subreddit and look at the Author of that submission
+    kind: submission
+    rules:
+      - name: attr10all
+        kind: attribution
+        author:
+          exclude:
+              # the key of the usernote type to look for https://github.com/toolbox-team/reddit-moderator-toolbox/wiki/Subreddit-Wikis%3A-usernotes#working-with-note-types
+              # rule will not run if current usernote on Author is of type 'gooduser'
+            - type: gooduser
+        criteria:
+          - threshold: '> 10%'
+            window: 90 days
+          - threshold: '> 10%'
+            window: 100
+    actions:
+      - kind: usernote
+        # the key of usernote type
+        # https://github.com/toolbox-team/reddit-moderator-toolbox/wiki/Subreddit-Wikis%3A-usernotes#working-with-note-types
+        type: spamwarn
+        # content is mustache templated
+        content: >-
+          Self Promotion: {{rules.attr10all.titlesDelim}}
+          {{rules.attr10sub.largestPercent}}%

docs/examples/userNotes/usernoteSP.yaml

@@ -0,0 +1,23 @@
+checks:
+  - name: Self Promo Activities
+    # check will run on a new submission in your subreddit and look at the Author of that submission
+    description: >-
+      Check if any of Author's aggregated submission origins are >10% of entire
+      history
+    kind: submission
+    rules:
+      - name: attr10all
+        kind: attribution
+        criteria:
+          - threshold: '> 10%'
+            window: 90 days
+          - threshold: '> 10%'
+            window: 100
+    actions:
+      - kind: usernote
+        # the key of usernote type
+        # https://github.com/toolbox-team/reddit-moderator-toolbox/wiki/Subreddit-Wikis%3A-usernotes#working-with-note-types
+        type: spamwarn
+        content: >-
+          Self Promotion: {{rules.attr10all.titlesDelim}}
+          {{rules.attr10sub.largestPercent}}%

docs/gettingStartedMod.md

@@ -14,8 +14,8 @@ This getting started guide is for **reddit moderators** -- that is, someone who
 
 Before continuing with this guide you should first make sure you understand how a ContextMod works. Please review this documentation:
 
-* [How It Works](/docs#how-it-works)
-* [Core Concepts](/docs#concepts)
+* [How It Works](/docs/README.md#how-it-works)
+* [Core Concepts](/docs/README.md#concepts)
 
 # Choose A Bot
 
@@ -36,15 +36,16 @@ If the Operator has communicated that **you should add a bot they control as a m
 
 ___
 
-Ensure that you are in communication with the **operator** for this bot. The bot **will not automatically accept a moderator invitation,** it must be manually done by the bot operator. This is an intentional barrier to ensure moderators and the operator are familiar with their respective needs and have some form of trust.
+Ensure that you are in communication with the **operator** of this bot. The bot **will only accept a moderator invitation if your subreddit has been whitelisted by the operator.** This is an intentional barrier to ensure moderators and the operator are familiar with their respective needs and have some form of trust.
 
 Now invite the bot to moderate your subreddit. The bot should have at least these permissions:
 
 * Manage Users
 * Manage Posts and Comments
 * Manage Flair
-
-Additionally, the bot must have the **Manage Wiki Pages** permission if you plan to use [Toolbox User Notes](https://www.reddit.com/r/toolbox/wiki/docs/usernotes). If you are not planning on using this feature and do not want the bot to have this permission then you **must** ensure the bot has visibility to the configuration wiki page (detailed below).
+* Manage Wiki Pages
+  * Required to read the moderator-only visible wiki page used to configure the bot
+  * Required to read/write to [Toolbox User Notes](https://www.reddit.com/r/toolbox/wiki/docs/usernotes)
 
 ## Bring Your Own Bot (BYOB)
 
@@ -60,7 +61,7 @@ If the operator has communicated that **they want to use a bot you control** thi
 
 **Cons:**
 
-* More setup required for both moderators and operators
+* You must have access to the credentials for the reddit account (bot)
 
 ___
 
@@ -72,15 +73,28 @@ Review the information shown on the invite link webpage and then follow the dire
 
 # Configuring the Bot
 
+The bot's behavior is defined using a configuration, like automoderator, that is stored in the **wiki** of each subreddit it moderates.
+
+The default location for this page is at `https://old.reddit.com/r/YOURSUBERDDIT/wiki/botconfig/contextbot`
+
 ## Setup wiki page
 
+The bot automatically tries to create its configuration wiki page. You can find the result of this in the log for your subreddit in the web interface.
+
+If this fails for some reason you can create the wiki page through the web interface by navigating to your subreddit's tab, opening the [built-in editor (click **View**)](/docs/screenshots/configBox.png), and following the directions in **Create configuration for...** link found there.
+
+If neither of the above approaches work, or you do not wish to use the web interface, expand the section below for directions on how to manually setup the wiki page:
+
+<details>
+
 * Visit the wiki page of the subreddit you want the bot to moderate
   * The default location the bot checks for a configuration is at `https://old.reddit.com/r/YOURSUBERDDIT/wiki/botconfig/contextbot`
   * If the page does not exist create it
 * Ensure the wiki page visibility is restricted
   * On the wiki page click **settings** (**Page settings** in new reddit)
   * Check the box for **Only mods may edit and view** and then **save**
-    * Alternatively, if you did not give the bot the **Manage Wiki Pages** permission then add it to the **allow users to edit page** setting
+
+</details>
 
 ## Procure a configuration
 
@@ -94,25 +108,46 @@ Visit the [Examples](https://github.com/FoxxMD/context-mod/tree/master/docs/exam
 
 After you have found a configuration to use as a starting point:
 
-* In a new tab open the github page for the configuration you want ([example](/docs/examples/repeatActivity/crosspostSpamming.json5))
-* Click the **Raw** button, then select all and copy all of the text to your clipboard.
+* Copy the URL for the configuration file EX `https://github.com/FoxxMD/context-mod/blob/master/docs/examples/subredditReady/freekarma.json5` and either:
+  * (Easiest) **Load** it into your [subreddit's built-in editor](#using-the-built-in-editor) and **Save**
+  * or on the file's page, click the **Raw** button, select all and copy to your clipboard, and [manually save to your wiki page](#manually-saving)
 
 ### Build Your Own Config
 
-Additionally, you can use [this schema editor](https://json-schema.app/view/%23?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Fcontext-mod%2Fmaster%2Fsrc%2FSchema%2FApp.json) to build your configuration. The editor features a ton of handy features:
+CM comes equipped with a [configuration explorer](https://json-schema.app/view/%23?url=https%3A%2F%2Fraw.githubusercontent.com%2FFoxxMD%2Fcontext-mod%2Fmaster%2Fsrc%2FSchema%2FApp.json) to help you see all available options, with descriptions and examples, that can be used in your configuration.
 
-* fully annotated configuration data/structure
-* generated examples in json/yaml
-* built-in editor that automatically validates your config
+To create or edit a configuration you should use **CM's buit-in editor** which features:
+* syntax validation and formatting
+* full configuration validation with error highlighting, hints, and fixes
+* hover over properties to see documentation and examples
 
-PROTIP: Find an example config to use as a starting point and then build on it using the editor.
+To use the editor either:
+* [use your subreddit's built-in editor](#using-the-built-in-editor)
+* or use the public editor at https://cm.foxxmd.dev/config
+
+PROTIP: Find an [example config](#using-an-example-config) to use as a starting point and then build on it using the editor.
 
 ## Saving Your Configuration
 
-* Open the wiki page you created in the [previous step](#setup-wiki-page) and click **edit**
+### Using the built-in Editor
+
+In the web interface each subreddit's tab has access to the built-in editor. Use this built-in editor to automatically create, load, or save the configuration for that subreddit's wiki.
+
+* Visit the tab for the subreddit you want to edit the configuration of
+* Open the [built-in editor by click **View**](/docs/screenshots/configBox.png)
+* Edit your configuration
+* Follow the directions on the **Save to r/..** link found at the top of the editor to automatically save your configuration
+
+### Manually Saving
+
+<details>
+
+* Open the wiki page you created in the [wiki setup step](#setup-wiki-page) and click **edit**
   * Copy-paste your configuration into the wiki text box
   * Save the edited wiki page
 
+</details>
+
 ___
 
 The bot automatically checks for new configurations on your wiki page every 5 minutes. If your operator has the web interface accessible you may login there and force the config to update on your subreddit.

docs/imageComparison.md

@@ -7,6 +7,7 @@ ContextMod supports comparing image content, for the purpose of detecting duplic
 
 To enable comparisons reference the example below (at the top-level of your rule) and configure as needed:
 
+JSON
 ```json5
 {
   "name": "ruleWithImageDetection",
@@ -31,9 +32,18 @@ To enable comparisons reference the example below (at the top-level of your rule
   },
   //
   // And above ^^^
-  ...
+  //...
 }
 ```
+YAML
+```yaml
+name: ruleWithImageDetection
+kind: recentActivity
+  enable: true
+  threshold: 5
+  fetchBehavior: extension
+
+```
 
 **Perceptual Hashing** (`hash`) and **Pixel Comparisons** (`pixel`) may be used at the same time. Refer to the documentation below to see how they interact.
 
@@ -114,10 +124,12 @@ To further configure hashing refer to this code block:
       // the higher the bits the more accurate the comparison
       //
       // NOTE: Hashes of different sizes (bits) cannot be compared. If you are caching hashes make sure all rules where results may be shared use the same bit count to ensure hashes can be compared. Otherwise hashes will be recomputed.
-      "bits": 32, // default is 32 if not defined
+      "bits": 32,
+      // default is 32 if not defined
       //
       // number of seconds to cache an image hash
-      "ttl": 60, // default is 60 if not defined
+      "ttl": 60,
+      // default is 60 if not defined
       //
       // "High Confidence" Threshold
       // If the difference in comparison is equal to or less than this number the images are considered the same and pixel comparison WILL NOT occur
@@ -139,8 +151,22 @@ To further configure hashing refer to this code block:
     //
     // And above ^^^
     //"pixel": {...}
-  },
+  }
   //...
+}
+```
+YAML
+```yaml
+name: ruleWithImageDetectionAndConfiguredHashing
+kind: recentActivity
+imageDetection:
+  enable: true
+  hash:
+    enable: true
+    bits: 32
+    ttl: 60
+    hardThreshold: 5
+    softThreshold: 0
 ```
 
 ## Pixel Comparison
@@ -184,18 +210,28 @@ To configure pixel comparisons refer to this code block:
 
 ```json5
 {
-    "name": "ruleWithImageDetectionAndPixelEnabled",
-    "kind": "recentActivity",
-    "imageDetection": {
-      //"hash": {...}
-      "pixel": {
-        // enable or disable pixel comparisons (disabled by default)
-        "enable": true,
-        // if the comparison difference percentage is equal to or less than this value the images are considered the same
-        //
-        // if not defined the value from imageDetection.threshold will be used
-        "threshold": 5
-      }
-    },
-    //...
+  "name": "ruleWithImageDetectionAndPixelEnabled",
+  "kind": "recentActivity",
+  "imageDetection": {
+    //"hash": {...}
+    "pixel": {
+      // enable or disable pixel comparisons (disabled by default)
+      "enable": true,
+      // if the comparison difference percentage is equal to or less than this value the images are considered the same
+      //
+      // if not defined the value from imageDetection.threshold will be used
+      "threshold": 5
+    }
+  },
+  //...
+}
+```
+YAML
+```yaml
+name: ruleWithImageDetectionAndPixelEnabled
+kind: recentActivity
+imageDetection:
+  pixel:
+    enable: true
+    threshold: 5
 ```

docs/operatorConfiguration.md

@@ -121,6 +121,16 @@ Below are examples of the minimum required config to run the application using a
 Using **FILE**
 <details>
 
+YAML
+```yaml
+bots:
+  - credentials:
+      clientId: f4b4df1c7b2
+      clientSecret: 34v5q1c56ub
+      refreshToken: 34_f1w1v4
+      accessToken: p75_1c467b2
+```
+JSON
 ```json5
 {
   "bots": [
@@ -175,6 +185,11 @@ An example of using multiple configuration levels together IE all are provided t
   }
 }
 ```
+YAML
+```yaml
+logging:
+  level: debug
+```
 
 </details>
 
@@ -220,6 +235,30 @@ See the [Architecture Docs](/docs/serverClientArchitecture.md) for more informat
 
 <details>
 
+YAML
+```yaml
+bots:
+  - credentials:
+      clientId: f4b4df1c7b2
+      clientSecret: 34v5q1c56ub
+      refreshToken: 34_f1w1v4
+      accessToken: p75_1c467b2
+web:
+  credentials:
+    clientId: f4b4df1c7b2
+    clientSecret: 34v5q1c56ub
+    redirectUri: 'http://localhost:8085/callback'
+  clients:
+      # server application running on this same CM instance
+    - host: 'localhost:8095'
+      secret: localSecret
+      # a server application running somewhere else
+    - host: 'mySecondContextMod.com:8095'
+      secret: anotherSecret
+api:
+  secret: localSecret
+```
+JSON
 ```json5
 {
   "bots": [
@@ -289,3 +328,14 @@ A caching object in the json configuration:
  }
 }
 ```
+YAML
+```yaml
+provider:
+  store: memory
+  ttl: 60
+  max: 500
+  host: localhost
+  port: 6379
+  auth_pass: null
+  db: 0
+```

package.json

@@ -35,6 +35,7 @@
     "cache-manager": "^3.4.4",
     "cache-manager-redis-store": "^2.0.0",
     "commander": "^8.0.0",
+    "comment-json": "^4.1.1",
     "cookie-parser": "^1.3.5",
     "dayjs": "^1.10.5",
     "deepmerge": "^4.2.2",
@@ -52,7 +53,6 @@
     "he": "^1.2.0",
     "http-proxy": "^1.18.1",
     "image-size": "^1.0.0",
-    "js-yaml": "^4.1.0",
     "json5": "^2.2.0",
     "jsonwebtoken": "^8.5.1",
     "leven": "^3.1.0",
@@ -68,6 +68,7 @@
     "passport-custom": "^1.1.1",
     "passport-jwt": "^4.0.0",
     "pixelmatch": "^5.2.1",
+    "pony-cause": "^1.1.1",
     "pretty-print-json": "^1.0.3",
     "safe-stable-stringify": "^1.1.1",
     "snoostorm": "^1.5.2",
@@ -82,6 +83,7 @@
     "winston-daily-rotate-file": "^4.5.5",
     "winston-duplex": "^0.1.1",
     "winston-transport": "^4.4.0",
+    "yaml": "2.0.0-10",
     "zlib": "^1.0.5"
   },
   "devDependencies": {
@@ -111,8 +113,9 @@
     "@types/string-similarity": "^4.0.0",
     "@types/tcp-port-used": "^1.0.0",
     "@types/triple-beam": "^1.3.2",
+    "ts-essentials": "^9.1.2",
     "ts-json-schema-generator": "^0.93.0",
-    "typescript-json-schema": "^0.50.1"
+    "typescript-json-schema": "~0.53"
   },
   "optionalDependencies": {
     "sharp": "^0.29.1"

src/Action/ActionFactory.ts

@@ -10,10 +10,11 @@ import ApproveAction, {ApproveActionConfig} from "./ApproveAction";
 import BanAction, {BanActionJson} from "./BanAction";
 import {MessageAction, MessageActionJson} from "./MessageAction";
 import {SubredditResources} from "../Subreddit/SubredditResources";
-import Snoowrap from "snoowrap";
+import {UserFlairAction, UserFlairActionJson} from './UserFlairAction';
+import {ExtendedSnoowrap} from '../Utils/SnoowrapClients';
 
 export function actionFactory
-(config: ActionJson, logger: Logger, subredditName: string, resources: SubredditResources, client: Snoowrap): Action {
+(config: ActionJson, logger: Logger, subredditName: string, resources: SubredditResources, client: ExtendedSnoowrap): Action {
     switch (config.kind) {
         case 'comment':
             return new CommentAction({...config as CommentActionJson, logger, subredditName, resources, client});
@@ -25,6 +26,8 @@ export function actionFactory
             return new ReportAction({...config as ReportActionJson, logger, subredditName, resources, client});
         case 'flair':
             return new FlairAction({...config as FlairActionJson, logger, subredditName, resources, client});
+        case 'userflair':
+            return new UserFlairAction({...config as UserFlairActionJson, logger, subredditName, resources, client});
         case 'approve':
             return new ApproveAction({...config as ApproveActionConfig, logger, subredditName, resources, client});
         case 'usernote':

src/Action/ApproveAction.ts

@@ -1,39 +1,84 @@
-import {ActionJson, ActionConfig} from "./index";
+import {ActionJson, ActionConfig, ActionOptions} from "./index";
 import Action from "./index";
-import Snoowrap, {Comment, Submission} from "snoowrap";
+import Snoowrap from "snoowrap";
 import {RuleResult} from "../Rule";
 import {ActionProcessResult} from "../Common/interfaces";
+import Submission from "snoowrap/dist/objects/Submission";
+import Comment from "snoowrap/dist/objects/Comment";
 
 export class ApproveAction extends Action {
+
+    targets: ApproveTarget[]
+
     getKind() {
         return 'Approve';
     }
 
+    constructor(options: ApproveOptions) {
+        super(options);
+        const {
+            targets = ['self']
+        } = options;
+
+        this.targets = targets;
+    }
+
     async process(item: Comment | Submission, ruleResults: RuleResult[], runtimeDryrun?: boolean): Promise<ActionProcessResult> {
         const dryRun = runtimeDryrun || this.dryRun;
-        //snoowrap typing issue, thinks comments can't be locked
-        // @ts-ignore
-        if (item.approved) {
-            this.logger.warn('Item is already approved');
-            return {
-                dryRun,
-                success: false,
-                result: 'Item is already approved'
+        const touchedEntities = [];
+
+        const realTargets = item instanceof Submission ? ['self'] : this.targets;
+
+        for(const target of realTargets) {
+            let targetItem = item;
+            if(target !== 'self' && item instanceof Comment) {
+                targetItem = await this.resources.getActivity(this.client.getSubmission(item.link_id));
+            }
+
+            // @ts-ignore
+            if (item.approved) {
+                const msg = `${target === 'self' ? 'Item' : 'Comment\'s parent Submission'} is already approved`;
+                this.logger.warn(msg);
+                return {
+                    dryRun,
+                    success: false,
+                    result: msg
+                }
+            }
+
+            if (!dryRun) {
+                // make sure we have an actual item and not just a plain object from cache
+                if(target !== 'self' && !(targetItem instanceof Submission)) {
+                    // @ts-ignore
+                    targetItem = await this.client.getSubmission((item as Comment).link_id).fetch();
+                }
+                // @ts-ignore
+                touchedEntities.push(await targetItem.approve());
             }
         }
-        if (!dryRun) {
-            // @ts-ignore
-            await item.approve();
-        }
+
         return {
             dryRun,
             success: true,
+            touchedEntities
         }
     }
 }
 
-export interface ApproveActionConfig extends ActionConfig {
+export type ApproveTarget = 'self' | 'parent';
 
+export interface ApproveOptions extends ApproveActionConfig, ActionOptions {}
+
+export interface ApproveActionConfig extends ActionConfig {
+    /**
+     * Specify which Activities to approve
+     *
+     * This setting is only applicable if the Activity being acted on is a **comment**. On a **submission** the setting does nothing
+     *
+     * * self => approve activity being checked (comment)
+     * * parent => approve parent (submission) of activity being checked (comment)
+     * */
+    targets?: ApproveTarget[]
 }
 
 /**

src/Action/BanAction.ts

@@ -39,6 +39,7 @@ export class BanAction extends Action {
         const renderedBody = content === undefined ? undefined : await renderContent(content, item, ruleResults, this.resources.userNotes);
         const renderedContent = renderedBody === undefined ? undefined : `${renderedBody}${await this.resources.generateFooter(item, this.footer)}`;
 
+        const touchedEntities = [];
         let banPieces = [];
         banPieces.push(`Message: ${renderedContent === undefined ? 'None' : `${renderedContent.length > 100 ? `\r\n${renderedContent}` : renderedContent}`}`);
         banPieces.push(`Reason:  ${this.reason || 'None'}`);
@@ -50,18 +51,20 @@ export class BanAction extends Action {
             // @ts-ignore
             const fetchedSub = await item.subreddit.fetch();
             const fetchedName = await item.author.name;
-            await fetchedSub.banUser({
+            const bannedUser = await fetchedSub.banUser({
                 name: fetchedName,
                 banMessage: renderedContent === undefined ? undefined : renderedContent,
                 banReason: this.reason,
                 banNote: this.note,
                 duration: this.duration
             });
+            touchedEntities.push(bannedUser);
         }
         return {
             dryRun,
             success: true,
-            result: `Banned ${item.author.name} ${durText}${this.reason !== undefined ? ` (${this.reason})` : ''}`
+            result: `Banned ${item.author.name} ${durText}${this.reason !== undefined ? ` (${this.reason})` : ''}`,
+            touchedEntities
         };
     }
 }

src/Action/CommentAction.ts

@@ -51,16 +51,19 @@ export class CommentAction extends Action {
                 result: 'Cannot comment because Item is archived'
             };
         }
+        const touchedEntities = [];
         let reply: Comment;
         if(!dryRun) {
             // @ts-ignore
            reply = await item.reply(renderedContent);
+           touchedEntities.push(reply);
         }
         if (this.lock) {
             if (!dryRun) {
                 // snoopwrap typing issue, thinks comments can't be locked
                 // @ts-ignore
                 await item.lock();
+                touchedEntities.push(item);
             }
         }
         if (this.distinguish && !dryRun) {
@@ -78,7 +81,8 @@ export class CommentAction extends Action {
         return {
             dryRun,
             success: true,
-            result: `${modifierStr}${this.lock ? ' - Locked Author\'s Activity - ' : ''}${truncateStringToLength(100)(body)}`
+            result: `${modifierStr}${this.lock ? ' - Locked Author\'s Activity - ' : ''}${truncateStringToLength(100)(body)}`,
+            touchedEntities,
         };
     }
 }

src/Action/LockAction.ts

@@ -11,6 +11,7 @@ export class LockAction extends Action {
 
     async process(item: Comment | Submission, ruleResults: RuleResult[], runtimeDryrun?: boolean): Promise<ActionProcessResult> {
         const dryRun = runtimeDryrun || this.dryRun;
+        const touchedEntities = [];
         //snoowrap typing issue, thinks comments can't be locked
         // @ts-ignore
         if (item.locked) {
@@ -25,10 +26,12 @@ export class LockAction extends Action {
             //snoowrap typing issue, thinks comments can't be locked
             // @ts-ignore
             await item.lock();
+            touchedEntities.push(item);
         }
         return {
             dryRun,
-            success: true
+            success: true,
+            touchedEntities
         }
     }
 }

src/Action/MessageAction.ts

@@ -12,7 +12,8 @@ import {
     REDDIT_ENTITY_REGEX_URL,
     truncateStringToLength
 } from "../util";
-import SimpleError from "../Utils/SimpleError";
+import {SimpleError} from "../Utils/Errors";
+import {ErrorWithCause} from "pony-cause";
 
 export class MessageAction extends Action {
     content: string;
@@ -65,10 +66,7 @@ export class MessageAction extends Action {
                     recipient = `/r/${entityData.name}`;
                 }
             } catch (err: any) {
-                this.logger.error(`'to' field for message was not in a valid format. See ${REDDIT_ENTITY_REGEX_URL} for valid examples`);
-                this.logger.error(err);
-                err.logged = true;
-                throw err;
+                throw new ErrorWithCause(`'to' field for message was not in a valid format. See ${REDDIT_ENTITY_REGEX_URL} for valid examples`, {cause: err});
             }
             if(recipient.includes('/r/') && this.asSubreddit) {
                 throw new SimpleError(`Cannot send a message as a subreddit to another subreddit. Requested recipient: ${recipient}`);

src/Action/RemoveAction.ts

@@ -1,4 +1,4 @@
-import {ActionJson, ActionConfig} from "./index";
+import {ActionJson, ActionConfig, ActionOptions} from "./index";
 import Action from "./index";
 import Snoowrap, {Comment, Submission} from "snoowrap";
 import {RuleResult} from "../Rule";
@@ -6,12 +6,23 @@ import {activityIsRemoved} from "../Utils/SnoowrapUtils";
 import {ActionProcessResult} from "../Common/interfaces";
 
 export class RemoveAction extends Action {
+    spam: boolean;
+
     getKind() {
         return 'Remove';
     }
 
+    constructor(options: RemoveOptions) {
+        super(options);
+        const {
+            spam = false,
+        } = options;
+        this.spam = spam;
+    }
+
     async process(item: Comment | Submission, ruleResults: RuleResult[], runtimeDryrun?: boolean): Promise<ActionProcessResult> {
         const dryRun = runtimeDryrun || this.dryRun;
+        const touchedEntities = [];
         // issue with snoowrap typings, doesn't think prop exists on Submission
         // @ts-ignore
         if (activityIsRemoved(item)) {
@@ -21,25 +32,33 @@ export class RemoveAction extends Action {
                 result: 'Item is already removed',
             }
         }
+        if (this.spam) {
+            this.logger.verbose('Marking as spam on removal');
+        }
         if (!dryRun) {
             // @ts-ignore
-            await item.remove();
+            await item.remove({spam: this.spam});
+            touchedEntities.push(item);
         }
 
         return {
             dryRun,
             success: true,
+            touchedEntities
         }
     }
 }
 
-export interface RemoveActionConfig extends ActionConfig {
+export interface RemoveOptions extends RemoveActionConfig, ActionOptions {
+}
 
+export interface RemoveActionConfig extends ActionConfig {
+    spam?: boolean
 }
 
 /**
  * Remove the Activity
  * */
 export interface RemoveActionJson extends RemoveActionConfig, ActionJson {
-kind: 'remove'
+    kind: 'remove'
 }

src/Action/ReportAction.ts

@@ -29,15 +29,20 @@ export class ReportAction extends Action {
         const renderedContent = await renderContent(content, item, ruleResults, this.resources.userNotes);
         this.logger.verbose(`Contents:\r\n${renderedContent}`);
         const truncatedContent = reportTrunc(renderedContent);
+        const touchedEntities = [];
         if(!dryRun) {
             // @ts-ignore
             await item.report({reason: truncatedContent});
+            // due to reddit not updating this in response (maybe)?? just increment stale activity
+            item.num_reports++;
+            touchedEntities.push(item);
         }
 
         return {
             dryRun,
             success: true,
-            result: truncatedContent
+            result: truncatedContent,
+            touchedEntities
         };
     }
 }

src/Action/SubmissionAction/FlairAction.ts

@@ -1,20 +1,23 @@
 import {SubmissionActionConfig} from "./index";
 import Action, {ActionJson, ActionOptions} from "../index";
-import Snoowrap, {Comment, Submission} from "snoowrap";
 import {RuleResult} from "../../Rule";
 import {ActionProcessResult} from "../../Common/interfaces";
+import Submission from 'snoowrap/dist/objects/Submission';
+import Comment from 'snoowrap/dist/objects/Comment';
 
 export class FlairAction extends Action {
     text: string;
     css: string;
+    flair_template_id: string;
 
     constructor(options: FlairActionOptions) {
         super(options);
-        if (options.text === undefined && options.css === undefined) {
-            throw new Error('Must define either text or css on FlairAction');
+        if (options.text === undefined && options.css === undefined && options.flair_template_id === undefined) {
+            throw new Error('Must define either text+css or flair_template_id on FlairAction');
         }
         this.text = options.text || '';
         this.css = options.css || '';
+        this.flair_template_id = options.flair_template_id || '';
     }
 
     getKind() {
@@ -34,8 +37,12 @@ export class FlairAction extends Action {
         this.logger.verbose(flairSummary);
         if (item instanceof Submission) {
             if(!this.dryRun) {
-                // @ts-ignore
-                await item.assignFlair({text: this.text, cssClass: this.css})
+                if (this.flair_template_id) {
+                    await item.selectFlair({flair_template_id: this.flair_template_id}).then(() => {});
+                } else {
+                    await item.assignFlair({text: this.text, cssClass: this.css}).then(() => {});
+                }
+
             }
         } else {
             this.logger.warn('Cannot flair Comment');
@@ -60,12 +67,16 @@ export class FlairAction extends Action {
 export interface FlairActionConfig extends SubmissionActionConfig {
     /**
      * The text of the flair to apply
-    * */
+     * */
     text?: string,
     /**
      * The text of the css class of the flair to apply
      * */
     css?: string,
+    /**
+     * Flair template ID to assign
+     * */
+    flair_template_id?: string,
 }
 
 export interface FlairActionOptions extends FlairActionConfig,ActionOptions {
@@ -76,5 +87,5 @@ export interface FlairActionOptions extends FlairActionConfig,ActionOptions {
  * Flair the Submission
  * */
 export interface FlairActionJson extends FlairActionConfig, ActionJson {
-kind: 'flair'
+    kind: 'flair'
 }

src/Action/UserFlairAction.ts

@@ -0,0 +1,109 @@
+import Action, {ActionConfig, ActionJson, ActionOptions} from './index';
+import {Comment, RedditUser, Submission} from 'snoowrap';
+import {RuleResult} from '../Rule';
+import {ActionProcessResult} from '../Common/interfaces';
+
+export class UserFlairAction extends Action {
+  text?: string;
+  css?: string;
+  flair_template_id?: string;
+
+  constructor(options: UserFlairActionOptions) {
+    super(options);
+
+    this.text = options.text === null || options.text === '' ? undefined : options.text;
+    this.css = options.css === null || options.css === '' ? undefined : options.css;
+    this.flair_template_id = options.flair_template_id === null || options.flair_template_id === '' ? undefined : options.flair_template_id;
+  }
+
+  getKind() {
+    return 'User Flair';
+  }
+
+  async process(item: Comment | Submission, ruleResults: RuleResult[], runtimeDryrun?: boolean): Promise<ActionProcessResult> {
+    const dryRun = runtimeDryrun || this.dryRun;
+    let flairParts = [];
+
+    if (this.flair_template_id !== undefined) {
+      flairParts.push(`Flair template ID: ${this.flair_template_id}`)
+      if(this.text !== undefined || this.css !== undefined) {
+        this.logger.warn('Text/CSS properties will be ignored since a flair template is specified');
+      }
+    } else {
+      if (this.text !== undefined) {
+        flairParts.push(`Text: ${this.text}`);
+      }
+      if (this.css !== undefined) {
+        flairParts.push(`CSS: ${this.css}`);
+      }
+    }
+
+    const flairSummary = flairParts.length === 0 ? 'Unflair user' : flairParts.join(' | ');
+    this.logger.verbose(flairSummary);
+
+    if (!this.dryRun) {
+      if (this.flair_template_id !== undefined) {
+        try {
+          // @ts-ignore
+          await this.client.assignUserFlairByTemplateId({
+            subredditName: item.subreddit.display_name,
+            flairTemplateId: this.flair_template_id,
+            username: item.author.name,
+          });
+        } catch (err: any) {
+          this.logger.error('Either the flair template ID is incorrect or you do not have permission to access it.');
+          throw err;
+        }
+      } else if (this.text === undefined && this.css === undefined) {
+        // @ts-ignore
+        await item.subreddit.deleteUserFlair(item.author.name);
+      } else {
+        // @ts-ignore
+        await item.author.assignFlair({
+          subredditName: item.subreddit.display_name,
+          cssClass: this.css,
+          text: this.text,
+        });
+      }
+    }
+
+    return {
+      dryRun,
+      success: true,
+      result: flairSummary,
+    }
+  }
+}
+
+/**
+ * Flair the Author of an Activity
+ *
+ * Leave all properties blank or null to remove a User's existing flair
+ * */
+export interface UserFlairActionConfig extends ActionConfig {
+  /**
+   * The text of the flair to apply
+   * */
+  text?: string,
+  /**
+   * The text of the css class of the flair to apply
+   * */
+  css?: string,
+  /**
+   * Flair template to pick.
+   *
+   * **Note:** If this template is used text/css are ignored
+   * */
+  flair_template_id?: string;
+}
+
+export interface UserFlairActionOptions extends UserFlairActionConfig, ActionOptions {
+
+}
+
+/**
+ * Flair the Submission
+ * */
+export interface UserFlairActionJson extends UserFlairActionConfig, ActionJson {
+  kind: 'userflair'
+}

src/Action/index.ts

@@ -1,17 +1,18 @@
-import Snoowrap, {Comment, Submission} from "snoowrap";
+import {Comment, Submission} from "snoowrap";
 import {Logger} from "winston";
 import {RuleResult} from "../Rule";
-import {SubredditResources} from "../Subreddit/SubredditResources";
+import {checkAuthorFilter, SubredditResources} from "../Subreddit/SubredditResources";
 import {ActionProcessResult, ActionResult, ChecksActivityState, TypedActivityStates} from "../Common/interfaces";
 import Author, {AuthorOptions} from "../Author/Author";
 import {mergeArr} from "../util";
 import LoggedError from "../Utils/LoggedError";
+import {ExtendedSnoowrap} from '../Utils/SnoowrapClients';
 
 export abstract class Action {
     name?: string;
     logger: Logger;
     resources: SubredditResources;
-    client: Snoowrap
+    client: ExtendedSnoowrap;
     authorIs: AuthorOptions;
     itemIs: TypedActivityStates;
     dryRun: boolean;
@@ -27,6 +28,7 @@ export abstract class Action {
             subredditName,
             dryRun = false,
             authorIs: {
+                excludeCondition = 'OR',
                 include = [],
                 exclude = [],
             } = {},
@@ -41,6 +43,7 @@ export abstract class Action {
         this.logger = logger.child({labels: [`Action ${this.getActionUniqueName()}`]}, mergeArr);
 
         this.authorIs = {
+            excludeCondition,
             exclude: exclude.map(x => new Author(x)),
             include: include.map(x => new Author(x)),
         }
@@ -71,27 +74,10 @@ export abstract class Action {
                 actRes.runReason = `Activity did not pass 'itemIs' test, Action not run`;
                 return actRes;
             }
-            if (this.authorIs.include !== undefined && this.authorIs.include.length > 0) {
-                for (const auth of this.authorIs.include) {
-                    if (await this.resources.testAuthorCriteria(item, auth)) {
-                        actRes.run = true;
-                        const results = await this.process(item, ruleResults, runtimeDryrun);
-                        return {...actRes, ...results};
-                    }
-                }
-                this.logger.verbose('Inclusive author criteria not matched, Action not run');
-                actRes.runReason = 'Inclusive author criteria not matched';
-                return actRes;
-            } else if (this.authorIs.exclude !== undefined && this.authorIs.exclude.length > 0) {
-                for (const auth of this.authorIs.exclude) {
-                    if (await this.resources.testAuthorCriteria(item, auth, false)) {
-                        actRes.run = true;
-                        const results = await this.process(item, ruleResults, runtimeDryrun);
-                        return {...actRes, ...results};
-                    }
-                }
-                this.logger.verbose('Exclusive author criteria not matched, Action not run');
-                actRes.runReason = 'Exclusive author criteria not matched';
+            const [authFilterResult, authFilterType] = await checkAuthorFilter(item, this.authorIs, this.resources, this.logger);
+            if(!authFilterResult) {
+                this.logger.verbose(`${authFilterType} author criteria not matched, Action not run`);
+                actRes.runReason = `${authFilterType} author criteria not matched`;
                 return actRes;
             }
 
@@ -114,8 +100,8 @@ export abstract class Action {
 export interface ActionOptions extends ActionConfig {
     logger: Logger;
     subredditName: string;
-    resources: SubredditResources
-    client: Snoowrap
+    resources: SubredditResources;
+    client: ExtendedSnoowrap;
 }
 
 export interface ActionConfig extends ChecksActivityState {
@@ -162,7 +148,7 @@ export interface ActionJson extends ActionConfig {
     /**
      * The type of action that will be performed
      */
-    kind: 'comment' | 'lock' | 'remove' | 'report' | 'approve' | 'ban' | 'flair' | 'usernote' | 'message'
+    kind: 'comment' | 'lock' | 'remove' | 'report' | 'approve' | 'ban' | 'flair' | 'usernote' | 'message' | 'userflair'
 }
 
 export const isActionJson = (obj: object): obj is ActionJson => {

src/App.ts

@@ -1,7 +1,7 @@
 import winston, {Logger} from "winston";
 import dayjs, {Dayjs} from "dayjs";
 import {getLogger} from "./Utils/loggerFactory";
-import {Invokee, OperatorConfig} from "./Common/interfaces";
+import {Invokee, OperatorConfig, OperatorConfigWithFileContext, OperatorFileConfig} from "./Common/interfaces";
 import Bot from "./Bot";
 import LoggedError from "./Utils/LoggedError";
 import {sleep} from "./util";
@@ -14,7 +14,10 @@ export class App {
 
     error: any;
 
-    constructor(config: OperatorConfig) {
+    config: OperatorConfig;
+    fileConfig: OperatorFileConfig;
+
+    constructor(config: OperatorConfigWithFileContext) {
         const {
             operator: {
                 name,
@@ -23,6 +26,11 @@ export class App {
             bots = [],
         } = config;
 
+        const {fileConfig, ...rest} = config;
+
+        this.config = rest;
+        this.fileConfig = fileConfig;
+
         this.logger = getLogger(config.logging);
 
         this.logger.info(`Operators: ${name.length === 0 ? 'None Specified' : name.join(', ')}`)

src/Author/Author.ts

@@ -1,5 +1,5 @@
 import {UserNoteCriteria} from "../Rule";
-import {CompareValue, CompareValueOrPercent, DurationComparor} from "../Common/interfaces";
+import {CompareValue, CompareValueOrPercent, DurationComparor, JoinOperands} from "../Common/interfaces";
 import {parseStringToRegex} from "../util";
 
 /**
@@ -12,7 +12,17 @@ export interface AuthorOptions {
      * */
     include?: AuthorCriteria[];
     /**
-     * Only runs if `include` is not present. Will "pass" if any of set of the AuthorCriteria **does not** pass
+     * * OR => if ANY exclude condition "does not" pass then the exclude test passes
+     * * AND => if ALL exclude conditions "do not" pass then the exclude test passes
+     *
+     * Defaults to OR
+     * @default OR
+     * */
+    excludeCondition?: JoinOperands
+    /**
+     * Only runs if `include` is not present. Each AuthorCriteria is comprised of conditions that the Author being checked must "not" pass. See excludeCondition for set behavior
+     *
+     * EX: `isMod: true, name: Automoderator` => Will pass if the Author IS NOT a mod and IS NOT named Automoderator
      * */
     exclude?: AuthorCriteria[];
 }
@@ -36,15 +46,20 @@ export interface AuthorCriteria {
      * */
     name?: string[],
     /**
-     * A list of (user) flair css class values from the subreddit to match against
+     * A (user) flair css class (or list of) from the subreddit to match against
      * @examples ["red"]
      * */
-    flairCssClass?: string[],
+    flairCssClass?: string | string[],
     /**
-     * A list of (user) flair text values from the subreddit to match against
+     * A (user) flair text value (or list of) from the subreddit to match against
      * @examples ["Approved"]
      * */
-    flairText?: string[],
+    flairText?: string | string[],
+
+    /**
+     * A (user) flair template id (or list of) from the subreddit to match against
+     * */
+    flairTemplate?: string | string[]
     /**
      * Is the author a moderator?
      * */
@@ -136,8 +151,12 @@ export class Author implements AuthorCriteria {
 
     constructor(options: AuthorCriteria) {
         this.name = options.name;
-        this.flairCssClass = options.flairCssClass;
-        this.flairText = options.flairText;
+        if(options.flairCssClass !== undefined) {
+            this.flairCssClass = typeof options.flairCssClass === 'string' ? [options.flairCssClass] : options.flairCssClass;
+        }
+        if(options.flairText !== undefined) {
+            this.flairText = typeof options.flairText === 'string' ? [options.flairText] : options.flairText;
+        }
         this.isMod = options.isMod;
         this.userNotes = options.userNotes;
         this.age = options.age;

src/Bot/index.ts

@@ -3,25 +3,35 @@ import {Logger} from "winston";
 import dayjs, {Dayjs} from "dayjs";
 import {Duration} from "dayjs/plugin/duration";
 import EventEmitter from "events";
-import {BotInstanceConfig, Invokee, PAUSED, RUNNING, STOPPED, SYSTEM, USER} from "../Common/interfaces";
+import {
+    BotInstanceConfig,
+    FilterCriteriaDefaults,
+    Invokee,
+    PAUSED,
+    PollOn,
+    RUNNING,
+    STOPPED,
+    SYSTEM,
+    USER
+} from "../Common/interfaces";
 import {
     createRetryHandler,
-    formatNumber,
+    formatNumber, getExceptionMessage,
     mergeArr,
     parseBool,
-    parseDuration,
-    parseSubredditName,
+    parseDuration, parseMatchMessage,
+    parseSubredditName, RetryOptions,
     sleep,
     snooLogWrapper
 } from "../util";
 import {Manager} from "../Subreddit/Manager";
 import {ExtendedSnoowrap, ProxiedSnoowrap} from "../Utils/SnoowrapClients";
-import {ModQueueStream, UnmoderatedStream} from "../Subreddit/Streams";
+import {CommentStream, ModQueueStream, SPoll, SubmissionStream, UnmoderatedStream} from "../Subreddit/Streams";
 import {BotResourcesManager} from "../Subreddit/SubredditResources";
 import LoggedError from "../Utils/LoggedError";
 import pEvent from "p-event";
-import SimpleError from "../Utils/SimpleError";
-import {isRateLimitError} from "../Utils/Errors";
+import {SimpleError, isRateLimitError, isRequestError, isScopeError, isStatusError, CMError} from "../Utils/Errors";
+import {ErrorWithCause} from "pony-cause";
 
 
 class Bot {
@@ -33,6 +43,7 @@ class Bot {
     running: boolean = false;
     subreddits: string[];
     excludeSubreddits: string[];
+    filterCriteriaDefaults?: FilterCriteriaDefaults
     subManagers: Manager[] = [];
     heartbeatInterval: number;
     nextHeartbeat: Dayjs = dayjs();
@@ -43,6 +54,7 @@ class Bot {
     nannyMode?: 'soft' | 'hard';
     nannyRunning: boolean = false;
     nextNannyCheck: Dayjs = dayjs().add(10, 'second');
+    sharedStreamRetryHandler: Function;
     nannyRetryHandler: Function;
     managerRetryHandler: Function;
     nextExpiration: Dayjs = dayjs();
@@ -51,7 +63,7 @@ class Bot {
     botAccount?: string;
     maxWorkers: number;
     startedAt: Dayjs = dayjs();
-    sharedModqueue: boolean = false;
+    sharedStreams: PollOn[] = [];
     streamListedOnce: string[] = [];
 
     stagger: number;
@@ -78,6 +90,7 @@ class Bot {
         const {
             notifications,
             name,
+            filterCriteriaDefaults,
             subreddits: {
                 names = [],
                 exclude = [],
@@ -98,7 +111,7 @@ class Bot {
                 debug,
             },
             polling: {
-                sharedMod,
+                shared = [],
                 stagger = 2000,
             },
             queue: {
@@ -123,7 +136,8 @@ class Bot {
         this.hardLimit = hardLimit;
         this.wikiLocation = wikiConfig;
         this.heartbeatInterval = heartbeatInterval;
-        this.sharedModqueue = sharedMod;
+        this.filterCriteriaDefaults = filterCriteriaDefaults;
+        this.sharedStreams = shared;
         if(name !== undefined) {
             this.botName = name;
         }
@@ -178,7 +192,7 @@ class Bot {
             this.client = proxy === undefined ? new ExtendedSnoowrap(creds) : new ProxiedSnoowrap({...creds, proxy});
             this.client.config({
                 warnings: true,
-                maxRetryAttempts: 5,
+                maxRetryAttempts: 2,
                 debug,
                 logger: snooLogWrapper(this.logger.child({labels: ['Snoowrap']}, mergeArr)),
                 continueAfterRatelimitError: false,
@@ -190,56 +204,12 @@ class Bot {
             }
         }
 
-        const retryHandler = createRetryHandler({maxRequestRetry: 8, maxOtherRetry: 1}, this.logger);
+        this.sharedStreamRetryHandler = createRetryHandler({maxRequestRetry: 8, maxOtherRetry: 2}, this.logger);
         this.nannyRetryHandler = createRetryHandler({maxRequestRetry: 5, maxOtherRetry: 1}, this.logger);
-        this.managerRetryHandler = createRetryHandler({maxRequestRetry: 5, maxOtherRetry: 10, waitOnRetry: false, clearRetryCountAfter: 2}, this.logger);
+        this.managerRetryHandler = createRetryHandler({maxRequestRetry: 8, maxOtherRetry: 8, waitOnRetry: false, clearRetryCountAfter: 2}, this.logger);
 
         this.stagger = stagger ?? 2000;
 
-        const modStreamErrorListener = (name: string) => async (err: any) => {
-            this.logger.error('Polling error occurred', err);
-            const shouldRetry = await retryHandler(err);
-            if(shouldRetry) {
-                defaultUnmoderatedStream.startInterval();
-            } else {
-                for(const m of this.subManagers) {
-                    if(m.modStreamCallbacks.size > 0) {
-                        m.notificationManager.handle('runStateChanged', `${name.toUpperCase()} Polling Stopped`, 'Encountered too many errors from Reddit while polling. Will try to restart on next heartbeat.');
-                    }
-                }
-                this.logger.error(`Mod stream ${name.toUpperCase()} encountered too many errors while polling. Will try to restart on next heartbeat.`);
-            }
-        }
-
-        const modStreamListingListener = (name: string) => async (listing: (Comment|Submission)[]) => {
-            // dole out in order they were received
-            if(!this.streamListedOnce.includes(name)) {
-                this.streamListedOnce.push(name);
-                return;
-            }
-            for(const i of listing) {
-                const foundManager = this.subManagers.find(x => x.subreddit.display_name === i.subreddit.display_name && x.modStreamCallbacks.get(name) !== undefined);
-                if(foundManager !== undefined) {
-                    foundManager.modStreamCallbacks.get(name)(i);
-                    if(stagger !== undefined) {
-                        await sleep(stagger);
-                    }
-                }
-            }
-
-        }
-
-        const defaultUnmoderatedStream = new UnmoderatedStream(this.client, {subreddit: 'mod', limit: 100, clearProcessed: { size: 100, retain: 100 }});
-        // @ts-ignore
-        defaultUnmoderatedStream.on('error', modStreamErrorListener('unmoderated'));
-        defaultUnmoderatedStream.on('listing', modStreamListingListener('unmoderated'));
-        const defaultModqueueStream = new ModQueueStream(this.client, {subreddit: 'mod', limit: 100, clearProcessed: { size: 100, retain: 100 }});
-        // @ts-ignore
-        defaultModqueueStream.on('error', modStreamErrorListener('modqueue'));
-        defaultModqueueStream.on('listing', modStreamListingListener('modqueue'));
-        this.cacheManager.modStreams.set('unmoderated', defaultUnmoderatedStream);
-        this.cacheManager.modStreams.set('modqueue', defaultModqueueStream);
-
         process.on('uncaughtException', (e) => {
             this.error = e;
         });
@@ -263,6 +233,37 @@ class Bot {
         });
     }
 
+    createSharedStreamErrorListener = (name: string) => async (err: any) => {
+        const shouldRetry = await this.sharedStreamRetryHandler(err);
+        if(shouldRetry) {
+            (this.cacheManager.modStreams.get(name) as SPoll<any>).startInterval(false, 'Within retry limits');
+        } else {
+            for(const m of this.subManagers) {
+                if(m.sharedStreamCallbacks.size > 0) {
+                    m.notificationManager.handle('runStateChanged', `${name.toUpperCase()} Polling Stopped`, 'Encountered too many errors from Reddit while polling. Will try to restart on next heartbeat.');
+                }
+            }
+            this.logger.error(`Mod stream ${name.toUpperCase()} encountered too many errors while polling. Will try to restart on next heartbeat.`);
+        }
+    }
+
+    createSharedStreamListingListener = (name: string) => async (listing: (Comment|Submission)[]) => {
+        // dole out in order they were received
+        if(!this.streamListedOnce.includes(name)) {
+            this.streamListedOnce.push(name);
+            return;
+        }
+        for(const i of listing) {
+            const foundManager = this.subManagers.find(x => x.subreddit.display_name === i.subreddit.display_name && x.sharedStreamCallbacks.get(name) !== undefined && x.eventsState.state === RUNNING);
+            if(foundManager !== undefined) {
+                foundManager.sharedStreamCallbacks.get(name)(i);
+                if(this.stagger !== undefined) {
+                    await sleep(this.stagger);
+                }
+            }
+        }
+    }
+
     async onTerminate(reason = 'The application was shutdown') {
         for(const m of this.subManagers) {
             await m.notificationManager.handle('runStateChanged', 'Application Shutdown', reason);
@@ -278,22 +279,16 @@ class Bot {
             if (initial) {
                 this.logger.error('An error occurred while trying to initialize the Reddit API Client which would prevent the entire application from running.');
             }
-            if (err.name === 'StatusCodeError') {
-                const authHeader = err.response.headers['www-authenticate'];
-                if (authHeader !== undefined && authHeader.includes('insufficient_scope')) {
-                    this.logger.error('Reddit responded with a 403 insufficient_scope. Please ensure you have chosen the correct scopes when authorizing your account.');
-                } else if (err.statusCode === 401) {
-                    this.logger.error('It is likely a credential is missing or incorrect. Check clientId, clientSecret, refreshToken, and accessToken');
-                } else if(err.statusCode === 400) {
-                    this.logger.error('Credentials may have been invalidated due to prior behavior. The error message may contain more information.');
-                }
-                this.logger.error(`Error Message: ${err.message}`);
-            } else {
-                this.logger.error(err);
-            }
-            this.error = `Error occurred while testing Reddit API client: ${err.message}`;
-            err.logged = true;
-            throw err;
+            const hint = getExceptionMessage(err, {
+                401: 'Likely a credential is missing or incorrect. Check clientId, clientSecret, refreshToken, and accessToken',
+                400: 'Credentials may have been invalidated manually or by reddit due to behavior',
+            });
+            let msg = `Error occurred while testing Reddit API client${hint !== undefined ? `: ${hint}` : ''}`;
+            this.error = msg;
+            const clientError = new CMError(msg, {cause: err});
+            clientError.logged = true;
+            this.logger.error(clientError);
+            throw clientError;
         }
     }
 
@@ -316,7 +311,7 @@ class Bot {
         while(!subListing.isFinished) {
             subListing = await subListing.fetchMore({amount: 100});
         }
-        availSubs = subListing;
+        availSubs = subListing.filter(x => x.display_name !== `u_${user.name}`);
 
         this.logger.info(`u/${user.name} is a moderator of these subreddits: ${availSubs.map(x => x.display_name_prefixed).join(', ')}`);
 
@@ -336,39 +331,175 @@ class Bot {
             }
         } else {
             if(this.excludeSubreddits.length > 0) {
-                this.logger.info(`Will run on all moderated subreddits but user-defined excluded: ${this.excludeSubreddits.join(', ')}`);
+                this.logger.info(`Will run on all moderated subreddits but own profile and user-defined excluded: ${this.excludeSubreddits.join(', ')}`);
                 const normalExcludes = this.excludeSubreddits.map(x => x.toLowerCase());
                 subsToRun = availSubs.filter(x => !normalExcludes.includes(x.display_name.toLowerCase()));
             } else {
                 this.logger.info(`No user-defined subreddit constraints detected, will run on all moderated subreddits EXCEPT own profile (${this.botAccount})`);
-                subsToRun = availSubs.filter(x => x.display_name_prefixed !== this.botAccount);
+                subsToRun = availSubs;
             }
         }
 
-        let subSchedule: Manager[] = [];
         // get configs for subs we want to run on and build/validate them
         for (const sub of subsToRun) {
-            const manager = new Manager(sub, this.client, this.logger, this.cacheManager, {dryRun: this.dryRun, sharedModqueue: this.sharedModqueue, wikiLocation: this.wikiLocation, botName: this.botName, maxWorkers: this.maxWorkers});
             try {
-                await manager.parseConfiguration('system', true, {suppressNotification: true});
+                this.subManagers.push(this.createManager(sub));
             } catch (err: any) {
-                if (!(err instanceof LoggedError)) {
-                    this.logger.error(`Config was not valid:`, {subreddit: sub.display_name_prefixed});
-                    this.logger.error(err, {subreddit: sub.display_name_prefixed});
-                }
+
             }
-            // all errors from managers will count towards bot-level retry count
-            manager.on('error', async (err) => await this.panicOnRetries(err));
-            subSchedule.push(manager);
         }
-        this.subManagers = subSchedule;
+        for(const m of this.subManagers) {
+            try {
+                await this.initManager(m);
+            } catch (err: any) {
+
+            }
+        }
+
+        this.parseSharedStreams();
+    }
+
+    parseSharedStreams() {
+
+        const sharedCommentsSubreddits = !this.sharedStreams.includes('newComm') ? [] : this.subManagers.filter(x => x.isPollingShared('newComm')).map(x => x.subreddit.display_name);
+        if (sharedCommentsSubreddits.length > 0) {
+            const stream = this.cacheManager.modStreams.get('newComm');
+            if (stream === undefined || stream.subreddit !== sharedCommentsSubreddits.join('+')) {
+                let processed;
+                if (stream !== undefined) {
+                    this.logger.info('Restarting SHARED COMMENT STREAM due to a subreddit config change');
+                    stream.end('Replacing with a new stream with updated subreddits');
+                    processed = stream.processed;
+                }
+                if (sharedCommentsSubreddits.length > 100) {
+                    this.logger.warn(`SHARED COMMENT STREAM => Reddit can only combine 100 subreddits for getting new Comments but this bot has ${sharedCommentsSubreddits.length}`);
+                }
+                const defaultCommentStream = new CommentStream(this.client, {
+                    subreddit: sharedCommentsSubreddits.join('+'),
+                    limit: 100,
+                    enforceContinuity: true,
+                    logger: this.logger,
+                    processed,
+                    label: 'Shared Polling'
+                });
+                // @ts-ignore
+                defaultCommentStream.on('error', this.createSharedStreamErrorListener('newComm'));
+                defaultCommentStream.on('listing', this.createSharedStreamListingListener('newComm'));
+                this.cacheManager.modStreams.set('newComm', defaultCommentStream);
+            }
+        } else {
+            const stream = this.cacheManager.modStreams.get('newComm');
+            if (stream !== undefined) {
+                stream.end('Determined no managers are listening on shared stream parsing');
+            }
+        }
+
+        const sharedSubmissionsSubreddits = !this.sharedStreams.includes('newSub') ? [] : this.subManagers.filter(x => x.isPollingShared('newSub')).map(x => x.subreddit.display_name);
+        if (sharedSubmissionsSubreddits.length > 0) {
+            const stream = this.cacheManager.modStreams.get('newSub');
+            if (stream === undefined || stream.subreddit !== sharedSubmissionsSubreddits.join('+')) {
+                let processed;
+                if (stream !== undefined) {
+                    this.logger.info('Restarting SHARED SUBMISSION STREAM due to a subreddit config change');
+                    stream.end('Replacing with a new stream with updated subreddits');
+                    processed = stream.processed;
+                }
+                if (sharedSubmissionsSubreddits.length > 100) {
+                    this.logger.warn(`SHARED SUBMISSION STREAM => Reddit can only combine 100 subreddits for getting new Submissions but this bot has ${sharedSubmissionsSubreddits.length}`);
+                }
+                const defaultSubStream = new SubmissionStream(this.client, {
+                    subreddit: sharedSubmissionsSubreddits.join('+'),
+                    limit: 100,
+                    enforceContinuity: true,
+                    logger: this.logger,
+                    processed,
+                    label: 'Shared Polling'
+                });
+                // @ts-ignore
+                defaultSubStream.on('error', this.createSharedStreamErrorListener('newSub'));
+                defaultSubStream.on('listing', this.createSharedStreamListingListener('newSub'));
+                this.cacheManager.modStreams.set('newSub', defaultSubStream);
+            }
+        } else {
+            const stream = this.cacheManager.modStreams.get('newSub');
+            if (stream !== undefined) {
+                stream.end('Determined no managers are listening on shared stream parsing');
+            }
+        }
+
+        const isUnmoderatedShared = !this.sharedStreams.includes('unmoderated') ? false : this.subManagers.some(x => x.isPollingShared('unmoderated'));
+        const unmoderatedstream = this.cacheManager.modStreams.get('unmoderated');
+        if (isUnmoderatedShared && unmoderatedstream === undefined) {
+            const defaultUnmoderatedStream = new UnmoderatedStream(this.client, {
+                subreddit: 'mod',
+                limit: 100,
+                logger: this.logger,
+                label: 'Shared Polling'
+            });
+            // @ts-ignore
+            defaultUnmoderatedStream.on('error', this.createSharedStreamErrorListener('unmoderated'));
+            defaultUnmoderatedStream.on('listing', this.createSharedStreamListingListener('unmoderated'));
+            this.cacheManager.modStreams.set('unmoderated', defaultUnmoderatedStream);
+        } else if (!isUnmoderatedShared && unmoderatedstream !== undefined) {
+            unmoderatedstream.end('Determined no managers are listening on shared stream parsing');
+        }
+
+        const isModqueueShared = !this.sharedStreams.includes('modqueue') ? false : this.subManagers.some(x => x.isPollingShared('modqueue'));
+        const modqueuestream = this.cacheManager.modStreams.get('modqueue');
+        if (isModqueueShared && modqueuestream === undefined) {
+            const defaultModqueueStream = new ModQueueStream(this.client, {
+                subreddit: 'mod',
+                limit: 100,
+                logger: this.logger,
+                label: 'Shared Polling'
+            });
+            // @ts-ignore
+            defaultModqueueStream.on('error', this.createSharedStreamErrorListener('modqueue'));
+            defaultModqueueStream.on('listing', this.createSharedStreamListingListener('modqueue'));
+            this.cacheManager.modStreams.set('modqueue', defaultModqueueStream);
+        } else if (isModqueueShared && modqueuestream !== undefined) {
+            modqueuestream.end('Determined no managers are listening on shared stream parsing');
+        }
+    }
+
+    async initManager(manager: Manager) {
+        try {
+            await manager.parseConfiguration('system', true, {suppressNotification: true, suppressChangeEvent: true});
+        } catch (err: any) {
+            if(err.logged !== true) {
+                const normalizedError = new ErrorWithCause(`Bot could not start manager because config was not valid`, {cause: err});
+                // @ts-ignore
+                this.logger.error(normalizedError, {subreddit: manager.subreddit.display_name_prefixed});
+            } else {
+                this.logger.error('Bot could not start manager because config was not valid', {subreddit: manager.subreddit.display_name_prefixed});
+            }
+        }
+    }
+
+    createManager(sub: Subreddit): Manager {
+        const manager = new Manager(sub, this.client, this.logger, this.cacheManager, {
+            dryRun: this.dryRun,
+            sharedStreams: this.sharedStreams,
+            wikiLocation: this.wikiLocation,
+            botName: this.botName as string,
+            maxWorkers: this.maxWorkers,
+            filterCriteriaDefaults: this.filterCriteriaDefaults,
+        });
+        // all errors from managers will count towards bot-level retry count
+        manager.on('error', async (err) => await this.panicOnRetries(err));
+        manager.on('configChange', async () => {
+           this.parseSharedStreams();
+           await this.runSharedStreams(false);
+        });
+        return manager;
     }
 
     // if the cumulative errors exceeds configured threshold then stop ALL managers as there is most likely something very bad happening
     async panicOnRetries(err: any) {
-        if(!this.managerRetryHandler(err)) {
+        if(!await this.managerRetryHandler(err)) {
+            this.logger.warn('Bot detected too many errors from managers within a short time. Stopping all managers and will try to restart on next heartbeat.');
             for(const m of this.subManagers) {
-                await m.stop();
+                await m.stop('system',{reason: 'Bot detected too many errors from all managers. Stopping all manager as a failsafe.'});
             }
         }
     }
@@ -384,14 +515,49 @@ class Bot {
         this.logger.info('Bot is stopped.');
     }
 
-    async runModStreams(notify = false) {
+    async checkModInvites() {
+        const subs: string[] = await this.cacheManager.getPendingSubredditInvites();
+        for (const name of subs) {
+            try {
+                // @ts-ignore
+                await this.client.getSubreddit(name).acceptModeratorInvite();
+                this.logger.info(`Accepted moderator invite for r/${name}!`);
+                await this.cacheManager.deletePendingSubredditInvite(name);
+                // @ts-ignore
+                const sub = await this.client.getSubreddit(name);
+                this.logger.info(`Attempting to add manager for r/${name}`);
+                try {
+                    const manager = this.createManager(sub);
+                    this.logger.info(`Starting manager for r/${name}`);
+                    this.subManagers.push(manager);
+                    await this.initManager(manager);
+                    await manager.start('system', {reason: 'Caused by creation due to moderator invite'});
+                    await this.runSharedStreams();
+                } catch (err: any) {
+                    if (!(err instanceof LoggedError)) {
+                        this.logger.error(err);
+                    }
+                }
+            } catch (err: any) {
+                if (err.message.includes('NO_INVITE_FOUND')) {
+                    this.logger.warn(`No pending moderation invite for r/${name} was found`);
+                } else if (isStatusError(err) && err.statusCode === 403) {
+                    this.logger.error(`Error occurred while checking r/${name} for a pending moderation invite. It is likely that this bot does not have the 'modself' oauth permission. Error: ${err.message}`);
+                } else {
+                    this.logger.error(`Error occurred while checking r/${name} for a pending moderation invite. Error: ${err.message}`);
+                }
+            }
+        }
+    }
+
+    async runSharedStreams(notify = false) {
         for(const [k,v] of this.cacheManager.modStreams) {
-            if(!v.running && this.subManagers.some(x => x.modStreamCallbacks.get(k) !== undefined)) {
+            if(!v.running && this.subManagers.some(x => x.sharedStreamCallbacks.get(k) !== undefined)) {
                 v.startInterval();
-                this.logger.info(`Starting default ${k.toUpperCase()} mod stream`);
+                this.logger.info(`Starting ${k.toUpperCase()} shared polling`);
                 if(notify) {
                     for(const m of this.subManagers) {
-                        if(m.modStreamCallbacks.size > 0) {
+                        if(m.sharedStreamCallbacks.size > 0) {
                             await m.notificationManager.handle('runStateChanged', `${k.toUpperCase()} Polling Started`, 'Polling was successfully restarted on heartbeat.');
                         }
                     }
@@ -402,6 +568,8 @@ class Bot {
     }
 
     async runManagers(causedBy: Invokee = 'system') {
+        this.running = true;
+
         if(this.subManagers.every(x => !x.validConfigLoaded)) {
             this.logger.warn('All managers have invalid configs!');
             this.error = 'All managers have invalid configs';
@@ -413,11 +581,11 @@ class Bot {
             }
         }
 
-        await this.runModStreams();
+        await this.runSharedStreams();
 
-        this.running = true;
         this.nextNannyCheck = dayjs().add(10, 'second');
         this.nextHeartbeat = dayjs().add(this.heartbeatInterval, 'second');
+        await this.checkModInvites();
         await this.healthLoop();
     }
 
@@ -432,13 +600,14 @@ class Bot {
                     await this.runApiNanny();
                     this.nextNannyCheck = dayjs().add(10, 'second');
                 } catch (err: any) {
-                    this.logger.info('Delaying next nanny check for 2 minutes due to emitted error');
-                    this.nextNannyCheck = dayjs().add(120, 'second');
+                    this.logger.info('Delaying next nanny check for 4 minutes due to emitted error');
+                    this.nextNannyCheck = dayjs().add(240, 'second');
                 }
             }
             if(dayjs().isSameOrAfter(this.nextHeartbeat)) {
                 try {
                     await this.heartbeat();
+                    await this.checkModInvites();
                 } catch (err: any) {
                     this.logger.error(`Error occurred during heartbeat check: ${err.message}`);
                 }
@@ -492,9 +661,11 @@ class Bot {
                     }
                 }
             } catch (err: any) {
-                this.logger.info('Stopping event polling to prevent activity processing queue from backing up. Will be restarted when config update succeeds.')
-                await s.stopEvents('system', {reason: 'Invalid config will cause events to pile up in queue. Will be restarted when config update succeeds (next heartbeat).'});
-                if(!(err instanceof LoggedError)) {
+                if(s.eventsState.state === RUNNING) {
+                    this.logger.info('Stopping event polling to prevent activity processing queue from backing up. Will be restarted when config update succeeds.')
+                    await s.stopEvents('system', {reason: 'Invalid config will cause events to pile up in queue. Will be restarted when config update succeeds (next heartbeat).'});
+                }
+                if(err.logged !== true) {
                     this.logger.error(err, {subreddit: s.displayLabel});
                 }
                 if(this.nextHeartbeat !== undefined) {
@@ -502,7 +673,7 @@ class Bot {
                 }
             }
         }
-        await this.runModStreams(true);
+        await this.runSharedStreams(true);
     }
 
     async runApiNanny() {
@@ -570,6 +741,10 @@ class Bot {
                     m.notificationManager.handle('runStateChanged', 'Hard Limit Triggered', `Hard Limit of ${this.hardLimit} hit (API Remaining: ${this.client.ratelimitRemaining}). Subreddit event polling has been paused.`, 'system', 'warn');
                 }
 
+                for(const [k,v] of this.cacheManager.modStreams) {
+                    v.end('Hard limit cutoff');
+                }
+
                 this.nannyMode = 'hard';
                 return;
             }
@@ -636,6 +811,7 @@ class Bot {
                         await m.startEvents('system', {reason: 'API Nanny has been turned off due to better API conditions'});
                     }
                 }
+                await this.runSharedStreams(true);
                 this.nannyMode = undefined;
             }
 

src/Check/index.ts

@@ -28,8 +28,11 @@ import * as RuleSchema from '../Schema/Rule.json';
 import * as RuleSetSchema from '../Schema/RuleSet.json';
 import * as ActionSchema from '../Schema/Action.json';
 import {ActionObjectJson, RuleJson, RuleObjectJson, ActionJson as ActionTypeJson} from "../Common/types";
-import {SubredditResources} from "../Subreddit/SubredditResources";
-import {Author, AuthorCriteria, AuthorOptions} from "../Author/Author";
+import {checkAuthorFilter, SubredditResources} from "../Subreddit/SubredditResources";
+import {Author, AuthorCriteria, AuthorOptions} from '..';
+import {ExtendedSnoowrap} from '../Utils/SnoowrapClients';
+import {isRateLimitError} from "../Utils/Errors";
+import {ErrorWithCause} from "pony-cause";
 
 const checkLogName = truncateStringToLength(25);
 
@@ -42,15 +45,12 @@ export abstract class Check implements ICheck {
     rules: Array<RuleSet | Rule> = [];
     logger: Logger;
     itemIs: TypedActivityStates;
-    authorIs: {
-        include: AuthorCriteria[],
-        exclude: AuthorCriteria[]
-    };
+    authorIs: AuthorOptions;
     cacheUserResult: Required<UserResultCacheOptions>;
     dryRun?: boolean;
     notifyOnTrigger: boolean;
     resources: SubredditResources;
-    client: Snoowrap;
+    client: ExtendedSnoowrap;
 
     constructor(options: CheckOptions) {
         const {
@@ -68,6 +68,7 @@ export abstract class Check implements ICheck {
             itemIs = [],
             authorIs: {
                 include = [],
+                excludeCondition,
                 exclude = [],
             } = {},
             dryRun,
@@ -88,6 +89,7 @@ export abstract class Check implements ICheck {
         this.condition = condition;
         this.itemIs = itemIs;
         this.authorIs = {
+            excludeCondition,
             exclude: exclude.map(x => new Author(x)),
             include: include.map(x => new Author(x)),
         }
@@ -158,7 +160,7 @@ export abstract class Check implements ICheck {
         runStats.push(`${this.actions.length} Actions`);
         // not sure if this should be info or verbose
         this.logger.info(`=${this.enabled ? 'Enabled' : 'Disabled'}= ${type.toUpperCase()} (${this.condition})${this.notifyOnTrigger ? ' ||Notify on Trigger|| ' : ''} => ${runStats.join(' | ')}${this.description !== undefined ? ` => ${this.description}` : ''}`);
-        if (this.rules.length === 0 && this.itemIs.length === 0 && this.authorIs.exclude.length === 0 && this.authorIs.include.length === 0) {
+        if (this.rules.length === 0 && this.itemIs.length === 0 && this.authorIs.exclude?.length === 0 && this.authorIs.include?.length === 0) {
             this.logger.warn('No rules, item tests, or author test found -- this check will ALWAYS PASS!');
         }
         let ruleSetIndex = 1;
@@ -201,30 +203,9 @@ export abstract class Check implements ICheck {
                 this.logger.verbose(`${FAIL} => Item did not pass 'itemIs' test`);
                 return [false, allRuleResults];
             }
-            let authorPass = null;
-            if (this.authorIs.include !== undefined && this.authorIs.include.length > 0) {
-                for (const auth of this.authorIs.include) {
-                    if (await this.resources.testAuthorCriteria(item, auth)) {
-                        authorPass = true;
-                        break;
-                    }
-                }
-                if (!authorPass) {
-                    this.logger.verbose(`${FAIL} => Inclusive author criteria not matched`);
-                    return Promise.resolve([false, allRuleResults]);
-                }
-            }
-            if (authorPass === null && this.authorIs.exclude !== undefined && this.authorIs.exclude.length > 0) {
-                for (const auth of this.authorIs.exclude) {
-                    if (await this.resources.testAuthorCriteria(item, auth, false)) {
-                        authorPass = true;
-                        break;
-                    }
-                }
-                if (!authorPass) {
-                    this.logger.verbose(`${FAIL} =>  Exclusive author criteria not matched`);
-                    return Promise.resolve([false, allRuleResults]);
-                }
+            const [authFilterResult, authFilterType] = await checkAuthorFilter(item, this.authorIs, this.resources, this.logger);
+            if(!authFilterResult) {
+                return Promise.resolve([false, allRuleResults]);
             }
 
             if (this.rules.length === 0) {
@@ -269,9 +250,7 @@ export abstract class Check implements ICheck {
             this.logger.info(`${PASS} => Rules: ${resultsSummary(allResults, this.condition)}`);
             return [true, allRuleResults];
         } catch (e: any) {
-            e.logged = true;
-            this.logger.warn(`Running rules failed due to uncaught exception`, e);
-            throw e;
+            throw new ErrorWithCause('Running rules failed due to error', {cause: e});
         }
     }
 
@@ -345,13 +324,13 @@ export interface ICheck extends JoinCondition, ChecksActivityState {
 }
 
 export interface CheckOptions extends ICheck {
-    rules: Array<IRuleSet | IRule>
-    actions: ActionConfig[]
-    logger: Logger
-    subredditName: string
-    notifyOnTrigger?: boolean
-    resources: SubredditResources
-    client: Snoowrap
+    rules: Array<IRuleSet | IRule>;
+    actions: ActionConfig[];
+    logger: Logger;
+    subredditName: string;
+    notifyOnTrigger?: boolean;
+    resources: SubredditResources;
+    client: ExtendedSnoowrap;
     cacheUserResult?: UserResultCacheOptions;
 }
 

src/Common/Config/AbstractConfigDocument.ts

@@ -0,0 +1,27 @@
+import {ConfigFormat} from "../types";
+
+export interface ConfigDocumentInterface<DocumentType> {
+    format: ConfigFormat;
+    parsed: DocumentType
+    //parsingError: Error | string;
+    raw: string;
+    location?: string;
+    toString(): string;
+    toJS(): object;
+}
+
+abstract class AbstractConfigDocument<DocumentType> implements ConfigDocumentInterface<DocumentType> {
+    public abstract format: ConfigFormat;
+    public abstract parsed: DocumentType;
+    //public abstract parsingError: Error | string;
+
+
+    constructor(public raw: string, public location?: string) {
+    }
+
+
+    public abstract toString(): string;
+    public abstract toJS(): object;
+}
+
+export default AbstractConfigDocument;

src/Common/Config/JsonConfigDocument.ts

@@ -0,0 +1,30 @@
+import AbstractConfigDocument from "./AbstractConfigDocument";
+import {stringify, parse} from 'comment-json';
+import JSON5 from 'json5';
+import {ConfigFormat} from "../types";
+import {OperatorJsonConfig} from "../interfaces";
+
+class JsonConfigDocument extends AbstractConfigDocument<OperatorJsonConfig> {
+
+    public parsed: OperatorJsonConfig;
+    protected cleanParsed: OperatorJsonConfig;
+    public format: ConfigFormat;
+
+    public constructor(raw: string, location?: string) {
+        super(raw, location);
+        this.parsed = parse(raw);
+        this.cleanParsed = JSON5.parse(raw);
+        this.format = 'json';
+    }
+
+    public toJS(): OperatorJsonConfig {
+        return this.cleanParsed;
+    }
+
+    public toString(): string {
+        return stringify(this.parsed, null, 1);
+    }
+
+}
+
+export default JsonConfigDocument;

src/Common/Config/Operator/index.ts

@@ -0,0 +1,54 @@
+import YamlConfigDocument from "../YamlConfigDocument";
+import JsonConfigDocument from "../JsonConfigDocument";
+import {YAMLMap, YAMLSeq} from "yaml";
+import {BotInstanceJsonConfig, OperatorJsonConfig} from "../../interfaces";
+import {assign} from 'comment-json';
+
+export interface OperatorConfigDocumentInterface {
+    addBot(botData: BotInstanceJsonConfig): void;
+    toJS(): OperatorJsonConfig;
+}
+
+export class YamlOperatorConfigDocument extends YamlConfigDocument implements OperatorConfigDocumentInterface {
+    addBot(botData: BotInstanceJsonConfig) {
+        const bots = this.parsed.get('bots') as YAMLSeq;
+        if (bots === undefined) {
+            this.parsed.add({key: 'bots', value: [botData]});
+        } else if (botData.name !== undefined) {
+            // overwrite if we find an existing
+            const existingIndex = bots.items.findIndex(x => (x as YAMLMap).get('name') === botData.name);
+            if (existingIndex !== -1) {
+                this.parsed.setIn(['bots', existingIndex], botData);
+            } else {
+                this.parsed.addIn(['bots'], botData);
+            }
+        } else {
+            this.parsed.addIn(['bots'], botData);
+        }
+    }
+
+    toJS(): OperatorJsonConfig  {
+        return super.toJS();
+    }
+}
+
+export class JsonOperatorConfigDocument extends JsonConfigDocument implements OperatorConfigDocumentInterface {
+    addBot(botData: BotInstanceJsonConfig) {
+        if (this.parsed.bots === undefined) {
+            this.parsed.bots = [botData];
+        } else if (botData.name !== undefined) {
+            const existingIndex = this.parsed.bots.findIndex(x => x.name === botData.name);
+            if (existingIndex !== -1) {
+                this.parsed.bots[existingIndex] = assign(this.parsed.bots[existingIndex], botData);
+            } else {
+                this.parsed.bots.push(botData);
+            }
+        } else {
+            this.parsed.bots.push(botData);
+        }
+    }
+
+    toJS(): OperatorJsonConfig {
+        return super.toJS();
+    }
+}

src/Common/Config/YamlConfigDocument.ts

@@ -0,0 +1,24 @@
+import AbstractConfigDocument from "./AbstractConfigDocument";
+import {Document, parseDocument} from 'yaml';
+import {ConfigFormat} from "../types";
+
+class YamlConfigDocument extends AbstractConfigDocument<Document> {
+
+    public parsed: Document;
+    public format: ConfigFormat;
+
+    public constructor(raw: string, location?: string) {
+        super(raw, location);
+        this.parsed = parseDocument(raw);
+        this.format = 'yaml';
+    }
+    public toJS(): object {
+        return this.parsed.toJS();
+    }
+
+    public toString(): string {
+        return this.parsed.toString();
+    }
+}
+
+export default YamlConfigDocument;

src/Common/ImageData.ts

@@ -3,9 +3,9 @@ import {Submission} from "snoowrap/dist/objects";
 import {URL} from "url";
 import {absPercentDifference, getSharpAsync, isValidImageURL} from "../util";
 import sizeOf from "image-size";
-import SimpleError from "../Utils/SimpleError";
 import {Sharp} from "sharp";
 import {blockhash} from "./blockhash/blockhash";
+import {SimpleError} from "../Utils/Errors";
 
 export interface ImageDataOptions {
     width?: number,

src/Common/defaults.ts

@@ -1,7 +1,7 @@
-import {HistoricalStats} from "./interfaces";
+import {HistoricalStats, FilterCriteriaDefaults} from "./interfaces";
 
 export const cacheOptDefaults = {ttl: 60, max: 500, checkPeriod: 600};
-export const cacheTTLDefaults = {authorTTL: 60, userNotesTTL: 300, wikiTTL: 300, submissionTTL: 60, commentTTL: 60, filterCriteriaTTL: 60, subredditTTL: 600};
+export const cacheTTLDefaults = {authorTTL: 60, userNotesTTL: 300, wikiTTL: 300, submissionTTL: 60, commentTTL: 60, filterCriteriaTTL: 60, subredditTTL: 600, selfTTL: 60};
 export const historicalDefaults: HistoricalStats = {
     eventsCheckedTotal: 0,
     eventsActionedTotal: 0,
@@ -29,3 +29,13 @@ export const createHistoricalDefaults = (): HistoricalStats => {
         actionsRun: new Map(),
     };
 }
+
+export const filterCriteriaDefault: FilterCriteriaDefaults = {
+    authorIs: {
+        exclude: [
+            {
+                isMod: true
+            }
+        ]
+    }
+}

src/Common/interfaces.ts

@@ -5,6 +5,17 @@ import Poll from "snoostorm/out/util/Poll";
 import Snoowrap from "snoowrap";
 import {RuleResult} from "../Rule";
 import {IncomingMessage} from "http";
+import Submission from "snoowrap/dist/objects/Submission";
+import Comment from "snoowrap/dist/objects/Comment";
+import RedditUser from "snoowrap/dist/objects/RedditUser";
+import {AuthorCriteria, AuthorOptions} from "../Author/Author";
+import {ConfigFormat} from "./types";
+import AbstractConfigDocument, {ConfigDocumentInterface} from "./Config/AbstractConfigDocument";
+import {Document as YamlDocument} from 'yaml';
+import {JsonOperatorConfigDocument, YamlOperatorConfigDocument} from "./Config/Operator";
+import {ConsoleTransportOptions} from "winston/lib/winston/transports";
+import {DailyRotateFileTransportOptions} from "winston-daily-rotate-file";
+import {DuplexTransportOptions} from "winston-duplex/dist/DuplexTransport";
 
 /**
  * An ISO 8601 Duration
@@ -486,38 +497,6 @@ export type PollOn = 'unmoderated' | 'modqueue' | 'newSub' | 'newComm';
 export interface PollingOptionsStrong extends PollingOptions {
     limit: number,
     interval: number,
-    clearProcessed: ClearProcessedOptions
-}
-
-/**
- * For very long-running, high-volume subreddits clearing the list of processed activities helps manage memory bloat
- *
- * All of these options have default values based on the limit and/or interval set for polling options on each subreddit stream. They only need to modified if the defaults are not sufficient.
- *
- * If both `after` and `size` are defined whichever is hit first will trigger the list to clear. `after` will be reset after ever clear.
- * */
-export interface ClearProcessedOptions {
-    /**
-     * An interval the processed list should be cleared after.
-     *
-     * * EX `9 days`
-     * * EX `3 months`
-     * * EX `5 minutes`
-     * @pattern ^\s*(?<time>\d+)\s*(?<unit>days?|weeks?|months?|years?|hours?|minutes?|seconds?|milliseconds?)\s*$
-     * */
-    after?: string,
-    /**
-     * Number of activities found in processed list after which the list should be cleared.
-     *
-     * Defaults to the `limit` value from `PollingOptions`
-     * */
-    size?: number,
-    /**
-     * The number of activities to retain in processed list after clearing.
-     *
-     * Defaults to `limit` value from `PollingOptions`
-     * */
-    retain?: number,
 }
 
 export interface PollingDefaults {
@@ -591,8 +570,6 @@ export interface PollingOptions extends PollingDefaults {
      *
      * */
     pollOn: 'unmoderated' | 'modqueue' | 'newSub' | 'newComm'
-
-    clearProcessed?: ClearProcessedOptions
 }
 
 export interface TTLConfig {
@@ -670,6 +647,24 @@ export interface TTLConfig {
      * @default 60
      * */
     filterCriteriaTTL?: number | boolean;
+
+    /**
+     * Amount of time, in seconds, an Activity that the bot has acted on or created will be ignored if found during polling
+     *
+     * This is useful to prevent the bot from checking Activities it *just* worked on or a product of the checks. Examples:
+     *
+     * * Ignore comments created through an Action
+     * * Ignore Activity polled from modqueue that the bot just reported
+     *
+     * This value should be at least as long as the longest polling interval for modqueue/newComm
+     *
+     * * If `0` or `true` will cache indefinitely (not recommended)
+     * * If `false` will not cache
+     *
+     * @examples [50]
+     * @default 50
+     * */
+    selfTTL?: number | boolean
 }
 
 export interface CacheConfig extends TTLConfig {
@@ -834,6 +829,13 @@ export interface ManagerOptions {
     notifications?: NotificationConfig
 
     credentials?: ThirdPartyCredentialsJsonConfig
+
+    /**
+     * Set the default filter criteria for all checks. If this property is specified it will override any defaults passed from the bot's config
+     *
+     * Default behavior is to exclude all mods and automoderator from checks
+     * */
+    filterCriteriaDefaults?: FilterCriteriaDefaults
 }
 
 /**
@@ -909,6 +911,20 @@ export interface ActivityState {
     distinguished?: boolean
     approved?: boolean
     score?: CompareValue
+    /**
+     * A string containing a comparison operator and a value to compare against
+     *
+     * The syntax is `(< OR > OR <= OR >=) <number>`
+     *
+     * * EX `> 2`  => greater than 2 total reports
+     *
+     * Defaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:
+     *
+     * * EX `> 3 mod` => greater than 3 mod reports
+     * * EX `>= 1 user` => greater than 1 user report
+     *
+     * @pattern ^\s*(>|>=|<|<=)\s*(\d+)\s*(%?)(.*)$
+     * */
     reports?: CompareValue
     age?: DurationComparor
 }
@@ -930,8 +946,9 @@ export interface SubmissionState extends ActivityState {
      * */
     title?: string
 
-    link_flair_text?: string
-    link_flair_css_class?: string
+    link_flair_text?: string | string[]
+    link_flair_css_class?: string | string[]
+    flairTemplate?: string | string[]
 }
 
 // properties calculated/derived by CM -- not provided as plain values by reddit
@@ -1057,6 +1074,92 @@ export interface RegExResult {
 }
 
 type LogLevel = "error" | "warn" | "info" | "verbose" | "debug";
+
+export type LogConsoleOptions = Pick<ConsoleTransportOptions, 'silent' | 'eol' | 'stderrLevels' | 'consoleWarnLevels'> & {
+    level?: LogLevel
+}
+
+export type LogFileOptions = Omit<DailyRotateFileTransportOptions, 'stream' | 'handleRejections' | 'options' | 'handleExceptions' | 'format' | 'log' | 'logv' | 'close' | 'dirname'> & {
+    level?: LogLevel
+    /**
+     * The absolute path to a directory where rotating log files should be stored.
+     *
+     * * If not present or `null` or `false` no log files will be created
+     * * If `true` logs will be stored at `[working directory]/logs`
+     *
+     * * ENV => `LOG_DIR`
+     * * ARG => `--logDir [dir]`
+     *
+     * @examples ["/var/log/contextmod"]
+     * */
+    dirname?: string | boolean | null
+}
+
+// export type StrongFileOptions = LogFileOptions & {
+//     dirname?: string
+// }
+
+export type LogStreamOptions = Omit<DuplexTransportOptions, 'name' | 'stream' | 'handleRejections' | 'handleExceptions' | 'format' | 'log' | 'logv' | 'close'> & {
+    level?: LogLevel
+}
+
+export interface LoggingOptions  {
+    /**
+     * The minimum log level to output. The log level set will output logs at its level **and all levels above it:**
+     *
+     *  * `error`
+     *  * `warn`
+     *  * `info`
+     *  * `verbose`
+     *  * `debug`
+     *
+     *  Note: `verbose` will display *a lot* of information on the status/result of run rules/checks/actions etc. which is very useful for testing configurations. Once your bot is stable changing the level to `info` will reduce log noise.
+     *
+     *  * ENV => `LOG_LEVEL`
+     *  * ARG => `--logLevel <level>`
+     *
+     *  @default "verbose"
+     *  @examples ["verbose"]
+     * */
+    level?: LogLevel,
+    /**
+     * **DEPRECATED** - Use `file.dirname` instead
+     * The absolute path to a directory where rotating log files should be stored.
+     *
+     * * If not present or `null` or `false` no log files will be created
+     * * If `true` logs will be stored at `[working directory]/logs`
+     *
+     * * ENV => `LOG_DIR`
+     * * ARG => `--logDir [dir]`
+     *
+     * @examples ["/var/log/contextmod"]
+     * @deprecated
+     * @see logging.file.dirname
+     * */
+    path?: string | boolean | null
+
+    /**
+     * Options for Rotating File logging
+     * */
+    file?: LogFileOptions
+    /**
+     * Options for logging to api/web
+     * */
+    stream?: LogStreamOptions
+    /**
+     * Options for logging to console
+     * */
+    console?: LogConsoleOptions
+}
+
+export type StrongLoggingOptions = Required<Pick<LoggingOptions, 'stream' | 'console' | 'file'>> & {
+    level?: LogLevel
+};
+
+export type LoggerFactoryOptions = StrongLoggingOptions & {
+    additionalTransports?: any[]
+    defaultLabel?: string
+}
 /**
  * Available cache providers
  * */
@@ -1072,6 +1175,7 @@ export type StrongCache = {
     submissionTTL: number | boolean,
     commentTTL: number | boolean,
     subredditTTL: number | boolean,
+    selfTTL: number | boolean,
     filterCriteriaTTL: number | boolean,
     provider: CacheOptions
     actionedEventsMax?: number,
@@ -1184,6 +1288,7 @@ export interface Notifier {
 export interface ManagerStateChangeOption {
     reason?: string
     suppressNotification?: boolean
+    suppressChangeEvent?: boolean
 }
 
 /**
@@ -1289,6 +1394,53 @@ export interface WebCredentials {
     redirectUri?: string,
 }
 
+export interface SnoowrapOptions {
+    /**
+     * Proxy all requests to Reddit's API through this endpoint
+     *
+     * * ENV => `PROXY`
+     * * ARG => `--proxy <proxyEndpoint>`
+     *
+     * @examples ["http://localhost:4443"]
+     * */
+    proxy?: string,
+    /**
+     * Manually set the debug status for snoowrap
+     *
+     * When snoowrap has `debug: true` it will log the http status response of reddit api requests to at the `debug` level
+     *
+     * * Set to `true` to always output
+     * * Set to `false` to never output
+     *
+     * If not present or `null` will be set based on `logLevel`
+     *
+     * * ENV => `SNOO_DEBUG`
+     * * ARG => `--snooDebug`
+     * */
+    debug?: boolean,
+}
+
+export type FilterCriteriaDefaultBehavior = 'replace' | 'merge';
+
+export interface FilterCriteriaDefaults {
+    itemIs?: TypedActivityStates
+    /**
+     * Determine how itemIs defaults behave when itemIs is present on the check
+     *
+     * * merge => adds defaults to check's itemIs
+     * * replace => check itemIs will replace defaults (no defaults used)
+     * */
+    itemIsBehavior?: FilterCriteriaDefaultBehavior
+    /**
+     * Determine how authorIs defaults behave when authorIs is present on the check
+     *
+     * * merge => merges defaults with check's authorIs
+     * * replace => check authorIs will replace defaults (no defaults used)
+     * */
+    authorIs?: AuthorOptions
+    authorIsBehavior?: FilterCriteriaDefaultBehavior
+}
+
 /**
  * The configuration for an **individual reddit account** ContextMod will run as a bot.
  *
@@ -1309,33 +1461,20 @@ export interface BotInstanceJsonConfig {
     notifications?: NotificationConfig
 
     /**
-     * Settings to control some [Snoowrap](https://github.com/not-an-aardvark/snoowrap) behavior
+     * Settings to control some [Snoowrap](https://github.com/not-an-aardvark/snoowrap) behavior.
+     *
+     * Overrides any defaults provided at top-level operator config.
+     *
+     * Set to an empty object to "ignore" any top-level config
      * */
-    snoowrap?: {
-        /**
-         * Proxy all requests to Reddit's API through this endpoint
-         *
-         * * ENV => `PROXY`
-         * * ARG => `--proxy <proxyEndpoint>`
-         *
-         * @examples ["http://localhost:4443"]
-         * */
-        proxy?: string,
-        /**
-         * Manually set the debug status for snoowrap
-         *
-         * When snoowrap has `debug: true` it will log the http status response of reddit api requests to at the `debug` level
-         *
-         * * Set to `true` to always output
-         * * Set to `false` to never output
-         *
-         * If not present or `null` will be set based on `logLevel`
-         *
-         * * ENV => `SNOO_DEBUG`
-         * * ARG => `--snooDebug`
-         * */
-        debug?: boolean,
-    }
+    snoowrap?: SnoowrapOptions
+
+    /**
+     * Define the default behavior for all filter criteria on all checks in all subreddits
+     *
+     * Defaults to exclude mods and automoderator from checks
+     * */
+    filterCriteriaDefaults?: FilterCriteriaDefaults
 
     /**
      * Settings related to bot behavior for subreddits it is managing
@@ -1404,18 +1543,31 @@ export interface BotInstanceJsonConfig {
      * */
     polling?: PollingDefaults & {
         /**
-         * If set to `true` all subreddits polling unmoderated/modqueue with default polling settings will share a request to "r/mod"
-         * otherwise each subreddit will poll its own mod view
+         * DEPRECATED: See `shared`
+         *
+         *  Using the ENV or ARG will sett `unmoderated` and `modqueue` on `shared`
          *
          * * ENV => `SHARE_MOD`
          * * ARG => `--shareMod`
          *
          * @default false
+         * @deprecated
          * */
         sharedMod?: boolean,
 
         /**
-         * If sharing a mod stream stagger pushing relevant Activities to individual subreddits.
+         * Set which polling sources should be shared among subreddits using default polling settings for that source
+         *
+         * * For `unmoderated and `modqueue` the bot will poll on **r/mod** for new activities
+         * * For `newSub` and `newComm` all subreddits sharing the source will be combined to poll like **r/subreddit1+subreddit2/new**
+         *
+         * If set to `true` all polling sources will be shared,  otherwise specify which sourcs should be shared as a list
+         *
+         * */
+        shared?: PollOn[] | true,
+
+        /**
+         * If sharing a stream staggers pushing relevant Activities to individual subreddits.
          *
          * Useful when running many subreddits and rules are potentially cpu/memory/traffic heavy -- allows spreading out load
          * */
@@ -1518,38 +1670,7 @@ export interface OperatorJsonConfig {
     /**
      * Settings to configure global logging defaults
      * */
-    logging?: {
-        /**
-         * The minimum log level to output. The log level set will output logs at its level **and all levels above it:**
-         *
-         *  * `error`
-         *  * `warn`
-         *  * `info`
-         *  * `verbose`
-         *  * `debug`
-         *
-         *  Note: `verbose` will display *a lot* of information on the status/result of run rules/checks/actions etc. which is very useful for testing configurations. Once your bot is stable changing the level to `info` will reduce log noise.
-         *
-         *  * ENV => `LOG_LEVEL`
-         *  * ARG => `--logLevel <level>`
-         *
-         *  @default "verbose"
-         *  @examples ["verbose"]
-         * */
-        level?: LogLevel,
-        /**
-         * The absolute path to a directory where rotating log files should be stored.
-         *
-         * * If not present or `null` no log files will be created
-         * * If `true` logs will be stored at `[working directory]/logs`
-         *
-         * * ENV => `LOG_DIR`
-         * * ARG => `--logDir [dir]`
-         *
-         * @examples ["/var/log/contextmod"]
-         * */
-        path?: string,
-    },
+    logging?: LoggingOptions,
 
     /**
      * Settings to configure the default caching behavior globally
@@ -1558,6 +1679,11 @@ export interface OperatorJsonConfig {
      * */
     caching?: OperatorCacheConfig
 
+    /**
+     * Set global snoowrap options as well as default snoowrap config for all bots that don't specify their own
+     * */
+    snoowrap?: SnoowrapOptions
+
     bots?: BotInstanceJsonConfig[]
 
     /**
@@ -1722,7 +1848,7 @@ export interface BotInstanceConfig extends BotInstanceJsonConfig {
         heartbeatInterval: number,
     },
     polling: {
-        sharedMod: boolean,
+        shared: PollOn[],
         stagger?: number,
         limit: number,
         interval: number,
@@ -1744,10 +1870,7 @@ export interface OperatorConfig extends OperatorJsonConfig {
         display?: string,
     },
     notifications?: NotificationConfig
-    logging: {
-        level: LogLevel,
-        path?: string,
-    },
+    logging: StrongLoggingOptions,
     caching: StrongCache,
     web: {
         port: number,
@@ -1774,6 +1897,15 @@ export interface OperatorConfig extends OperatorJsonConfig {
     credentials: ThirdPartyCredentialsJsonConfig
 }
 
+export interface OperatorFileConfig {
+    document: YamlOperatorConfigDocument | JsonOperatorConfigDocument
+    isWriteable?: boolean
+}
+
+export interface OperatorConfigWithFileContext extends OperatorConfig {
+    fileConfig: OperatorFileConfig
+}
+
 //export type OperatorConfig = Required<OperatorJsonConfig>;
 
 interface CacheTypeStat {
@@ -1801,20 +1933,18 @@ export interface LogInfo {
     bot?: string
 }
 
-export interface ActionResult {
+export interface ActionResult extends ActionProcessResult {
     kind: string,
     name: string,
     run: boolean,
     runReason?: string,
-    dryRun: boolean,
-    success: boolean,
-    result?: string,
 }
 
 export interface ActionProcessResult {
     success: boolean,
     dryRun: boolean,
     result?: string
+    touchedEntities?: (Submission | Comment | RedditUser | string)[]
 }
 
 export interface ActionedEvent {
@@ -1843,14 +1973,6 @@ export interface RedditEntity {
     type: RedditEntityType
 }
 
-export interface StatusCodeError extends Error {
-    name: 'StatusCodeError',
-    statusCode: number,
-    message: string,
-    response: IncomingMessage,
-    error: Error
-}
-
 export interface HistoricalStatsDisplay extends HistoricalStats {
     checksRunTotal: number
     checksFromCacheTotal: number
@@ -1954,3 +2076,27 @@ export interface StringComparisonOptions {
     lengthWeight?: number,
     transforms?: ((str: string) => string)[]
 }
+
+export interface FilterCriteriaPropertyResult<T> {
+    property: keyof T
+    expected: (string | boolean | number)[]
+    found?: string | boolean | number | null
+    passed?: null | boolean
+    reason?: string
+    behavior: FilterBehavior
+}
+
+export interface FilterCriteriaResult<T> {
+    behavior: FilterBehavior
+    criteria: T//AuthorCriteria | TypedActivityStates
+    propertyResults: FilterCriteriaPropertyResult<T>[]
+    passed: boolean
+}
+
+export type FilterBehavior = 'include' | 'exclude'
+
+export interface FilterResult<T> {
+    criteriaResults: FilterCriteriaResult<T>[]
+    join: JoinOperands
+    passed: boolean
+}

src/Common/types.ts

@@ -3,6 +3,7 @@ import {RepeatActivityJSONConfig} from "../Rule/RepeatActivityRule";
 import {AuthorRuleJSONConfig} from "../Rule/AuthorRule";
 import {AttributionJSONConfig} from "../Rule/AttributionRule";
 import {FlairActionJson} from "../Action/SubmissionAction/FlairAction";
+import {UserFlairActionJson} from "../Action/UserFlairAction";
 import {CommentActionJson} from "../Action/CommentAction";
 import {ReportActionJson} from "../Action/ReportAction";
 import {LockActionJson} from "../Action/LockAction";
@@ -18,7 +19,7 @@ import {RepostRuleJSONConfig} from "../Rule/RepostRule";
 export type RuleJson = RecentActivityRuleJSONConfig | RepeatActivityJSONConfig | AuthorRuleJSONConfig | AttributionJSONConfig | HistoryJSONConfig | RegexRuleJSONConfig | RepostRuleJSONConfig | string;
 export type RuleObjectJson = Exclude<RuleJson, string>
 
-export type ActionJson = CommentActionJson | FlairActionJson | ReportActionJson | LockActionJson | RemoveActionJson | ApproveActionJson | BanActionJson | UserNoteActionJson | MessageActionJson | string;
+export type ActionJson = CommentActionJson | FlairActionJson | ReportActionJson | LockActionJson | RemoveActionJson | ApproveActionJson | BanActionJson | UserNoteActionJson | MessageActionJson | UserFlairActionJson | string;
 export type ActionObjectJson = Exclude<ActionJson, string>;
 
 // borrowed from https://github.com/jabacchetta/set-random-interval/blob/master/src/index.ts
@@ -27,3 +28,5 @@ export type SetRandomInterval = (
     minDelay: number,
     maxDelay: number,
 ) => { clear: () => void };
+
+export type ConfigFormat = 'json' | 'yaml';

src/Common/typings/support.d.ts

@@ -0,0 +1,26 @@
+declare module 'snoowrap/dist/errors' {
+
+    export interface InvalidUserError extends Error {
+
+    }
+    export interface NoCredentialsError extends Error {
+
+    }
+    export interface InvalidMethodCallError extends Error {
+
+    }
+
+    export interface RequestError extends Error {
+        statusCode: number,
+        response: http.IncomingMessage
+        error: Error
+    }
+
+    export interface StatusCodeError extends RequestError {
+        name: 'StatusCodeError',
+    }
+
+    export interface RateLimitError extends RequestError {
+        name: 'RateLimitError',
+    }
+}

src/ConfigBuilder.ts

@@ -1,12 +1,12 @@
 import {Logger} from "winston";
 import {
     buildCacheOptionsFromProvider, buildCachePrefix,
-    createAjvFactory,
+    createAjvFactory, fileOrDirectoryIsWriteable,
     mergeArr,
     normalizeName,
     overwriteMerge,
-    parseBool, randomId,
-    readConfigFile,
+    parseBool, parseFromJsonOrYamlToObject, randomId,
+    readConfigFile, removeFromSourceIfKeysExistsInDestination,
     removeUndefinedKeys
 } from "./util";
 import {CommentCheck} from "./Check/CommentCheck";
@@ -31,19 +31,34 @@ import {
     CacheOptions,
     BotInstanceJsonConfig,
     BotInstanceConfig,
-    RequiredWebRedditCredentials, RedditCredentials, BotCredentialsJsonConfig, BotCredentialsConfig
+    RequiredWebRedditCredentials,
+    RedditCredentials,
+    BotCredentialsJsonConfig,
+    BotCredentialsConfig,
+    FilterCriteriaDefaults, TypedActivityStates, OperatorFileConfig
 } from "./Common/interfaces";
 import {isRuleSetJSON, RuleSetJson, RuleSetObjectJson} from "./Rule/RuleSet";
 import deepEqual from "fast-deep-equal";
-import {ActionJson, ActionObjectJson, RuleJson, RuleObjectJson} from "./Common/types";
+import {ActionJson, ActionObjectJson, ConfigFormat, RuleJson, RuleObjectJson} from "./Common/types";
 import {isActionJson} from "./Action";
 import {getLogger} from "./Utils/loggerFactory";
 import {GetEnvVars} from 'env-cmd';
 import {operatorConfig} from "./Utils/CommandConfig";
 import merge from 'deepmerge';
 import * as process from "process";
-import {cacheOptDefaults, cacheTTLDefaults} from "./Common/defaults";
+import {cacheOptDefaults, cacheTTLDefaults, filterCriteriaDefault} from "./Common/defaults";
 import objectHash from "object-hash";
+import {AuthorCriteria, AuthorOptions} from "./Author/Author";
+import path from 'path';
+import {
+    JsonOperatorConfigDocument,
+    OperatorConfigDocumentInterface,
+    YamlOperatorConfigDocument
+} from "./Common/Config/Operator";
+import {ConfigDocumentInterface} from "./Common/Config/AbstractConfigDocument";
+import {Document as YamlDocument} from "yaml";
+import {SimpleError} from "./Utils/Errors";
+import {ErrorWithCause} from "pony-cause";
 
 export interface ConfigBuilderOptions {
     logger: Logger,
@@ -115,22 +130,51 @@ export class ConfigBuilder {
         return validConfig as JSONConfig;
     }
 
-    parseToStructured(config: JSONConfig): CheckStructuredJson[] {
+    parseToStructured(config: JSONConfig, filterCriteriaDefaultsFromBot?: FilterCriteriaDefaults): CheckStructuredJson[] {
         let namedRules: Map<string, RuleObjectJson> = new Map();
         let namedActions: Map<string, ActionObjectJson> = new Map();
-        const {checks = []} = config;
+        const {checks = [], filterCriteriaDefaults} = config;
         for (const c of checks) {
             const {rules = []} = c;
             namedRules = extractNamedRules(rules, namedRules);
             namedActions = extractNamedActions(c.actions, namedActions);
         }
 
+        const filterDefs = filterCriteriaDefaults ?? filterCriteriaDefaultsFromBot;
+        const {
+            authorIsBehavior = 'merge',
+            itemIsBehavior = 'merge',
+            authorIs: authorIsDefault = {},
+            itemIs: itemIsDefault = []
+        } = filterDefs || {};
+
         const structuredChecks: CheckStructuredJson[] = [];
         for (const c of checks) {
-            const {rules = []} = c;
+            const {rules = [], authorIs = {}, itemIs = []} = c;
             const strongRules = insertNamedRules(rules, namedRules);
             const strongActions = insertNamedActions(c.actions, namedActions);
-            const strongCheck = {...c, rules: strongRules, actions: strongActions} as CheckStructuredJson;
+
+            let derivedAuthorIs: AuthorOptions = authorIsDefault;
+            if (authorIsBehavior === 'merge') {
+                derivedAuthorIs = merge.all([authorIs, authorIsDefault], {arrayMerge: removeFromSourceIfKeysExistsInDestination});
+            } else if (Object.keys(authorIs).length > 0) {
+                derivedAuthorIs = authorIs;
+            }
+
+            let derivedItemIs: TypedActivityStates = itemIsDefault;
+            if (itemIsBehavior === 'merge') {
+                derivedItemIs = [...itemIs, ...itemIsDefault];
+            } else if (itemIs.length > 0) {
+                derivedItemIs = itemIs;
+            }
+
+            const strongCheck = {
+                ...c,
+                authorIs: derivedAuthorIs,
+                itemIs: derivedItemIs,
+                rules: strongRules,
+                actions: strongActions
+            } as CheckStructuredJson;
             structuredChecks.push(strongCheck);
         }
 
@@ -146,10 +190,6 @@ export const buildPollingOptions = (values: (string | PollingOptions)[]): Pollin
                 pollOn: v as PollOn,
                 interval: DEFAULT_POLLING_INTERVAL,
                 limit: DEFAULT_POLLING_LIMIT,
-                clearProcessed: {
-                    size: DEFAULT_POLLING_LIMIT,
-                    retain: DEFAULT_POLLING_LIMIT,
-                }
             });
         } else {
             const {
@@ -157,14 +197,12 @@ export const buildPollingOptions = (values: (string | PollingOptions)[]): Pollin
                 interval = DEFAULT_POLLING_INTERVAL,
                 limit = DEFAULT_POLLING_LIMIT,
                 delayUntil,
-                clearProcessed = {size: limit, retain: limit},
             } = v;
             opts.push({
                 pollOn: p as PollOn,
                 interval,
                 limit,
                 delayUntil,
-                clearProcessed
             });
         }
     }
@@ -281,8 +319,6 @@ export const parseDefaultBotInstanceFromArgs = (args: any): BotInstanceJsonConfi
         heartbeat,
         hardLimit,
         authorTTL,
-        snooProxy,
-        snooDebug,
         sharedMod,
         caching,
     } = args || {};
@@ -294,10 +330,6 @@ export const parseDefaultBotInstanceFromArgs = (args: any): BotInstanceJsonConfi
             accessToken,
             refreshToken,
         },
-        snoowrap: {
-            proxy: snooProxy,
-            debug: snooDebug,
-        },
         subreddits: {
             names: subreddits,
             wikiConfig,
@@ -305,7 +337,7 @@ export const parseDefaultBotInstanceFromArgs = (args: any): BotInstanceJsonConfi
             heartbeatInterval: heartbeat,
         },
         polling: {
-            sharedMod,
+            shared: sharedMod ? ['unmoderated', 'modqueue'] : undefined,
         },
         nanny: {
             softLimit,
@@ -330,6 +362,8 @@ export const parseOpConfigFromArgs = (args: any): OperatorJsonConfig => {
         mode,
         caching,
         authorTTL,
+        snooProxy,
+        snooDebug,
     } = args || {};
 
     const data = {
@@ -340,12 +374,25 @@ export const parseOpConfigFromArgs = (args: any): OperatorJsonConfig => {
         },
         logging: {
             level: logLevel,
-            path: logDir === true ? `${process.cwd()}/logs` : undefined,
+            file: {
+                level: logLevel,
+                dirName: logDir,
+            },
+            stream: {
+                level: logLevel,
+            },
+            console: {
+                level: logLevel,
+            }
         },
         caching: {
             provider: caching,
             authorTTL
         },
+        snoowrap: {
+            proxy: snooProxy,
+            debug: snooDebug,
+        },
         web: {
             enabled: web,
             port,
@@ -401,12 +448,8 @@ export const parseDefaultBotInstanceFromEnv = (): BotInstanceJsonConfig => {
             dryRun: parseBool(process.env.DRYRUN, undefined),
             heartbeatInterval: process.env.HEARTBEAT !== undefined ? parseInt(process.env.HEARTBEAT) : undefined,
         },
-        snoowrap: {
-            proxy: process.env.PROXY,
-            debug: parseBool(process.env.SNOO_DEBUG, undefined),
-        },
         polling: {
-            sharedMod: parseBool(process.env.SHARE_MOD),
+            shared: parseBool(process.env.SHARE_MOD) ? ['unmoderated', 'modqueue'] : undefined,
         },
         nanny: {
             softLimit: process.env.SOFT_LIMIT !== undefined ? parseInt(process.env.SOFT_LIMIT) : undefined,
@@ -424,9 +467,17 @@ export const parseOpConfigFromEnv = (): OperatorJsonConfig => {
             display: process.env.OPERATOR_DISPLAY
         },
         logging: {
-            // @ts-ignore
             level: process.env.LOG_LEVEL,
-            path: process.env.LOG_DIR === 'true' ? `${process.cwd()}/logs` : undefined,
+            file: {
+                level: process.env.LOG_LEVEL,
+                dirname: process.env.LOG_DIR,
+            },
+            stream: {
+                level: process.env.LOG_LEVEL,
+            },
+            console: {
+                level: process.env.LOG_LEVEL,
+            }
         },
         caching: {
             provider: {
@@ -435,6 +486,10 @@ export const parseOpConfigFromEnv = (): OperatorJsonConfig => {
             },
             authorTTL: process.env.AUTHOR_TTL !== undefined ? parseInt(process.env.AUTHOR_TTL) : undefined
         },
+        snoowrap: {
+            proxy: process.env.PROXY,
+            debug: parseBool(process.env.SNOO_DEBUG, undefined),
+        },
         web: {
             port: process.env.PORT !== undefined ? parseInt(process.env.PORT) : undefined,
             session: {
@@ -448,9 +503,9 @@ export const parseOpConfigFromEnv = (): OperatorJsonConfig => {
             },
         },
         credentials: {
-          youtube: {
-              apiKey: process.env.YOUTUBE_API_KEY
-          }
+            youtube: {
+                apiKey: process.env.YOUTUBE_API_KEY
+            }
         }
     }
 
@@ -463,12 +518,26 @@ export const parseOpConfigFromEnv = (): OperatorJsonConfig => {
 // Actual ENVs (from environment)
 // json config
 // args from cli
-export const parseOperatorConfigFromSources = async (args: any): Promise<OperatorJsonConfig> => {
-    const {logLevel = process.env.LOG_LEVEL, logDir = process.env.LOG_DIR || false} = args || {};
+export const parseOperatorConfigFromSources = async (args: any): Promise<[OperatorJsonConfig, OperatorFileConfig]> => {
+    const {logLevel = process.env.LOG_LEVEL ?? 'debug', logDir = process.env.LOG_DIR} = args || {};
     const envPath = process.env.OPERATOR_ENV;
+    const initLoggerOptions = {
+        level: logLevel,
+        console: {
+            level: logLevel
+        },
+        file: {
+            level: logLevel,
+            dirname: logDir,
+        },
+        stream: {
+            level: logLevel
+        }
+    }
 
     // create a pre config logger to help with debugging
-    const initLogger = getLogger({logLevel, logDir: logDir === true ? `${process.cwd()}/logs` : logDir}, 'init');
+    // default to debug if nothing is provided
+    const initLogger = getLogger(initLoggerOptions, 'init');
 
     try {
         const vars = await GetEnvVars({
@@ -494,24 +563,90 @@ export const parseOperatorConfigFromSources = async (args: any): Promise<Operato
         //swallow silently for now 😬
     }
 
-    const {operatorConfig = process.env.OPERATOR_CONFIG} = args;
+    const {operatorConfig = (process.env.OPERATOR_CONFIG ?? path.resolve(__dirname, '../config.yaml'))} = args;
     let configFromFile: OperatorJsonConfig = {};
-    if (operatorConfig !== undefined) {
-        let rawConfig;
-        try {
-            rawConfig = await readConfigFile(operatorConfig, {log: initLogger}) as object;
-        } catch (err: any) {
-            initLogger.error('Cannot continue app startup because operator config file was not parseable.');
-            err.logged = true;
-            throw err;
+    let fileConfigFormat: ConfigFormat | undefined = undefined;
+    let fileConfig: object = {};
+    let rawConfig: string = '';
+    let configDoc: YamlOperatorConfigDocument | JsonOperatorConfigDocument;
+    let writeable = false;
+    try {
+        writeable = await fileOrDirectoryIsWriteable(operatorConfig);
+    } catch (e) {
+        initLogger.warn(`Issue while parsing operator config file location: ${e} \n This is only a problem if you do not have a config file but are planning on adding bots interactively.`);
+    }
+
+    try {
+        const [rawConfigValue, format] = await readConfigFile(operatorConfig, {log: initLogger});
+        rawConfig = rawConfigValue ?? '';
+        fileConfigFormat = format as ConfigFormat;
+    } catch (err: any) {
+        const {code} = err;
+        if (code === 'ENOENT') {
+            initLogger.warn('No operator config file found but will continue');
+            if (err.extension !== undefined) {
+                fileConfigFormat = err.extension
+            }
+        } else {
+            throw new ErrorWithCause('Cannot continue app startup because operator config file exists but was not parseable.', {cause: err});
         }
+    }
+    const [format, doc, jsonErr, yamlErr] = parseFromJsonOrYamlToObject(rawConfig, {
+        location: operatorConfig,
+        jsonDocFunc: (content, location) => new JsonOperatorConfigDocument(content, location),
+        yamlDocFunc: (content, location) => new YamlOperatorConfigDocument(content, location)
+    });
+
+
+    if (format !== undefined && fileConfigFormat === undefined) {
+        fileConfigFormat = 'yaml';
+    }
+
+    if (doc === undefined && rawConfig !== '') {
+        initLogger.error(`Could not parse file contents at ${operatorConfig} as JSON or YAML (likely it is ${fileConfigFormat}):`);
+        initLogger.error(jsonErr);
+        initLogger.error(yamlErr);
+        throw new SimpleError(`Could not parse file contents at ${operatorConfig} as JSON or YAML`);
+    } else if (doc === undefined && rawConfig === '') {
+        // create an empty doc
+        if(fileConfigFormat === 'json') {
+            configDoc = new JsonOperatorConfigDocument('{}', operatorConfig);
+        } else {
+            configDoc = new YamlOperatorConfigDocument('', operatorConfig);
+            configDoc.parsed = new YamlDocument({});
+        }
+        configFromFile = {};
+    } else {
+        configDoc = doc as (YamlOperatorConfigDocument | JsonOperatorConfigDocument);
+
         try {
-            configFromFile = validateJson(rawConfig, operatorSchema, initLogger) as OperatorJsonConfig;
+            configFromFile = validateJson(configDoc.toJS(), operatorSchema, initLogger) as OperatorJsonConfig;
+            const {
+                bots = [],
+                logging: {
+                    path = undefined
+                } = {}
+            } = configFromFile || {};
+            if(path !== undefined) {
+                initLogger.warn(`'path' property in top-level 'logging' object is DEPRECATED and will be removed in next minor version. Use 'logging.file.dirname' instead`);
+            }
+            for (const b of bots) {
+                const {
+                    polling: {
+                        sharedMod
+                    } = {}
+                } = b;
+                if (sharedMod !== undefined) {
+                    initLogger.warn(`'sharedMod' bot config property is DEPRECATED and will be removed in next minor version. Use 'shared' property instead (see docs)`);
+                    break;
+                }
+            }
         } catch (err: any) {
             initLogger.error('Cannot continue app startup because operator config file was not valid.');
             throw err;
         }
     }
+
     const opConfigFromArgs = parseOpConfigFromArgs(args);
     const opConfigFromEnv = parseOpConfigFromEnv();
 
@@ -538,7 +673,10 @@ export const parseOperatorConfigFromSources = async (args: any): Promise<Operato
         botInstances = botInstancesFromFile.map(x => merge.all([defaultBotInstance, x], {arrayMerge: overwriteMerge}));
     }
 
-    return removeUndefinedKeys({...mergedConfig, bots: botInstances}) as OperatorJsonConfig;
+    return [removeUndefinedKeys({...mergedConfig, bots: botInstances}) as OperatorJsonConfig, {
+        document: configDoc,
+        isWriteable: writeable
+    }];
 }
 
 export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): OperatorConfig => {
@@ -551,6 +689,9 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
         logging: {
             level = 'verbose',
             path,
+            file = {},
+            console = {},
+            stream = {},
         } = {},
         caching: opCache,
         web: {
@@ -568,6 +709,7 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
             credentials: webCredentials,
             operators,
         } = {},
+        snoowrap: snoowrapOp = {},
         api: {
             port: apiPort = 8095,
             secret: apiSecret = randomId(),
@@ -623,156 +765,14 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
         }
     }
 
-    let hydratedBots: BotInstanceConfig[] = bots.map(x => {
-        const {
-            name: botName,
-            polling: {
-                sharedMod = false,
-                stagger,
-                limit = 100,
-                interval = 30,
-            } = {},
-            queue: {
-                maxWorkers = 1,
-            } = {},
-            caching,
-            nanny: {
-                softLimit = 250,
-                hardLimit = 50
-            } = {},
-            snoowrap = {},
-            credentials = {},
-            subreddits: {
-                names = [],
-                exclude = [],
-                wikiConfig = 'botconfig/contextbot',
-                dryRun,
-                heartbeatInterval = 300,
-            } = {},
-        } = x;
-
-        let botCache: StrongCache;
-        let botActionedEventsDefault: number;
-
-        if (caching === undefined) {
-
-            botCache = {
-                ...cacheTTLDefaults,
-                actionedEventsDefault: opActionedEventsDefault,
-                actionedEventsMax: opActionedEventsMax,
-                provider: {...defaultProvider}
-            };
-        } else {
-            const {
-                provider,
-                actionedEventsMax = opActionedEventsMax,
-                actionedEventsDefault = opActionedEventsDefault,
-                ...restConfig
-            } = caching;
-
-            botActionedEventsDefault = actionedEventsDefault;
-            if (actionedEventsMax !== undefined) {
-                botActionedEventsDefault = Math.min(actionedEventsDefault, actionedEventsMax);
-            }
-
-            if (typeof provider === 'string') {
-                botCache = {
-                    ...cacheTTLDefaults,
-                    ...restConfig,
-                    actionedEventsDefault: botActionedEventsDefault,
-                    provider: {
-                        store: provider as CacheProvider,
-                        ...cacheOptDefaults
-                    }
-                }
-            } else {
-                const {ttl = 60, max = 500, store = 'memory', ...rest} = provider || {};
-                botCache = {
-                    ...cacheTTLDefaults,
-                    ...restConfig,
-                    actionedEventsDefault: botActionedEventsDefault,
-                    actionedEventsMax,
-                    provider: {
-                        store,
-                        ...cacheOptDefaults,
-                        ...rest,
-                    },
-                }
-            }
-        }
-
-        let botCreds: BotCredentialsConfig;
-
-        if((credentials as any).clientId !== undefined) {
-            const creds = credentials as RedditCredentials;
-            const {
-                clientId: ci,
-                clientSecret: cs,
-                ...restCred
-            } = creds;
-            botCreds = {
-                reddit: {
-                        clientId: (ci as string),
-                        clientSecret: (cs as string),
-                        ...restCred,
-                }
-            }
-        } else {
-            const creds = credentials as BotCredentialsJsonConfig;
-            const {
-                reddit: {
-                    clientId: ci,
-                    clientSecret: cs,
-                    ...restRedditCreds
-                },
-                ...rest
-            } = creds;
-            botCreds = {
-                reddit: {
-                    clientId: (ci as string),
-                    clientSecret: (cs as string),
-                    ...restRedditCreds,
-                },
-                ...rest
-            }
-        }
-
-        if (botCache.provider.prefix === undefined || botCache.provider.prefix === defaultProvider.prefix) {
-            // need to provide unique prefix to bot
-            botCache.provider.prefix = buildCachePrefix([botCache.provider.prefix, 'bot', (botName || objectHash.sha1(botCreds))]);
-        }
-
-        return {
-            name: botName,
-            snoowrap,
-            subreddits: {
-                names,
-                exclude,
-                wikiConfig,
-                heartbeatInterval,
-                dryRun,
-            },
-            credentials: botCreds,
-            caching: botCache,
-            polling: {
-                sharedMod,
-                stagger,
-                limit,
-                interval,
-            },
-            queue: {
-                maxWorkers,
-            },
-            nanny: {
-                softLimit,
-                hardLimit
-            }
-        }
-
-    });
-
     const defaultOperators = typeof name === 'string' ? [name] : name;
 
+    const {
+        dirname = path,
+        ...fileRest
+    } = file;
+
+
     const config: OperatorConfig = {
         mode,
         operator: {
@@ -781,7 +781,19 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
         },
         logging: {
             level,
-            path
+            file: {
+                level: level,
+                dirname,
+                ...fileRest,
+            },
+            stream: {
+                level: level,
+                ...stream,
+            },
+            console: {
+                level: level,
+                ...console,
+            }
         },
         caching: cache,
         web: {
@@ -807,9 +819,175 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
             secret: apiSecret,
             friendly
         },
-        bots: hydratedBots,
+        bots: [],
         credentials,
     };
 
+    config.bots = bots.map(x => buildBotConfig(x, config));
+
     return config;
 }
+
+export const buildBotConfig = (data: BotInstanceJsonConfig, opConfig: OperatorConfig): BotInstanceConfig => {
+    const {
+        snoowrap: snoowrapOp,
+        caching: {
+            actionedEventsMax: opActionedEventsMax,
+            actionedEventsDefault: opActionedEventsDefault = 25,
+            provider: defaultProvider,
+        } = {}
+    } = opConfig;
+    const {
+        name: botName,
+        filterCriteriaDefaults = filterCriteriaDefault,
+        polling: {
+            sharedMod,
+            shared = [],
+            stagger,
+            limit = 100,
+            interval = 30,
+        } = {},
+        queue: {
+            maxWorkers = 1,
+        } = {},
+        caching,
+        nanny: {
+            softLimit = 250,
+            hardLimit = 50
+        } = {},
+        snoowrap = snoowrapOp,
+        credentials = {},
+        subreddits: {
+            names = [],
+            exclude = [],
+            wikiConfig = 'botconfig/contextbot',
+            dryRun,
+            heartbeatInterval = 300,
+        } = {},
+    } = data;
+
+    let botCache: StrongCache;
+    let botActionedEventsDefault: number;
+
+    if (caching === undefined) {
+
+        botCache = {
+            ...cacheTTLDefaults,
+            actionedEventsDefault: opActionedEventsDefault,
+            actionedEventsMax: opActionedEventsMax,
+            provider: {...defaultProvider as CacheOptions}
+        };
+    } else {
+        const {
+            provider,
+            actionedEventsMax = opActionedEventsMax,
+            actionedEventsDefault = opActionedEventsDefault,
+            ...restConfig
+        } = caching;
+
+        botActionedEventsDefault = actionedEventsDefault;
+        if (actionedEventsMax !== undefined) {
+            botActionedEventsDefault = Math.min(actionedEventsDefault, actionedEventsMax);
+        }
+
+        if (typeof provider === 'string') {
+            botCache = {
+                ...cacheTTLDefaults,
+                ...restConfig,
+                actionedEventsDefault: botActionedEventsDefault,
+                provider: {
+                    store: provider as CacheProvider,
+                    ...cacheOptDefaults
+                }
+            }
+        } else {
+            const {ttl = 60, max = 500, store = 'memory', ...rest} = provider || {};
+            botCache = {
+                ...cacheTTLDefaults,
+                ...restConfig,
+                actionedEventsDefault: botActionedEventsDefault,
+                actionedEventsMax,
+                provider: {
+                    store,
+                    ...cacheOptDefaults,
+                    ...rest,
+                },
+            }
+        }
+    }
+
+    let botCreds: BotCredentialsConfig;
+
+    if ((credentials as any).clientId !== undefined) {
+        const creds = credentials as RedditCredentials;
+        const {
+            clientId: ci,
+            clientSecret: cs,
+            ...restCred
+        } = creds;
+        botCreds = {
+            reddit: {
+                clientId: (ci as string),
+                clientSecret: (cs as string),
+                ...restCred,
+            }
+        }
+    } else {
+        const creds = credentials as BotCredentialsJsonConfig;
+        const {
+            reddit: {
+                clientId: ci,
+                clientSecret: cs,
+                ...restRedditCreds
+            },
+            ...rest
+        } = creds;
+        botCreds = {
+            reddit: {
+                clientId: (ci as string),
+                clientSecret: (cs as string),
+                ...restRedditCreds,
+            },
+            ...rest
+        }
+    }
+
+    if (botCache.provider.prefix === undefined || botCache.provider.prefix === (defaultProvider as CacheOptions).prefix) {
+        // need to provide unique prefix to bot
+        botCache.provider.prefix = buildCachePrefix([botCache.provider.prefix, 'bot', (botName || objectHash.sha1(botCreds))]);
+    }
+
+    let realShared = shared === true ? ['unmoderated', 'modqueue', 'newComm', 'newSub'] : shared;
+    if (sharedMod === true) {
+        realShared.push('unmoderated');
+        realShared.push('modqueue');
+    }
+
+    return {
+        name: botName,
+        snoowrap: snoowrap || {},
+        filterCriteriaDefaults,
+        subreddits: {
+            names,
+            exclude,
+            wikiConfig,
+            heartbeatInterval,
+            dryRun,
+        },
+        credentials: botCreds,
+        caching: botCache,
+        polling: {
+            shared: [...new Set(realShared)] as PollOn[],
+            stagger,
+            limit,
+            interval,
+        },
+        queue: {
+            maxWorkers,
+        },
+        nanny: {
+            softLimit,
+            hardLimit
+        }
+    }
+}

src/Rule/AttributionRule.ts

@@ -14,8 +14,8 @@ import {
     PASS
 } from "../util";
 import { Comment } from "snoowrap/dist/objects";
-import SimpleError from "../Utils/SimpleError";
 import as from "async";
+import {SimpleError} from "../Utils/Errors";
 
 
 export interface AttributionCriteria {

src/Rule/AuthorRule.ts

@@ -2,6 +2,7 @@ import {Rule, RuleJSONConfig, RuleOptions, RuleResult} from "./index";
 import {Comment} from "snoowrap";
 import Submission from "snoowrap/dist/objects/Submission";
 import {Author, AuthorCriteria} from "../Author/Author";
+import {checkAuthorFilter} from "../Subreddit/SubredditResources";
 
 /**
  * Checks the author of the Activity against AuthorCriteria. This differs from a Rule's AuthorOptions as this is a full Rule and will only pass/fail, not skip.
@@ -59,20 +60,8 @@ export class AuthorRule extends Rule {
     }
 
     protected async process(item: Comment | Submission): Promise<[boolean, RuleResult]> {
-        if (this.include.length > 0) {
-            for (const auth of this.include) {
-                if (await this.resources.testAuthorCriteria(item, auth)) {
-                    return Promise.resolve([true, this.getResult(true)]);
-                }
-            }
-            return Promise.resolve([false, this.getResult(false)]);
-        }
-        for (const auth of this.exclude) {
-            if (await this.resources.testAuthorCriteria(item, auth, false)) {
-                return Promise.resolve([true, this.getResult(true)]);
-            }
-        }
-        return Promise.resolve([false, this.getResult(false)]);
+        const [result, filterType] = await checkAuthorFilter(item, {include: this.include, exclude: this.exclude}, this.resources, this.logger);
+        return Promise.resolve([result, this.getResult(result)]);
     }
 }
 

src/Rule/RecentActivityRule.ts

@@ -1,6 +1,7 @@
 import {Rule, RuleJSONConfig, RuleOptions, RulePremise, RuleResult} from "./index";
-import {Comment, VoteableContent} from "snoowrap";
+import {VoteableContent} from "snoowrap";
 import Submission from "snoowrap/dist/objects/Submission";
+import Comment from "snoowrap/dist/objects/Comment";
 import as from 'async';
 import pMap from 'p-map';
 // @ts-ignore
@@ -23,7 +24,7 @@ import {
     parseSubredditName,
     parseUsableLinkIdentifier,
     PASS, sleep,
-    toStrongSubredditState
+    toStrongSubredditState, windowToActivityWindowCriteria
 } from "../util";
 import {
     ActivityWindow,
@@ -43,7 +44,7 @@ const parseLink = parseUsableLinkIdentifier();
 export class RecentActivityRule extends Rule {
     window: ActivityWindowType;
     thresholds: ActivityThreshold[];
-    useSubmissionAsReference: boolean;
+    useSubmissionAsReference: boolean | undefined;
     imageDetection: StrongImageDetection
     lookAt?: 'comments' | 'submissions';
 
@@ -51,7 +52,7 @@ export class RecentActivityRule extends Rule {
         super(options);
         const {
             window = 15,
-            useSubmissionAsReference = true,
+            useSubmissionAsReference,
             imageDetection,
             lookAt,
         } = options || {};
@@ -115,20 +116,53 @@ export class RecentActivityRule extends Rule {
     async process(item: Submission | Comment): Promise<[boolean, RuleResult]> {
         let activities;
 
+        // ACID is a bitch
+        // reddit may not return the activity being checked in the author's recent history due to availability/consistency issues or *something*
+        // so make sure we add it in if config is checking the same type and it isn't included
+        // TODO refactor this for SubredditState everywhere branch
+        let shouldIncludeSelf = true;
+        const strongWindow = windowToActivityWindowCriteria(this.window);
+        const {
+            subreddits: {
+                include = [],
+                exclude = []
+            } = {}
+        } = strongWindow;
+        if (include.length > 0 && !include.some(x => x.toLocaleLowerCase() === item.subreddit.display_name.toLocaleLowerCase())) {
+            shouldIncludeSelf = false;
+        } else if (exclude.length > 0 && exclude.some(x => x.toLocaleLowerCase() === item.subreddit.display_name.toLocaleLowerCase())) {
+            shouldIncludeSelf = false;
+        }
+
         switch (this.lookAt) {
             case 'comments':
                 activities = await this.resources.getAuthorComments(item.author, {window: this.window});
+                if (shouldIncludeSelf && item instanceof Comment && !activities.some(x => x.name === item.name)) {
+                    activities.unshift(item);
+                }
                 break;
             case 'submissions':
                 activities = await this.resources.getAuthorSubmissions(item.author, {window: this.window});
+                if (shouldIncludeSelf && item instanceof Submission && !activities.some(x => x.name === item.name)) {
+                    activities.unshift(item);
+                }
                 break;
             default:
                 activities = await this.resources.getAuthorActivities(item.author, {window: this.window});
+                if (shouldIncludeSelf && !activities.some(x => x.name === item.name)) {
+                    activities.unshift(item);
+                }
                 break;
         }
 
         let viableActivity = activities;
-        if (this.useSubmissionAsReference) {
+        // if config does not specify reference then we set the default based on whether the item is a submission or not
+        // -- this is essentially the same as defaulting reference to true BUT eliminates noisy "can't use comment as reference" log statement when item is a comment
+        let inferredSubmissionAsRef = this.useSubmissionAsReference;
+        if(inferredSubmissionAsRef === undefined) {
+            inferredSubmissionAsRef = isSubmission(item);
+        }
+        if (inferredSubmissionAsRef) {
             if (!asSubmission(item)) {
                 this.logger.warn('Cannot use post as reference because triggered item is not a Submission');
             } else if (item.is_self) {
@@ -310,34 +344,6 @@ export class RecentActivityRule extends Rule {
                 }
             }
 
-            for (const activity of viableActivity) {
-                if (asSubmission(activity) && submissionState !== undefined) {
-                    if (!(await this.resources.testItemCriteria(activity, [submissionState]))) {
-                        continue;
-                    }
-                } else if (commentState !== undefined) {
-                    if (!(await this.resources.testItemCriteria(activity, [commentState]))) {
-                        continue;
-                    }
-                }
-                let inSubreddits = false;
-                for (const ss of subStates) {
-                    const res = await this.resources.testSubredditCriteria(activity, ss);
-                    if (res) {
-                        inSubreddits = true;
-                        break;
-                    }
-                }
-                if (inSubreddits) {
-                    currCount++;
-                    combinedKarma += activity.score;
-                    const pSub = getActivitySubredditName(activity);
-                    if (!presentSubs.includes(pSub)) {
-                        presentSubs.push(pSub);
-                    }
-                }
-            }
-
             const {operator, value, isPercent} = parseGenericValueOrPercentComparison(threshold);
             let sum = {
                 subsWithActivity: presentSubs,
@@ -421,6 +427,7 @@ export class RecentActivityRule extends Rule {
                 threshold,
                 testValue,
                 karmaThreshold,
+                combinedKarma,
             }
         };
     }
@@ -501,6 +508,16 @@ interface RecentActivityConfig extends ActivityWindow, ReferenceSubmission {
     thresholds: ActivityThreshold[],
 
     imageDetection?: ImageDetection
+
+    /**
+     * When Activity is a submission should we only include activities that are other submissions with the same content?
+     *
+     * * When the Activity is a submission this defaults to **true**
+     * * When the Activity is a comment it is ignored (not relevant)
+     *
+     * @default true
+     * */
+    useSubmissionAsReference?: boolean
 }
 
 export interface RecentActivityRuleOptions extends RecentActivityConfig, RuleOptions {

src/Rule/RegexRule.ts

@@ -11,7 +11,7 @@ import {
     ActivityWindowType, JoinOperands,
 } from "../Common/interfaces";
 import dayjs from 'dayjs';
-import SimpleError from "../Utils/SimpleError";
+import {SimpleError} from "../Utils/Errors";
 
 export interface RegexCriteria {
     /**
@@ -152,7 +152,8 @@ export class RegexRule extends Rule {
             }, []);
 
             // check regex
-            const reg = parseStringToRegex(regex, 'g');
+            const regexContent = await this.resources.getContent(regex);
+            const reg = parseStringToRegex(regexContent, 'g');
             if(reg === undefined) {
                 throw new SimpleError(`Value given for regex on Criteria ${name} was not valid: ${regex}`);
             }
@@ -257,7 +258,7 @@ export class RegexRule extends Rule {
             const critResults = {
                 criteria: {
                     name,
-                    regex,
+                    regex: regex !== regexContent ? `${regex} from ${regexContent}` : regex,
                     testOn,
                     matchThreshold,
                     activityMatchThreshold,

src/Rule/index.ts

@@ -2,7 +2,7 @@ import Snoowrap, {Comment} from "snoowrap";
 import Submission from "snoowrap/dist/objects/Submission";
 import {Logger} from "winston";
 import {findResultByPremise, mergeArr} from "../util";
-import {SubredditResources} from "../Subreddit/SubredditResources";
+import {checkAuthorFilter, SubredditResources} from "../Subreddit/SubredditResources";
 import {ChecksActivityState, TypedActivityStates} from "../Common/interfaces";
 import Author, {AuthorOptions} from "../Author/Author";
 
@@ -65,6 +65,7 @@ export abstract class Rule implements IRule, Triggerable {
             name = this.getKind(),
             logger,
             authorIs: {
+                excludeCondition = 'OR',
                 include = [],
                 exclude = [],
             } = {},
@@ -78,6 +79,7 @@ export abstract class Rule implements IRule, Triggerable {
         this.client = client;
 
         this.authorIs = {
+            excludeCondition,
             exclude: exclude.map(x => new Author(x)),
             include: include.map(x => new Author(x)),
         }
@@ -99,23 +101,10 @@ export abstract class Rule implements IRule, Triggerable {
                 this.logger.verbose(`(Skipped) Item did not pass 'itemIs' test`);
                 return Promise.resolve([null, this.getResult(null, {result: `Item did not pass 'itemIs' test`})]);
             }
-            if (this.authorIs.include !== undefined && this.authorIs.include.length > 0) {
-                for (const auth of this.authorIs.include) {
-                    if (await this.resources.testAuthorCriteria(item, auth)) {
-                        return this.process(item);
-                    }
-                }
-                this.logger.verbose('(Skipped) Inclusive author criteria not matched');
-                return Promise.resolve([null, this.getResult(null, {result: 'Inclusive author criteria not matched'})]);
-            }
-            if (this.authorIs.exclude !== undefined && this.authorIs.exclude.length > 0) {
-                for (const auth of this.authorIs.exclude) {
-                    if (await this.resources.testAuthorCriteria(item, auth, false)) {
-                        return this.process(item);
-                    }
-                }
-                this.logger.verbose('(Skipped) Exclusive author criteria not matched');
-                return Promise.resolve([null, this.getResult(null, {result: 'Exclusive author criteria not matched'})]);
+            const [authFilterResult, authFilterType] = await checkAuthorFilter(item, this.authorIs, this.resources, this.logger);
+            if(!authFilterResult) {
+                this.logger.verbose(`(Skipped) ${authFilterType} Author criteria not matched`);
+                return Promise.resolve([null, this.getResult(null, {result: `${authFilterType} author criteria not matched`})]);
             }
         } catch (err: any) {
             this.logger.error('Error occurred during Rule pre-process checks');

src/Schema/Action.json

@@ -50,24 +50,52 @@
                     ]
                 },
                 "flairCssClass": {
-                    "description": "A list of (user) flair css class values from the subreddit to match against",
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "A (user) flair css class (or list of) from the subreddit to match against",
                     "examples": [
                         "red"
+                    ]
+                },
+                "flairTemplate": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
                     ],
-                    "items": {
-                        "type": "string"
-                    },
-                    "type": "array"
+                    "description": "A (user) flair template id (or list of) from the subreddit to match against"
                 },
                 "flairText": {
-                    "description": "A list of (user) flair text values from the subreddit to match against",
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "A (user) flair text value (or list of) from the subreddit to match against",
                     "examples": [
                         "Approved"
-                    ],
-                    "items": {
-                        "type": "string"
-                    },
-                    "type": "array"
+                    ]
                 },
                 "isMod": {
                     "description": "Is the author a moderator?",
@@ -131,12 +159,21 @@
             ],
             "properties": {
                 "exclude": {
-                    "description": "Only runs if `include` is not present. Will \"pass\" if any of set of the AuthorCriteria **does not** pass",
+                    "description": "Only runs if `include` is not present. Each AuthorCriteria is comprised of conditions that the Author being checked must \"not\" pass. See excludeCondition for set behavior\n\nEX: `isMod: true, name: Automoderator` => Will pass if the Author IS NOT a mod and IS NOT named Automoderator",
                     "items": {
                         "$ref": "#/definitions/AuthorCriteria"
                     },
                     "type": "array"
                 },
+                "excludeCondition": {
+                    "default": "OR",
+                    "description": "* OR => if ANY exclude condition \"does not\" pass then the exclude test passes\n* AND => if ALL exclude conditions \"do not\" pass then the exclude test passes\n\nDefaults to OR",
+                    "enum": [
+                        "AND",
+                        "OR"
+                    ],
+                    "type": "string"
+                },
                 "include": {
                     "description": "Will \"pass\" if any set of AuthorCriteria passes",
                     "items": {
@@ -189,7 +226,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
@@ -240,14 +277,47 @@
                 "filtered": {
                     "type": "boolean"
                 },
+                "flairTemplate": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
+                },
                 "is_self": {
                     "type": "boolean"
                 },
                 "link_flair_css_class": {
-                    "type": "string"
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
                 },
                 "link_flair_text": {
-                    "type": "string"
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
                 },
                 "locked": {
                     "type": "boolean"
@@ -263,7 +333,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
@@ -391,6 +461,7 @@
                 "message",
                 "remove",
                 "report",
+                "userflair",
                 "usernote"
             ],
             "type": "string"

src/Schema/App.json

@@ -222,6 +222,17 @@
                     ],
                     "pattern": "^[a-zA-Z]([\\w -]*[\\w])?$",
                     "type": "string"
+                },
+                "targets": {
+                    "description": "Specify which Activities to approve\n\nThis setting is only applicable if the Activity being acted on is a **comment**. On a **submission** the setting does nothing\n\n* self => approve activity being checked (comment)\n* parent => approve parent (submission) of activity being checked (comment)",
+                    "items": {
+                        "enum": [
+                            "parent",
+                            "self"
+                        ],
+                        "type": "string"
+                    },
+                    "type": "array"
                 }
             },
             "required": [
@@ -235,8 +246,7 @@
                     "default": "undefined",
                     "description": "This list determines which categories of domains should be aggregated on. All aggregated domains will be tested against `threshold`\n\n* If `media` is included then aggregate author's submission history which reddit recognizes as media (youtube, vimeo, etc.)\n* If `redditMedia` is included then aggregate on author's submissions history which are media hosted on reddit: galleries, videos, and images (i.redd.it / v.redd.it)\n* If `self` is included then aggregate on author's submission history which are self-post (`self.[subreddit]`) or domain is `reddit.com`\n* If `link` is included then aggregate author's submission history which is external links and not recognized as `media` by reddit\n\nIf nothing is specified or list is empty (default) rule will only aggregate on `link` and `media` (ignores reddit-hosted content and self-posts)",
                     "examples": [
-                        [
-                        ]
+                        []
                     ],
                     "items": {
                         "enum": [
@@ -269,8 +279,7 @@
                 },
                 "domains": {
                     "default": [
-                        [
-                        ]
+                        []
                     ],
                     "description": "A list of domains whose Activities will be tested against `threshold`.\n\nThe values are tested as partial strings so you do not need to include full URLs, just the part that matters.\n\nEX `[\"youtube\"]` will match submissions with the domain `https://youtube.com/c/aChannel`\nEX `[\"youtube.com/c/bChannel\"]` will NOT match submissions with the domain `https://youtube.com/c/aChannel`\n\nIf you wish to aggregate on self-posts for a subreddit use the syntax `self.[subreddit]` EX `self.AskReddit`\n\n**If this Rule is part of a Check for a Submission and you wish to aggregate on the domain of the Submission use the special string `AGG:SELF`**\n\nIf nothing is specified or list is empty (default) aggregate using `aggregateOn`",
                     "items": {
@@ -505,24 +514,52 @@
                     ]
                 },
                 "flairCssClass": {
-                    "description": "A list of (user) flair css class values from the subreddit to match against",
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "A (user) flair css class (or list of) from the subreddit to match against",
                     "examples": [
                         "red"
+                    ]
+                },
+                "flairTemplate": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
                     ],
-                    "items": {
-                        "type": "string"
-                    },
-                    "type": "array"
+                    "description": "A (user) flair template id (or list of) from the subreddit to match against"
                 },
                 "flairText": {
-                    "description": "A list of (user) flair text values from the subreddit to match against",
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "A (user) flair text value (or list of) from the subreddit to match against",
                     "examples": [
                         "Approved"
-                    ],
-                    "items": {
-                        "type": "string"
-                    },
-                    "type": "array"
+                    ]
                 },
                 "isMod": {
                     "description": "Is the author a moderator?",
@@ -586,12 +623,21 @@
             ],
             "properties": {
                 "exclude": {
-                    "description": "Only runs if `include` is not present. Will \"pass\" if any of set of the AuthorCriteria **does not** pass",
+                    "description": "Only runs if `include` is not present. Each AuthorCriteria is comprised of conditions that the Author being checked must \"not\" pass. See excludeCondition for set behavior\n\nEX: `isMod: true, name: Automoderator` => Will pass if the Author IS NOT a mod and IS NOT named Automoderator",
                     "items": {
                         "$ref": "#/definitions/AuthorCriteria"
                     },
                     "type": "array"
                 },
+                "excludeCondition": {
+                    "default": "OR",
+                    "description": "* OR => if ANY exclude condition \"does not\" pass then the exclude test passes\n* AND => if ALL exclude conditions \"do not\" pass then the exclude test passes\n\nDefaults to OR",
+                    "enum": [
+                        "AND",
+                        "OR"
+                    ],
+                    "type": "string"
+                },
                 "include": {
                     "description": "Will \"pass\" if any set of AuthorCriteria passes",
                     "items": {
@@ -856,6 +902,17 @@
                     ],
                     "description": "The cache provider and, optionally, a custom configuration for that provider\n\nIf not present or `null` provider will be `memory`.\n\nTo specify another `provider` but use its default configuration set this property to a string of one of the available providers: `memory`, `redis`, or `none`"
                 },
+                "selfTTL": {
+                    "default": 50,
+                    "description": "Amount of time, in seconds, an Activity that the bot has acted on or created will be ignored if found during polling\n\nThis is useful to prevent the bot from checking Activities it *just* worked on or a product of the checks. Examples:\n\n* Ignore comments created through an Action\n* Ignore Activity polled from modqueue that the bot just reported\n\nThis value should be at least as long as the longest polling interval for modqueue/newComm\n\n* If `0` or `true` will cache indefinitely (not recommended)\n* If `false` will not cache",
+                    "examples": [
+                        50
+                    ],
+                    "type": [
+                        "number",
+                        "boolean"
+                    ]
+                },
                 "submissionTTL": {
                     "default": 60,
                     "description": "Amount of time, in seconds, a submission should be cached\n\n* If `0` or `true` will cache indefinitely (not recommended)\n* If `false` will not cache",
@@ -904,8 +961,7 @@
             "type": "object"
         },
         "CacheOptions": {
-            "additionalProperties": {
-            },
+            "additionalProperties": {},
             "description": "Configure granular settings for a cache provider with this object",
             "properties": {
                 "auth_pass": {
@@ -970,25 +1026,6 @@
             ],
             "type": "string"
         },
-        "ClearProcessedOptions": {
-            "description": "For very long-running, high-volume subreddits clearing the list of processed activities helps manage memory bloat\n\nAll of these options have default values based on the limit and/or interval set for polling options on each subreddit stream. They only need to modified if the defaults are not sufficient.\n\nIf both `after` and `size` are defined whichever is hit first will trigger the list to clear. `after` will be reset after ever clear.",
-            "properties": {
-                "after": {
-                    "description": "An interval the processed list should be cleared after.\n\n* EX `9 days`\n* EX `3 months`\n* EX `5 minutes`",
-                    "pattern": "^\\s*(?<time>\\d+)\\s*(?<unit>days?|weeks?|months?|years?|hours?|minutes?|seconds?|milliseconds?)\\s*$",
-                    "type": "string"
-                },
-                "retain": {
-                    "description": "The number of activities to retain in processed list after clearing.\n\nDefaults to `limit` value from `PollingOptions`",
-                    "type": "number"
-                },
-                "size": {
-                    "description": "Number of activities found in processed list after which the list should be cleared.\n\nDefaults to the `limit` value from `PollingOptions`",
-                    "type": "number"
-                }
-            },
-            "type": "object"
-        },
         "CommentActionJson": {
             "description": "Reply to the Activity. For a submission the reply will be a top-level comment.",
             "properties": {
@@ -1118,6 +1155,9 @@
                             {
                                 "$ref": "#/definitions/FlairActionJson"
                             },
+                            {
+                                "$ref": "#/definitions/UserFlairActionJson"
+                            },
                             {
                                 "$ref": "#/definitions/CommentActionJson"
                             },
@@ -1319,7 +1359,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
@@ -1422,6 +1462,61 @@
             },
             "type": "object"
         },
+        "FilterCriteriaDefaults": {
+            "properties": {
+                "authorIs": {
+                    "$ref": "#/definitions/AuthorOptions",
+                    "description": "Determine how authorIs defaults behave when authorIs is present on the check\n\n* merge => merges defaults with check's authorIs\n* replace => check authorIs will replace defaults (no defaults used)",
+                    "examples": [
+                        {
+                            "include": [
+                                {
+                                    "flairText": [
+                                        "Contributor",
+                                        "Veteran"
+                                    ]
+                                },
+                                {
+                                    "isMod": true
+                                }
+                            ]
+                        }
+                    ]
+                },
+                "authorIsBehavior": {
+                    "enum": [
+                        "merge",
+                        "replace"
+                    ],
+                    "type": "string"
+                },
+                "itemIs": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "$ref": "#/definitions/SubmissionState"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "items": {
+                                "$ref": "#/definitions/CommentState"
+                            },
+                            "type": "array"
+                        }
+                    ]
+                },
+                "itemIsBehavior": {
+                    "description": "Determine how itemIs defaults behave when itemIs is present on the check\n\n* merge => adds defaults to check's itemIs\n* replace => check itemIs will replace defaults (no defaults used)",
+                    "enum": [
+                        "merge",
+                        "replace"
+                    ],
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
         "FlairActionJson": {
             "description": "Flair the Submission",
             "properties": {
@@ -1465,6 +1560,10 @@
                     ],
                     "type": "boolean"
                 },
+                "flair_template_id": {
+                    "description": "Flair template ID to assign",
+                    "type": "string"
+                },
                 "itemIs": {
                     "anyOf": [
                         {
@@ -2092,10 +2191,6 @@
                 }
             ],
             "properties": {
-                "clearProcessed": {
-                    "$ref": "#/definitions/ClearProcessedOptions",
-                    "description": "For very long-running, high-volume subreddits clearing the list of processed activities helps manage memory bloat\n\nAll of these options have default values based on the limit and/or interval set for polling options on each subreddit stream. They only need to modified if the defaults are not sufficient.\n\nIf both `after` and `size` are defined whichever is hit first will trigger the list to clear. `after` will be reset after ever clear."
-                },
                 "delayUntil": {
                     "description": "Delay processing Activity until it is `N` seconds old\n\nUseful if there are other bots that may process an Activity and you want this bot to run first/last/etc.\n\nIf the Activity is already `N` seconds old when it is initially retrieved no refresh of the Activity occurs (no API request is made) and it is immediately processed.",
                     "type": "number"
@@ -2215,7 +2310,7 @@
                 },
                 "useSubmissionAsReference": {
                     "default": true,
-                    "description": "If activity is a Submission and is a link (not self-post) then only look at Submissions that contain this link, otherwise consider all activities.",
+                    "description": "When Activity is a submission should we only include activities that are other submissions with the same content?\n\n* When the Activity is a submission this defaults to **true**\n* When the Activity is a comment it is ignored (not relevant)",
                     "type": "boolean"
                 },
                 "window": {
@@ -2540,6 +2635,9 @@
                     ],
                     "pattern": "^[a-zA-Z]([\\w -]*[\\w])?$",
                     "type": "string"
+                },
+                "spam": {
+                    "type": "boolean"
                 }
             },
             "required": [
@@ -3192,6 +3290,9 @@
                             {
                                 "$ref": "#/definitions/FlairActionJson"
                             },
+                            {
+                                "$ref": "#/definitions/UserFlairActionJson"
+                            },
                             {
                                 "$ref": "#/definitions/CommentActionJson"
                             },
@@ -3377,14 +3478,47 @@
                 "filtered": {
                     "type": "boolean"
                 },
+                "flairTemplate": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
+                },
                 "is_self": {
                     "type": "boolean"
                 },
                 "link_flair_css_class": {
-                    "type": "string"
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
                 },
                 "link_flair_text": {
-                    "type": "string"
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
                 },
                 "locked": {
                     "type": "boolean"
@@ -3400,7 +3534,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
@@ -3467,8 +3601,7 @@
             "type": "object"
         },
         "ThirdPartyCredentialsJsonConfig": {
-            "additionalProperties": {
-            },
+            "additionalProperties": {},
             "properties": {
                 "youtube": {
                     "properties": {
@@ -3493,6 +3626,95 @@
             ],
             "type": "string"
         },
+        "UserFlairActionJson": {
+            "description": "Flair the Submission",
+            "properties": {
+                "authorIs": {
+                    "$ref": "#/definitions/AuthorOptions",
+                    "description": "If present then these Author criteria are checked before running the Action. If criteria fails then the Action is not run.",
+                    "examples": [
+                        {
+                            "include": [
+                                {
+                                    "flairText": [
+                                        "Contributor",
+                                        "Veteran"
+                                    ]
+                                },
+                                {
+                                    "isMod": true
+                                }
+                            ]
+                        }
+                    ]
+                },
+                "css": {
+                    "description": "The text of the css class of the flair to apply",
+                    "type": "string"
+                },
+                "dryRun": {
+                    "default": false,
+                    "description": "If `true` the Action will not make the API request to Reddit to perform its action.",
+                    "examples": [
+                        false,
+                        true
+                    ],
+                    "type": "boolean"
+                },
+                "enable": {
+                    "default": true,
+                    "description": "If set to `false` the Action will not be run",
+                    "examples": [
+                        true
+                    ],
+                    "type": "boolean"
+                },
+                "flair_template_id": {
+                    "description": "Flair template to pick.\n\n**Note:** If this template is used text/css are ignored",
+                    "type": "string"
+                },
+                "itemIs": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "$ref": "#/definitions/SubmissionState"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "items": {
+                                "$ref": "#/definitions/CommentState"
+                            },
+                            "type": "array"
+                        }
+                    ],
+                    "description": "A list of criteria to test the state of the `Activity` against before running the Action.\n\nIf any set of criteria passes the Action will be run."
+                },
+                "kind": {
+                    "description": "The type of action that will be performed",
+                    "enum": [
+                        "userflair"
+                    ],
+                    "type": "string"
+                },
+                "name": {
+                    "description": "An optional, but highly recommended, friendly name for this Action. If not present will default to `kind`.\n\nCan only contain letters, numbers, underscore, spaces, and dashes",
+                    "examples": [
+                        "myDescriptiveAction"
+                    ],
+                    "pattern": "^[a-zA-Z]([\\w -]*[\\w])?$",
+                    "type": "string"
+                },
+                "text": {
+                    "description": "The text of the flair to apply",
+                    "type": "string"
+                }
+            },
+            "required": [
+                "kind"
+            ],
+            "type": "object"
+        },
         "UserNoteActionJson": {
             "description": "Add a Toolbox User Note to the Author of this Activity",
             "properties": {
@@ -3688,6 +3910,10 @@
             ],
             "type": "boolean"
         },
+        "filterCriteriaDefaults": {
+            "$ref": "#/definitions/FilterCriteriaDefaults",
+            "description": "Set the default filter criteria for all checks. If this property is specified it will override any defaults passed from the bot's config\n\nDefault behavior is to exclude all mods and automoderator from checks"
+        },
         "footer": {
             "anyOf": [
                 {

src/Schema/OperatorConfig.json

@@ -1,6 +1,189 @@
 {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "definitions": {
+        "AuthorCriteria": {
+            "additionalProperties": false,
+            "description": "Criteria with which to test against the author of an Activity. The outcome of the test is based on:\n\n1. All present properties passing and\n2. If a property is a list then any value from the list matching",
+            "examples": [
+                {
+                    "flairText": [
+                        "Contributor",
+                        "Veteran"
+                    ],
+                    "isMod": true,
+                    "name": [
+                        "FoxxMD",
+                        "AnotherUser"
+                    ]
+                }
+            ],
+            "minProperties": 1,
+            "properties": {
+                "age": {
+                    "description": "Test the age of the Author's account (when it was created) against this comparison\n\nThe syntax is `(< OR > OR <= OR >=) <number> <unit>`\n\n* EX `> 100 days` => Passes if Author's account is older than 100 days\n* EX `<= 2 months` => Passes if Author's account is younger than or equal to 2 months\n\nUnit must be one of [DayJS Duration units](https://day.js.org/docs/en/durations/creating)\n\n[See] https://regexr.com/609n8 for example",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(days?|weeks?|months?|years?|hours?|minutes?|seconds?|milliseconds?)\\s*$",
+                    "type": "string"
+                },
+                "commentKarma": {
+                    "description": "A string containing a comparison operator and a value to compare karma against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 100`  => greater than 100 comment karma\n* EX `<= 75%` => comment karma is less than or equal to 75% of **all karma**",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "description": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "An (array of) string/regular expression to test contents of an Author's profile description against\n\nIf no flags are specified then the **insensitive** flag is used by default\n\nIf using an array then if **any** value in the array passes the description test passes",
+                    "examples": [
+                        [
+                            "/test$/i",
+                            "look for this string literal"
+                        ]
+                    ]
+                },
+                "flairCssClass": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "A (user) flair css class (or list of) from the subreddit to match against",
+                    "examples": [
+                        "red"
+                    ]
+                },
+                "flairTemplate": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "A (user) flair template id (or list of) from the subreddit to match against"
+                },
+                "flairText": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "A (user) flair text value (or list of) from the subreddit to match against",
+                    "examples": [
+                        "Approved"
+                    ]
+                },
+                "isMod": {
+                    "description": "Is the author a moderator?",
+                    "type": "boolean"
+                },
+                "linkKarma": {
+                    "description": "A string containing a comparison operator and a value to compare link karma against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign]`\n\n* EX `> 100`  => greater than 100 link karma\n* EX `<= 75%` => link karma is less than or equal to 75% of **all karma**",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "name": {
+                    "description": "A list of reddit usernames (case-insensitive) to match against. Do not include the \"u/\" prefix\n\n EX to match against /u/FoxxMD and /u/AnotherUser use [\"FoxxMD\",\"AnotherUser\"]",
+                    "examples": [
+                        "FoxxMD",
+                        "AnotherUser"
+                    ],
+                    "items": {
+                        "type": "string"
+                    },
+                    "type": "array"
+                },
+                "shadowBanned": {
+                    "description": "Is the author shadowbanned?\n\nThis is determined by trying to retrieve the author's profile. If a 404 is returned it is likely they are shadowbanned",
+                    "type": "boolean"
+                },
+                "totalKarma": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "userNotes": {
+                    "description": "A list of UserNote properties to check against the User Notes attached to this Author in this Subreddit (must have Toolbox enabled and used User Notes at least once)",
+                    "items": {
+                        "$ref": "#/definitions/UserNoteCriteria"
+                    },
+                    "type": "array"
+                },
+                "verified": {
+                    "description": "Does Author's account have a verified email?",
+                    "type": "boolean"
+                }
+            },
+            "type": "object"
+        },
+        "AuthorOptions": {
+            "description": "If present then these Author criteria are checked before running the rule. If criteria fails then the rule is skipped.",
+            "examples": [
+                {
+                    "include": [
+                        {
+                            "flairText": [
+                                "Contributor",
+                                "Veteran"
+                            ]
+                        },
+                        {
+                            "isMod": true
+                        }
+                    ]
+                }
+            ],
+            "properties": {
+                "exclude": {
+                    "description": "Only runs if `include` is not present. Each AuthorCriteria is comprised of conditions that the Author being checked must \"not\" pass. See excludeCondition for set behavior\n\nEX: `isMod: true, name: Automoderator` => Will pass if the Author IS NOT a mod and IS NOT named Automoderator",
+                    "items": {
+                        "$ref": "#/definitions/AuthorCriteria"
+                    },
+                    "type": "array"
+                },
+                "excludeCondition": {
+                    "default": "OR",
+                    "description": "* OR => if ANY exclude condition \"does not\" pass then the exclude test passes\n* AND => if ALL exclude conditions \"do not\" pass then the exclude test passes\n\nDefaults to OR",
+                    "enum": [
+                        "AND",
+                        "OR"
+                    ],
+                    "type": "string"
+                },
+                "include": {
+                    "description": "Will \"pass\" if any set of AuthorCriteria passes",
+                    "items": {
+                        "$ref": "#/definitions/AuthorCriteria"
+                    },
+                    "type": "array"
+                }
+            },
+            "type": "object"
+        },
         "BotConnection": {
             "description": "Configuration required to connect to a CM Server",
             "properties": {
@@ -58,6 +241,10 @@
                         }
                     ]
                 },
+                "filterCriteriaDefaults": {
+                    "$ref": "#/definitions/FilterCriteriaDefaults",
+                    "description": "Define the default behavior for all filter criteria on all checks in all subreddits\n\nDefaults to exclude mods and automoderator from checks"
+                },
                 "name": {
                     "type": "string"
                 },
@@ -94,13 +281,36 @@
                         },
                         {
                             "properties": {
+                                "shared": {
+                                    "anyOf": [
+                                        {
+                                            "items": {
+                                                "enum": [
+                                                    "modqueue",
+                                                    "newComm",
+                                                    "newSub",
+                                                    "unmoderated"
+                                                ],
+                                                "type": "string"
+                                            },
+                                            "type": "array"
+                                        },
+                                        {
+                                            "enum": [
+                                                true
+                                            ],
+                                            "type": "boolean"
+                                        }
+                                    ],
+                                    "description": "Set which polling sources should be shared among subreddits using default polling settings for that source\n\n* For `unmoderated and `modqueue` the bot will poll on **r/mod** for new activities\n* For `newSub` and `newComm` all subreddits sharing the source will be combined to poll like **r/subreddit1+subreddit2/new**\n\nIf set to `true` all polling sources will be shared,  otherwise specify which sourcs should be shared as a list"
+                                },
                                 "sharedMod": {
                                     "default": false,
-                                    "description": "If set to `true` all subreddits polling unmoderated/modqueue with default polling settings will share a request to \"r/mod\"\notherwise each subreddit will poll its own mod view\n\n* ENV => `SHARE_MOD`\n* ARG => `--shareMod`",
+                                    "description": "DEPRECATED: See `shared`\n\n Using the ENV or ARG will sett `unmoderated` and `modqueue` on `shared`\n\n* ENV => `SHARE_MOD`\n* ARG => `--shareMod`",
                                     "type": "boolean"
                                 },
                                 "stagger": {
-                                    "description": "If sharing a mod stream stagger pushing relevant Activities to individual subreddits.\n\nUseful when running many subreddits and rules are potentially cpu/memory/traffic heavy -- allows spreading out load",
+                                    "description": "If sharing a stream staggers pushing relevant Activities to individual subreddits.\n\nUseful when running many subreddits and rules are potentially cpu/memory/traffic heavy -- allows spreading out load",
                                     "type": "number"
                                 }
                             },
@@ -124,21 +334,8 @@
                     "type": "object"
                 },
                 "snoowrap": {
-                    "description": "Settings to control some [Snoowrap](https://github.com/not-an-aardvark/snoowrap) behavior",
-                    "properties": {
-                        "debug": {
-                            "description": "Manually set the debug status for snoowrap\n\nWhen snoowrap has `debug: true` it will log the http status response of reddit api requests to at the `debug` level\n\n* Set to `true` to always output\n* Set to `false` to never output\n\nIf not present or `null` will be set based on `logLevel`\n\n* ENV => `SNOO_DEBUG`\n* ARG => `--snooDebug`",
-                            "type": "boolean"
-                        },
-                        "proxy": {
-                            "description": "Proxy all requests to Reddit's API through this endpoint\n\n* ENV => `PROXY`\n* ARG => `--proxy <proxyEndpoint>`",
-                            "examples": [
-                                "http://localhost:4443"
-                            ],
-                            "type": "string"
-                        }
-                    },
-                    "type": "object"
+                    "$ref": "#/definitions/SnoowrapOptions",
+                    "description": "Settings to control some [Snoowrap](https://github.com/not-an-aardvark/snoowrap) behavior.\n\nOverrides any defaults provided at top-level operator config.\n\nSet to an empty object to \"ignore\" any top-level config"
                 },
                 "subreddits": {
                     "description": "Settings related to bot behavior for subreddits it is managing",
@@ -200,8 +397,7 @@
             "type": "object"
         },
         "CacheOptions": {
-            "additionalProperties": {
-            },
+            "additionalProperties": {},
             "description": "Configure granular settings for a cache provider with this object",
             "properties": {
                 "auth_pass": {
@@ -266,6 +462,73 @@
             ],
             "type": "string"
         },
+        "CommentState": {
+            "description": "Different attributes a `Comment` can be in. Only include a property if you want to check it.",
+            "examples": [
+                {
+                    "op": true,
+                    "removed": false
+                }
+            ],
+            "properties": {
+                "age": {
+                    "description": "A duration and how to compare it against a value\n\nThe syntax is `(< OR > OR <= OR >=) <number> <unit>` EX `> 100 days`, `<= 2 months`\n\n* EX `> 100 days` => Passes if the date being compared is before 100 days ago\n* EX `<= 2 months` => Passes if the date being compared is after or equal to 2 months\n\nUnit must be one of [DayJS Duration units](https://day.js.org/docs/en/durations/creating)\n\n[See] https://regexr.com/609n8 for example",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(days|weeks|months|years|hours|minutes|seconds|milliseconds)\\s*$",
+                    "type": "string"
+                },
+                "approved": {
+                    "type": "boolean"
+                },
+                "deleted": {
+                    "type": "boolean"
+                },
+                "depth": {
+                    "description": "The (nested) level of a comment.\n\n* 0 mean the comment is at top-level (replying to submission)\n* non-zero, Nth value means the comment has N parent comments",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(days|weeks|months|years|hours|minutes|seconds|milliseconds)\\s*$",
+                    "type": "string"
+                },
+                "distinguished": {
+                    "type": "boolean"
+                },
+                "filtered": {
+                    "type": "boolean"
+                },
+                "locked": {
+                    "type": "boolean"
+                },
+                "op": {
+                    "description": "Is this Comment Author also the Author of the Submission this comment is in?",
+                    "type": "boolean"
+                },
+                "removed": {
+                    "type": "boolean"
+                },
+                "reports": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "score": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "spam": {
+                    "type": "boolean"
+                },
+                "stickied": {
+                    "type": "boolean"
+                },
+                "submissionState": {
+                    "description": "A list of SubmissionState attributes to test the Submission this comment is in",
+                    "items": {
+                        "$ref": "#/definitions/SubmissionState"
+                    },
+                    "type": "array"
+                }
+            },
+            "type": "object"
+        },
         "DiscordProviderConfig": {
             "properties": {
                 "name": {
@@ -288,6 +551,172 @@
             ],
             "type": "object"
         },
+        "FilterCriteriaDefaults": {
+            "properties": {
+                "authorIs": {
+                    "$ref": "#/definitions/AuthorOptions",
+                    "description": "Determine how authorIs defaults behave when authorIs is present on the check\n\n* merge => merges defaults with check's authorIs\n* replace => check authorIs will replace defaults (no defaults used)",
+                    "examples": [
+                        {
+                            "include": [
+                                {
+                                    "flairText": [
+                                        "Contributor",
+                                        "Veteran"
+                                    ]
+                                },
+                                {
+                                    "isMod": true
+                                }
+                            ]
+                        }
+                    ]
+                },
+                "authorIsBehavior": {
+                    "enum": [
+                        "merge",
+                        "replace"
+                    ],
+                    "type": "string"
+                },
+                "itemIs": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "$ref": "#/definitions/SubmissionState"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "items": {
+                                "$ref": "#/definitions/CommentState"
+                            },
+                            "type": "array"
+                        }
+                    ]
+                },
+                "itemIsBehavior": {
+                    "description": "Determine how itemIs defaults behave when itemIs is present on the check\n\n* merge => adds defaults to check's itemIs\n* replace => check itemIs will replace defaults (no defaults used)",
+                    "enum": [
+                        "merge",
+                        "replace"
+                    ],
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
+        "LoggingOptions": {
+            "properties": {
+                "console": {
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/Pick<Transports.ConsoleTransportOptions,\"silent\"|\"eol\"|\"stderrLevels\"|\"consoleWarnLevels\">"
+                        },
+                        {
+                            "properties": {
+                                "level": {
+                                    "enum": [
+                                        "debug",
+                                        "error",
+                                        "info",
+                                        "verbose",
+                                        "warn"
+                                    ],
+                                    "type": "string"
+                                }
+                            },
+                            "type": "object"
+                        }
+                    ],
+                    "description": "Options for logging to console"
+                },
+                "file": {
+                    "allOf": [
+                        {
+                            "properties": {
+                                "dirname": {
+                                    "description": "The absolute path to a directory where rotating log files should be stored.\n\n* If not present or `null` or `false` no log files will be created\n* If `true` logs will be stored at `[working directory]/logs`\n\n* ENV => `LOG_DIR`\n* ARG => `--logDir [dir]`",
+                                    "examples": [
+                                        "/var/log/contextmod"
+                                    ],
+                                    "type": [
+                                        "null",
+                                        "string",
+                                        "boolean"
+                                    ]
+                                },
+                                "level": {
+                                    "enum": [
+                                        "debug",
+                                        "error",
+                                        "info",
+                                        "verbose",
+                                        "warn"
+                                    ],
+                                    "type": "string"
+                                }
+                            },
+                            "type": "object"
+                        },
+                        {
+                            "$ref": "#/definitions/Omit<DailyRotateFileTransportOptions,\"stream\"|\"dirname\"|\"options\"|\"handleRejections\"|\"format\"|\"handleExceptions\"|\"log\"|\"logv\"|\"close\">"
+                        }
+                    ],
+                    "description": "Options for Rotating File logging"
+                },
+                "level": {
+                    "default": "verbose",
+                    "description": "The minimum log level to output. The log level set will output logs at its level **and all levels above it:**\n\n * `error`\n * `warn`\n * `info`\n * `verbose`\n * `debug`\n\n Note: `verbose` will display *a lot* of information on the status/result of run rules/checks/actions etc. which is very useful for testing configurations. Once your bot is stable changing the level to `info` will reduce log noise.\n\n * ENV => `LOG_LEVEL`\n * ARG => `--logLevel <level>`",
+                    "enum": [
+                        "debug",
+                        "error",
+                        "info",
+                        "verbose",
+                        "warn"
+                    ],
+                    "examples": [
+                        "verbose"
+                    ],
+                    "type": "string"
+                },
+                "path": {
+                    "description": "**DEPRECATED** - Use `file.dirname` instead\nThe absolute path to a directory where rotating log files should be stored.\n\n* If not present or `null` or `false` no log files will be created\n* If `true` logs will be stored at `[working directory]/logs`\n\n* ENV => `LOG_DIR`\n* ARG => `--logDir [dir]`",
+                    "examples": [
+                        "/var/log/contextmod"
+                    ],
+                    "type": [
+                        "null",
+                        "string",
+                        "boolean"
+                    ]
+                },
+                "stream": {
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/Omit<DuplexTransportOptions,\"name\"|\"stream\"|\"handleRejections\"|\"format\"|\"handleExceptions\"|\"log\"|\"logv\"|\"close\">"
+                        },
+                        {
+                            "properties": {
+                                "level": {
+                                    "enum": [
+                                        "debug",
+                                        "error",
+                                        "info",
+                                        "verbose",
+                                        "warn"
+                                    ],
+                                    "type": "string"
+                                }
+                            },
+                            "type": "object"
+                        }
+                    ],
+                    "description": "Options for logging to api/web"
+                }
+            },
+            "type": "object"
+        },
         "NotificationConfig": {
             "properties": {
                 "events": {
@@ -353,6 +782,90 @@
             ],
             "type": "object"
         },
+        "Omit<DailyRotateFileTransportOptions,\"stream\"|\"dirname\"|\"options\"|\"handleRejections\"|\"format\"|\"handleExceptions\"|\"log\"|\"logv\"|\"close\">": {
+            "properties": {
+                "auditFile": {
+                    "description": "A string representing the name of the name of the audit file. (default: './hash-audit.json')",
+                    "type": "string"
+                },
+                "createSymlink": {
+                    "description": "Create a tailable symlink to the current active log file. (default: false)",
+                    "type": "boolean"
+                },
+                "datePattern": {
+                    "description": "A string representing the moment.js date format to be used for rotating. The meta characters used in this string will dictate the frequency of the file rotation. For example, if your datePattern is simply 'HH' you will end up with 24 log files that are picked up and appended to every day. (default 'YYYY-MM-DD')",
+                    "type": "string"
+                },
+                "eol": {
+                    "type": "string"
+                },
+                "extension": {
+                    "description": "A string representing an extension to be added to the filename, if not included in the filename property. (default: '')",
+                    "type": "string"
+                },
+                "filename": {
+                    "description": "Filename to be used to log to. This filename can include the %DATE% placeholder which will include the formatted datePattern at that point in the filename. (default: 'winston.log.%DATE%)",
+                    "type": "string"
+                },
+                "frequency": {
+                    "description": "A string representing the frequency of rotation. (default: 'custom')",
+                    "type": "string"
+                },
+                "json": {
+                    "type": "boolean"
+                },
+                "level": {
+                    "type": "string"
+                },
+                "maxFiles": {
+                    "description": "Maximum number of logs to keep. If not set, no logs will be removed. This can be a number of files or number of days. If using days, add 'd' as the suffix. (default: null)",
+                    "type": [
+                        "string",
+                        "number"
+                    ]
+                },
+                "maxSize": {
+                    "description": "Maximum size of the file after which it will rotate. This can be a number of bytes, or units of kb, mb, and gb. If using the units, add 'k', 'm', or 'g' as the suffix. The units need to directly follow the number. (default: null)",
+                    "type": [
+                        "string",
+                        "number"
+                    ]
+                },
+                "silent": {
+                    "type": "boolean"
+                },
+                "symlinkName": {
+                    "description": "The name of the tailable symlink. (default: 'current.log')",
+                    "type": "string"
+                },
+                "utc": {
+                    "description": "A boolean whether or not to generate file name from \"datePattern\" in UTC format. (default: false)",
+                    "type": "boolean"
+                },
+                "zippedArchive": {
+                    "description": "A boolean to define whether or not to gzip archived log files. (default 'false')",
+                    "type": "boolean"
+                }
+            },
+            "type": "object"
+        },
+        "Omit<DuplexTransportOptions,\"name\"|\"stream\"|\"handleRejections\"|\"format\"|\"handleExceptions\"|\"log\"|\"logv\"|\"close\">": {
+            "properties": {
+                "dump": {
+                    "type": "boolean"
+                },
+                "eol": {
+                    "type": "string"
+                },
+                "level": {
+                    "type": "string"
+                },
+                "silent": {
+                    "type": "boolean"
+                }
+            },
+            "type": "object"
+        },
         "OperatorCacheConfig": {
             "properties": {
                 "actionedEventsDefault": {
@@ -414,6 +927,17 @@
                     ],
                     "description": "The cache provider and, optionally, a custom configuration for that provider\n\nIf not present or `null` provider will be `memory`.\n\nTo specify another `provider` but use its default configuration set this property to a string of one of the available providers: `memory`, `redis`, or `none`"
                 },
+                "selfTTL": {
+                    "default": 50,
+                    "description": "Amount of time, in seconds, an Activity that the bot has acted on or created will be ignored if found during polling\n\nThis is useful to prevent the bot from checking Activities it *just* worked on or a product of the checks. Examples:\n\n* Ignore comments created through an Action\n* Ignore Activity polled from modqueue that the bot just reported\n\nThis value should be at least as long as the longest polling interval for modqueue/newComm\n\n* If `0` or `true` will cache indefinitely (not recommended)\n* If `false` will not cache",
+                    "examples": [
+                        50
+                    ],
+                    "type": [
+                        "number",
+                        "boolean"
+                    ]
+                },
                 "submissionTTL": {
                     "default": 60,
                     "description": "Amount of time, in seconds, a submission should be cached\n\n* If `0` or `true` will cache indefinitely (not recommended)\n* If `false` will not cache",
@@ -461,6 +985,29 @@
             },
             "type": "object"
         },
+        "Pick<Transports.ConsoleTransportOptions,\"silent\"|\"eol\"|\"stderrLevels\"|\"consoleWarnLevels\">": {
+            "properties": {
+                "consoleWarnLevels": {
+                    "items": {
+                        "type": "string"
+                    },
+                    "type": "array"
+                },
+                "eol": {
+                    "type": "string"
+                },
+                "silent": {
+                    "type": "boolean"
+                },
+                "stderrLevels": {
+                    "items": {
+                        "type": "string"
+                    },
+                    "type": "array"
+                }
+            },
+            "type": "object"
+        },
         "PollingDefaults": {
             "properties": {
                 "delayUntil": {
@@ -529,9 +1076,131 @@
             },
             "type": "object"
         },
-        "ThirdPartyCredentialsJsonConfig": {
-            "additionalProperties": {
+        "SnoowrapOptions": {
+            "properties": {
+                "debug": {
+                    "description": "Manually set the debug status for snoowrap\n\nWhen snoowrap has `debug: true` it will log the http status response of reddit api requests to at the `debug` level\n\n* Set to `true` to always output\n* Set to `false` to never output\n\nIf not present or `null` will be set based on `logLevel`\n\n* ENV => `SNOO_DEBUG`\n* ARG => `--snooDebug`",
+                    "type": "boolean"
+                },
+                "proxy": {
+                    "description": "Proxy all requests to Reddit's API through this endpoint\n\n* ENV => `PROXY`\n* ARG => `--proxy <proxyEndpoint>`",
+                    "examples": [
+                        "http://localhost:4443"
+                    ],
+                    "type": "string"
+                }
             },
+            "type": "object"
+        },
+        "SubmissionState": {
+            "description": "Different attributes a `Submission` can be in. Only include a property if you want to check it.",
+            "examples": [
+                {
+                    "over_18": true,
+                    "removed": false
+                }
+            ],
+            "properties": {
+                "age": {
+                    "description": "A duration and how to compare it against a value\n\nThe syntax is `(< OR > OR <= OR >=) <number> <unit>` EX `> 100 days`, `<= 2 months`\n\n* EX `> 100 days` => Passes if the date being compared is before 100 days ago\n* EX `<= 2 months` => Passes if the date being compared is after or equal to 2 months\n\nUnit must be one of [DayJS Duration units](https://day.js.org/docs/en/durations/creating)\n\n[See] https://regexr.com/609n8 for example",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(days|weeks|months|years|hours|minutes|seconds|milliseconds)\\s*$",
+                    "type": "string"
+                },
+                "approved": {
+                    "type": "boolean"
+                },
+                "deleted": {
+                    "type": "boolean"
+                },
+                "distinguished": {
+                    "type": "boolean"
+                },
+                "filtered": {
+                    "type": "boolean"
+                },
+                "flairTemplate": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
+                },
+                "is_self": {
+                    "type": "boolean"
+                },
+                "link_flair_css_class": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
+                },
+                "link_flair_text": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
+                },
+                "locked": {
+                    "type": "boolean"
+                },
+                "over_18": {
+                    "description": "NSFW",
+                    "type": "boolean"
+                },
+                "pinned": {
+                    "type": "boolean"
+                },
+                "removed": {
+                    "type": "boolean"
+                },
+                "reports": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "score": {
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
+                    "type": "string"
+                },
+                "spam": {
+                    "type": "boolean"
+                },
+                "spoiler": {
+                    "type": "boolean"
+                },
+                "stickied": {
+                    "type": "boolean"
+                },
+                "title": {
+                    "description": "A valid regular expression to match against the title of the submission",
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
+        "ThirdPartyCredentialsJsonConfig": {
+            "additionalProperties": {},
             "properties": {
                 "youtube": {
                     "properties": {
@@ -547,6 +1216,43 @@
             },
             "type": "object"
         },
+        "UserNoteCriteria": {
+            "properties": {
+                "count": {
+                    "default": ">= 1",
+                    "description": "Number of occurrences of this type. Ignored if `search` is `current`\n\nA string containing a comparison operator and/or a value to compare number of occurrences against\n\nThe syntax is `(< OR > OR <= OR >=) <number>[percent sign] [ascending|descending]`",
+                    "examples": [
+                        ">= 1"
+                    ],
+                    "pattern": "^\\s*(?<opStr>>|>=|<|<=)\\s*(?<value>\\d+)\\s*(?<percent>%?)\\s*(?<extra>asc.*|desc.*)*$",
+                    "type": "string"
+                },
+                "search": {
+                    "default": "current",
+                    "description": "How to test the notes for this Author:\n\n### current\n\nOnly the most recent note is checked for `type`\n\n### total\n\nThe `count` comparison of `type` must be found within all notes\n\n* EX `count: > 3`   => Must have more than 3 notes of `type`, total\n* EX `count: <= 25%` => Must have 25% or less of notes of `type`, total\n\n### consecutive\n\nThe `count` **number** of `type` notes must be found in a row.\n\nYou may also specify the time-based order in which to search the notes by specifying `ascending (asc)` or `descending (desc)` in the `count` value. Default is `descending`\n\n* EX `count: >= 3` => Must have 3 or more notes of `type` consecutively, in descending order\n* EX `count: < 2`  => Must have less than 2 notes of `type` consecutively, in descending order\n* EX `count: > 4 asc` => Must have greater than 4 notes of `type` consecutively, in ascending order",
+                    "enum": [
+                        "consecutive",
+                        "current",
+                        "total"
+                    ],
+                    "examples": [
+                        "current"
+                    ],
+                    "type": "string"
+                },
+                "type": {
+                    "description": "User Note type key to search for",
+                    "examples": [
+                        "spamwarn"
+                    ],
+                    "type": "string"
+                }
+            },
+            "required": [
+                "type"
+            ],
+            "type": "object"
+        },
         "WebCredentials": {
             "description": "Separate credentials for the web interface can be provided when also running the api.\n\nAll properties not specified will default to values given in ENV/ARG credential properties\n\nRefer to the [required credentials table](https://github.com/FoxxMD/context-mod/blob/master/docs/operatorConfiguration.md#minimum-required-configuration) to see what is necessary for the web interface.",
             "examples": [
@@ -620,32 +1326,8 @@
             "$ref": "#/definitions/ThirdPartyCredentialsJsonConfig"
         },
         "logging": {
-            "description": "Settings to configure global logging defaults",
-            "properties": {
-                "level": {
-                    "default": "verbose",
-                    "description": "The minimum log level to output. The log level set will output logs at its level **and all levels above it:**\n\n * `error`\n * `warn`\n * `info`\n * `verbose`\n * `debug`\n\n Note: `verbose` will display *a lot* of information on the status/result of run rules/checks/actions etc. which is very useful for testing configurations. Once your bot is stable changing the level to `info` will reduce log noise.\n\n * ENV => `LOG_LEVEL`\n * ARG => `--logLevel <level>`",
-                    "enum": [
-                        "debug",
-                        "error",
-                        "info",
-                        "verbose",
-                        "warn"
-                    ],
-                    "examples": [
-                        "verbose"
-                    ],
-                    "type": "string"
-                },
-                "path": {
-                    "description": "The absolute path to a directory where rotating log files should be stored.\n\n* If not present or `null` no log files will be created\n* If `true` logs will be stored at `[working directory]/logs`\n\n* ENV => `LOG_DIR`\n* ARG => `--logDir [dir]`",
-                    "examples": [
-                        "/var/log/contextmod"
-                    ],
-                    "type": "string"
-                }
-            },
-            "type": "object"
+            "$ref": "#/definitions/LoggingOptions",
+            "description": "Settings to configure global logging defaults"
         },
         "mode": {
             "default": "all",
@@ -694,6 +1376,10 @@
             },
             "type": "object"
         },
+        "snoowrap": {
+            "$ref": "#/definitions/SnoowrapOptions",
+            "description": "Set global snoowrap options as well as default snoowrap config for all bots that don't specify their own"
+        },
         "web": {
             "description": "Settings for the web interface",
             "properties": {

src/Schema/Rule.json

@@ -184,8 +184,7 @@
                     "default": "undefined",
                     "description": "This list determines which categories of domains should be aggregated on. All aggregated domains will be tested against `threshold`\n\n* If `media` is included then aggregate author's submission history which reddit recognizes as media (youtube, vimeo, etc.)\n* If `redditMedia` is included then aggregate on author's submissions history which are media hosted on reddit: galleries, videos, and images (i.redd.it / v.redd.it)\n* If `self` is included then aggregate on author's submission history which are self-post (`self.[subreddit]`) or domain is `reddit.com`\n* If `link` is included then aggregate author's submission history which is external links and not recognized as `media` by reddit\n\nIf nothing is specified or list is empty (default) rule will only aggregate on `link` and `media` (ignores reddit-hosted content and self-posts)",
                     "examples": [
-                        [
-                        ]
+                        []
                     ],
                     "items": {
                         "enum": [
@@ -218,8 +217,7 @@
                 },
                 "domains": {
                     "default": [
-                        [
-                        ]
+                        []
                     ],
                     "description": "A list of domains whose Activities will be tested against `threshold`.\n\nThe values are tested as partial strings so you do not need to include full URLs, just the part that matters.\n\nEX `[\"youtube\"]` will match submissions with the domain `https://youtube.com/c/aChannel`\nEX `[\"youtube.com/c/bChannel\"]` will NOT match submissions with the domain `https://youtube.com/c/aChannel`\n\nIf you wish to aggregate on self-posts for a subreddit use the syntax `self.[subreddit]` EX `self.AskReddit`\n\n**If this Rule is part of a Check for a Submission and you wish to aggregate on the domain of the Submission use the special string `AGG:SELF`**\n\nIf nothing is specified or list is empty (default) aggregate using `aggregateOn`",
                     "items": {
@@ -454,24 +452,52 @@
                     ]
                 },
                 "flairCssClass": {
-                    "description": "A list of (user) flair css class values from the subreddit to match against",
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "A (user) flair css class (or list of) from the subreddit to match against",
                     "examples": [
                         "red"
+                    ]
+                },
+                "flairTemplate": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
                     ],
-                    "items": {
-                        "type": "string"
-                    },
-                    "type": "array"
+                    "description": "A (user) flair template id (or list of) from the subreddit to match against"
                 },
                 "flairText": {
-                    "description": "A list of (user) flair text values from the subreddit to match against",
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "A (user) flair text value (or list of) from the subreddit to match against",
                     "examples": [
                         "Approved"
-                    ],
-                    "items": {
-                        "type": "string"
-                    },
-                    "type": "array"
+                    ]
                 },
                 "isMod": {
                     "description": "Is the author a moderator?",
@@ -535,12 +561,21 @@
             ],
             "properties": {
                 "exclude": {
-                    "description": "Only runs if `include` is not present. Will \"pass\" if any of set of the AuthorCriteria **does not** pass",
+                    "description": "Only runs if `include` is not present. Each AuthorCriteria is comprised of conditions that the Author being checked must \"not\" pass. See excludeCondition for set behavior\n\nEX: `isMod: true, name: Automoderator` => Will pass if the Author IS NOT a mod and IS NOT named Automoderator",
                     "items": {
                         "$ref": "#/definitions/AuthorCriteria"
                     },
                     "type": "array"
                 },
+                "excludeCondition": {
+                    "default": "OR",
+                    "description": "* OR => if ANY exclude condition \"does not\" pass then the exclude test passes\n* AND => if ALL exclude conditions \"do not\" pass then the exclude test passes\n\nDefaults to OR",
+                    "enum": [
+                        "AND",
+                        "OR"
+                    ],
+                    "type": "string"
+                },
                 "include": {
                     "description": "Will \"pass\" if any set of AuthorCriteria passes",
                     "items": {
@@ -666,7 +701,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
@@ -1148,7 +1183,7 @@
                 },
                 "useSubmissionAsReference": {
                     "default": true,
-                    "description": "If activity is a Submission and is a link (not self-post) then only look at Submissions that contain this link, otherwise consider all activities.",
+                    "description": "When Activity is a submission should we only include activities that are other submissions with the same content?\n\n* When the Activity is a submission this defaults to **true**\n* When the Activity is a comment it is ignored (not relevant)",
                     "type": "boolean"
                 },
                 "window": {
@@ -1913,14 +1948,47 @@
                 "filtered": {
                     "type": "boolean"
                 },
+                "flairTemplate": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
+                },
                 "is_self": {
                     "type": "boolean"
                 },
                 "link_flair_css_class": {
-                    "type": "string"
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
                 },
                 "link_flair_text": {
-                    "type": "string"
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
                 },
                 "locked": {
                     "type": "boolean"
@@ -1936,7 +2004,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },

src/Schema/RuleSet.json

@@ -158,8 +158,7 @@
                     "default": "undefined",
                     "description": "This list determines which categories of domains should be aggregated on. All aggregated domains will be tested against `threshold`\n\n* If `media` is included then aggregate author's submission history which reddit recognizes as media (youtube, vimeo, etc.)\n* If `redditMedia` is included then aggregate on author's submissions history which are media hosted on reddit: galleries, videos, and images (i.redd.it / v.redd.it)\n* If `self` is included then aggregate on author's submission history which are self-post (`self.[subreddit]`) or domain is `reddit.com`\n* If `link` is included then aggregate author's submission history which is external links and not recognized as `media` by reddit\n\nIf nothing is specified or list is empty (default) rule will only aggregate on `link` and `media` (ignores reddit-hosted content and self-posts)",
                     "examples": [
-                        [
-                        ]
+                        []
                     ],
                     "items": {
                         "enum": [
@@ -192,8 +191,7 @@
                 },
                 "domains": {
                     "default": [
-                        [
-                        ]
+                        []
                     ],
                     "description": "A list of domains whose Activities will be tested against `threshold`.\n\nThe values are tested as partial strings so you do not need to include full URLs, just the part that matters.\n\nEX `[\"youtube\"]` will match submissions with the domain `https://youtube.com/c/aChannel`\nEX `[\"youtube.com/c/bChannel\"]` will NOT match submissions with the domain `https://youtube.com/c/aChannel`\n\nIf you wish to aggregate on self-posts for a subreddit use the syntax `self.[subreddit]` EX `self.AskReddit`\n\n**If this Rule is part of a Check for a Submission and you wish to aggregate on the domain of the Submission use the special string `AGG:SELF`**\n\nIf nothing is specified or list is empty (default) aggregate using `aggregateOn`",
                     "items": {
@@ -428,24 +426,52 @@
                     ]
                 },
                 "flairCssClass": {
-                    "description": "A list of (user) flair css class values from the subreddit to match against",
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "A (user) flair css class (or list of) from the subreddit to match against",
                     "examples": [
                         "red"
+                    ]
+                },
+                "flairTemplate": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
                     ],
-                    "items": {
-                        "type": "string"
-                    },
-                    "type": "array"
+                    "description": "A (user) flair template id (or list of) from the subreddit to match against"
                 },
                 "flairText": {
-                    "description": "A list of (user) flair text values from the subreddit to match against",
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ],
+                    "description": "A (user) flair text value (or list of) from the subreddit to match against",
                     "examples": [
                         "Approved"
-                    ],
-                    "items": {
-                        "type": "string"
-                    },
-                    "type": "array"
+                    ]
                 },
                 "isMod": {
                     "description": "Is the author a moderator?",
@@ -509,12 +535,21 @@
             ],
             "properties": {
                 "exclude": {
-                    "description": "Only runs if `include` is not present. Will \"pass\" if any of set of the AuthorCriteria **does not** pass",
+                    "description": "Only runs if `include` is not present. Each AuthorCriteria is comprised of conditions that the Author being checked must \"not\" pass. See excludeCondition for set behavior\n\nEX: `isMod: true, name: Automoderator` => Will pass if the Author IS NOT a mod and IS NOT named Automoderator",
                     "items": {
                         "$ref": "#/definitions/AuthorCriteria"
                     },
                     "type": "array"
                 },
+                "excludeCondition": {
+                    "default": "OR",
+                    "description": "* OR => if ANY exclude condition \"does not\" pass then the exclude test passes\n* AND => if ALL exclude conditions \"do not\" pass then the exclude test passes\n\nDefaults to OR",
+                    "enum": [
+                        "AND",
+                        "OR"
+                    ],
+                    "type": "string"
+                },
                 "include": {
                     "description": "Will \"pass\" if any set of AuthorCriteria passes",
                     "items": {
@@ -640,7 +675,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },
@@ -1122,7 +1157,7 @@
                 },
                 "useSubmissionAsReference": {
                     "default": true,
-                    "description": "If activity is a Submission and is a link (not self-post) then only look at Submissions that contain this link, otherwise consider all activities.",
+                    "description": "When Activity is a submission should we only include activities that are other submissions with the same content?\n\n* When the Activity is a submission this defaults to **true**\n* When the Activity is a comment it is ignored (not relevant)",
                     "type": "boolean"
                 },
                 "window": {
@@ -1887,14 +1922,47 @@
                 "filtered": {
                     "type": "boolean"
                 },
+                "flairTemplate": {
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
+                },
                 "is_self": {
                     "type": "boolean"
                 },
                 "link_flair_css_class": {
-                    "type": "string"
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
                 },
                 "link_flair_text": {
-                    "type": "string"
+                    "anyOf": [
+                        {
+                            "items": {
+                                "type": "string"
+                            },
+                            "type": "array"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
                 },
                 "locked": {
                     "type": "boolean"
@@ -1910,7 +1978,7 @@
                     "type": "boolean"
                 },
                 "reports": {
-                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 100`  => greater than 100",
+                    "description": "A string containing a comparison operator and a value to compare against\n\nThe syntax is `(< OR > OR <= OR >=) <number>`\n\n* EX `> 2`  => greater than 2 total reports\n\nDefaults to TOTAL reports on an Activity. Suffix the value with the report type to check that type:\n\n* EX `> 3 mod` => greater than 3 mod reports\n* EX `>= 1 user` => greater than 1 user report",
                     "pattern": "^\\s*(>|>=|<|<=)\\s*(\\d+)\\s*(%?)(.*)$",
                     "type": "string"
                 },

src/Subreddit/Manager.ts

@@ -1,24 +1,32 @@
-import Snoowrap, {Comment, Subreddit} from "snoowrap";
+import Snoowrap, {Comment, Subreddit, WikiPage} from "snoowrap";
 import {Logger} from "winston";
 import {SubmissionCheck} from "../Check/SubmissionCheck";
 import {CommentCheck} from "../Check/CommentCheck";
 import {
-    cacheStats, createHistoricalStatsDisplay,
+    cacheStats,
+    createHistoricalStatsDisplay,
     createRetryHandler,
-    determineNewResults, findLastIndex, formatNumber,
-    mergeArr, parseFromJsonOrYamlToObject, pollingInfo, resultsSummary, sleep, totalFromMapStats, triggeredIndicator,
+    determineNewResults,
+    findLastIndex,
+    formatNumber, likelyJson5,
+    mergeArr,
+    parseFromJsonOrYamlToObject,
+    parseRedditEntity,
+    pollingInfo,
+    resultsSummary,
+    sleep,
+    totalFromMapStats,
+    triggeredIndicator,
 } from "../util";
-import {Poll} from "snoostorm";
-import pEvent from "p-event";
 import {RuleResult} from "../Rule";
 import {ConfigBuilder, buildPollingOptions} from "../ConfigBuilder";
 import {
     ActionedEvent,
     ActionResult,
     DEFAULT_POLLING_INTERVAL,
-    DEFAULT_POLLING_LIMIT, Invokee,
+    DEFAULT_POLLING_LIMIT, FilterCriteriaDefaults, Invokee,
     ManagerOptions, ManagerStateChangeOption, ManagerStats, PAUSED,
-    PollingOptionsStrong, ResourceStats, RUNNING, RunState, STOPPED, SYSTEM, USER
+    PollingOptionsStrong, PollOn, RUNNING, RunState, STOPPED, SYSTEM, USER
 } from "../Common/interfaces";
 import Submission from "snoowrap/dist/objects/Submission";
 import {activityIsRemoved, itemContentPeek} from "../Utils/SnoowrapUtils";
@@ -38,10 +46,10 @@ import {queue, QueueObject} from 'async';
 import {JSONConfig} from "../JsonConfig";
 import {CheckStructuredJson} from "../Check";
 import NotificationManager from "../Notification/NotificationManager";
-import action from "../Web/Server/routes/authenticated/user/action";
 import {createHistoricalDefaults, historicalDefaults} from "../Common/defaults";
 import {ExtendedSnoowrap} from "../Utils/SnoowrapClients";
-import {isRateLimitError} from "../Utils/Errors";
+import {CMError, isRateLimitError, isStatusError} from "../Utils/Errors";
+import {ErrorWithCause} from "pony-cause";
 
 export interface RunningState {
     state: RunState,
@@ -53,6 +61,7 @@ export interface runCheckOptions {
     delayUntil?: number,
     dryRun?: boolean,
     refresh?: boolean,
+    force?: boolean,
 }
 
 export interface CheckTask {
@@ -62,7 +71,7 @@ export interface CheckTask {
 }
 
 export interface RuntimeManagerOptions extends ManagerOptions {
-    sharedModqueue?: boolean;
+    sharedStreams?: PollOn[];
     wikiLocation?: string;
     botName: string;
     maxWorkers: number;
@@ -86,13 +95,15 @@ export class Manager extends EventEmitter {
     wikiLocation: string;
     lastWikiRevision?: DayjsObj
     lastWikiCheck: DayjsObj = dayjs();
+    wikiFormat: ('yaml' | 'json') = 'yaml';
+    filterCriteriaDefaults?: FilterCriteriaDefaults
     //wikiUpdateRunning: boolean = false;
 
-    streamListedOnce: string[] = [];
-    streams: SPoll<Snoowrap.Submission | Snoowrap.Comment>[] = [];
-    modStreamCallbacks: Map<string, any> = new Map();
+    streams: Map<string, SPoll<Snoowrap.Submission | Snoowrap.Comment>> = new Map();
+    sharedStreamCallbacks: Map<string, any> = new Map();
+    pollingRetryHandler: Function;
     dryRun?: boolean;
-    sharedModqueue: boolean;
+    sharedStreams: PollOn[];
     cacheManager: BotResourcesManager;
     globalDryRun?: boolean;
     queue: QueueObject<CheckTask>;
@@ -130,6 +141,8 @@ export class Manager extends EventEmitter {
 
     notificationManager: NotificationManager;
 
+    modPermissions?: string[]
+
     // use by api nanny to slow event consumption
     delayBy?: number;
 
@@ -184,7 +197,7 @@ export class Manager extends EventEmitter {
     constructor(sub: Subreddit, client: ExtendedSnoowrap, logger: Logger, cacheManager: BotResourcesManager, opts: RuntimeManagerOptions = {botName: 'ContextMod', maxWorkers: 1}) {
         super();
 
-        const {dryRun, sharedModqueue = false, wikiLocation = 'botconfig/contextbot', botName, maxWorkers} = opts;
+        const {dryRun, sharedStreams = [], wikiLocation = 'botconfig/contextbot', botName, maxWorkers, filterCriteriaDefaults} = opts;
         this.displayLabel = opts.nickname || `${sub.display_name_prefixed}`;
         const getLabels = this.getCurrentLabels;
         const getDisplay = this.getDisplay;
@@ -200,7 +213,9 @@ export class Manager extends EventEmitter {
         }, mergeArr);
         this.globalDryRun = dryRun;
         this.wikiLocation = wikiLocation;
-        this.sharedModqueue = sharedModqueue;
+        this.filterCriteriaDefaults = filterCriteriaDefaults;
+        this.sharedStreams = sharedStreams;
+        this.pollingRetryHandler = createRetryHandler({maxRequestRetry: 3, maxOtherRetry: 2}, this.logger);
         this.subreddit = sub;
         this.client = client;
         this.botName = botName;
@@ -255,6 +270,18 @@ export class Manager extends EventEmitter {
         })(this), 10000);
     }
 
+    protected async getModPermissions(): Promise<string[]> {
+        if(this.modPermissions !== undefined) {
+            return this.modPermissions as string[];
+        }
+        this.logger.debug('Retrieving mod permissions for bot');
+        const userInfo = parseRedditEntity(this.botName, 'user');
+        const mods = this.subreddit.getModerators({name: userInfo.name});
+        // @ts-ignore
+        this.modPermissions = mods[0].mod_permissions;
+        return this.modPermissions as string[];
+    }
+
     protected getMaxWorkers(subMaxWorkers?: number) {
         let maxWorkers = this.globalMaxWorkers;
 
@@ -333,7 +360,7 @@ export class Manager extends EventEmitter {
         return q;
     }
 
-    protected async parseConfigurationFromObject(configObj: object) {
+    protected async parseConfigurationFromObject(configObj: object, suppressChangeEvent: boolean = false) {
         try {
             const configBuilder = new ConfigBuilder({logger: this.logger});
             const validJson = configBuilder.validateJson(configObj);
@@ -393,7 +420,7 @@ export class Manager extends EventEmitter {
 
             const commentChecks: Array<CommentCheck> = [];
             const subChecks: Array<SubmissionCheck> = [];
-            const structuredChecks = configBuilder.parseToStructured(validJson);
+            const structuredChecks = configBuilder.parseToStructured(validJson, this.filterCriteriaDefaults);
 
             // TODO check that bot has permissions for subreddit for all specified actions
             // can find permissions in this.subreddit.mod_permissions
@@ -423,6 +450,19 @@ export class Manager extends EventEmitter {
                 this.logger.info(checkSummary);
             }
             this.validConfigLoaded = true;
+            if(this.eventsState.state === RUNNING) {
+                // need to update polling, potentially
+                await this.buildPolling();
+                for(const stream of this.streams.values()) {
+                    if(!stream.running) {
+                        this.logger.debug(`Starting Polling for ${stream.name.toUpperCase()} ${stream.frequency / 1000}s interval`);
+                        stream.startInterval();
+                    }
+                }
+            }
+            if(!suppressChangeEvent) {
+                this.emit('configChange');
+            }
         } catch (err: any) {
             this.validConfigLoaded = false;
             throw err;
@@ -430,15 +470,51 @@ export class Manager extends EventEmitter {
     }
 
     async parseConfiguration(causedBy: Invokee = 'system', force: boolean = false, options?: ManagerStateChangeOption) {
-        const {reason, suppressNotification = false} = options || {};
+        const {reason, suppressNotification = false, suppressChangeEvent = false} = options || {};
         //this.wikiUpdateRunning = true;
         this.lastWikiCheck = dayjs();
 
         try {
             let sourceData: string;
+            let wiki: WikiPage;
             try {
-                // @ts-ignore
-                const wiki = await this.subreddit.getWikiPage(this.wikiLocation).fetch();
+                try {
+                    // @ts-ignore
+                    wiki = await this.subreddit.getWikiPage(this.wikiLocation).fetch();
+                } catch (err: any) {
+                    if(isStatusError(err) && err.statusCode === 404) {
+                        // see if we can create the page
+                        if (!this.client.scope.includes('wikiedit')) {
+                            throw new ErrorWithCause(`Page does not exist and could not be created because Bot does not have oauth permission 'wikiedit'`, {cause: err});
+                        }
+                        const modPermissions = await this.getModPermissions();
+                        if (!modPermissions.includes('all') && !modPermissions.includes('wiki')) {
+                            throw new ErrorWithCause(`Page does not exist and could not be created because Bot not have mod permissions for creating wiki pages. Must have 'all' or 'wiki'`, {cause: err});
+                        }
+                        if(!this.client.scope.includes('modwiki')) {
+                            throw new ErrorWithCause(`Bot COULD create wiki config page but WILL NOT because it does not have the oauth permissions 'modwiki' which is required to set page visibility and editing permissions. Safety first!`, {cause: err});
+                        }
+                        // @ts-ignore
+                        wiki = await this.subreddit.getWikiPage(this.wikiLocation).edit({
+                            text: '',
+                            reason: 'Empty configuration created for ContextMod'
+                        });
+                        this.logger.info(`Wiki page at ${this.wikiLocation} did not exist so bot created it!`);
+
+                        // 0 = use subreddit wiki permissions
+                        // 1 = only approved wiki contributors
+                        // 2 = only mods may edit and view
+                        // @ts-ignore
+                        await this.subreddit.getWikiPage(this.wikiLocation).editSettings({
+                            permissionLevel: 2,
+                            // don't list this page on r/[subreddit]/wiki/pages
+                            listed: false,
+                        });
+                        this.logger.info('Bot set wiki page visibility to MODS ONLY');
+                    } else {
+                        throw err;
+                    }
+                }
                 const revisionDate = dayjs.unix(wiki.revision_date);
                 if (!force && this.validConfigLoaded && (this.lastWikiRevision !== undefined && this.lastWikiRevision.isSame(revisionDate))) {
                     // nothing to do, we already have this revision
@@ -465,26 +541,37 @@ export class Manager extends EventEmitter {
                 this.lastWikiRevision = revisionDate;
                 sourceData = await wiki.content_md;
             } catch (err: any) {
-                const msg = `Could not read wiki configuration. Please ensure the page https://reddit.com${this.subreddit.url}wiki/${this.wikiLocation} exists and is readable -- error: ${err.message}`;
-                this.logger.error(msg);
-                throw new ConfigParseError(msg);
+                let hint = '';
+                if(isStatusError(err) && err.statusCode === 403) {
+                    hint = ` -- HINT: Either the page is restricted to mods only and the bot's reddit account does have the mod permission 'all' or 'wiki' OR the bot does not have the 'wikiread' oauth permission`;
+                }
+                const msg = `Could not read wiki configuration. Please ensure the page https://reddit.com${this.subreddit.url}wiki/${this.wikiLocation} exists and is readable${hint}`;
+                throw new ErrorWithCause(msg, {cause: err});
             }
 
-            if (sourceData === '') {
+            if (sourceData.replace('\r\n', '').trim() === '') {
                 this.logger.error(`Wiki page contents was empty`);
                 throw new ConfigParseError('Wiki page contents was empty');
             }
 
-            const [configObj, jsonErr, yamlErr] = parseFromJsonOrYamlToObject(sourceData);
+            const [format, configObj, jsonErr, yamlErr] = parseFromJsonOrYamlToObject(sourceData);
+            this.wikiFormat = format;
 
             if (configObj === undefined) {
-                this.logger.error(`Could not parse wiki page contents as JSON or YAML:`);
-                this.logger.error(jsonErr);
-                this.logger.error(yamlErr);
+                this.logger.error(`Could not parse wiki page contents as JSON or YAML. Looks like it should be ${this.wikiFormat}?`);
+                if (this.wikiFormat === 'json') {
+                    this.logger.error(jsonErr);
+                    this.logger.error('Check DEBUG output for yaml error');
+                    this.logger.debug(yamlErr);
+                } else {
+                    this.logger.error(yamlErr);
+                    this.logger.error('Check DEBUG output for json error');
+                    this.logger.debug(jsonErr);
+                }
                 throw new ConfigParseError('Could not parse wiki page contents as JSON or YAML')
             }
 
-            await this.parseConfigurationFromObject(configObj);
+            await this.parseConfigurationFromObject(configObj.toJS(), suppressChangeEvent);
             this.logger.info('Checks updated');
 
             if(!suppressNotification) {
@@ -493,8 +580,12 @@ export class Manager extends EventEmitter {
 
             return true;
         } catch (err: any) {
+            const error = new ErrorWithCause('Failed to parse subreddit configuration', {cause: err});
+            // @ts-ignore
+           //error.logged = true;
+            this.logger.error(error);
             this.validConfigLoaded = false;
-            throw err;
+            throw error;
         }
     }
 
@@ -502,6 +593,18 @@ export class Manager extends EventEmitter {
         const checks = checkType === 'Comment' ? this.commentChecks : this.submissionChecks;
         let item = activity;
         const itemId = await item.id;
+
+        if(await this.resources.hasRecentSelf(item)) {
+            const {force = false} = options || {};
+            let recentMsg = `Found in Activities recently (last ${this.resources.selfTTL} seconds) modified/created by this bot`;
+            if(force) {
+                this.logger.debug(`${recentMsg} but will run anyway because "force" option was true.`);
+            } else {
+                this.logger.debug(`${recentMsg} so will skip running.`);
+                return;
+            }
+        }
+
         let allRuleResults: RuleResult[] = [];
         const itemIdentifier = `${checkType === 'Submission' ? 'SUB' : 'COM'} ${itemId}`;
         this.currentLabels = [itemIdentifier];
@@ -611,6 +714,7 @@ export class Manager extends EventEmitter {
                     if (e.logged !== true) {
                         this.logger.warn(`Running rules for Check ${check.name} failed due to uncaught exception`, e);
                     }
+                    this.emit('error', e);
                 }
 
                 if (triggered) {
@@ -623,6 +727,11 @@ export class Manager extends EventEmitter {
                         actionedEvent.ruleSummary = resultsSummary(currentResults, check.condition);
                     }
                     runActions = await check.runActions(item, currentResults.filter(x => x.triggered), dryRun);
+                    // we only can about report and comment actions since those can produce items for newComm and modqueue
+                    const recentCandidates = runActions.filter(x => ['report','comment'].includes(x.kind.toLocaleLowerCase())).map(x => x.touchedEntities === undefined ? [] : x.touchedEntities).flat();
+                    for(const recent of recentCandidates) {
+                        await this.resources.setRecentSelf(recent as (Submission|Comment));
+                    }
                     actionsRun = runActions.length;
 
                     if(check.notifyOnTrigger) {
@@ -670,132 +779,192 @@ export class Manager extends EventEmitter {
         }
     }
 
-    async buildPolling() {
-        // give current handle() time to stop
-        //await sleep(1000);
+    isPollingShared(streamName: string): boolean {
+        const pollOption = this.pollOptions.find(x => x.pollOn === streamName);
+        return pollOption !== undefined && pollOption.limit === DEFAULT_POLLING_LIMIT && pollOption.interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(streamName as PollOn);
+    }
 
-        const retryHandler = createRetryHandler({maxRequestRetry: 3, maxOtherRetry: 1}, this.logger);
+    async buildPolling() {
+
+        const sources: PollOn[] = ['unmoderated', 'modqueue', 'newComm', 'newSub'];
 
         const subName = this.subreddit.display_name;
 
-        for (const pollOpt of this.pollOptions) {
-            const {
-                pollOn,
-                limit,
-                interval,
-                delayUntil,
-                clearProcessed,
-            } = pollOpt;
-            let stream: SPoll<Snoowrap.Submission | Snoowrap.Comment>;
-            let modStreamType: string | undefined;
+        for (const source of sources) {
 
-            switch (pollOn) {
-                case 'unmoderated':
-                    if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedModqueue) {
-                        modStreamType = 'unmoderated';
-                        // use default mod stream from resources
-                        stream = this.cacheManager.modStreams.get('unmoderated') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
-                    } else {
-                        stream = new UnmoderatedStream(this.client, {
-                            subreddit: this.subreddit.display_name,
-                            limit: limit,
-                            pollTime: interval * 1000,
-                            clearProcessed,
-                        });
-                    }
-                    break;
-                case 'modqueue':
-                    if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL) {
-                        modStreamType = 'modqueue';
-                        // use default mod stream from resources
-                        stream = this.cacheManager.modStreams.get('modqueue') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
-                    } else {
-                        stream = new ModQueueStream(this.client, {
-                            subreddit: this.subreddit.display_name,
-                            limit: limit,
-                            pollTime: interval * 1000,
-                            clearProcessed
-                        });
-                    }
-                    break;
-                case 'newSub':
-                    stream = new SubmissionStream(this.client, {
-                        subreddit: this.subreddit.display_name,
-                        limit: limit,
-                        pollTime: interval * 1000,
-                        clearProcessed
-                    });
-                    break;
-                case 'newComm':
-                    stream = new CommentStream(this.client, {
-                        subreddit: this.subreddit.display_name,
-                        limit: limit,
-                        pollTime: interval * 1000,
-                        clearProcessed
-                    });
-                    break;
+            if (!sources.includes(source)) {
+                this.logger.error(`'${source}' is not a valid polling source. Valid sources: unmoderated | modqueue | newComm | newSub`);
+                continue;
             }
 
-            stream.once('listing', async (listing) => {
-                if (!this.streamListedOnce.includes(pollOn)) {
-                    // warning if poll event could potentially miss activities
-                    if (this.commentChecks.length === 0 && ['unmoderated', 'modqueue', 'newComm'].some(x => x === pollOn)) {
-                        this.logger.warn(`Polling '${pollOn}' may return Comments but no comments checks were configured.`);
-                    }
-                    if (this.submissionChecks.length === 0 && ['unmoderated', 'modqueue', 'newSub'].some(x => x === pollOn)) {
-                        this.logger.warn(`Polling '${pollOn}' may return Submissions but no submission checks were configured.`);
-                    }
-                    this.streamListedOnce.push(pollOn);
+            const pollOpt = this.pollOptions.find(x => x.pollOn.toLowerCase() === source.toLowerCase());
+            if (pollOpt === undefined) {
+                if(this.sharedStreamCallbacks.has(source)) {
+                    this.logger.debug(`Removing listener for shared polling on ${source.toUpperCase()} because it no longer exists in config`);
+                    this.sharedStreamCallbacks.delete(source);
                 }
-            });
-
-            const onItem = async (item: Comment | Submission) => {
-                if (!this.streamListedOnce.includes(pollOn)) {
-                    return;
+                const existingStream = this.streams.get(source);
+                if (existingStream !== undefined) {
+                    this.logger.debug(`Stopping polling on ${source.toUpperCase()} because it no longer exists in config`);
+                    existingStream.end();
+                    this.streams.delete(source);
                 }
-                if (item.subreddit.display_name !== subName || this.eventsState.state !== RUNNING) {
-                    return;
-                }
-                let checkType: 'Submission' | 'Comment' | undefined;
-                if (item instanceof Submission) {
-                    if (this.submissionChecks.length > 0) {
-                        checkType = 'Submission';
-                    }
-                } else if (this.commentChecks.length > 0) {
-                    checkType = 'Comment';
-                }
-                if (checkType !== undefined) {
-                    this.firehose.push({checkType, activity: item, options: {delayUntil}})
-                }
-            };
-
-            if (modStreamType !== undefined) {
-                this.modStreamCallbacks.set(pollOn, onItem);
             } else {
-                stream.on('item', onItem);
-                // @ts-ignore
-                stream.on('error', async (err: any) => {
 
-                    this.emit('error', err);
+                const {
+                    limit,
+                    interval,
+                    delayUntil,
+                } = pollOpt;
+                let stream: SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                let modStreamType: string | undefined;
 
-                    if(isRateLimitError(err)) {
-                        this.logger.error('Encountered rate limit while polling! Bot is all out of requests :( Stopping subreddit queue and polling.');
-                        await this.stop();
+                switch (source) {
+                    case 'unmoderated':
+                        if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
+                            modStreamType = 'unmoderated';
+                            // use default mod stream from resources
+                            stream = this.cacheManager.modStreams.get('unmoderated') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                        } else {
+                            stream = new UnmoderatedStream(this.client, {
+                                subreddit: this.subreddit.display_name,
+                                limit: limit,
+                                pollTime: interval * 1000,
+                                logger: this.logger,
+                            });
+                        }
+                        break;
+                    case 'modqueue':
+                        if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
+                            modStreamType = 'modqueue';
+                            // use default mod stream from resources
+                            stream = this.cacheManager.modStreams.get('modqueue') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                        } else {
+                            stream = new ModQueueStream(this.client, {
+                                subreddit: this.subreddit.display_name,
+                                limit: limit,
+                                pollTime: interval * 1000,
+                                logger: this.logger,
+                            });
+                        }
+                        break;
+                    case 'newSub':
+                        if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
+                            modStreamType = 'newSub';
+                            // use default mod stream from resources
+                            stream = this.cacheManager.modStreams.get('newSub') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                        } else {
+                            stream = new SubmissionStream(this.client, {
+                                subreddit: this.subreddit.display_name,
+                                limit: limit,
+                                pollTime: interval * 1000,
+                                logger: this.logger,
+                            });
+                        }
+                        break;
+                    case 'newComm':
+                        if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
+                            modStreamType = 'newComm';
+                            // use default mod stream from resources
+                            stream = this.cacheManager.modStreams.get('newComm') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
+                        } else {
+                            stream = new CommentStream(this.client, {
+                                subreddit: this.subreddit.display_name,
+                                limit: limit,
+                                pollTime: interval * 1000,
+                                logger: this.logger,
+                            });
+                        }
+                        break;
+                }
+
+                if (stream === undefined) {
+                    this.logger.error(`Should have found polling source for '${source}' but it did not exist for some reason!`);
+                    continue;
+                }
+
+                const onItem = async (item: Comment | Submission) => {
+                    if (item.subreddit.display_name !== subName || this.eventsState.state !== RUNNING) {
+                        return;
                     }
-                    this.logger.error('Polling error occurred', err);
-                    const shouldRetry = await retryHandler(err);
-                    if (shouldRetry) {
-                        stream.startInterval();
+                    let checkType: 'Submission' | 'Comment' | undefined;
+                    if (item instanceof Submission) {
+                        if (this.submissionChecks.length > 0) {
+                            checkType = 'Submission';
+                        }
+                    } else if (this.commentChecks.length > 0) {
+                        checkType = 'Comment';
+                    }
+                    if (checkType !== undefined) {
+                        this.firehose.push({checkType, activity: item, options: {delayUntil}})
+                    }
+                };
+
+                if (modStreamType !== undefined) {
+                    let removedOwn = false;
+                    const existingStream = this.streams.get(source);
+                    if(existingStream !== undefined) {
+                        existingStream.end();
+                        this.streams.delete(source);
+                        removedOwn = true;
+                    }
+                    if(!this.sharedStreamCallbacks.has(source)) {
+                        stream.once('listing', this.noChecksWarning(source));
+                        this.logger.debug(`${removedOwn ? 'Stopped own polling and replace with ' : 'Set '}listener on shared polling ${source}`);
+                    }
+                    this.sharedStreamCallbacks.set(source, onItem);
+                } else {
+                    let ownPollingMsgParts: string[] = [];
+                    let removedShared = false;
+                    if(this.sharedStreamCallbacks.has(source)) {
+                        removedShared = true;
+                        this.sharedStreamCallbacks.delete(source);
+                        ownPollingMsgParts.push('removed shared polling listener');
+                    }
+
+                    const existingStream = this.streams.get(source);
+                    let processed;
+                    if (existingStream !== undefined) {
+                        ownPollingMsgParts.push('replaced existing');
+                        processed = existingStream.processed;
+                        existingStream.end();
                     } else {
-                        this.logger.warn('Stopping subreddit processing/polling due to too many errors');
-                        await this.stop();
+                        ownPollingMsgParts.push('create new');
+                        stream.once('listing', this.noChecksWarning(source));
                     }
-                });
-                this.streams.push(stream);
+
+                    this.logger.debug(`Polling ${source.toUpperCase()} => ${ownPollingMsgParts.join('and')} dedicated stream`);
+
+                    stream.on('item', onItem);
+                    // @ts-ignore
+                    stream.on('error', async (err: any) => {
+
+                        this.emit('error', err);
+
+                        const shouldRetry = await this.pollingRetryHandler(err);
+                        if (shouldRetry) {
+                            stream.startInterval(false, 'Within retry limits');
+                        } else {
+                            this.logger.warn('Stopping subreddit processing/polling due to too many errors');
+                            await this.stop();
+                        }
+                    });
+
+                    this.streams.set(source, stream);
+                }
             }
         }
     }
 
+    noChecksWarning = (source: PollOn) => (listing: any) => {
+        if (this.commentChecks.length === 0 && ['modqueue', 'newComm'].some(x => x === source)) {
+            this.logger.warn(`Polling '${source.toUpperCase()}' may return Comments but no comments checks were configured.`);
+        }
+        if (this.submissionChecks.length === 0 && ['unmoderated', 'modqueue', 'newSub'].some(x => x === source)) {
+            this.logger.warn(`Polling '${source.toUpperCase()}' may return Submissions but no submission checks were configured.`);
+        }
+    }
+
     startQueue(causedBy: Invokee = 'system', options?: ManagerStateChangeOption) {
         const {reason, suppressNotification = false} = options || {};
         if(this.queueState.state === RUNNING) {
@@ -921,7 +1090,10 @@ export class Manager extends EventEmitter {
                 this.logger.warn('No submission or comment checks found!');
             }
 
-            for (const s of this.streams) {
+            if (this.streams.size > 0) {
+                this.logger.debug(`Starting own streams => ${[...this.streams.values()].map(x => `${x.name.toUpperCase()} ${x.frequency / 1000}s interval`).join(' | ')}`)
+            }
+            for (const s of this.streams.values()) {
                 s.startInterval();
             }
             this.startedAt = dayjs();
@@ -946,7 +1118,7 @@ export class Manager extends EventEmitter {
                 state: PAUSED,
                 causedBy
             };
-            for(const s of this.streams) {
+            for(const s of this.streams.values()) {
                 s.end();
             }
             if(causedBy === USER) {
@@ -963,15 +1135,10 @@ export class Manager extends EventEmitter {
     stopEvents(causedBy: Invokee = 'system', options?: ManagerStateChangeOption) {
         const {reason, suppressNotification = false} = options || {};
         if(this.eventsState.state !== STOPPED) {
-            for (const s of this.streams) {
+            for (const s of this.streams.values()) {
                 s.end();
             }
-            this.streams = [];
-            // for (const [k, v] of this.modStreamCallbacks) {
-            //     const stream = this.cacheManager.modStreams.get(k) as Poll<Snoowrap.Submission | Snoowrap.Comment>;
-            //     stream.removeListener('item', v);
-            // }
-            this.modStreamCallbacks = new Map();
+            this.streams = new Map();
             this.startedAt = undefined;
             this.logger.info(`Events STOPPED by ${causedBy}`);
             this.eventsState = {

src/Subreddit/Streams.ts

@@ -1,111 +1,151 @@
 import {Poll, SnooStormOptions} from "snoostorm"
-import Snoowrap from "snoowrap";
+import Snoowrap, {Listing} from "snoowrap";
 import {EventEmitter} from "events";
 import {PollConfiguration} from "snoostorm/out/util/Poll";
-import {ClearProcessedOptions, DEFAULT_POLLING_INTERVAL} from "../Common/interfaces";
-import dayjs, {Dayjs} from "dayjs";
-import { Duration } from "dayjs/plugin/duration";
-import {parseDuration, setRandomInterval, sleep} from "../util";
+import {DEFAULT_POLLING_INTERVAL} from "../Common/interfaces";
+import {mergeArr, parseDuration, random} from "../util";
+import { Logger } from "winston";
+import {ErrorWithCause} from "pony-cause";
 
 type Awaitable<T> = Promise<T> | T;
 
-interface RCBPollingOptions extends SnooStormOptions {
+interface RCBPollingOptions<T> extends SnooStormOptions {
     subreddit: string,
-    clearProcessed?: ClearProcessedOptions
+    enforceContinuity?: boolean
+    logger: Logger
+    name?: string,
+    processed?: Set<T[keyof T]>
+    label?: string
 }
 
-interface RCBPollConfiguration<T> extends PollConfiguration<T> {
-    clearProcessed?: ClearProcessedOptions
+interface RCBPollConfiguration<T> extends PollConfiguration<T>,RCBPollingOptions<T> {
+    get: () => Promise<Listing<T>>
 }
 
 export class SPoll<T extends object> extends Poll<T> {
     identifier: keyof T;
-    getter: () => Awaitable<T[]>;
+    getter: () => Promise<Listing<T>>;
     frequency;
     running: boolean = false;
-    clearProcessedDuration?: Duration;
-    clearProcessedSize?: number;
-    clearProcessedAfter?: Dayjs;
-    retainProcessed: number = 0;
+    // intention of newStart is to make polling behavior such that only "new" items AFTER polling has started get emitted
+    // -- that is, we don't want to emit the items we immediately fetch on a fresh poll start since they existed "before" polling started
+    newStart: boolean = true;
+    enforceContinuity: boolean;
     randInterval?: { clear: () => void };
+    name: string = 'Reddit Stream';
+    logger: Logger;
+    subreddit: string;
 
     constructor(options: RCBPollConfiguration<T>) {
         super(options);
-        this.identifier = options.identifier;
-        this.getter = options.get;
-        this.frequency = options.frequency;
         const {
-            after,
-            size,
-            retain = 0,
-        } = options.clearProcessed || {};
-        if(after !== undefined) {
-            this.clearProcessedDuration = parseDuration(after);
-        }
-        this.clearProcessedSize = size;
-        this.retainProcessed = retain;
-        if (this.clearProcessedDuration !== undefined) {
-            this.clearProcessedAfter = dayjs().add(this.clearProcessedDuration.asSeconds(), 's');
+            identifier,
+            get,
+            frequency,
+            enforceContinuity = false,
+            logger,
+            name,
+            subreddit,
+            label = 'Polling',
+            processed
+        } = options;
+        this.subreddit = subreddit;
+        this.name = name !== undefined ? name : this.name;
+        this.logger = logger.child({labels: [label, this.name]}, mergeArr)
+        this.identifier = identifier;
+        this.getter = get;
+        this.frequency = frequency;
+        this.enforceContinuity = enforceContinuity;
+
+        // if we pass in processed on init the intention is to "continue" from where the previous stream left off
+        // WITHOUT new start behavior
+        if (processed !== undefined) {
+            this.processed = processed;
+            this.newStart = false;
         }
+
         clearInterval(this.interval);
     }
 
-    startInterval = () => {
-        this.running = true;
-        this.randInterval = setRandomInterval((function (self) {
+    createInterval = () => {
+        this.interval = setTimeout((function (self) {
             return async () => {
                 try {
-                    // DEBUGGING
-                    //
-                    // Removing processed clearing to see if it fixes weird, duplicate/delayed comment processing behavior
-                    //
-                    // clear the tracked, processed activity ids after a set period or number of activities have been processed
-                    // because when RCB is long-running and has streams from high-volume subreddits this list never gets smaller...
-
-                    // so clear if after time period
-                    // if ((self.clearProcessedAfter !== undefined && dayjs().isSameOrAfter(self.clearProcessedAfter))
-                    //     // or clear if processed list is larger than defined max allowable size (default setting, 2 * polling option limit)
-                    //     || (self.clearProcessedSize !== undefined && self.processed.size >= self.clearProcessedSize)) {
-                    //     if (self.retainProcessed === 0) {
-                    //         self.processed = new Set();
-                    //     } else {
-                    //         // retain some processed so we have continuity between processed list resets -- this is default behavior and retains polling option limit # of activities
-                    //         // we can slice from the set here because ID order is guaranteed for Set object so list is oldest -> newest
-                    //         // -- retain last LIMIT number of activities (or all if retain # is larger than list due to user config error)
-                    //         self.processed = new Set(Array.from(self.processed).slice(Math.max(0, self.processed.size - self.retainProcessed)));
-                    //     }
-                    //     // reset time interval if there is one
-                    //     if (self.clearProcessedAfter !== undefined && self.clearProcessedDuration !== undefined) {
-                    //         self.clearProcessedAfter = dayjs().add(self.clearProcessedDuration.asSeconds(), 's');
-                    //     }
-                    // }
-                    const batch = await self.getter();
+                    self.logger.debug('Polling...');
+                    let batch = await self.getter();
                     const newItems: T[] = [];
-                    for (const item of batch) {
-                        const id = item[self.identifier];
-                        if (self.processed.has(id)) continue;
+                    let anyAlreadySeen = false;
+                    let page = 1;
+                    // initial iteration should always run
+                    // but only continue iterating if stream enforces continuity and we've only seen new items so far
+                    while(page === 1 || (self.enforceContinuity && !self.newStart && !anyAlreadySeen)) {
+                        if(page !== 1) {
+                            self.logger.debug(`Did not find any already seen activities and continuity is enforced. This probably means there were more new items than 1 api call can return. Fetching next page (${page})...`);
+                            // @ts-ignore
+                            batch = await batch.fetchMore({amount: 100});
+                        }
+                        if(batch.length === 0 || batch.isFinished) {
+                            // if nothing is returned we don't want to end up in an endless loop!
+                            anyAlreadySeen = true;
+                        }
+                        for (const item of batch) {
+                            const id = item[self.identifier];
+                            if (self.processed.has(id)) {
+                                anyAlreadySeen = true;
+                                continue;
+                            }
 
-                        // Emit for new items and add it to the list
-                        newItems.push(item);
-                        self.processed.add(id);
-                        self.emit("item", item);
+                            // Emit for new items and add it to the list
+                            newItems.push(item);
+                            self.processed.add(id);
+                            // but don't emit on new start since we are "buffering" already existing activities
+                            if(!self.newStart) {
+                                self.emit("item", item);
+                            }
+                        }
+                        page++;
                     }
-
-                    // Emit the new listing of all new items
-                    self.emit("listing", newItems);
+                    const newItemMsg = `Found ${newItems.length} new items out of ${batch.length} returned`;
+                    if(self.newStart) {
+                        self.logger.debug(`${newItemMsg} but will ignore all on first start.`);
+                        self.emit("listing", []);
+                    } else {
+                        self.logger.debug(newItemMsg);
+                        // Emit the new listing of all new items
+                        self.emit("listing", newItems);
+                    }
+                    // no longer new start on n+1 interval
+                    self.newStart = false;
+                    // if everything succeeded then create a new timeout
+                    self.createInterval();
                 } catch (err: any) {
+                    self.running = false;
+                    self.logger.error(new ErrorWithCause('Polling Interval stopped due to error encountered', {cause: err}));
                     self.emit('error', err);
-                    self.end();
                 }
             }
-        })(this), this.frequency - 1, this.frequency + 1);
+        })(this), random(this.frequency - 1, this.frequency + 1));
     }
 
-    end = () => {
-        this.running = false;
-        if(this.randInterval !== undefined) {
-            this.randInterval.clear();
+    // allow controlling newStart state
+    startInterval = (newStartState?: boolean, msg?: string) => {
+        this.running = true;
+        if(newStartState !== undefined) {
+            this.newStart = newStartState;
         }
+        const startMsg = `Polling Interval Started${msg !== undefined ? `: ${msg}` : ''}`;
+        this.logger.debug(startMsg)
+        this.createInterval();
+    }
+
+    end = (reason?: string) => {
+        let msg ='Stopping Polling Interval';
+        if(reason !== undefined) {
+            msg += `: ${reason}`;
+        }
+        this.logger.debug(msg);
+        this.running = false;
+        this.newStart = true;
         super.end();
     }
 }
@@ -113,12 +153,13 @@ export class SPoll<T extends object> extends Poll<T> {
 export class UnmoderatedStream extends SPoll<Snoowrap.Submission | Snoowrap.Comment> {
     constructor(
         client: Snoowrap,
-        options: RCBPollingOptions) {
+        options: RCBPollingOptions<Snoowrap.Submission | Snoowrap.Comment>) {
         super({
             frequency: options.pollTime || DEFAULT_POLLING_INTERVAL * 1000,
             get: async () => client.getSubreddit(options.subreddit).getUnmoderated(options),
             identifier: "id",
-            clearProcessed: options.clearProcessed
+            name: 'Unmoderated',
+            ...options,
         });
     }
 }
@@ -126,12 +167,13 @@ export class UnmoderatedStream extends SPoll<Snoowrap.Submission | Snoowrap.Comm
 export class ModQueueStream extends SPoll<Snoowrap.Submission | Snoowrap.Comment> {
     constructor(
         client: Snoowrap,
-        options: RCBPollingOptions) {
+        options: RCBPollingOptions<Snoowrap.Submission | Snoowrap.Comment>) {
         super({
             frequency: options.pollTime || DEFAULT_POLLING_INTERVAL * 1000,
             get: async () => client.getSubreddit(options.subreddit).getModqueue(options),
             identifier: "id",
-            clearProcessed: options.clearProcessed
+            name: 'Modqueue',
+            ...options,
         });
     }
 }
@@ -139,12 +181,13 @@ export class ModQueueStream extends SPoll<Snoowrap.Submission | Snoowrap.Comment
 export class SubmissionStream extends SPoll<Snoowrap.Submission | Snoowrap.Comment> {
     constructor(
         client: Snoowrap,
-        options: RCBPollingOptions) {
+        options: RCBPollingOptions<Snoowrap.Submission | Snoowrap.Comment>) {
         super({
             frequency: options.pollTime || DEFAULT_POLLING_INTERVAL * 1000,
             get: async () => client.getNew(options.subreddit, options),
             identifier: "id",
-            clearProcessed: options.clearProcessed
+            name: 'Submission',
+            ...options,
         });
     }
 }
@@ -152,12 +195,13 @@ export class SubmissionStream extends SPoll<Snoowrap.Submission | Snoowrap.Comme
 export class CommentStream extends SPoll<Snoowrap.Submission | Snoowrap.Comment> {
     constructor(
         client: Snoowrap,
-        options: RCBPollingOptions) {
+        options: RCBPollingOptions<Snoowrap.Submission | Snoowrap.Comment>) {
         super({
             frequency: options.pollTime || DEFAULT_POLLING_INTERVAL * 1000,
             get: async () => client.getNewComments(options.subreddit, options),
             identifier: "id",
-            clearProcessed: options.clearProcessed
+            name: 'Comment',
+            ...options,
         });
     }
 }

src/Subreddit/SubredditResources.ts

@@ -13,12 +13,27 @@ import as from 'async';
 import fetch from 'node-fetch';
 import {
     asSubmission,
-    buildCacheOptionsFromProvider, buildCachePrefix,
-    cacheStats, compareDurationValue, comparisonTextOp, createCacheManager, createHistoricalStatsDisplay,
-    formatNumber, getActivityAuthorName, getActivitySubredditName, isStrongSubredditState,
-    mergeArr, parseDurationComparison,
-    parseExternalUrl, parseGenericValueComparison, parseRedditEntity,
-    parseWikiContext, shouldCacheSubredditStateCriteriaResult, subredditStateIsNameOnly, toStrongSubredditState
+    buildCacheOptionsFromProvider,
+    buildCachePrefix,
+    cacheStats,
+    compareDurationValue,
+    comparisonTextOp,
+    createCacheManager,
+    createHistoricalStatsDisplay, FAIL,
+    fetchExternalUrl, filterCriteriaSummary,
+    formatNumber,
+    getActivityAuthorName,
+    getActivitySubredditName,
+    isStrongSubredditState, isSubmission,
+    mergeArr,
+    parseDurationComparison,
+    parseExternalUrl,
+    parseGenericValueComparison,
+    parseRedditEntity,
+    parseWikiContext, PASS,
+    shouldCacheSubredditStateCriteriaResult,
+    subredditStateIsNameOnly,
+    toStrongSubredditState
 } from "../util";
 import LoggedError from "../Utils/LoggedError";
 import {
@@ -40,12 +55,12 @@ import {
     HistoricalStats,
     HistoricalStatUpdateData,
     SubredditHistoricalStats,
-    SubredditHistoricalStatsDisplay, ThirdPartyCredentialsJsonConfig,
+    SubredditHistoricalStatsDisplay, ThirdPartyCredentialsJsonConfig, FilterCriteriaResult,
 } from "../Common/interfaces";
 import UserNotes from "./UserNotes";
 import Mustache from "mustache";
 import he from "he";
-import {AuthorCriteria} from "../Author/Author";
+import {AuthorCriteria, AuthorOptions} from "../Author/Author";
 import {SPoll} from "./Streams";
 import {Cache} from 'cache-manager';
 import {Submission, Comment, Subreddit} from "snoowrap/dist/objects";
@@ -90,6 +105,7 @@ export class SubredditResources {
     protected submissionTTL: number | false = cacheTTLDefaults.submissionTTL;
     protected commentTTL: number | false = cacheTTLDefaults.commentTTL;
     protected filterCriteriaTTL: number | false = cacheTTLDefaults.filterCriteriaTTL;
+    public selfTTL: number | false = cacheTTLDefaults.selfTTL;
     name: string;
     protected logger: Logger;
     userNotes: UserNotes;
@@ -119,6 +135,7 @@ export class SubredditResources {
                 authorTTL,
                 wikiTTL,
                 filterCriteriaTTL,
+                selfTTL,
                 submissionTTL,
                 commentTTL,
                 subredditTTL,
@@ -144,6 +161,7 @@ export class SubredditResources {
         this.subredditTTL = subredditTTL === true ? 0 : subredditTTL;
         this.wikiTTL = wikiTTL === true ? 0 : wikiTTL;
         this.filterCriteriaTTL = filterCriteriaTTL === true ? 0 : filterCriteriaTTL;
+        this.selfTTL = selfTTL === true ? 0 : selfTTL;
         this.subreddit = subreddit;
         this.thirdPartyCredentials = thirdPartyCredentials;
         this.name = name;
@@ -167,7 +185,7 @@ export class SubredditResources {
             this.stats.cache.userNotes.requests++;
             this.stats.cache.userNotes.miss += miss ? 1 : 0;
         }
-        this.userNotes = new UserNotes(userNotesTTL, this.subreddit, this.logger, this.cache, cacheUseCB)
+        this.userNotes = new UserNotes(userNotesTTL, this.subreddit, this.client, this.logger, this.cache, cacheUseCB)
 
         if(this.cacheType === 'memory' && this.cacheSettingsHash !== 'default') {
             const min = Math.min(...([this.wikiTTL, this.authorTTL, this.submissionTTL, this.commentTTL, this.filterCriteriaTTL].filter(x => typeof x === 'number' && x !== 0) as number[]));
@@ -391,6 +409,50 @@ export class SubredditResources {
         }
     }
 
+    async hasActivity(item: Submission | Comment) {
+        const hash = asSubmission(item) ? `sub-${item.name}` : `comm-${item.name}`;
+        const res = await this.cache.get(hash);
+        return res !== undefined && res !== null;
+    }
+
+    // @ts-ignore
+    async getRecentSelf(item: Submission | Comment): Promise<(Submission | Comment | undefined)> {
+        const hash = asSubmission(item) ? `sub-recentSelf-${item.name}` : `comm-recentSelf-${item.name}`;
+        const res = await this.cache.get(hash);
+        if(res === null) {
+            return undefined;
+        }
+        return res as (Submission | Comment | undefined);
+    }
+
+    async setRecentSelf(item: Submission | Comment) {
+        if(this.selfTTL !== false) {
+            const hash = asSubmission(item) ? `sub-recentSelf-${item.name}` : `comm-recentSelf-${item.name}`;
+            // @ts-ignore
+            await this.cache.set(hash, item, {ttl: this.selfTTL});
+        }
+        return;
+    }
+    /**
+    * Returns true if the activity being checked was recently acted on/created by the bot and has not changed since that time
+    * */
+    async hasRecentSelf(item: Submission | Comment) {
+        const recent = await this.getRecentSelf(item) as (Submission | Comment | undefined);
+        if (recent !== undefined) {
+            return item.num_reports === recent.num_reports;
+
+            // can't really used edited since its only ever updated once with no timestamp
+            // if(item.num_reports !== recent.num_reports) {
+            //     return false;
+            // }
+            // if(!asSubmission(item)) {
+            //     return item.edited === recent.edited;
+            // }
+            // return true;
+        }
+        return false;
+    }
+
     // @ts-ignore
     async getSubreddit(item: Submission | Comment) {
         try {
@@ -546,8 +608,7 @@ export class SubredditResources {
             }
         } else {
             try {
-                const response = await fetch(extUrl as string);
-                wikiContent = await response.text();
+                wikiContent = await fetchExternalUrl(extUrl as string, this.logger);
             } catch (err: any) {
                 const msg = `Error occurred while trying to fetch the url ${extUrl}`;
                 this.logger.error(msg, err);
@@ -664,7 +725,7 @@ export class SubredditResources {
         return await this.isSubreddit(await this.getSubreddit(item), state, this.logger);
     }
 
-    async testAuthorCriteria(item: (Comment | Submission), authorOpts: AuthorCriteria, include = true) {
+    async testAuthorCriteria(item: (Comment | Submission), authorOpts: AuthorCriteria, include = true): Promise<FilterCriteriaResult<AuthorCriteria>> {
         if (this.filterCriteriaTTL !== false) {
             // in the criteria check we only actually use the `item` to get the author flair
             // which will be the same for the entire subreddit
@@ -677,17 +738,18 @@ export class SubredditResources {
             await this.stats.cache.authorCrit.identifierRequestCount.set(hash, (await this.stats.cache.authorCrit.identifierRequestCount.wrap(hash, () => 0) as number) + 1);
             this.stats.cache.authorCrit.requestTimestamps.push(Date.now());
             this.stats.cache.authorCrit.requests++;
-            let miss = false;
-            const cachedAuthorTest = await this.cache.wrap(hash, async () => {
-                miss = true;
-                return await testAuthorCriteria(item, authorOpts, include, this.userNotes);
-            }, {ttl: this.filterCriteriaTTL});
-            if (!miss) {
+
+            // need to check shape of result to invalidate old result type
+            let cachedAuthorTest: FilterCriteriaResult<AuthorCriteria> = await this.cache.get(hash) as FilterCriteriaResult<AuthorCriteria>;
+            if(cachedAuthorTest !== null && cachedAuthorTest !== undefined && typeof cachedAuthorTest === 'object') {
                 this.logger.debug(`Cache Hit: Author Check on ${userName} (Hash ${hash})`);
+                return cachedAuthorTest;
             } else {
                 this.stats.cache.authorCrit.miss++;
+                cachedAuthorTest = await testAuthorCriteria(item, authorOpts, include, this.userNotes);
+                await this.cache.set(hash, cachedAuthorTest, {ttl: this.filterCriteriaTTL});
+                return cachedAuthorTest;
             }
-            return cachedAuthorTest;
         }
 
         return await testAuthorCriteria(item, authorOpts, include, this.userNotes);
@@ -717,7 +779,7 @@ export class SubredditResources {
                 const cachedItem = await this.cache.get(hash);
                 if (cachedItem !== undefined && cachedItem !== null) {
                     this.logger.debug(`Cache Hit: Item Check on ${item.name} (Hash ${hash})`);
-                    return cachedItem as boolean;
+                    //return cachedItem as boolean;
                 }
                 const itemResult = await this.isItem(item, states, this.logger);
                 this.stats.cache.itemCrit.miss++;
@@ -734,8 +796,8 @@ export class SubredditResources {
         return await this.isItem(i, activityStates, this.logger);
     }
 
-    async isSubreddit (subreddit: Subreddit, stateCriteria: SubredditState | StrongSubredditState, logger: Logger) {
-        delete stateCriteria.stateDescription;
+    async isSubreddit (subreddit: Subreddit, stateCriteriaRaw: SubredditState | StrongSubredditState, logger: Logger) {
+        const {stateDescription, ...stateCriteria} = stateCriteriaRaw;
 
         if (Object.keys(stateCriteria).length === 0) {
             return true;
@@ -811,7 +873,7 @@ export class SubredditResources {
                     if (crit[k] !== undefined) {
                         switch (k) {
                             case 'submissionState':
-                                if(!(item instanceof Comment)) {
+                                if(isSubmission(item)) {
                                     log.warn('`submissionState` is not allowed in `itemIs` criteria when the main Activity is a Submission');
                                     continue;
                                 }
@@ -836,13 +898,30 @@ export class SubredditResources {
                                 break;
                             case 'reports':
                                 if (!item.can_mod_post) {
-                                    log.debug(`Cannot test for reports on Activity in a subreddit bot account is not a moderato Activist. Skipping criteria...`);
+                                    log.debug(`Cannot test for reports on Activity in a subreddit bot account is not a moderator of. Skipping criteria...`);
                                     break;
                                 }
                                 const reportCompare = parseGenericValueComparison(crit[k] as string);
-                                if(!comparisonTextOp(item.num_reports, reportCompare.operator, reportCompare.value)) {
+                                let reportType = 'total';
+                                if(reportCompare.extra !== undefined && reportCompare.extra.trim() !== '') {
+                                    const requestedType = reportCompare.extra.toLocaleLowerCase().trim();
+                                    if(requestedType.includes('mod')) {
+                                        reportType = 'mod';
+                                    } else if(requestedType.includes('user')) {
+                                        reportType = 'user';
+                                    } else {
+                                        log.warn(`Did not recognize the report type "${requestedType}" -- can only use "mod" or "user". Will default to TOTAL reports`);
+                                    }
+                                }
+                                let reportNum = item.num_reports;
+                                if(reportType === 'user') {
+                                    reportNum = item.user_reports.length;
+                                } else {
+                                    reportNum = item.mod_reports.length;
+                                }
+                                if(!comparisonTextOp(reportNum, reportCompare.operator, reportCompare.value)) {
                                     // @ts-ignore
-                                    log.debug(`Failed: Expected => ${k}:${crit[k]} | Found => ${k}:${item.num_reports}`)
+                                    log.debug(`Failed: Expected => ${k}:${crit[k]} ${reportType} reports | Found => ${k}:${reportNum} ${reportType} reports`)
                                     return false
                                 }
                                 break;
@@ -913,7 +992,7 @@ export class SubredditResources {
                                 }
                                 break;
                             case 'op':
-                                if(item instanceof Submission) {
+                                if(isSubmission(item)) {
                                     log.warn(`On a Submission the 'op' property will always be true. Did you mean to use this on a comment instead?`);
                                     break;
                                 }
@@ -925,7 +1004,7 @@ export class SubredditResources {
                                 }
                                 break;
                             case 'depth':
-                                if(item instanceof Submission) {
+                                if(isSubmission(item)) {
                                     log.warn(`Cannot test for 'depth' on a Submission`);
                                     break;
                                 }
@@ -937,6 +1016,36 @@ export class SubredditResources {
                                     return false
                                 }
                                 break;
+                            case 'flairTemplate':
+                            case 'link_flair_text':
+                            case 'link_flair_css_class':
+                                if(asSubmission(item)) {
+                                    const subCrit = crit as SubmissionState;
+                                    let propertyValue: string | null;
+                                    if(k === 'flairTemplate') {
+                                        propertyValue = await item.link_flair_template_id;
+                                    } else {
+                                        propertyValue = await item[k];
+                                    }
+                                    const expectedValues = typeof subCrit[k] === 'string' ? [subCrit[k]] : (subCrit[k] as string[]);
+                                    const VALUEPass = () => {
+                                        for (const c of expectedValues) {
+                                            if (c === propertyValue) {
+                                                return true;
+                                            }
+                                        }
+                                        return false;
+                                    };
+                                    const valueResult = VALUEPass();
+                                    if(!valueResult) {
+                                        log.debug(`Failed: Expected => ${k} ${expectedValues.join(' OR ')} | Found => ${k}:${propertyValue}`);
+                                        return false;
+                                    }
+                                    break;
+                                } else {
+                                    log.warn(`Cannot test for ${k} on Comment`);
+                                    break;
+                                }
                             default:
                                 // @ts-ignore
                                 if (item[k] !== undefined) {
@@ -1064,6 +1173,7 @@ export class BotResourcesManager {
                 submissionTTL,
                 subredditTTL,
                 filterCriteriaTTL,
+                selfTTL,
                 provider,
                 actionedEventsMax,
                 actionedEventsDefault,
@@ -1080,7 +1190,7 @@ export class BotResourcesManager {
         this.cacheHash = objectHash.sha1(relevantCacheSettings);
         this.defaultCacheConfig = caching;
         this.defaultThirdPartyCredentials = thirdParty;
-        this.ttlDefaults = {authorTTL, userNotesTTL, wikiTTL, commentTTL, submissionTTL, filterCriteriaTTL, subredditTTL};
+        this.ttlDefaults = {authorTTL, userNotesTTL, wikiTTL, commentTTL, submissionTTL, filterCriteriaTTL, subredditTTL, selfTTL};
 
         const options = provider;
         this.cacheType = options.store;
@@ -1178,4 +1288,108 @@ export class BotResourcesManager {
 
         return resource;
     }
+
+    async getPendingSubredditInvites(): Promise<(string[])> {
+        const subredditNames = await this.defaultCache.get(`modInvites`);
+        if (subredditNames !== undefined && subredditNames !== null) {
+            return subredditNames as string[];
+        }
+        return [];
+    }
+
+    async addPendingSubredditInvite(subreddit: string): Promise<void> {
+        let subredditNames = await this.defaultCache.get(`modInvites`) as (string[] | undefined | null);
+        if (subredditNames === undefined || subredditNames === null) {
+            subredditNames = [];
+        }
+        subredditNames.push(subreddit);
+        await this.defaultCache.set(`modInvites`, subredditNames, {ttl: 0});
+        return;
+    }
+
+    async deletePendingSubredditInvite(subreddit: string): Promise<void> {
+        let subredditNames = await this.defaultCache.get(`modInvites`) as (string[] | undefined | null);
+        if (subredditNames === undefined || subredditNames === null) {
+            subredditNames = [];
+        }
+        subredditNames = subredditNames.filter(x => x !== subreddit);
+        await this.defaultCache.set(`modInvites`, subredditNames, {ttl: 0});
+        return;
+    }
+
+    async clearPendingSubredditInvites(): Promise<void> {
+        await this.defaultCache.del(`modInvites`);
+        return;
+    }
+}
+
+export const checkAuthorFilter = async (item: (Submission | Comment), filter: AuthorOptions, resources: SubredditResources, logger: Logger): Promise<[boolean, ('inclusive' | 'exclusive' | undefined)]> => {
+    const authLogger = logger.child({labels: ['Author Filter']}, mergeArr);
+    const {
+        include = [],
+        excludeCondition = 'AND',
+        exclude = [],
+    } = filter;
+    let authorPass = null;
+    if (include.length > 0) {
+        let index = 1;
+        for (const auth of include) {
+            const critResult = await resources.testAuthorCriteria(item, auth);
+            const [summary, details] = filterCriteriaSummary(critResult);
+            if (critResult.passed) {
+                authLogger.verbose(`${PASS} => Inclusive Author Criteria ${index} => ${summary}`);
+                authLogger.debug(`Criteria Details: \n${details.join('\n')}`);
+                return [true, 'inclusive'];
+            } else {
+                authLogger.debug(`${FAIL} => Inclusive Author Criteria ${index} => ${summary}`);
+                authLogger.debug(`Criteria Details: \n${details.join('\n')}`);
+            }
+            index++;
+        }
+        authLogger.verbose(`${FAIL} => No Inclusive Author Criteria matched`);
+        return [false, 'inclusive'];
+    }
+    if (exclude.length > 0) {
+        let index = 1;
+        const summaries: string[] = [];
+        for (const auth of exclude) {
+            const critResult = await resources.testAuthorCriteria(item, auth, false);
+            const [summary, details] = filterCriteriaSummary(critResult);
+            if (critResult.passed) {
+                if(excludeCondition === 'OR') {
+                    authLogger.verbose(`${PASS} (OR) => Exclusive Author Criteria ${index} => ${summary}`);
+                    authLogger.debug(`Criteria Details: \n${details.join('\n')}`);
+                    authorPass = true;
+                    break;
+                }
+                summaries.push(summary);
+                authLogger.debug(`${PASS} (AND) => Exclusive Author Criteria ${index} => ${summary}`);
+                authLogger.debug(`Criteria Details: \n${details.join('\n')}`);
+            } else if (!critResult.passed) {
+                if(excludeCondition === 'AND') {
+                    authLogger.verbose(`${FAIL} (AND) => Exclusive Author Criteria ${index} => ${summary}`);
+                    authLogger.debug(`Criteria Details: \n${details.join('\n')}`);
+                    authorPass = false;
+                    break;
+                }
+                summaries.push(summary);
+                authLogger.debug(`${FAIL} (OR) => Exclusive Author Criteria ${index} => ${summary}`);
+                authLogger.debug(`Criteria Details: \n${details.join('\n')}`);
+            }
+            index++;
+        }
+        if(excludeCondition === 'AND' && authorPass === null) {
+            authorPass = true;
+        }
+        if (authorPass !== true) {
+            if(excludeCondition === 'OR') {
+                authLogger.verbose(`${FAIL} => Exclusive author criteria not matched => ${summaries.length === 1 ? `${summaries[0]}` : '(many, see debug)'}`);
+            }
+            return [false, 'exclusive']
+        } else if(excludeCondition === 'AND') {
+            authLogger.verbose(`${PASS} => Exclusive author criteria matched => ${summaries.length === 1 ? `${summaries[0]}` : '(many, see debug)'}`);
+        }
+        return [true, 'exclusive'];
+    }
+    return [true, undefined];
 }

src/Subreddit/UserNotes.ts

@@ -1,5 +1,5 @@
 import dayjs, {Dayjs} from "dayjs";
-import {Comment, RedditUser, WikiPage} from "snoowrap";
+import Snoowrap, {Comment, RedditUser, WikiPage} from "snoowrap";
 import {
     COMMENT_URL_ID,
     deflateUserNotes, getActivityAuthorName,
@@ -57,7 +57,7 @@ export type UserNotesConstants = Pick<any, "users" | "warnings">;
 export class UserNotes {
     notesTTL: number | false;
     subreddit: Subreddit;
-    wiki: WikiPage;
+    client: Snoowrap;
     moderators?: RedditUser[];
     logger: Logger;
     identifier: string;
@@ -70,14 +70,14 @@ export class UserNotes {
     debounceCB: any;
     batchCount: number = 0;
 
-    constructor(ttl: number | boolean, subreddit: Subreddit, logger: Logger, cache: Cache, cacheCB: Function) {
+    constructor(ttl: number | boolean, subreddit: Subreddit, client: Snoowrap, logger: Logger, cache: Cache, cacheCB: Function) {
         this.notesTTL = ttl === true ? 0 : ttl;
         this.subreddit = subreddit;
         this.logger = logger;
-        this.wiki = subreddit.getWikiPage('usernotes');
         this.identifier = `${this.subreddit.display_name}-usernotes`;
         this.cache = cache;
         this.cacheCB = cacheCB;
+        this.client = client;
     }
 
     async getUserNotes(user: RedditUser): Promise<UserNote[]> {
@@ -172,8 +172,8 @@ export class UserNotes {
             //     this.saveDebounce = undefined;
             // }
             // @ts-ignore
-            this.wiki = await this.subreddit.getWikiPage('usernotes').fetch();
-            const wikiContent = this.wiki.content_md;
+            const wiki = this.client.getSubreddit(this.subreddit.display_name).getWikiPage('usernotes');
+            const wikiContent = await wiki.content_md;
             // TODO don't handle for versions lower than 6
             const userNotes = JSON.parse(wikiContent);
 
@@ -197,6 +197,7 @@ export class UserNotes {
         const blob = deflateUserNotes(payload.blob);
         const wikiPayload = {text: JSON.stringify({...payload, blob}), reason: 'ContextBot edited usernotes'};
         try {
+            const wiki = this.client.getSubreddit(this.subreddit.display_name).getWikiPage('usernotes');
             if (this.notesTTL !== false) {
                 // DISABLED for now because if it fails throws an uncaught rejection
                 // and need to figured out how to handle this other than just logging (want to interrupt action flow too?)
@@ -226,12 +227,12 @@ export class UserNotes {
                 // this.logger.debug(`Saving Usernotes has been debounced for 5 seconds (${this.batchCount} batched)`)
 
                 // @ts-ignore
-                await this.subreddit.getWikiPage('usernotes').edit(wikiPayload);
+                await wiki.edit(wikiPayload);
                 await this.cache.set(this.identifier, payload, {ttl: this.notesTTL});
                 this.users = new Map();
             } else {
                 // @ts-ignore
-                this.wiki = await this.subreddit.getWikiPage('usernotes').edit(wikiPayload);
+                await wiki.edit(wikiPayload);
             }
 
             return payload as RawUserNotesPayload;

src/Utils/CommandConfig.ts

@@ -56,7 +56,7 @@ export const port = new commander.Option('-p, --port <port>', 'Port for web serv
 export const sharedMod = new commander.Option('-q, --shareMod', `If enabled then all subreddits using the default settings to poll "unmoderated" or "modqueue" will retrieve results from a shared request to /r/mod (default: process.env.SHARE_MOD || false)`)
     .argParser(parseBool);
 
-export const operatorConfig = new commander.Option('-c, --operatorConfig <path>', 'An absolute path to a JSON file to load all parameters from (default: process.env.OPERATOR_CONFIG)');
+export const operatorConfig = new commander.Option('-c, --operatorConfig <path>', 'An absolute path to a YAML/JSON file to load all parameters from (default: process.env.OPERATOR_CONFIG | CWD/config.yaml)');
 
 export const getUniversalWebOptions = (): commander.Option[] => {
     return [

src/Utils/Errors.ts

@@ -1,18 +1,39 @@
-import {StatusCodeError} from "../Common/interfaces";
+import {RateLimitError, RequestError, StatusCodeError} from 'snoowrap/dist/errors';
+import ExtendableError from "es6-error";
+import {ErrorWithCause} from "pony-cause";
 
 
-export const isRateLimitError = (err: any) => {
-    return typeof err === 'object' && err.name === 'RateLimitError';
+export const isRateLimitError = (err: any): err is RateLimitError => {
+    return isRequestError(err) && err.name === 'RateLimitError';
 }
 
 export const isScopeError = (err: any): boolean => {
-    if(typeof err === 'object' && err.name === 'StatusCodeError' && err.response !== undefined) {
+    if(isStatusError(err)) {
         const authHeader = err.response.headers['www-authenticate'];
         return authHeader !== undefined && authHeader.includes('insufficient_scope');
     }
     return false;
 }
 
-export const isStatusError = (err: any): err is StatusCodeError => {
-    return typeof err === 'object' && err.name === 'StatusCodeError' && err.response !== undefined;
+export const getScopeError = (err: any): string | undefined => {
+    if(isScopeError(err)) {
+        return err.response.headers['www-authenticate'];
+    }
+    return undefined;
+}
+
+export const isStatusError = (err: any): err is StatusCodeError => {
+    return isRequestError(err) && err.name === 'StatusCodeError';
+}
+
+export const isRequestError = (err: any): err is RequestError => {
+    return typeof err === 'object' && err.response !== undefined;
+}
+
+export class SimpleError extends ExtendableError {
+
+}
+
+export class CMError extends ErrorWithCause {
+    logged: boolean = false;
 }

src/Utils/SimpleError.ts

@@ -1,7 +0,0 @@
-import ExtendableError from "es6-error";
-
-class SimpleError extends ExtendableError {
-
-}
-
-export default SimpleError;

src/Utils/SnoowrapUtils.ts

@@ -8,30 +8,31 @@ import he from "he";
 import {RuleResult, UserNoteCriteria} from "../Rule";
 import {
     ActivityWindowType, CommentState, DomainInfo,
-    DurationVal,
+    DurationVal, FilterCriteriaPropertyResult, FilterCriteriaResult,
     SubmissionState,
     TypedActivityStates
 } from "../Common/interfaces";
 import {
+    asUserNoteCriteria,
     compareDurationValue,
-    comparisonTextOp, escapeRegex, getActivityAuthorName,
-    isActivityWindowCriteria,
+    comparisonTextOp, escapeRegex, formatNumber, getActivityAuthorName,
+    isActivityWindowCriteria, isUserNoteCriteria,
     normalizeName,
     parseDuration,
     parseDurationComparison,
     parseGenericValueComparison,
     parseGenericValueOrPercentComparison,
     parseRuleResultsToMarkdownSummary, parseStringToRegex,
-    parseSubredditName,
-    truncateStringToLength, windowToActivityWindowCriteria
+    parseSubredditName, removeUndefinedKeys,
+    truncateStringToLength, userNoteCriteriaSummary, windowToActivityWindowCriteria
 } from "../util";
 import UserNotes from "../Subreddit/UserNotes";
 import {Logger} from "winston";
 import InvalidRegexError from "./InvalidRegexError";
-import SimpleError from "./SimpleError";
 import {AuthorCriteria} from "../Author/Author";
 import {URL} from "url";
-import {isStatusError} from "./Errors";
+import {SimpleError, isStatusError} from "./Errors";
+import {Dictionary, ElementOf, SafeDictionary} from "ts-essentials";
 
 export const BOT_LINK = 'https://www.reddit.com/r/ContextModBot/comments/otz396/introduction_to_contextmodbot';
 
@@ -354,24 +355,54 @@ export const renderContent = async (template: string, data: (Submission | Commen
     return he.decode(rendered);
 }
 
-export const testAuthorCriteria = async (item: (Comment | Submission), authorOpts: AuthorCriteria, include = true, userNotes: UserNotes) => {
-    const {shadowBanned, ...rest} = authorOpts;
+type AuthorCritPropHelper = SafeDictionary<FilterCriteriaPropertyResult<AuthorCriteria>, keyof AuthorCriteria>;
+type RequiredAuthorCrit = Required<AuthorCriteria>;
 
-    if(shadowBanned !== undefined) {
+export const testAuthorCriteria = async (item: (Comment | Submission), authorOpts: AuthorCriteria, include = true, userNotes: UserNotes): Promise<FilterCriteriaResult<AuthorCriteria>> => {
+
+
+    const definedAuthorOpts = (removeUndefinedKeys(authorOpts) as RequiredAuthorCrit);
+
+    const propResultsMap = Object.entries(definedAuthorOpts).reduce((acc: AuthorCritPropHelper, [k, v]) => {
+        const key = (k as keyof AuthorCriteria);
+        let ex;
+        if (Array.isArray(v)) {
+            ex = v.map(x => {
+                if (asUserNoteCriteria(x)) {
+                    return userNoteCriteriaSummary(x);
+                }
+                return x;
+            });
+        } else {
+            ex = [v];
+        }
+        acc[key] = {
+            property: key,
+            expected: ex,
+            behavior: include ? 'include' : 'exclude',
+        };
+        return acc;
+    }, {});
+
+    const {shadowBanned} = authorOpts;
+
+    if (shadowBanned !== undefined) {
         try {
             // @ts-ignore
             await item.author.fetch();
             // user is not shadowbanned
             // if criteria specifies they SHOULD be shadowbanned then return false now
-            if(shadowBanned) {
-                return false;
+            if (shadowBanned) {
+                propResultsMap.shadowBanned!.found = false;
+                propResultsMap.shadowBanned!.passed = false;
             }
         } catch (err: any) {
-            if(isStatusError(err) && err.statusCode === 404) {
+            if (isStatusError(err) && err.statusCode === 404) {
                 // user is shadowbanned
                 // if criteria specifies they should not be shadowbanned then return false now
-                if(!shadowBanned) {
-                    return false;
+                if (!shadowBanned) {
+                    propResultsMap.shadowBanned!.found = true;
+                    propResultsMap.shadowBanned!.passed = false;
                 }
             } else {
                 throw err;
@@ -379,17 +410,30 @@ export const testAuthorCriteria = async (item: (Comment | Submission), authorOpt
         }
     }
 
-    try {
-        const authorName = getActivityAuthorName(item.author);
 
-        for (const k of Object.keys(rest)) {
-            // @ts-ignore
-            if (authorOpts[k] !== undefined) {
+
+    if (propResultsMap.shadowBanned === undefined || propResultsMap.shadowBanned.passed === undefined) {
+        try {
+            const authorName = getActivityAuthorName(item.author);
+
+            const keys = Object.keys(propResultsMap) as (keyof AuthorCriteria)[]
+
+            let shouldContinue = true;
+            for (const k of keys) {
+                if (k === 'shadowBanned') {
+                    // we have already taken care of this with shadowban check above
+                    continue;
+                }
+
+                const authorOptVal = definedAuthorOpts[k];
+
+                //if (authorOpts[k] !== undefined) {
                 switch (k) {
                     case 'name':
+                        const nameVal = authorOptVal as RequiredAuthorCrit['name'];
                         const authPass = () => {
-                            // @ts-ignore
-                            for (const n of authorOpts[k]) {
+
+                            for (const n of nameVal) {
                                 if (n.toLowerCase() === authorName.toLowerCase()) {
                                     return true;
                                 }
@@ -397,8 +441,10 @@ export const testAuthorCriteria = async (item: (Comment | Submission), authorOpt
                             return false;
                         }
                         const authResult = authPass();
-                        if ((include && !authResult) || (!include && authResult)) {
-                            return false;
+                        propResultsMap.name!.found = authorName;
+                        propResultsMap.name!.passed = !((include && !authResult) || (!include && authResult));
+                        if (!propResultsMap.name!.passed) {
+                            shouldContinue = false;
                         }
                         break;
                     case 'flairCssClass':
@@ -413,8 +459,10 @@ export const testAuthorCriteria = async (item: (Comment | Submission), authorOpt
                             return false;
                         }
                         const cssResult = cssPass();
-                        if ((include && !cssResult) || (!include && cssResult)) {
-                            return false;
+                        propResultsMap.flairCssClass!.found = css;
+                        propResultsMap.flairCssClass!.passed = !((include && !cssResult) || (!include && cssResult));
+                        if (!propResultsMap.flairCssClass!.passed) {
+                            shouldContinue = false;
                         }
                         break;
                     case 'flairText':
@@ -429,68 +477,103 @@ export const testAuthorCriteria = async (item: (Comment | Submission), authorOpt
                             return false;
                         };
                         const textResult = textPass();
-                        if ((include && !textResult) || (!include && textResult)) {
+                        propResultsMap.flairText!.found = text;
+                        propResultsMap.flairText!.passed = !((include && !textResult) || (!include && textResult));
+                        if (!propResultsMap.flairText!.passed) {
+                            shouldContinue = false;
+                        }
+                        break;
+                    case 'flairTemplate':
+                        const templateId = await item.author_flair_template_id;
+                        const templatePass = () => {
+                            // @ts-ignore
+                            for (const c of authorOpts[k]) {
+                                if (c === templateId) {
+                                    return true;
+                                }
+                            }
                             return false;
+                        };
+                        const templateResult = templatePass();
+                        propResultsMap.flairTemplate!.found = templateId;
+                        propResultsMap.flairTemplate!.passed = !((include && !templateResult) || (!include && templateResult));
+                        if (!propResultsMap.flairTemplate!.passed) {
+                            shouldContinue = false;
                         }
                         break;
                     case 'isMod':
                         const mods: RedditUser[] = await item.subreddit.getModerators();
                         const isModerator = mods.some(x => x.name === authorName);
                         const modMatch = authorOpts.isMod === isModerator;
-                        if ((include && !modMatch) || (!include && modMatch)) {
-                            return false;
+                        propResultsMap.isMod!.found = isModerator;
+                        propResultsMap.isMod!.passed = !((include && !modMatch) || (!include && modMatch));
+                        if (!propResultsMap.isMod!.passed) {
+                            shouldContinue = false;
                         }
                         break;
                     case 'age':
-                        const ageTest = compareDurationValue(parseDurationComparison(await authorOpts.age as string), dayjs.unix(await item.author.created));
-                        if ((include && !ageTest) || (!include && ageTest)) {
-                            return false;
+                        const authorAge = dayjs.unix(await item.author.created);
+                        const ageTest = compareDurationValue(parseDurationComparison(await authorOpts.age as string), authorAge);
+                        propResultsMap.age!.found = authorAge.fromNow(true);
+                        propResultsMap.age!.passed = !((include && !ageTest) || (!include && ageTest));
+                        if (!propResultsMap.age!.passed) {
+                            shouldContinue = false;
                         }
                         break;
                     case 'linkKarma':
+                        // @ts-ignore
+                        const tk = await item.author.total_karma as number;
                         const lkCompare = parseGenericValueOrPercentComparison(await authorOpts.linkKarma as string);
                         let lkMatch;
                         if (lkCompare.isPercent) {
-                            // @ts-ignore
-                            const tk = await item.author.total_karma as number;
+
                             lkMatch = comparisonTextOp(item.author.link_karma / tk, lkCompare.operator, lkCompare.value / 100);
                         } else {
                             lkMatch = comparisonTextOp(item.author.link_karma, lkCompare.operator, lkCompare.value);
                         }
-                        if ((include && !lkMatch) || (!include && lkMatch)) {
-                            return false;
+                        propResultsMap.linkKarma!.found = tk;
+                        propResultsMap.linkKarma!.passed = !((include && !lkMatch) || (!include && lkMatch));
+                        if (!propResultsMap.linkKarma!.passed) {
+                            shouldContinue = false;
                         }
                         break;
                     case 'commentKarma':
+                        // @ts-ignore
+                        const ck = await item.author.total_karma as number;
                         const ckCompare = parseGenericValueOrPercentComparison(await authorOpts.commentKarma as string);
                         let ckMatch;
                         if (ckCompare.isPercent) {
-                            // @ts-ignore
-                            const ck = await item.author.total_karma as number;
                             ckMatch = comparisonTextOp(item.author.comment_karma / ck, ckCompare.operator, ckCompare.value / 100);
                         } else {
                             ckMatch = comparisonTextOp(item.author.comment_karma, ckCompare.operator, ckCompare.value);
                         }
-                        if ((include && !ckMatch) || (!include && ckMatch)) {
-                            return false;
+                        propResultsMap.commentKarma!.found = ck;
+                        propResultsMap.commentKarma!.passed = !((include && !ckMatch) || (!include && ckMatch));
+                        if (!propResultsMap.commentKarma!.passed) {
+                            shouldContinue = false;
                         }
                         break;
                     case 'totalKarma':
+                        // @ts-ignore
+                        const totalKarma = await item.author.total_karma as number;
                         const tkCompare = parseGenericValueComparison(await authorOpts.totalKarma as string);
                         if (tkCompare.isPercent) {
                             throw new SimpleError(`'totalKarma' value on AuthorCriteria cannot be a percentage`);
                         }
-                        // @ts-ignore
-                        const totalKarma = await item.author.total_karma as number;
                         const tkMatch = comparisonTextOp(totalKarma, tkCompare.operator, tkCompare.value);
-                        if ((include && !tkMatch) || (!include && tkMatch)) {
-                            return false;
+                        propResultsMap.totalKarma!.found = totalKarma;
+                        propResultsMap.totalKarma!.passed = !((include && !tkMatch) || (!include && tkMatch));
+                        if (!propResultsMap.totalKarma!.passed) {
+                            shouldContinue = false;
                         }
                         break;
                     case 'verified':
-                        const vMatch = await item.author.has_verified_mail === authorOpts.verified as boolean;
-                        if ((include && !vMatch) || (!include && vMatch)) {
-                            return false;
+                        const verified = await item.author.has_verified_mail;
+                        const vMatch = verified === authorOpts.verified as boolean;
+                        propResultsMap.verified!.found = verified;
+                        propResultsMap.verified!.passed = !((include && !vMatch) || (!include && vMatch));
+                        if (!propResultsMap.verified!.passed) {
+                            shouldContinue = false;
                         }
                         break;
                     case 'description':
@@ -498,25 +581,32 @@ export const testAuthorCriteria = async (item: (Comment | Submission), authorOpt
                         const desc = await item.author.subreddit?.display_name.public_description;
                         const dVals = authorOpts[k] as string[];
                         let passed = false;
-                        for(const val of dVals) {
+                        let passReg;
+                        for (const val of dVals) {
                             let reg = parseStringToRegex(val, 'i');
-                            if(reg === undefined) {
+                            if (reg === undefined) {
                                 reg = parseStringToRegex(`/.*${escapeRegex(val.trim())}.*/`, 'i');
-                                if(reg === undefined) {
+                                if (reg === undefined) {
                                     throw new SimpleError(`Could not convert 'description' value to a valid regex: ${authorOpts[k] as string}`);
                                 }
                             }
-                            if(reg.test(desc)) {
+                            if (reg.test(desc)) {
                                 passed = true;
+                                passReg = reg.toString();
                                 break;
                             }
                         }
-                        if(!passed) {
-                            return false;
+                        propResultsMap.description!.found = typeof desc === 'string' ? truncateStringToLength(50)(desc) : desc;
+                        propResultsMap.description!.passed = !((include && !passed) || (!include && passed));
+                        if (!propResultsMap.description!.passed) {
+                            shouldContinue = false;
+                        } else {
+                            propResultsMap.description!.reason = `Matched with: ${passReg as string}`;
                         }
                         break;
                     case 'userNotes':
                         const notes = await userNotes.getUserNotes(item.author);
+                        let foundNoteResult: string[] = [];
                         const notePass = () => {
                             for (const noteCriteria of authorOpts[k] as UserNoteCriteria[]) {
                                 const {count = '>= 1', search = 'current', type} = noteCriteria;
@@ -529,8 +619,14 @@ export const testAuthorCriteria = async (item: (Comment | Submission), authorOpt
                                 const order = extra.includes('asc') ? 'ascending' : 'descending';
                                 switch (search) {
                                     case 'current':
-                                        if (notes.length > 0 && notes[notes.length - 1].noteType === type) {
-                                            return true;
+                                        if (notes.length > 0) {
+                                            const currentNoteType = notes[notes.length - 1].noteType;
+                                            foundNoteResult.push(`Current => ${currentNoteType}`);
+                                            if (currentNoteType === type) {
+                                                return true;
+                                            }
+                                        } else {
+                                            foundNoteResult.push('No notes present');
                                         }
                                         break;
                                     case 'consecutive':
@@ -549,39 +645,64 @@ export const testAuthorCriteria = async (item: (Comment | Submission), authorOpt
                                             if (isPercent) {
                                                 throw new SimpleError(`When comparing UserNotes with 'consecutive' search 'count' cannot be a percentage. Given: ${count}`);
                                             }
+                                            foundNoteResult.push(`Found ${currCount} ${type} consecutively`);
                                             if (comparisonTextOp(currCount, operator, value)) {
                                                 return true;
                                             }
                                         }
                                         break;
                                     case 'total':
+                                        const filteredNotes = notes.filter(x => x.noteType === type);
                                         if (isPercent) {
-                                            if (comparisonTextOp(notes.filter(x => x.noteType === type).length / notes.length, operator, value / 100)) {
+                                            // avoid divide by zero
+                                            const percent = notes.length === 0 ? 0 : filteredNotes.length / notes.length;
+                                            foundNoteResult.push(`${formatNumber(percent)}% are ${type}`);
+                                            if (comparisonTextOp(percent, operator, value / 100)) {
+                                                return true;
+                                            }
+                                        } else {
+                                            foundNoteResult.push(`${filteredNotes.length} are ${type}`);
+                                            if (comparisonTextOp(notes.filter(x => x.noteType === type).length, operator, value)) {
                                                 return true;
                                             }
-                                        } else if (comparisonTextOp(notes.filter(x => x.noteType === type).length, operator, value)) {
-                                            return true;
                                         }
+                                        break;
                                 }
                             }
                             return false;
                         }
                         const noteResult = notePass();
-                        if ((include && !noteResult) || (!include && noteResult)) {
-                            return false;
+                        propResultsMap.userNotes!.found = foundNoteResult.join(' | ');
+                        propResultsMap.userNotes!.passed = !((include && !noteResult) || (!include && noteResult));
+                        if (!propResultsMap.userNotes!.passed) {
+                            shouldContinue = false;
                         }
                         break;
                 }
+                //}
+                if (!shouldContinue) {
+                    break;
+                }
+            }
+        } catch (err: any) {
+            if (isStatusError(err) && err.statusCode === 404) {
+                throw new SimpleError('Reddit returned a 404 while trying to retrieve User profile. It is likely this user is shadowbanned.');
+            } else {
+                throw err;
             }
         }
-        return true;
-    } catch (err: any) {
-        if(isStatusError(err) && err.statusCode === 404) {
-            throw new SimpleError('Reddit returned a 404 while trying to retrieve User profile. It is likely this user is shadowbanned.');
-        } else {
-            throw err;
-        }
     }
+
+    // gather values and determine overall passed
+    const propResults = Object.values(propResultsMap);
+    const passed = propResults.filter(x => typeof x.passed === 'boolean').every(x => x.passed === true);
+
+    return {
+        behavior: include ? 'include' : 'exclude',
+        criteria: authorOpts,
+        propertyResults: propResults,
+        passed,
+    };
 }
 
 export interface ItemContent {

src/Utils/loggerFactory.ts

@@ -1,19 +1,28 @@
 import {labelledFormat, logLevels} from "../util";
 import winston, {Logger} from "winston";
 import {DuplexTransport} from "winston-duplex";
+import {LoggerFactoryOptions} from "../Common/interfaces";
+import process from "process";
+import path from "path";
 
 const {transports} = winston;
 
-export const getLogger = (options: any, name = 'app'): Logger => {
+export const getLogger = (options: LoggerFactoryOptions, name = 'app'): Logger => {
     if(!winston.loggers.has(name)) {
         const {
-            path,
             level,
             additionalTransports = [],
             defaultLabel = 'App',
+            file: {
+                dirname,
+                ...fileRest
+            },
+            console,
+            stream
         } = options || {};
 
         const consoleTransport = new transports.Console({
+            ...console,
             handleExceptions: true,
             handleRejections: true,
         });
@@ -28,21 +37,39 @@ export const getLogger = (options: any, name = 'app'): Logger => {
                     objectMode: true,
                 },
                 name: 'duplex',
-                dump: false,
                 handleExceptions: true,
                 handleRejections: true,
+                ...stream,
+                dump: false,
             }),
             ...additionalTransports,
         ];
 
-        if (path !== undefined && path !== '') {
+        if (dirname !== undefined && dirname !== '' && dirname !== null) {
+
+            let realDir: string | undefined;
+            if(typeof dirname === 'boolean') {
+                if(!dirname) {
+                    realDir = undefined;
+                } else {
+                    realDir = path.resolve(__dirname, '../../logs')
+                }
+            } else if(dirname === 'true') {
+                realDir = path.resolve(__dirname, '../../logs')
+            } else if(dirname === 'false') {
+                realDir = undefined;
+            } else {
+                realDir = dirname;
+            }
+
             const rotateTransport = new winston.transports.DailyRotateFile({
-                dirname: path,
                 createSymlink: true,
                 symlinkName: 'contextBot-current.log',
                 filename: 'contextBot-%DATE%.log',
                 datePattern: 'YYYY-MM-DD',
                 maxSize: '5m',
+                dirname: realDir,
+                ...fileRest,
                 handleExceptions: true,
                 handleRejections: true,
             });

src/Web/Client/index.ts

@@ -9,10 +9,10 @@ import {Strategy as CustomStrategy} from 'passport-custom';
 import {OperatorConfig, BotConnection, LogInfo} from "../../Common/interfaces";
 import {
     buildCachePrefix,
-    createCacheManager, filterLogBySubreddit,
+    createCacheManager, defaultFormat, filterLogBySubreddit,
     formatLogLineToHtml,
     intersect, isLogLineMinLevel,
-    LogEntry, parseFromJsonOrYamlToObject, parseInstanceLogInfoName, parseInstanceLogName,
+    LogEntry, parseInstanceLogInfoName, parseInstanceLogName, parseRedditEntity,
     parseSubredditLogName, permissions,
     randomId, sleep, triggeredIndicator
 } from "../../util";
@@ -24,7 +24,6 @@ import EventEmitter from "events";
 import stream, {Readable, Writable, Transform} from "stream";
 import winston from "winston";
 import tcpUsed from "tcp-port-used";
-import SimpleError from "../../Utils/SimpleError";
 import http from "http";
 import jwt from 'jsonwebtoken';
 import {Server as SocketServer} from "socket.io";
@@ -39,17 +38,33 @@ import {prettyPrintJson} from "pretty-print-json";
 import DelimiterStream from 'delimiter-stream';
 import {pipeline} from 'stream/promises';
 import {defaultBotStatus} from "../Common/defaults";
-import {booleanMiddle} from "../Common/middleware";
+import {arrayMiddle, booleanMiddle} from "../Common/middleware";
 import {BotInstance, CMInstance} from "../interfaces";
 import { URL } from "url";
 import {MESSAGE} from "triple-beam";
 import Autolinker from "autolinker";
 import path from "path";
+import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients";
+import ClientUser from "../Common/User/ClientUser";
+import {BotStatusResponse} from "../Common/interfaces";
+import {TransformableInfo} from "logform";
+import {SimpleError} from "../../Utils/Errors";
+import {ErrorWithCause} from "pony-cause";
 
 const emitter = new EventEmitter();
 
 const app = addAsync(express());
-app.use(bodyParser.json());
+const jsonParser = bodyParser.json();
+
+// do not modify body if we are proxying it to server
+app.use((req, res, next) => {
+    if(req.url.indexOf('/api') !== 0) {
+        jsonParser(req, res, next);
+    } else {
+        next();
+    }
+});
+
 app.use(bodyParser.urlencoded({extended: false}));
 //app.use(cookieParser());
 app.set('views', `${__dirname}/../assets/views`);
@@ -58,6 +73,8 @@ app.use('/public', express.static(`${__dirname}/../assets/public`));
 app.use('/monaco', express.static(`${__dirname}/../../../node_modules/monaco-editor/`));
 app.use('/schemas', express.static(`${__dirname}/../../Schema/`));
 
+const userAgent = `web:contextBot:web`;
+
 const proxy = httpProxy.createProxyServer({
     ws: true,
     //hostRewrite: true,
@@ -70,24 +87,12 @@ declare module 'express-session' {
         sort?: string,
         level?: string,
         state?: string,
+        scope?: string[],
         botId?: string,
         authBotId?: string,
     }
 }
 
-// declare global {
-//     namespace Express {
-//         interface User {
-//             name: string
-//             subreddits: string[]
-//             machine?: boolean
-//             isOperator?: boolean
-//             realManagers?: string[]
-//             moderatedManagers?: string[]
-//         }
-//     }
-// }
-
 interface ConnectedUserInfo {
     level?: string,
     user?: string,
@@ -183,13 +188,14 @@ const webClient = async (options: OperatorConfig) => {
     * */
 
     passport.serializeUser(async function (data: any, done) {
-        const {user, subreddits} = data;
+        const {user, subreddits, scope, token} = data;
         //await webCache.set(`userSession-${user}`, { subreddits: subreddits.map((x: Subreddit) => x.display_name), isOperator: webOps.includes(user.toLowerCase()) }, {ttl: provider.ttl as number});
-        done(null, { subreddits: subreddits.map((x: Subreddit) => x.display_name), isOperator: webOps.includes(user.toLowerCase()), name: user });
+        done(null, { subreddits: subreddits.map((x: Subreddit) => x.display_name), isOperator: webOps.includes(user.toLowerCase()), name: user, scope, token, tokenExpiresAt: dayjs().unix() + (60 * 60) });
     });
 
-    passport.deserializeUser(async function (obj, done) {
-        done(null, obj as Express.User);
+    passport.deserializeUser(async function (obj: any, done) {
+        const user = new ClientUser(obj.name, obj.subreddits, {token: obj.token, scope: obj.scope, webOperator: obj.isOperator, tokenExpiresAt: obj.tokenExpiresAt});
+        done(null, user);
         // const data = await webCache.get(`userSession-${obj}`) as object;
         // if (data === undefined) {
         //     done('Not Found');
@@ -214,16 +220,20 @@ const webClient = async (options: OperatorConfig) => {
             } else if (req.session.state !== state) {
                 return done('Unexpected <b>state</b> value returned');
             }
-            const client = await Snoowrap.fromAuthCode({
-                userAgent: `web:contextBot:web`,
+            const client = await ExtendedSnoowrap.fromAuthCode({
+                userAgent,
                 clientId,
                 clientSecret,
                 redirectUri: redirectUri as string,
                 code: code as string,
             });
             const user = await client.getMe().name as string;
-            const subs = await client.getModeratedSubreddits();
-            return done(null, {user, subreddits: subs});
+            let subs = await client.getModeratedSubreddits({count: 100});
+            while(!subs.isFinished) {
+                subs = await subs.fetchMore({amount: 100});
+            }
+            io.to(req.session.id).emit('authStatus', {canSaveWiki: req.session.scope?.includes('wikiedit')});
+            return done(null, {user, subreddits: subs, scope: req.session.scope, token: client.accessToken});
         }
     ));
 
@@ -256,14 +266,24 @@ const webClient = async (options: OperatorConfig) => {
         }
     }
 
-    app.getAsync('/login', async (req, res, next) => {
+    const scopeMiddle = arrayMiddle(['scope']);
+    const successMiddle = booleanMiddle([{name: 'closeOnSuccess', defaultVal: undefined, required: false}]);
+    app.getAsync('/login', scopeMiddle, successMiddle, async (req, res, next) => {
         if (redirectUri === undefined) {
             return res.render('error', {error: `No <b>redirectUri</b> was specified through environmental variables or program argument. This must be provided in order to use the web interface.`});
         }
+        const {query: { scope: reqScopes = [], closeOnSuccess } } = req;
+        const scope = [...new Set(['identity', 'mysubreddits', ...(reqScopes as string[])])];
         req.session.state = randomId();
+        req.session.scope = scope;
+        // @ts-ignore
+        if(closeOnSuccess === true) {
+            // @ts-ignore
+            req.session.closeOnSuccess = closeOnSuccess;
+        }
         const authUrl = Snoowrap.getAuthUrl({
             clientId,
-            scope: ['identity', 'mysubreddits'],
+            scope: scope,
             redirectUri: redirectUri as string,
             permanent: false,
             state: req.session.state,
@@ -293,7 +313,7 @@ const webClient = async (options: OperatorConfig) => {
             // @ts-ignore
             const invite = await webCache.get(`invite:${req.session.inviteId}`) as InviteData;
             const client = await Snoowrap.fromAuthCode({
-                userAgent: `web:contextBot:web`,
+                userAgent,
                 clientId: invite.clientId,
                 clientSecret: invite.clientSecret,
                 redirectUri: invite.redirectUri,
@@ -301,32 +321,39 @@ const webClient = async (options: OperatorConfig) => {
             });
             // @ts-ignore
             const user = await client.getMe();
+            const userName = `u/${user.name}`;
             // @ts-ignore
             await webCache.del(`invite:${req.session.inviteId}`);
             let data: any = {
                 accessToken: client.accessToken,
                 refreshToken: client.refreshToken,
+                userName,
             };
             if(invite.instance !== undefined) {
                 const bot = cmInstances.find(x => x.friendly === invite.instance);
                 if(bot !== undefined) {
                     const botPayload: any = {
+                        overwrite: invite.overwrite === true,
+                        name: userName,
                         credentials: {
-                            accessToken: client.accessToken,
-                            refreshToken: client.refreshToken,
-                            clientId: invite.clientId,
-                            clientSecret: invite.clientSecret,
+                            reddit: {
+                                accessToken: client.accessToken,
+                                refreshToken: client.refreshToken,
+                                clientId: invite.clientId,
+                                clientSecret: invite.clientSecret,
+                            }
                         }
                     };
-                    if(invite.subreddit !== undefined) {
-                        botPayload.subreddits =  {names: [invite.subreddit]};
+                    if(invite.subreddits !== undefined && invite.subreddits.length > 0) {
+                        botPayload.subreddits =  {names: invite.subreddits};
                     }
                     const botAddResult: any = await addBot(bot, {name: invite.creator}, botPayload);
-                    let msg = botAddResult.success ? 'Bot successfully added to running instance' : 'An error occurred while adding the bot to the instance';
-                    if(botAddResult.success) {
-                        msg = `${msg}. ${botAddResult.stored === false ? 'Additionally, the bot was not stored in config so the operator will need to add it manually to persist after a restart.' : ''}`;
-                    }
-                    data.addResult = msg;
+                    // stored
+                    // success
+                    data = {...data, ...botAddResult};
+                    // @ts-ignore
+                    req.session.destroy();
+                    req.logout();
                 }
             }
             return res.render('callback', data);
@@ -345,7 +372,15 @@ const webClient = async (options: OperatorConfig) => {
                 if(e !== undefined) {
                     return res.render('error', {error: err});
                 }
-                return res.redirect('/');
+                // @ts-ignore
+                const useCloseRedir: boolean = req.session.closeOnSuccess as any
+                // @ts-ignore
+                delete req.session.closeOnSuccess;
+                if(useCloseRedir === true) {
+                    return res.render('close');
+                } else {
+                    return res.redirect('/');
+                }
             });
         })(req, res, next);
     });
@@ -360,12 +395,13 @@ const webClient = async (options: OperatorConfig) => {
     let token = randomId();
     interface InviteData {
         permissions: string[],
-        subreddit?: string,
+        subreddits?: string,
         instance?: string,
         clientId: string
         clientSecret: string
         redirectUri: string
         creator: string
+        overwrite?: boolean
     }
 
     const helperAuthed = async (req: express.Request, res: express.Response, next: Function) => {
@@ -381,7 +417,7 @@ const webClient = async (options: OperatorConfig) => {
                     '<div>or as an argument: <span class="font-mono">--operator YourRedditUsername</span></div>'});
         }
         // or if there is an operator and current user is operator
-        if(req.user.isOperator) {
+        if(req.user?.clientData?.webOperator) {
             return next();
         } else {
             return res.render('error', {error: 'You must be an <b>Operator</b> to access this route.'});
@@ -393,7 +429,8 @@ const webClient = async (options: OperatorConfig) => {
             redirectUri,
             clientId,
             clientSecret,
-            token: req.isAuthenticated() && req.user.isOperator ? token : undefined
+            token: req.isAuthenticated() && req.user?.clientData?.webOperator ? token : undefined,
+            instances: cmInstances.filter(x => req.user?.isInstanceOperator(x)).map(x => x.friendly),
         });
     });
 
@@ -421,7 +458,7 @@ const webClient = async (options: OperatorConfig) => {
             clientSecret: ce,
             redirect: redir,
             instance,
-            subreddit,
+            subreddits,
             code,
         } = req.body as any;
 
@@ -446,7 +483,7 @@ const webClient = async (options: OperatorConfig) => {
             clientSecret: (ce || clientSecret).trim(),
             redirectUri: redir.trim(),
             instance,
-            subreddit,
+            subreddits: subreddits.trim() === '' ? [] : subreddits.split(',').map((x: string) => parseRedditEntity(x).name),
             creator: (req.user as Express.User).name,
         }, {ttl: invitesMaxAge * 1000});
         return res.send(inviteId);
@@ -491,6 +528,8 @@ const webClient = async (options: OperatorConfig) => {
 
     const cmInstances: CMInstance[] = [];
     let init = false;
+    const formatter = defaultFormat();
+    const formatTransform = formatter.transform as (info: TransformableInfo, opts?: any) => TransformableInfo;
 
     let server: http.Server,
         io: SocketServer;
@@ -531,7 +570,9 @@ const webClient = async (options: OperatorConfig) => {
                             limit: sessionData.limit,
                             sort: sessionData.sort,
                             level: sessionData.level,
-                            stream: true
+                            stream: true,
+                            streamObjects: true,
+                            formatted: false,
                         }
                     });
 
@@ -555,8 +596,24 @@ const webClient = async (options: OperatorConfig) => {
                         }
                     });
 
+
                     delim.on('data', (c: any) => {
-                        io.to(sessionId).emit('log', formatLogLineToHtml(c.toString()));
+                        const logObj = JSON.parse(c) as LogInfo;
+                        let subredditMessage;
+                        let allMessage;
+                        if(logObj.subreddit !== undefined) {
+                            const {subreddit, bot, ...rest} = logObj
+                            // @ts-ignore
+                            subredditMessage = formatLogLineToHtml(formatter.transform(rest)[MESSAGE], rest.timestamp);
+                        }
+                        if(logObj.bot !== undefined) {
+                            const {bot, ...rest} = logObj
+                            // @ts-ignore
+                            allMessage = formatLogLineToHtml(formatter.transform(rest)[MESSAGE], rest.timestamp);
+                        }
+                        // @ts-ignore
+                        let formattedMessage = formatLogLineToHtml(formatter.transform(logObj)[MESSAGE], logObj.timestamp);
+                        io.to(sessionId).emit('log', {...logObj, subredditMessage, allMessage, formattedMessage});
                     });
 
                     gotStream.once('retry', retryFn);
@@ -574,9 +631,7 @@ const webClient = async (options: OperatorConfig) => {
         server = await app.listen(port);
         io = new SocketServer(server);
     } catch (err: any) {
-        logger.error('Error occurred while initializing web or socket.io server', err);
-        err.logged = true;
-        throw err;
+        throw new ErrorWithCause('[Web] Error occurred while initializing web or socket.io server', {cause: err});
     }
     logger.info(`Web UI started: http://localhost:${port}`, {label: ['Web']});
 
@@ -590,20 +645,16 @@ const webClient = async (options: OperatorConfig) => {
             return res.status(404).render('error', {error: msg});
         }
 
-        const user = req.user as Express.User;
-
-        const isOperator = instance.operators.includes(user.name);
-        const canAccessBot = isOperator || intersect(user.subreddits, instance.subreddits).length > 0;
-        if (!user.isOperator && !canAccessBot) {
+        if (!req.user?.clientData?.webOperator && !req.user?.canAccessInstance(instance)) {
             return res.status(404).render('error', {error: msg});
         }
 
-        if (req.params.subreddit !== undefined && !isOperator && !user.subreddits.includes(req.params.subreddit)) {
+        if (req.params.subreddit !== undefined && !req.user?.isInstanceOperator(instance) && !req.user?.subreddits.includes(req.params.subreddit)) {
             return res.status(404).render('error', {error: msg});
         }
         req.instance = instance;
         req.session.botId = instance.friendly;
-        if(canAccessBot) {
+        if(req.user?.canAccessInstance(instance)) {
             req.session.authBotId = instance.friendly;
         }
         return next();
@@ -628,15 +679,11 @@ const webClient = async (options: OperatorConfig) => {
             return res.status(404).render('error', {error: msg});
         }
 
-        const user = req.user as Express.User;
-
-        const isOperator = instance.operators.includes(user.name);
-        const canAccessBot = isOperator || intersect(user.subreddits, botInstance.subreddits).length > 0;
-        if (!user.isOperator && !canAccessBot) {
+        if (!req.user?.clientData?.webOperator && !req.user?.canAccessBot(botInstance)) {
             return res.status(404).render('error', {error: msg});
         }
 
-        if (req.params.subreddit !== undefined && !isOperator && !user.subreddits.includes(req.params.subreddit)) {
+        if (req.params.subreddit !== undefined && !req.user?.isInstanceOperator(instance) && !req.user?.subreddits.includes(req.params.subreddit)) {
             return res.status(404).render('error', {error: msg});
         }
         req.bot = botInstance;
@@ -736,12 +783,12 @@ const webClient = async (options: OperatorConfig) => {
         const level = req.session.level;
 
         const shownInstances = cmInstances.reduce((acc: CMInstance[], curr) => {
-            const isBotOperator = curr.operators.map(x => x.toLowerCase()).includes(user.name.toLowerCase());
-            if(user.isOperator) {
+            const isBotOperator = req.user?.isInstanceOperator(curr);
+            if(user?.clientData?.webOperator) {
                 // @ts-ignore
                 return acc.concat({...curr, canAccessLocation: true, isOperator: isBotOperator});
             }
-            if(!isBotOperator && intersect(user.subreddits, curr.subreddits).length === 0) {
+            if(!isBotOperator && !req.user?.canAccessInstance(curr)) {
                 return acc;
             }
             // @ts-ignore
@@ -768,7 +815,7 @@ const webClient = async (options: OperatorConfig) => {
             return res.render('offline', {
                 instances: shownInstances,
                 instanceId: (req.instance as CMInstance).friendly,
-                isOperator: instance.operators.includes((req.user as Express.User).name),
+                isOperator: req.user?.isInstanceOperator(instance),
                 // @ts-ignore
                 logs: filterLogBySubreddit(instanceLogMap, [instance.friendly], {limit, sort, level, allLogName: 'web', allLogsParser: parseInstanceLogInfoName }).get(instance.friendly),
                 logSettings: {
@@ -796,12 +843,39 @@ const webClient = async (options: OperatorConfig) => {
         //     return acc.concat({...curr, isOperator: instanceOperator});
         // },[]);
 
+        const isOp = req.user?.isInstanceOperator(instance);
+
         res.render('status', {
             instances: shownInstances,
-            bots: resp.bots,
+            bots: resp.bots.map((x: BotStatusResponse) => {
+                const {subreddits = []} = x;
+                const subredditsWithSimpleLogs = subreddits.map(y => {
+                    let transformedLogs: string[];
+                    if(y.name === 'All') {
+                        // only need to remove bot name here
+                        transformedLogs = (y.logs as LogInfo[]).map((z: LogInfo) => {
+                           const {bot, ...rest} = z;
+                           // @ts-ignore
+                           return formatLogLineToHtml(formatter.transform(rest)[MESSAGE] as string, rest.timestamp);
+                        });
+                    } else {
+                        transformedLogs = (y.logs as LogInfo[]).map((z: LogInfo) => {
+                            const {bot, subreddit, ...rest} = z;
+                            // @ts-ignore
+                            return formatLogLineToHtml(formatter.transform(rest)[MESSAGE] as string, rest.timestamp);
+                        });
+                    }
+                    y.logs = transformedLogs;
+                    return y;
+                });
+                return {...x, subreddits: subredditsWithSimpleLogs};
+            }),
             botId: (req.instance as CMInstance).friendly,
             instanceId: (req.instance as CMInstance).friendly,
-            isOperator: instance.operators.includes((req.user as Express.User).name),
+            isOperator: isOp,
+            system: isOp ? {
+                logs: resp.system.logs,
+                } : undefined,
             operators: instance.operators.join(', '),
             operatorDisplay: instance.operatorDisplay,
             logSettings: {
@@ -812,27 +886,60 @@ const webClient = async (options: OperatorConfig) => {
         });
     });
 
-    app.getAsync('/config', async (req: express.Request, res: express.Response) => {
+    app.getAsync('/bot/invites', defaultSession, async (req: express.Request, res: express.Response) => {
+        res.render('modInvites', {
+            title: `Pending Moderation Invites`,
+        });
+    });
+
+    app.getAsync('/config', defaultSession, async (req: express.Request, res: express.Response) => {
         const {format = 'json'} = req.query as any;
         res.render('config', {
             title: `Configuration Editor`,
             format,
+            canSave: req.user?.clientData?.scope?.includes('wikiedit') && req.user?.clientData?.tokenExpiresAt !== undefined && dayjs.unix(req.user?.clientData.tokenExpiresAt).isAfter(dayjs())
         });
     });
 
-    app.getAsync('/config/content', [ensureAuthenticatedApi, defaultSession, instanceWithPermissions, botWithPermissions, createUserToken], async (req: express.Request, res: express.Response) => {
+    app.postAsync('/config', [ensureAuthenticatedApi, defaultSession, instanceWithPermissions, botWithPermissions], async (req: express.Request, res: express.Response) => {
         const {subreddit} = req.query as any;
-        const resp = await got.get(`${(req.instance as CMInstance).normalUrl}/config`, {
-            headers: {
-                'Authorization': `Bearer ${req.token}`,
-            },
-            searchParams: {
-                subreddit,
-                bot: req.bot?.botName
-            }
-        }).text();
+        const {location, data, create = false} = req.body as any;
 
-        return res.send(resp);
+        const client = new ExtendedSnoowrap({
+            userAgent,
+            clientId,
+            clientSecret,
+            accessToken: req.user?.clientData?.token
+        });
+
+        try {
+            // @ts-ignore
+            const wiki = await client.getSubreddit(subreddit).getWikiPage(location);
+            await wiki.edit({
+                text: data,
+                reason: create ? 'Created Config through CM Web' : 'Updated through CM Web'
+            });
+        } catch (err: any) {
+            res.status(500);
+            return res.send(err.message);
+        }
+
+        if(create) {
+            try {
+                // @ts-ignore
+                await client.getSubreddit(subreddit).getWikiPage(location).editSettings({
+                    permissionLevel: 2,
+                    // don't list this page on r/[subreddit]/wiki/pages
+                    listed: false,
+                });
+            } catch (err: any) {
+                res.status(500);
+                return res.send(`Successfully created wiki page for configuration but encountered error while setting visibility. You should manually set the wiki page visibility on reddit. \r\n Error: ${err.message}`);
+            }
+        }
+
+        res.status(200);
+        return res.send();
     });
 
     app.getAsync('/events', [ensureAuthenticatedApi, defaultSession, instanceWithPermissions, botWithPermissions, createUserToken], async (req: express.Request, res: express.Response) => {
@@ -955,7 +1062,7 @@ const webClient = async (options: OperatorConfig) => {
             // setup general web log event
             const webLogListener = (log: string) => {
                 const subName = parseSubredditLogName(log);
-                if((subName === undefined || user.isOperator) && isLogLineMinLevel(log, session.level as string)) {
+                if((subName === undefined || user.clientData?.webOperator === true) && isLogLineMinLevel(log, session.level as string)) {
                     io.to(session.id).emit('webLog', formatLogLineToHtml(log));
                 }
             }
@@ -1024,8 +1131,9 @@ const webClient = async (options: OperatorConfig) => {
         try {
             const token = createToken(bot, userPayload);
             const resp = await got.post(`${bot.normalUrl}/bot`, {
-                body: botPayload,
+                body: JSON.stringify(botPayload),
                 headers: {
+                    'Content-Type': 'application/json',
                     'Authorization': `Bearer ${token}`,
                 }
             }).json() as object;
@@ -1051,7 +1159,7 @@ const webClient = async (options: OperatorConfig) => {
                 if(lastCheck > 15) {
                     shouldCheck = true;
                 }
-            } else if(lastCheck > 300) {
+            } else if(lastCheck > 60) {
                 shouldCheck = true;
             }
         }
@@ -1086,7 +1194,9 @@ const webClient = async (options: OperatorConfig) => {
                     }
                 }).json() as CMInstance;
 
-                botStat = {...botStat, ...resp, online: true};
+                const {bots, ...restResp} = resp;
+
+                botStat = {...botStat, ...restResp, bots: bots.map(x => ({...x, instance: botStat})), online: true};
                 const sameNameIndex = cmInstances.findIndex(x => x.friendly === botStat.friendly);
                 if(sameNameIndex > -1 && sameNameIndex !== existingClientIndex) {
                     logger.warn(`Client returned a friendly name that is not unique (${botStat.friendly}), will fallback to host as friendly (${botStat.normalUrl})`);