From 63696b746e9fa200972522dd4232f97eb3046383 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 4 Feb 2022 15:06:21 -0500 Subject: [PATCH] feat(flow): Implement basic flow control structures #73 * Add Run and postCheckBehavior config structures to schema and interfaces * Implement parsing from config and initial flow logic for running on activities in manager --- src/Check/index.ts | 14 +- src/Common/interfaces.ts | 46 +++ src/ConfigBuilder.ts | 115 ++++--- src/JsonConfig.ts | 11 +- src/Run/index.ts | 107 ++++++ src/Schema/App.json | 221 ++++++++++++- src/Schema/OperatorConfig.json | 48 +++ src/Subreddit/Manager.ts | 312 +++++++++++++----- .../routes/authenticated/user/action.ts | 2 - .../Server/routes/authenticated/user/index.ts | 2 +- src/index.ts | 5 +- src/util.ts | 2 +- 12 files changed, 739 insertions(+), 146 deletions(-) create mode 100644 src/Run/index.ts diff --git a/src/Check/index.ts b/src/Check/index.ts index f59630d..ae9dd2d 100644 --- a/src/Check/index.ts +++ b/src/Check/index.ts @@ -16,11 +16,11 @@ import { truncateStringToLength } from "../util"; import { - ActionResult, + ActionResult, ActivityType, ChecksActivityState, CommentState, JoinCondition, - JoinOperands, + JoinOperands, PostBehavior, PostBehaviorTypes, SubmissionState, TypedActivityStates, UserResultCache } from "../Common/interfaces"; @@ -51,6 +51,8 @@ export abstract class Check implements ICheck { notifyOnTrigger: boolean; resources: SubredditResources; client: ExtendedSnoowrap; + postTrigger: PostBehaviorTypes; + postFail: PostBehaviorTypes; constructor(options: CheckOptions) { const { @@ -65,6 +67,8 @@ export abstract class Check implements ICheck { notifyOnTrigger = false, subredditName, cacheUserResult = {}, + postTrigger = 'nextRun', + postFail = 'next', itemIs = [], authorIs: { include = [], @@ -93,6 +97,8 @@ export abstract class Check implements ICheck { exclude: exclude.map(x => new Author(x)), include: include.map(x => new Author(x)), } + this.postTrigger = postTrigger; + this.postFail = postFail; this.cacheUserResult = { ...userResultCacheDefault, ...cacheUserResult @@ -279,7 +285,7 @@ export abstract class Check implements ICheck { } } -export interface ICheck extends JoinCondition, ChecksActivityState { +export interface ICheck extends JoinCondition, ChecksActivityState, PostBehavior { /** * Friendly name for this Check EX "crosspostSpamCheck" * @@ -339,7 +345,7 @@ export interface CheckJson extends ICheck { * The type of event (new submission or new comment) this check should be run against * @examples ["submission", "comment"] */ - kind: 'submission' | 'comment' + kind: ActivityType /** * A list of Rules to run. * diff --git a/src/Common/interfaces.ts b/src/Common/interfaces.ts index 10ff9e9..434a912 100644 --- a/src/Common/interfaces.ts +++ b/src/Common/interfaces.ts @@ -16,6 +16,7 @@ import {JsonOperatorConfigDocument, YamlOperatorConfigDocument} from "./Config/O import {ConsoleTransportOptions} from "winston/lib/winston/transports"; import {DailyRotateFileTransportOptions} from "winston-daily-rotate-file"; import {DuplexTransportOptions} from "winston-duplex/dist/DuplexTransport"; +import {CommentCheckJson, SubmissionCheckJson} from "../Check"; /** * An ISO 8601 Duration @@ -836,6 +837,16 @@ export interface ManagerOptions { * Default behavior is to exclude all mods and automoderator from checks * */ filterCriteriaDefaults?: FilterCriteriaDefaults + + /** + * Set the default post-check behavior for all checks. If this property is specified it will override any defaults passed from the bot's config + * + * Default behavior is: + * + * * postFail => next + * * postTrigger => nextRun + * */ + postCheckBehaviorDefaults?: PostBehavior } /** @@ -1476,6 +1487,8 @@ export interface BotInstanceJsonConfig { * */ filterCriteriaDefaults?: FilterCriteriaDefaults + postCheckBehaviorDefaults?: PostBehavior + /** * Settings related to bot behavior for subreddits it is managing * */ @@ -2153,3 +2166,36 @@ export interface TextMatchOptions { **/ caseSensitive?: boolean } + +export type ActivityCheckJson = SubmissionCheckJson | CommentCheckJson; + +export type GotoPath = `goto:${string}`; +/** + * The possible behaviors that can occur after a check has run + * + * * next => continue to next Check/Run + * * stop => stop CM lifecycle for this activity (immediately end) + * * nextRun => skip any remaining Checks in this Run and start the next Run + * * goto:[path] => specify a run[.check] to jump to + * + * */ +export type PostBehaviorTypes = 'next' | 'stop' | 'nextRun' | GotoPath; + +export interface PostBehavior { + /** + * Do this behavior if a Check is triggered + * + * @default nextRun + * @example ["nextRun"] + * */ + postTrigger?: PostBehaviorTypes + /** + * Do this behavior if a Check is NOT triggered + * + * @default next + * @example ["next"] + * */ + postFail?: PostBehaviorTypes +} + +export type ActivityType = 'submission' | 'comment'; diff --git a/src/ConfigBuilder.ts b/src/ConfigBuilder.ts index 614a040..1cb1884 100644 --- a/src/ConfigBuilder.ts +++ b/src/ConfigBuilder.ts @@ -35,7 +35,7 @@ import { RedditCredentials, BotCredentialsJsonConfig, BotCredentialsConfig, - FilterCriteriaDefaults, TypedActivityStates, OperatorFileConfig + FilterCriteriaDefaults, TypedActivityStates, OperatorFileConfig, PostBehavior } from "./Common/interfaces"; import {isRuleSetJSON, RuleSetJson, RuleSetObjectJson} from "./Rule/RuleSet"; import deepEqual from "fast-deep-equal"; @@ -59,6 +59,7 @@ import {ConfigDocumentInterface} from "./Common/Config/AbstractConfigDocument"; import {Document as YamlDocument} from "yaml"; import {SimpleError} from "./Utils/Errors"; import {ErrorWithCause} from "pony-cause"; +import {RunStructuredJson} from "./Run"; export interface ConfigBuilderOptions { logger: Logger, @@ -130,55 +131,79 @@ export class ConfigBuilder { return validConfig as JSONConfig; } - parseToStructured(config: JSONConfig, filterCriteriaDefaultsFromBot?: FilterCriteriaDefaults): CheckStructuredJson[] { + parseToStructured(config: JSONConfig, filterCriteriaDefaultsFromBot?: FilterCriteriaDefaults, postCheckBehaviorDefaultsFromBot: PostBehavior = {}): RunStructuredJson[] { let namedRules: Map = new Map(); let namedActions: Map = new Map(); - const {checks = [], filterCriteriaDefaults} = config; - for (const c of checks) { - const {rules = []} = c; - namedRules = extractNamedRules(rules, namedRules); - namedActions = extractNamedActions(c.actions, namedActions); + const {checks = [], runs = [], filterCriteriaDefaults, postCheckBehaviorDefaults} = config; + + if(checks.length > 0 && runs.length > 0) { + // cannot have both checks and runs at top-level + throw new Error(`Subreddit configuration cannot contain both 'checks' and 'runs' at top-level.`); } - const filterDefs = filterCriteriaDefaults ?? filterCriteriaDefaultsFromBot; - const { - authorIsBehavior = 'merge', - itemIsBehavior = 'merge', - authorIs: authorIsDefault = {}, - itemIs: itemIsDefault = [] - } = filterDefs || {}; - - const structuredChecks: CheckStructuredJson[] = []; - for (const c of checks) { - const {rules = [], authorIs = {}, itemIs = []} = c; - const strongRules = insertNamedRules(rules, namedRules); - const strongActions = insertNamedActions(c.actions, namedActions); - - let derivedAuthorIs: AuthorOptions = authorIsDefault; - if (authorIsBehavior === 'merge') { - derivedAuthorIs = merge.all([authorIs, authorIsDefault], {arrayMerge: removeFromSourceIfKeysExistsInDestination}); - } else if (Object.keys(authorIs).length > 0) { - derivedAuthorIs = authorIs; - } - - let derivedItemIs: TypedActivityStates = itemIsDefault; - if (itemIsBehavior === 'merge') { - derivedItemIs = [...itemIs, ...itemIsDefault]; - } else if (itemIs.length > 0) { - derivedItemIs = itemIs; - } - - const strongCheck = { - ...c, - authorIs: derivedAuthorIs, - itemIs: derivedItemIs, - rules: strongRules, - actions: strongActions - } as CheckStructuredJson; - structuredChecks.push(strongCheck); + const realRuns = runs; + if(checks.length > 0) { + realRuns.push({name: 'Run1', checks: checks}); } - return structuredChecks; + for(const r of realRuns) { + for (const c of r.checks) { + const {rules = []} = c; + namedRules = extractNamedRules(rules, namedRules); + namedActions = extractNamedActions(c.actions, namedActions); + } + } + + const structuredRuns: RunStructuredJson[] = []; + + for(const r of realRuns) { + + const {filterCriteriaDefaults: filterCriteriaDefaultsFromRun, postFail, postTrigger } = r; + + const filterDefs = filterCriteriaDefaultsFromRun ?? (filterCriteriaDefaults ?? filterCriteriaDefaultsFromBot); + const { + authorIsBehavior = 'merge', + itemIsBehavior = 'merge', + authorIs: authorIsDefault = {}, + itemIs: itemIsDefault = [] + } = filterDefs || {}; + + const structuredChecks: CheckStructuredJson[] = []; + for (const c of r.checks) { + const {rules = [], authorIs = {}, itemIs = []} = c; + const strongRules = insertNamedRules(rules, namedRules); + const strongActions = insertNamedActions(c.actions, namedActions); + + let derivedAuthorIs: AuthorOptions = authorIsDefault; + if (authorIsBehavior === 'merge') { + derivedAuthorIs = merge.all([authorIs, authorIsDefault], {arrayMerge: removeFromSourceIfKeysExistsInDestination}); + } else if (Object.keys(authorIs).length > 0) { + derivedAuthorIs = authorIs; + } + + let derivedItemIs: TypedActivityStates = itemIsDefault; + if (itemIsBehavior === 'merge') { + derivedItemIs = [...itemIs, ...itemIsDefault]; + } else if (itemIs.length > 0) { + derivedItemIs = itemIs; + } + + const postCheckBehaviors = Object.assign({}, postCheckBehaviorDefaultsFromBot, removeUndefinedKeys({postFail, postTrigger})); + + const strongCheck = { + ...c, + authorIs: derivedAuthorIs, + itemIs: derivedItemIs, + rules: strongRules, + actions: strongActions, + ...postCheckBehaviors + } as CheckStructuredJson; + structuredChecks.push(strongCheck); + } + structuredRuns.push({...r, checks: structuredChecks}); + } + + return structuredRuns; } } @@ -840,6 +865,7 @@ export const buildBotConfig = (data: BotInstanceJsonConfig, opConfig: OperatorCo const { name: botName, filterCriteriaDefaults = filterCriteriaDefault, + postCheckBehaviorDefaults, polling: { sharedMod, shared = [], @@ -967,6 +993,7 @@ export const buildBotConfig = (data: BotInstanceJsonConfig, opConfig: OperatorCo name: botName, snoowrap: snoowrap || {}, filterCriteriaDefaults, + postCheckBehaviorDefaults, subreddits: { names, exclude, diff --git a/src/JsonConfig.ts b/src/JsonConfig.ts index 2208e24..3391210 100644 --- a/src/JsonConfig.ts +++ b/src/JsonConfig.ts @@ -1,5 +1,6 @@ import {CheckJson, CommentCheckJson, SubmissionCheckJson} from "./Check"; -import {ManagerOptions} from "./Common/interfaces"; +import {ActivityCheckJson, ManagerOptions} from "./Common/interfaces"; +import {RunJson} from "./Run"; export interface JSONConfig extends ManagerOptions { /** @@ -12,5 +13,11 @@ export interface JSONConfig extends ManagerOptions { * When a check "passes", and actions are performed, then all subsequent checks are skipped. * @minItems 1 * */ - checks: Array + checks?: ActivityCheckJson[] + + /** + * A list of sets of Checks to run + * @minItems 1 + * */ + runs?: RunJson[] } diff --git a/src/Run/index.ts b/src/Run/index.ts new file mode 100644 index 0000000..06b097c --- /dev/null +++ b/src/Run/index.ts @@ -0,0 +1,107 @@ +import {Check, CheckStructuredJson} from "../Check"; +import {ActivityCheckJson, FilterCriteriaDefaults, PostBehavior, PostBehaviorTypes} from "../Common/interfaces"; +import {SubmissionCheck} from "../Check/SubmissionCheck"; +import {CommentCheck} from "../Check/CommentCheck"; +import {Logger} from "winston"; +import {mergeArr} from "../util"; +import {SubredditResources} from "../Subreddit/SubredditResources"; +import {ExtendedSnoowrap} from "../Utils/SnoowrapClients"; + +export class Run { + name: string; + submissionChecks: SubmissionCheck[] = []; + commentChecks: CommentCheck[] = []; + postFail?: PostBehaviorTypes; + postTrigger?: PostBehaviorTypes; + filterCriteriaDefaults?: FilterCriteriaDefaults + logger: Logger; + client: ExtendedSnoowrap; + subreddtName: string; + resources: SubredditResources; + dryRun?: boolean; + + constructor(options: RunOptions) { + const { + name, + checks = [], + + postFail, + postTrigger, + filterCriteriaDefaults, + logger, + resources, + client, + subredditName, + dryRun, + } = options; + this.name = name; + this.logger = logger.child({labels: [`RUN ${name}`]}, mergeArr); + this.resources = resources; + this.client = client; + this.subreddtName = subredditName; + this.postFail = postFail; + this.postTrigger = postTrigger; + this.filterCriteriaDefaults = filterCriteriaDefaults; + this.dryRun = dryRun; + + for(const c of checks) { + const checkConfig = { + ...c, + dryRun: this.dryRun || c.dryRun, + logger: this.logger, + subredditName: this.subreddtName, + resources: this.resources, + client: this.client, + }; + if (c.kind === 'comment') { + this.commentChecks.push(new CommentCheck(checkConfig)); + } else if (c.kind === 'submission') { + this.submissionChecks.push(new SubmissionCheck(checkConfig)); + } + } + } +} + +export interface IRun extends PostBehavior { + /** + * Friendly name for this Run EX "flairsRun" + * + * Can only contain letters, numbers, underscore, spaces, and dashes + * + * @pattern ^[a-zA-Z]([\w -]*[\w])?$ + * @examples ["myNewRun"] + * */ + name?: string + /** + * Set the default filter criteria for all checks. If this property is specified it will override any defaults passed from the bot's config + * + * Default behavior is to exclude all mods and automoderator from checks + * */ + filterCriteriaDefaults?: FilterCriteriaDefaults + + /** + * Use this option to override the `dryRun` setting for all Actions of all Checks in this Run + * + * @examples [false, true] + * */ + dryRun?: boolean; +} + +export interface RunOptions extends IRun { + // submissionChecks?: SubmissionCheck[] + // commentChecks?: CommentCheck[] + checks: CheckStructuredJson[] + name: string + logger: Logger + resources: SubredditResources + client: ExtendedSnoowrap + subredditName: string; +} + +export interface RunJson extends IRun { + checks: ActivityCheckJson[] +} + +export interface RunStructuredJson extends RunJson { + checks: CheckStructuredJson[] +} diff --git a/src/Schema/App.json b/src/Schema/App.json index 81f7601..bc622c2 100644 --- a/src/Schema/App.json +++ b/src/Schema/App.json @@ -1274,6 +1274,46 @@ "description": "If notifications are configured and this is `true` then an `eventActioned` event will be sent when this check is triggered.", "type": "boolean" }, + "postFail": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "enum": [ + "next", + "nextRun", + "stop" + ], + "type": "string" + } + ], + "default": "next", + "description": "Do this behavior if a Check is NOT triggered" + }, + "postTrigger": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "enum": [ + "next", + "nextRun", + "stop" + ], + "type": "string" + } + ], + "default": "nextRun", + "description": "Do this behavior if a Check is triggered" + }, "rules": { "description": "A list of Rules to run.\n\nIf `Rule` objects are triggered based on `condition` then `actions` will be performed.\n\nCan be `Rule`, `RuleSet`, or the `name` of any **named** `Rule` in your subreddit's configuration.\n\n**If `rules` is an empty array or not present then `actions` are performed immediately.**", "items": { @@ -2227,6 +2267,51 @@ ], "type": "object" }, + "PostBehavior": { + "properties": { + "postFail": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "enum": [ + "next", + "nextRun", + "stop" + ], + "type": "string" + } + ], + "default": "next", + "description": "Do this behavior if a Check is NOT triggered" + }, + "postTrigger": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "enum": [ + "next", + "nextRun", + "stop" + ], + "type": "string" + } + ], + "default": "nextRun", + "description": "Do this behavior if a Check is triggered" + } + }, + "type": "object" + }, "RecentActivityRuleJSONConfig": { "description": "Checks a user's history for any Activity (Submission/Comment) in the subreddits specified in thresholds\n\nAvailable data for [Action templating](https://github.com/FoxxMD/context-mod#action-templating):\n\n```\nsummary => comma-deliminated list of subreddits that hit the threshold and their count EX subredditA(1), subredditB(4),...\nsubCount => Total number of subreddits that hit the threshold\ntotalCount => Total number of all activity occurrences in subreddits\n```", "properties": { @@ -3181,6 +3266,87 @@ ], "type": "object" }, + "RunJson": { + "properties": { + "checks": { + "items": { + "anyOf": [ + { + "$ref": "#/definitions/SubmissionCheckJson" + }, + { + "$ref": "#/definitions/CommentCheckJson" + } + ] + }, + "type": "array" + }, + "dryRun": { + "description": "Use this option to override the `dryRun` setting for all Actions of all Checks in this Run", + "examples": [ + false, + true + ], + "type": "boolean" + }, + "filterCriteriaDefaults": { + "$ref": "#/definitions/FilterCriteriaDefaults", + "description": "Set the default filter criteria for all checks. If this property is specified it will override any defaults passed from the bot's config\n\nDefault behavior is to exclude all mods and automoderator from checks" + }, + "name": { + "description": "Friendly name for this Run EX \"flairsRun\"\n\nCan only contain letters, numbers, underscore, spaces, and dashes", + "examples": [ + "myNewRun" + ], + "pattern": "^[a-zA-Z]([\\w -]*[\\w])?$", + "type": "string" + }, + "postFail": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "enum": [ + "next", + "nextRun", + "stop" + ], + "type": "string" + } + ], + "default": "next", + "description": "Do this behavior if a Check is NOT triggered" + }, + "postTrigger": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "enum": [ + "next", + "nextRun", + "stop" + ], + "type": "string" + } + ], + "default": "nextRun", + "description": "Do this behavior if a Check is triggered" + } + }, + "required": [ + "checks" + ], + "type": "object" + }, "SearchAndReplaceRegExp": { "properties": { "replace": { @@ -3426,6 +3592,46 @@ "description": "If notifications are configured and this is `true` then an `eventActioned` event will be sent when this check is triggered.", "type": "boolean" }, + "postFail": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "enum": [ + "next", + "nextRun", + "stop" + ], + "type": "string" + } + ], + "default": "next", + "description": "Do this behavior if a Check is NOT triggered" + }, + "postTrigger": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "enum": [ + "next", + "nextRun", + "stop" + ], + "type": "string" + } + ], + "default": "nextRun", + "description": "Do this behavior if a Check is triggered" + }, "rules": { "description": "A list of Rules to run.\n\nIf `Rule` objects are triggered based on `condition` then `actions` will be performed.\n\nCan be `Rule`, `RuleSet`, or the `name` of any **named** `Rule` in your subreddit's configuration.\n\n**If `rules` is an empty array or not present then `actions` are performed immediately.**", "items": { @@ -3971,6 +4177,10 @@ }, "type": "array" }, + "postCheckBehaviorDefaults": { + "$ref": "#/definitions/PostBehavior", + "description": "Set the default post-check behavior for all checks. If this property is specified it will override any defaults passed from the bot's config\n\nDefault behavior is:\n\n* postFail => next\n* postTrigger => nextRun" + }, "queue": { "properties": { "maxWorkers": { @@ -3984,11 +4194,16 @@ } }, "type": "object" + }, + "runs": { + "description": "A list of sets of Checks to run", + "items": { + "$ref": "#/definitions/RunJson" + }, + "minItems": 1, + "type": "array" } }, - "required": [ - "checks" - ], "type": "object" } diff --git a/src/Schema/OperatorConfig.json b/src/Schema/OperatorConfig.json index ee27ceb..b369991 100644 --- a/src/Schema/OperatorConfig.json +++ b/src/Schema/OperatorConfig.json @@ -319,6 +319,9 @@ ], "description": "Settings related to default polling configurations for subreddits" }, + "postCheckBehaviorDefaults": { + "$ref": "#/definitions/PostBehavior" + }, "queue": { "description": "Settings related to default configurations for queue behavior for subreddits", "properties": { @@ -1033,6 +1036,51 @@ }, "type": "object" }, + "PostBehavior": { + "properties": { + "postFail": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "enum": [ + "next", + "nextRun", + "stop" + ], + "type": "string" + } + ], + "default": "next", + "description": "Do this behavior if a Check is NOT triggered" + }, + "postTrigger": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "enum": [ + "next", + "nextRun", + "stop" + ], + "type": "string" + } + ], + "default": "nextRun", + "description": "Do this behavior if a Check is triggered" + } + }, + "type": "object" + }, "RedditCredentials": { "description": "Credentials required for the bot to interact with Reddit's API\n\nThese credentials will provided to both the API and Web interface unless otherwise specified with the `web.credentials` property\n\nRefer to the [required credentials table](https://github.com/FoxxMD/context-mod/blob/master/docs/operatorConfiguration.md#minimum-required-configuration) to see what is necessary to run the bot.", "examples": [ diff --git a/src/Subreddit/Manager.ts b/src/Subreddit/Manager.ts index f5b5c25..9df9b7e 100644 --- a/src/Subreddit/Manager.ts +++ b/src/Subreddit/Manager.ts @@ -3,13 +3,14 @@ import {Logger} from "winston"; import {SubmissionCheck} from "../Check/SubmissionCheck"; import {CommentCheck} from "../Check/CommentCheck"; import { + asSubmission, cacheStats, createHistoricalStatsDisplay, createRetryHandler, determineNewResults, findLastIndex, - formatNumber, likelyJson5, - mergeArr, + formatNumber, isSubmission, likelyJson5, + mergeArr, normalizeName, parseFromJsonOrYamlToObject, parseRedditEntity, pollingInfo, @@ -26,7 +27,7 @@ import { DEFAULT_POLLING_INTERVAL, DEFAULT_POLLING_LIMIT, FilterCriteriaDefaults, Invokee, ManagerOptions, ManagerStateChangeOption, ManagerStats, PAUSED, - PollingOptionsStrong, PollOn, RUNNING, RunState, STOPPED, SYSTEM, USER + PollingOptionsStrong, PollOn, PostBehavior, PostBehaviorTypes, RUNNING, RunState, STOPPED, SYSTEM, USER } from "../Common/interfaces"; import Submission from "snoowrap/dist/objects/Submission"; import {activityIsRemoved, itemContentPeek} from "../Utils/SnoowrapUtils"; @@ -44,12 +45,14 @@ import dayjs, {Dayjs as DayjsObj} from "dayjs"; import Action from "../Action"; import {queue, QueueObject} from 'async'; import {JSONConfig} from "../JsonConfig"; -import {CheckStructuredJson} from "../Check"; +import {Check, CheckStructuredJson} from "../Check"; import NotificationManager from "../Notification/NotificationManager"; import {createHistoricalDefaults, historicalDefaults} from "../Common/defaults"; import {ExtendedSnoowrap} from "../Utils/SnoowrapClients"; import {CMError, isRateLimitError, isStatusError} from "../Utils/Errors"; import {ErrorWithCause} from "pony-cause"; +import {Run} from "../Run"; +import got from "got"; export interface RunningState { state: RunState, @@ -65,7 +68,6 @@ export interface runCheckOptions { } export interface CheckTask { - checkType: ('Comment' | 'Submission'), activity: (Submission | Comment), options?: runCheckOptions } @@ -89,14 +91,20 @@ export class Manager extends EventEmitter { logger: Logger; botName: string; pollOptions: PollingOptionsStrong[] = []; - submissionChecks!: SubmissionCheck[]; - commentChecks!: CommentCheck[]; + get submissionChecks() { + return this.runs.map(x => x.submissionChecks).flat(); + } + get commentChecks() { + return this.runs.map(x => x.commentChecks).flat(); + } + runs: Run[] = [] resources!: SubredditResources; wikiLocation: string; lastWikiRevision?: DayjsObj lastWikiCheck: DayjsObj = dayjs(); wikiFormat: ('yaml' | 'json') = 'yaml'; filterCriteriaDefaults?: FilterCriteriaDefaults + postCheckBehaviorDefaults?: PostBehavior //wikiUpdateRunning: boolean = false; streams: Map> = new Map(); @@ -197,7 +205,7 @@ export class Manager extends EventEmitter { constructor(sub: Subreddit, client: ExtendedSnoowrap, logger: Logger, cacheManager: BotResourcesManager, opts: RuntimeManagerOptions = {botName: 'ContextMod', maxWorkers: 1}) { super(); - const {dryRun, sharedStreams = [], wikiLocation = 'botconfig/contextbot', botName, maxWorkers, filterCriteriaDefaults} = opts; + const {dryRun, sharedStreams = [], wikiLocation = 'botconfig/contextbot', botName, maxWorkers, filterCriteriaDefaults, postCheckBehaviorDefaults} = opts; this.displayLabel = opts.nickname || `${sub.display_name_prefixed}`; const getLabels = this.getCurrentLabels; const getDisplay = this.getDisplay; @@ -214,6 +222,7 @@ export class Manager extends EventEmitter { this.globalDryRun = dryRun; this.wikiLocation = wikiLocation; this.filterCriteriaDefaults = filterCriteriaDefaults; + this.postCheckBehaviorDefaults = postCheckBehaviorDefaults; this.sharedStreams = sharedStreams; this.pollingRetryHandler = createRetryHandler({maxRequestRetry: 3, maxOtherRetry: 2}, this.logger); this.subreddit = sub; @@ -341,7 +350,7 @@ export class Manager extends EventEmitter { try { const itemMeta = this.queuedItemsMeta[queuedItemIndex]; this.queuedItemsMeta.splice(queuedItemIndex, 1, {...itemMeta, state: 'processing'}); - await this.runChecks(task.checkType, task.activity, {...task.options, refresh: itemMeta.shouldRefresh}); + await this.runChecks(task.activity, {...task.options, refresh: itemMeta.shouldRefresh}); } finally { // always remove item meta regardless of success or failure since we are done with it meow this.queuedItemsMeta.splice(queuedItemIndex, 1); @@ -360,6 +369,14 @@ export class Manager extends EventEmitter { return q; } + public getCommentChecks() { + return this.runs.map(x => x.commentChecks); + } + + public getSubmissionChecks() { + return this.runs.map(x => x.commentChecks); + } + protected async parseConfigurationFromObject(configObj: object, suppressChangeEvent: boolean = false) { try { const configBuilder = new ConfigBuilder({logger: this.logger}); @@ -416,35 +433,50 @@ export class Manager extends EventEmitter { this.resources.setLogger(this.logger); this.logger.info('Subreddit-specific options updated'); - this.logger.info('Building Checks...'); + this.logger.info('Building Runs and Checks...'); - const commentChecks: Array = []; - const subChecks: Array = []; - const structuredChecks = configBuilder.parseToStructured(validJson, this.filterCriteriaDefaults); + const structuredRuns = configBuilder.parseToStructured(validJson, this.filterCriteriaDefaults, this.postCheckBehaviorDefaults); + + let runs: Run[] = []; // TODO check that bot has permissions for subreddit for all specified actions // can find permissions in this.subreddit.mod_permissions - for (const jCheck of structuredChecks) { - const checkConfig = { - ...jCheck, - dryRun: this.dryRun || jCheck.dryRun, + let index = 1; + for (const r of structuredRuns) { + const {name = `Run${index}`, ...rest} = r; + const run = new Run({ + name, + ...rest, logger: this.logger, - subredditName: this.subreddit.display_name, resources: this.resources, - client: this.client, - }; - if (jCheck.kind === 'comment') { - commentChecks.push(new CommentCheck(checkConfig)); - } else if (jCheck.kind === 'submission') { - subChecks.push(new SubmissionCheck(checkConfig)); - } + subredditName: this.subreddit.display_name, + client: this.client + }); + runs.push(run); + index++; + } + + // make sure run names are unique + const rNames: string[] = []; + for(const r of runs) { + if(rNames.includes(normalizeName(r.name))) { + throw new Error(`Rule names must be unique. Duplicate name detected: ${r.name}`); + } + rNames.push(normalizeName(r.name)); + } + + this.runs = runs; + const runSummary = `Found ${runs.length} Runs with ${this.submissionChecks.length + this.commentChecks.length} Checks`; + + if(this.runs.length === 0) { + this.logger.warn(runSummary); + } else { + this.logger.info(runSummary); } - this.submissionChecks = subChecks; - this.commentChecks = commentChecks; const checkSummary = `Found Checks -- Submission: ${this.submissionChecks.length} | Comment: ${this.commentChecks.length}`; - if (subChecks.length === 0 && commentChecks.length === 0) { + if (this.submissionChecks.length === 0 && this.commentChecks.length === 0) { this.logger.warn(checkSummary); } else { this.logger.info(checkSummary); @@ -589,8 +621,8 @@ export class Manager extends EventEmitter { } } - async runChecks(checkType: ('Comment' | 'Submission'), activity: (Submission | Comment), options?: runCheckOptions): Promise { - const checks = checkType === 'Comment' ? this.commentChecks : this.submissionChecks; + async runChecks(activity: (Submission | Comment), options?: runCheckOptions): Promise { + const checkType = isSubmission(activity) ? 'Submission' : 'Comment'; let item = activity; const itemId = await item.id; @@ -670,7 +702,7 @@ export class Manager extends EventEmitter { item = await activity.refresh(); } - if (item instanceof Submission) { + if (asSubmission(item)) { if (await item.removed_by_category === 'deleted') { this.logger.warn('Submission was deleted, cannot process.'); return; @@ -680,72 +712,180 @@ export class Manager extends EventEmitter { return; } - for (const check of checks) { - if (checkNames.length > 0 && !checkNames.map(x => x.toLowerCase()).some(x => x === check.name.toLowerCase())) { - this.logger.warn(`Check ${check.name} not in array of requested checks to run, skipping...`); + let continueRunIteration = true; + let runIndex = 0; + let gotoContext: string = ''; + while(continueRunIteration && (runIndex < this.runs.length || gotoContext !== '')) { + let currRun: Run; + if(gotoContext !== '') { + const [runName] = gotoContext.split('.'); + const gotoIndex = this.runs.findIndex(x => normalizeName(x.name) === normalizeName(runName)); + if(gotoIndex !== -1) { + if(gotoIndex > runIndex) { + this.logger.debug(`Fast forwarding Run iteration to ${this.runs[gotoIndex].name}`, {leaf: 'GOTO'}); + } else if(gotoIndex < runIndex) { + this.logger.debug(`Rewinding Run iteration to ${this.runs[gotoIndex].name}`, {leaf: 'GOTO'}); + } else { + this.logger.debug(`Did not iterate to next Run due to GOTO specifying same run`, {leaf: 'GOTO'}); + } + currRun = this.runs[gotoIndex]; + runIndex = gotoIndex; + if(!gotoContext.includes('.')) { + // goto completed, no check + gotoContext = ''; + } + } else { + throw new Error(`GOTO specified a Run that could not be found: ${runName}`); + } + } else { + currRun = this.runs[runIndex]; + } + + if(isSubmission(item)) { + if(currRun.submissionChecks.length === 0) { + currRun.logger.debug('Skipping b/c Run did not contain any submission Checks'); + continue; + } + } else if(currRun.commentChecks.length === 0) { + currRun.logger.debug('Skipping b/c Run did not contain any comment Checks'); continue; } - if(!check.enabled) { - this.logger.info(`Check ${check.name} not run because it is not enabled, skipping...`); - continue; - } - checksRunNames.push(check.name); - checksRun++; - triggered = false; - let isFromCache = false; - let currentResults: RuleResult[] = []; - try { - const [checkTriggered, checkResults, fromCache = false] = await check.runRules(item, allRuleResults); - isFromCache = fromCache; - if(!fromCache) { - await check.setCacheResult(item, {result: checkTriggered, ruleResults: checkResults}); + + const checks = isSubmission(item) ? currRun.submissionChecks : currRun.commentChecks; + let continueCheckIteration = true; + let checkIndex = 0; + while(continueCheckIteration && checkIndex < checks.length) { + let check: Check; + if(gotoContext !== '') { + const [runName, checkName] = gotoContext.split('.'); + const gotoIndex = checks.findIndex(x => normalizeName(x.name) === normalizeName(checkName)); + if(gotoIndex !== -1) { + if(gotoIndex > runIndex) { + this.logger.debug(`Fast forwarding Check iteration to ${checks[gotoIndex].name}`, {leaf: 'GOTO'}); + } else if(gotoIndex < runIndex) { + this.logger.debug(`Rewinding Check iteration to ${checks[gotoIndex].name}`, {leaf: 'GOTO'}); + } else { + this.logger.debug(`Did not iterate to next Check due to GOTO specifying same Check (you probably don't want to do this!)`, {leaf: 'GOTO'}); + } + check = checks[gotoIndex]; + checkIndex = gotoIndex; + gotoContext = ''; + } else { + throw new Error(`GOTO specified a Check that could not be found: ${checkName}`); + } } else { - cachedCheckNames.push(check.name); + check = checks[checkIndex]; } - currentResults = checkResults; - totalRulesRun += checkResults.length; - allRuleResults = allRuleResults.concat(determineNewResults(allRuleResults, checkResults)); - triggered = checkTriggered; - if(triggered && fromCache && !check.cacheUserResult.runActions) { - this.logger.info('Check was triggered but cache result options specified NOT to run actions...counting as check NOT triggered'); - triggered = false; - } - } catch (e: any) { - if (e.logged !== true) { - this.logger.warn(`Running rules for Check ${check.name} failed due to uncaught exception`, e); - } - this.emit('error', e); - } - if (triggered) { - triggeredCheckName = check.name; - actionedEvent.check = check.name; - actionedEvent.ruleResults = currentResults; - if(isFromCache) { - actionedEvent.ruleSummary = `Check result was found in cache: ${triggeredIndicator(true)}`; + if (checkNames.length > 0 && !checkNames.map(x => x.toLowerCase()).some(x => x === check.name.toLowerCase())) { + currRun.logger.warn(`Check ${check.name} not in array of requested checks to run, skipping...`); + checkIndex++; + continue; + } + if (!check.enabled) { + currRun.logger.info(`Check ${check.name} not run because it is not enabled, skipping...`); + checkIndex++; + continue; + } + checksRunNames.push(check.name); + checksRun++; + triggered = false; + let isFromCache = false; + let currentResults: RuleResult[] = []; + try { + const [checkTriggered, checkResults, fromCache = false] = await check.runRules(item, allRuleResults); + isFromCache = fromCache; + if (!fromCache) { + await check.setCacheResult(item, {result: checkTriggered, ruleResults: checkResults}); + } else { + cachedCheckNames.push(check.name); + } + currentResults = checkResults; + totalRulesRun += checkResults.length; + allRuleResults = allRuleResults.concat(determineNewResults(allRuleResults, checkResults)); + triggered = checkTriggered; + if (triggered && fromCache && !check.cacheUserResult.runActions) { + currRun.logger.info('Check was triggered but cache result options specified NOT to run actions...counting as check NOT triggered'); + triggered = false; + } + } catch (e: any) { + if (e.logged !== true) { + currRun.logger.warn(`Running rules for Check ${check.name} failed due to uncaught exception`, e); + } + this.emit('error', e); + } + + let behavior: PostBehaviorTypes; + let behaviorT: string; + + if (triggered) { + triggeredCheckName = check.name; + actionedEvent.check = check.name; + actionedEvent.ruleResults = currentResults; + if (isFromCache) { + actionedEvent.ruleSummary = `Check result was found in cache: ${triggeredIndicator(true)}`; + } else { + actionedEvent.ruleSummary = resultsSummary(currentResults, check.condition); + } + runActions = await check.runActions(item, currentResults.filter(x => x.triggered), dryRun); + // we only can about report and comment actions since those can produce items for newComm and modqueue + const recentCandidates = runActions.filter(x => ['report', 'comment'].includes(x.kind.toLocaleLowerCase())).map(x => x.touchedEntities === undefined ? [] : x.touchedEntities).flat(); + for (const recent of recentCandidates) { + await this.resources.setRecentSelf(recent as (Submission | Comment)); + } + actionsRun = runActions.length; + + if (check.notifyOnTrigger) { + const ar = runActions.map(x => x.name).join(', '); + this.notificationManager.handle('eventActioned', 'Check Triggered', `Check "${check.name}" was triggered on Event: \n\n ${ePeek} \n\n with the following actions run: ${ar}`); + } + behavior = check.postTrigger; + behaviorT = 'Trigger'; } else { - actionedEvent.ruleSummary = resultsSummary(currentResults, check.condition); + behavior = check.postFail; + behaviorT = 'Fail'; } - runActions = await check.runActions(item, currentResults.filter(x => x.triggered), dryRun); - // we only can about report and comment actions since those can produce items for newComm and modqueue - const recentCandidates = runActions.filter(x => ['report','comment'].includes(x.kind.toLocaleLowerCase())).map(x => x.touchedEntities === undefined ? [] : x.touchedEntities).flat(); - for(const recent of recentCandidates) { - await this.resources.setRecentSelf(recent as (Submission|Comment)); - } - actionsRun = runActions.length; - if(check.notifyOnTrigger) { - const ar = runActions.map(x => x.name).join(', '); - this.notificationManager.handle('eventActioned', 'Check Triggered', `Check "${check.name}" was triggered on Event: \n\n ${ePeek} \n\n with the following actions run: ${ar}`); + switch(behavior.toLowerCase()) { + case 'next': + check.logger.debug('Behavior => NEXT => run next check', {leaf: `Post Check ${behaviorT}`}); + checkIndex++; + break; + case 'nextrun': + check.logger.debug('Behavior => NEXT RUN => Skip remaining checks and go to next Run', {leaf: `Post Check ${behaviorT}`}); + continueCheckIteration = false; + break; + case 'stop': + check.logger.debug('Behavior => STOP => Immediately stop current Run and skip all remaining runs', {leaf: `Post Check ${behaviorT}`}); + continueRunIteration = false; + continueCheckIteration = false; + break; + default: + if(behavior.includes('goto:')) { + gotoContext = behavior.split(':')[1]; + check.logger.debug(`Behavior => GOTO => Set to ${gotoContext}`, {leaf: `Post Check ${behaviorT}`}); + if(!gotoContext.includes('.')) { + // no period means we are going directly to a run + continueCheckIteration = false; + } else { + const [runN, checkN] = gotoContext.split('.'); + if(runN !== '') { + // if run name is specified then also break check iteration + // OTHERWISE this is a special "in run" check path IE .check1 where we just want to continue iterating checks + continueCheckIteration = false; + } + } + } else { + throw new Error(`Post ${behaviorT} Behavior was not a valid value. Must be one of => next | nextRun | stop | goto:[path]`); + } } - break; } - } - if (!triggered) { - this.logger.info('No checks triggered'); + if (!triggered) { + this.logger.info('No checks triggered'); + } + runIndex++; } - } catch (err: any) { if (!(err instanceof LoggedError) && err.logged !== true) { this.logger.error('An unhandled error occurred while running checks', err); @@ -896,7 +1036,7 @@ export class Manager extends EventEmitter { checkType = 'Comment'; } if (checkType !== undefined) { - this.firehose.push({checkType, activity: item, options: {delayUntil}}) + this.firehose.push({activity: item, options: {delayUntil}}) } }; diff --git a/src/Web/Server/routes/authenticated/user/action.ts b/src/Web/Server/routes/authenticated/user/action.ts index 632bfa4..6a863c8 100644 --- a/src/Web/Server/routes/authenticated/user/action.ts +++ b/src/Web/Server/routes/authenticated/user/action.ts @@ -62,7 +62,6 @@ const action = async (req: Request, res: Response) => { const activities = await manager.subreddit.getUnmoderated({limit: 100}); for (const a of activities.reverse()) { await manager.firehose.push({ - checkType: a instanceof Submission ? 'Submission' : 'Comment', activity: a, options: { force: true, @@ -73,7 +72,6 @@ const action = async (req: Request, res: Response) => { const activities = await manager.subreddit.getModqueue({limit: 100}); for (const a of activities.reverse()) { await manager.firehose.push({ - checkType: a instanceof Submission ? 'Submission' : 'Comment', activity: a, options: { force: true diff --git a/src/Web/Server/routes/authenticated/user/index.ts b/src/Web/Server/routes/authenticated/user/index.ts index bea4d7d..bf99118 100644 --- a/src/Web/Server/routes/authenticated/user/index.ts +++ b/src/Web/Server/routes/authenticated/user/index.ts @@ -127,7 +127,7 @@ const action = async (req: Request, res: Response) => { // will run dryrun if specified or if running activity on subreddit it does not belong to const dr: boolean | undefined = (dryRun || manager.subreddit.display_name !== sub) ? true : undefined; manager.logger.info(`/u/${userName} running${dr === true ? ' DRY RUN ' : ' '}check on${manager.subreddit.display_name !== sub ? ' FOREIGN ACTIVITY ' : ' '}${url}`); - await manager.runChecks(activity instanceof Submission ? 'Submission' : 'Comment', activity, {dryRun: dr, force: true}) + await manager.runChecks(activity, {dryRun: dr, force: true}) } res.send('OK'); }; diff --git a/src/index.ts b/src/index.ts index d59a81d..8fd124e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -86,7 +86,7 @@ const program = new Command(); .allowUnknownOption() .description('Run check(s) on a specific activity', { activityIdentifier: 'Either a permalink URL or the ID of the Comment or Submission', - type: `If activityIdentifier is not a permalink URL then the type of activity ('comment' or 'submission'). May also specify 'submission' type when using a permalink to a comment to get the Submission`, + type: `No longer used`, bot: 'Specify the bot to try with using `bot.name` (from config) -- otherwise all bots will be built before the bot to be used can be determined' }); checkCommand = addOptions(checkCommand, getUniversalCLIOptions()); @@ -154,7 +154,7 @@ const program = new Command(); await b.buildManagers([sub]); if(b.subManagers.length > 0) { const manager = b.subManagers[0]; - await manager.runChecks(type === 'comment' ? 'Comment' : 'Submission', activity, {checkNames: checks}); + await manager.runChecks(activity, {checkNames: checks}); break; } } @@ -191,7 +191,6 @@ const program = new Command(); const activities = await manager.subreddit.getUnmoderated(); for (const a of activities.reverse()) { manager.firehose.push({ - checkType: a instanceof Submission ? 'Submission' : 'Comment', activity: a, options: {checkNames: checks} }); diff --git a/src/util.ts b/src/util.ts index 68fdfa5..b6aff77 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1577,7 +1577,7 @@ export const snooLogWrapper = (logger: Logger) => { * Cached activities lose type information when deserialized so need to check properties as well to see if the object is the shape of a Submission * */ export const isSubmission = (value: any) => { - return value instanceof Submission || value.domain !== undefined; + return value instanceof Submission || (value.id !== undefined && value.id.includes('t3_')) || value.domain !== undefined; } export const asSubmission = (value: any): value is Submission => {