Compare commits

...

6 Commits

Author SHA1 Message Date
FoxxMD
eee2a8e856 feat: Support multiple expressions for regex property 2022-11-22 16:03:50 -05:00
FoxxMD
e79779d980 feat: Implement templating for flair actions 2022-11-21 11:43:34 -05:00
FoxxMD
b094b72d4a docs: Update docker compose instructions
* Specify docker-compose minimum version
* Change commands to use new syntax
2022-11-21 11:29:14 -05:00
FoxxMD
d90e88360d feat: Add some author properties for templating 2022-11-17 13:16:10 -05:00
FoxxMD
9031f7fec8 refactor(polling): Improve resilience for polling source parsing
* Replace hard-coded polling sources with string constants
* Implement string to PollOn parsing which is case-insensitive and forgives mispelling
* Check that manager specifies only one of each polling source type when build config
2022-11-17 12:03:55 -05:00
FoxxMD
0a2b13e4c4 fix: Fix including self activities in Recent Activity without filtering
* Consolidate ACID check for author history results into authorActivities function so it can be used everywhere
* Remove self check in Recent Activity and used consolidated functionality so that filtering still occurs on current activity
2022-11-16 10:18:37 -05:00
15 changed files with 372 additions and 206 deletions

View File

@@ -24,6 +24,9 @@ services:
cache:
image: 'redis:7-alpine'
volumes:
# on linux will need to make sure this directory has correct permissions for container to access
- './data/cache:/data'
database:
image: 'mariadb:10.9.3'

View File

@@ -55,6 +55,8 @@ The included [`docker-compose.yml`](/docker-compose.yml) provides production-rea
#### Setup
The included `docker-compose.yml` file is written for **Docker Compose v2.**
For new installations copy [`config.yaml`](/docker/config/docker-compose/config.yaml) into a folder named `data` in the same folder `docker-compose.yml` will be run from. For users migrating their existing CM instances to docker-compose, copy your existing `config.yaml` into the same `data` folder.
Read through the comments in both `docker-compose.yml` and `config.yaml` and makes changes to any relevant settings (passwords, usernames, etc...). Ensure that any settings used in both files (EX mariaDB passwords) match.
@@ -62,13 +64,13 @@ Read through the comments in both `docker-compose.yml` and `config.yaml` and mak
To build and start CM:
```bash
docker-compose up -d
docker compose up -d
```
To include Grafana/Influx dependencies run:
```bash
docker-compose --profile full up -d
docker compose --profile full up -d
```
## Locally

View File

@@ -57,6 +57,29 @@ All Actions with `content` have access to this data:
| `title` | As comments => the body of the comment. As Submission => title | Test post please ignore |
| `shortTitle` | The same as `title` but truncated to 15 characters | test post pleas... |
#### Common Author
Additionally, `author` has these properties accessible:
| Name | Description | Example |
|----------------|-------------------------------------|----------|
| `age` | (Approximate) Age of account | 3 months |
| `linkKarma` | Amount of link karma | 10 |
| `commentKarma` | Amount of comment karma | 3 |
| `totalKarma` | Combined link+comment karma | 13 |
| `verified` | Does account have a verified email? | true |
NOTE: Accessing these properties may require an additional API call so use sparingly on high-volume comments
##### Example Usage
```
The user {{item.author}} has been a redditor for {{item.author.age}}
```
Produces:
> The user FoxxMD has been a redditor for 3 months
### Submissions
If the **Activity** is a Submission these additional properties are accessible:

View File

@@ -30,15 +30,14 @@ export class FlairAction extends Action {
async process(item: Comment | Submission, ruleResults: RuleResultEntity[], actionResults: ActionResultEntity[], options: runCheckOptions): Promise<ActionProcessResult> {
const dryRun = this.getRuntimeAwareDryrun(options);
let flairParts = [];
if(this.text !== '') {
flairParts.push(`Text: ${this.text}`);
}
if(this.css !== '') {
flairParts.push(`CSS: ${this.css}`);
}
if(this.flair_template_id !== '') {
flairParts.push(`Template: ${this.flair_template_id}`);
}
const renderedText = this.text === '' ? '' : await this.renderContent(this.text, item, ruleResults, actionResults) as string;
flairParts.push(`Text: ${renderedText === '' ? '(None)' : renderedText}`);
const renderedCss = this.css === '' ? '' : await this.renderContent(this.css, item, ruleResults, actionResults) as string;
flairParts.push(`CSS: ${renderedCss === '' ? '(None)' : renderedCss}`);
flairParts.push(`Template: ${this.flair_template_id === '' ? '(None)' : this.flair_template_id}`);
const flairSummary = flairParts.length === 0 ? 'No flair (unflaired)' : flairParts.join(' | ');
this.logger.verbose(flairSummary);
if (item instanceof Submission) {
@@ -51,9 +50,9 @@ export class FlairAction extends Action {
await item.assignFlair({flair_template_id: this.flair_template_id}).then(() => {});
item.link_flair_template_id = this.flair_template_id;
} else {
await item.assignFlair({text: this.text, cssClass: this.css}).then(() => {});
item.link_flair_css_class = this.css;
item.link_flair_text = this.text;
await item.assignFlair({text: renderedText, cssClass: renderedCss}).then(() => {});
item.link_flair_css_class = renderedCss;
item.link_flair_text = renderedText;
}
await this.resources.resetCacheForItem(item);
}

View File

@@ -26,6 +26,8 @@ export class UserFlairAction extends Action {
async process(item: Comment | Submission, ruleResults: RuleResultEntity[], actionResults: ActionResultEntity[], options: runCheckOptions): Promise<ActionProcessResult> {
const dryRun = this.getRuntimeAwareDryrun(options);
let flairParts = [];
let renderedText: string | undefined = undefined;
let renderedCss: string | undefined = undefined;
if (this.flair_template_id !== undefined) {
flairParts.push(`Flair template ID: ${this.flair_template_id}`)
@@ -34,10 +36,12 @@ export class UserFlairAction extends Action {
}
} else {
if (this.text !== undefined) {
flairParts.push(`Text: ${this.text}`);
renderedText = await this.renderContent(this.text, item, ruleResults, actionResults) as string;
flairParts.push(`Text: ${renderedText}`);
}
if (this.css !== undefined) {
flairParts.push(`CSS: ${this.css}`);
renderedCss = await this.renderContent(this.css, item, ruleResults, actionResults) as string;
flairParts.push(`CSS: ${renderedCss}`);
}
}
@@ -58,7 +62,7 @@ export class UserFlairAction extends Action {
this.logger.error('Either the flair template ID is incorrect or you do not have permission to access it.');
throw err;
}
} else if (this.text === undefined && this.css === undefined) {
} else if (renderedText === undefined && renderedCss === undefined) {
// @ts-ignore
await item.subreddit.deleteUserFlair(item.author.name);
item.author_flair_css_class = null;
@@ -68,11 +72,11 @@ export class UserFlairAction extends Action {
// @ts-ignore
await item.author.assignFlair({
subredditName: item.subreddit.display_name,
cssClass: this.css,
text: this.text,
cssClass: renderedCss,
text: renderedText,
});
item.author_flair_text = this.text ?? null;
item.author_flair_css_class = this.css ?? null;
item.author_flair_text = renderedText ?? null;
item.author_flair_css_class = renderedCss ?? null;
}
await this.resources.resetCacheForItem(item);
if(typeof item.author !== 'string') {

View File

@@ -46,7 +46,13 @@ import {RunStateType} from "../Common/Entities/RunStateType";
import {QueueRunState} from "../Common/Entities/EntityRunState/QueueRunState";
import {EventsRunState} from "../Common/Entities/EntityRunState/EventsRunState";
import {ManagerRunState} from "../Common/Entities/EntityRunState/ManagerRunState";
import {Invokee, PollOn} from "../Common/Infrastructure/Atomic";
import {
Invokee,
POLLING_COMMENTS, POLLING_MODQUEUE,
POLLING_SUBMISSIONS,
POLLING_UNMODERATED,
PollOn
} from "../Common/Infrastructure/Atomic";
import {FilterCriteriaDefaults} from "../Common/Infrastructure/Filters/FilterShapes";
import {snooLogWrapper} from "../Utils/loggerFactory";
import {InfluxClient} from "../Common/Influx/InfluxClient";
@@ -558,9 +564,9 @@ class Bot implements BotInstanceFunctions {
parseSharedStreams() {
const sharedCommentsSubreddits = !this.sharedStreams.includes('newComm') ? [] : this.subManagers.filter(x => x.isPollingShared('newComm')).map(x => x.subreddit.display_name);
const sharedCommentsSubreddits = !this.sharedStreams.includes(POLLING_COMMENTS) ? [] : this.subManagers.filter(x => x.isPollingShared(POLLING_COMMENTS)).map(x => x.subreddit.display_name);
if (sharedCommentsSubreddits.length > 0) {
const stream = this.cacheManager.modStreams.get('newComm');
const stream = this.cacheManager.modStreams.get(POLLING_COMMENTS);
if (stream === undefined || stream.subreddit !== sharedCommentsSubreddits.join('+')) {
let processed;
if (stream !== undefined) {
@@ -580,20 +586,20 @@ class Bot implements BotInstanceFunctions {
label: 'Shared Polling'
});
// @ts-ignore
defaultCommentStream.on('error', this.createSharedStreamErrorListener('newComm'));
defaultCommentStream.on('listing', this.createSharedStreamListingListener('newComm'));
this.cacheManager.modStreams.set('newComm', defaultCommentStream);
defaultCommentStream.on('error', this.createSharedStreamErrorListener(POLLING_COMMENTS));
defaultCommentStream.on('listing', this.createSharedStreamListingListener(POLLING_COMMENTS));
this.cacheManager.modStreams.set(POLLING_COMMENTS, defaultCommentStream);
}
} else {
const stream = this.cacheManager.modStreams.get('newComm');
const stream = this.cacheManager.modStreams.get(POLLING_COMMENTS);
if (stream !== undefined) {
stream.end('Determined no managers are listening on shared stream parsing');
}
}
const sharedSubmissionsSubreddits = !this.sharedStreams.includes('newSub') ? [] : this.subManagers.filter(x => x.isPollingShared('newSub')).map(x => x.subreddit.display_name);
const sharedSubmissionsSubreddits = !this.sharedStreams.includes(POLLING_SUBMISSIONS) ? [] : this.subManagers.filter(x => x.isPollingShared(POLLING_SUBMISSIONS)).map(x => x.subreddit.display_name);
if (sharedSubmissionsSubreddits.length > 0) {
const stream = this.cacheManager.modStreams.get('newSub');
const stream = this.cacheManager.modStreams.get(POLLING_SUBMISSIONS);
if (stream === undefined || stream.subreddit !== sharedSubmissionsSubreddits.join('+')) {
let processed;
if (stream !== undefined) {
@@ -613,19 +619,19 @@ class Bot implements BotInstanceFunctions {
label: 'Shared Polling'
});
// @ts-ignore
defaultSubStream.on('error', this.createSharedStreamErrorListener('newSub'));
defaultSubStream.on('listing', this.createSharedStreamListingListener('newSub'));
this.cacheManager.modStreams.set('newSub', defaultSubStream);
defaultSubStream.on('error', this.createSharedStreamErrorListener(POLLING_SUBMISSIONS));
defaultSubStream.on('listing', this.createSharedStreamListingListener(POLLING_SUBMISSIONS));
this.cacheManager.modStreams.set(POLLING_SUBMISSIONS, defaultSubStream);
}
} else {
const stream = this.cacheManager.modStreams.get('newSub');
const stream = this.cacheManager.modStreams.get(POLLING_SUBMISSIONS);
if (stream !== undefined) {
stream.end('Determined no managers are listening on shared stream parsing');
}
}
const isUnmoderatedShared = !this.sharedStreams.includes('unmoderated') ? false : this.subManagers.some(x => x.isPollingShared('unmoderated'));
const unmoderatedstream = this.cacheManager.modStreams.get('unmoderated');
const isUnmoderatedShared = !this.sharedStreams.includes(POLLING_UNMODERATED) ? false : this.subManagers.some(x => x.isPollingShared(POLLING_UNMODERATED));
const unmoderatedstream = this.cacheManager.modStreams.get(POLLING_UNMODERATED);
if (isUnmoderatedShared && unmoderatedstream === undefined) {
const defaultUnmoderatedStream = new UnmoderatedStream(this.client, {
subreddit: 'mod',
@@ -634,15 +640,15 @@ class Bot implements BotInstanceFunctions {
label: 'Shared Polling'
});
// @ts-ignore
defaultUnmoderatedStream.on('error', this.createSharedStreamErrorListener('unmoderated'));
defaultUnmoderatedStream.on('listing', this.createSharedStreamListingListener('unmoderated'));
this.cacheManager.modStreams.set('unmoderated', defaultUnmoderatedStream);
defaultUnmoderatedStream.on('error', this.createSharedStreamErrorListener(POLLING_UNMODERATED));
defaultUnmoderatedStream.on('listing', this.createSharedStreamListingListener(POLLING_UNMODERATED));
this.cacheManager.modStreams.set(POLLING_UNMODERATED, defaultUnmoderatedStream);
} else if (!isUnmoderatedShared && unmoderatedstream !== undefined) {
unmoderatedstream.end('Determined no managers are listening on shared stream parsing');
}
const isModqueueShared = !this.sharedStreams.includes('modqueue') ? false : this.subManagers.some(x => x.isPollingShared('modqueue'));
const modqueuestream = this.cacheManager.modStreams.get('modqueue');
const isModqueueShared = !this.sharedStreams.includes(POLLING_MODQUEUE) ? false : this.subManagers.some(x => x.isPollingShared(POLLING_MODQUEUE));
const modqueuestream = this.cacheManager.modStreams.get(POLLING_MODQUEUE);
if (isModqueueShared && modqueuestream === undefined) {
const defaultModqueueStream = new ModQueueStream(this.client, {
subreddit: 'mod',
@@ -651,9 +657,9 @@ class Bot implements BotInstanceFunctions {
label: 'Shared Polling'
});
// @ts-ignore
defaultModqueueStream.on('error', this.createSharedStreamErrorListener('modqueue'));
defaultModqueueStream.on('listing', this.createSharedStreamListingListener('modqueue'));
this.cacheManager.modStreams.set('modqueue', defaultModqueueStream);
defaultModqueueStream.on('error', this.createSharedStreamErrorListener(POLLING_MODQUEUE));
defaultModqueueStream.on('listing', this.createSharedStreamListingListener(POLLING_MODQUEUE));
this.cacheManager.modStreams.set(POLLING_MODQUEUE, defaultModqueueStream);
} else if (isModqueueShared && modqueuestream !== undefined) {
modqueuestream.end('Determined no managers are listening on shared stream parsing');
}

View File

@@ -111,6 +111,19 @@ export interface DurationObject {
export type JoinOperands = 'OR' | 'AND';
export type PollOn = 'unmoderated' | 'modqueue' | 'newSub' | 'newComm';
export const POLLING_UNMODERATED: PollOn = 'unmoderated';
export const POLLING_MODQUEUE: PollOn = 'modqueue';
export const POLLING_SUBMISSIONS: PollOn = 'newSub';
export const POLLING_COMMENTS: PollOn = 'newComm';
export const pollOnTypes: PollOn[] = [POLLING_UNMODERATED, POLLING_MODQUEUE, POLLING_SUBMISSIONS, POLLING_COMMENTS];
export const pollOnTypeMapping: Map<string, PollOn> = new Map([
['unmoderated', POLLING_UNMODERATED],
['modqueue', POLLING_MODQUEUE],
['newsub', POLLING_SUBMISSIONS],
['newcomm', POLLING_COMMENTS],
// be nice if user mispelled
['newcom', POLLING_COMMENTS]
]);
export type ModeratorNames = 'self' | 'automod' | 'automoderator' | string;
export type Invokee = 'system' | 'user';
export type RunState = 'running' | 'paused' | 'stopped';

View File

@@ -372,7 +372,7 @@ export interface PollingOptions extends PollingDefaults {
* * after they have been manually approved from modqueue
*
* */
pollOn: 'unmoderated' | 'modqueue' | 'newSub' | 'newComm'
pollOn: PollOn
}
export interface TTLConfig {
@@ -748,6 +748,10 @@ export interface RegExResult {
named: NamedGroup
}
export interface RegExResultWithTest extends RegExResult {
test: RegExp
}
export type StrongCache = {
authorTTL: number | boolean,
userNotesTTL: number | boolean,

View File

@@ -8,7 +8,7 @@ import {
overwriteMerge,
parseBool, parseExternalUrl, parseUrlContext, parseWikiContext, randomId,
readConfigFile,
removeUndefinedKeys, resolvePathFromEnvWithRelative, toStrongSharingACLConfig
removeUndefinedKeys, resolvePathFromEnvWithRelative, toPollOn, toStrongSharingACLConfig
} from "./util";
import Ajv, {Schema} from 'ajv';
@@ -74,8 +74,8 @@ import {ErrorWithCause} from "pony-cause";
import {RunConfigHydratedData, RunConfigData, RunConfigObject} from "./Run";
import {AuthorRuleConfig} from "./Rule/AuthorRule";
import {
CacheProvider, ConfigFormat, ConfigFragmentParseFunc,
PollOn
CacheProvider, ConfigFormat, ConfigFragmentParseFunc, POLLING_MODQUEUE, POLLING_UNMODERATED,
PollOn, pollOnTypes
} from "./Common/Infrastructure/Atomic";
import {
asFilterOptionsJson,
@@ -452,27 +452,31 @@ export class ConfigBuilder {
export const buildPollingOptions = (values: (string | PollingOptions)[]): PollingOptionsStrong[] => {
let opts: PollingOptionsStrong[] = [];
let rawOpts: PollingOptions;
for (const v of values) {
if (typeof v === 'string') {
opts.push({
pollOn: v as PollOn,
interval: DEFAULT_POLLING_INTERVAL,
limit: DEFAULT_POLLING_LIMIT,
});
rawOpts = {pollOn: v as PollOn}; // maybeee
} else {
const {
pollOn: p,
interval = DEFAULT_POLLING_INTERVAL,
limit = DEFAULT_POLLING_LIMIT,
delayUntil,
} = v;
opts.push({
pollOn: p as PollOn,
interval,
limit,
delayUntil,
});
rawOpts = v;
}
const {
pollOn: p,
interval = DEFAULT_POLLING_INTERVAL,
limit = DEFAULT_POLLING_LIMIT,
delayUntil,
} = rawOpts;
const pVal = toPollOn(p);
if (opts.some(x => x.pollOn === pVal)) {
throw new SimpleError(`Polling source ${pVal} cannot appear more than once in polling options`);
}
opts.push({
pollOn: pVal,
interval,
limit,
delayUntil,
});
}
return opts;
}
@@ -796,7 +800,7 @@ export const parseDefaultBotInstanceFromArgs = (args: any): BotInstanceJsonConfi
heartbeatInterval: heartbeat,
},
polling: {
shared: sharedMod ? ['unmoderated', 'modqueue'] : undefined,
shared: sharedMod ? [POLLING_UNMODERATED, POLLING_MODQUEUE] : undefined,
},
nanny: {
softLimit,
@@ -908,7 +912,7 @@ export const parseDefaultBotInstanceFromEnv = (): BotInstanceJsonConfig => {
heartbeatInterval: process.env.HEARTBEAT !== undefined ? parseInt(process.env.HEARTBEAT) : undefined,
},
polling: {
shared: parseBool(process.env.SHARE_MOD) ? ['unmoderated', 'modqueue'] : undefined,
shared: parseBool(process.env.SHARE_MOD) ? [POLLING_UNMODERATED, POLLING_MODQUEUE] : undefined,
},
nanny: {
softLimit: process.env.SOFT_LIMIT !== undefined ? parseInt(process.env.SOFT_LIMIT) : undefined,
@@ -1525,10 +1529,10 @@ export const buildBotConfig = (data: BotInstanceJsonConfig, opConfig: OperatorCo
botCache.provider.prefix = buildCachePrefix([botCache.provider.prefix, 'bot', (botName || objectHash.sha1(botCreds))]);
}
let realShared = shared === true ? ['unmoderated', 'modqueue', 'newComm', 'newSub'] : shared;
let realShared: PollOn[] = shared === true ? pollOnTypes : shared.map(toPollOn);
if (sharedMod === true) {
realShared.push('unmoderated');
realShared.push('modqueue');
realShared.push(POLLING_UNMODERATED);
realShared.push(POLLING_MODQUEUE);
}
const botLevelStatDefaults = {...statDefaultsFromOp, ...databaseStatisticsDefaults};
@@ -1566,7 +1570,7 @@ export const buildBotConfig = (data: BotInstanceJsonConfig, opConfig: OperatorCo
caching: botCache,
userAgent,
polling: {
shared: [...new Set(realShared)] as PollOn[],
shared: Array.from(new Set(realShared)),
stagger,
limit,
interval,

View File

@@ -126,28 +126,7 @@ export class RecentActivityRule extends Rule {
async process(item: Submission | Comment): Promise<[boolean, RuleResult]> {
let activities;
// ACID is a bitch
// reddit may not return the activity being checked in the author's recent history due to availability/consistency issues or *something*
// so make sure we add it in if config is checking the same type and it isn't included
// TODO refactor this for SubredditState everywhere branch
let shouldIncludeSelf = true;
const strongWindow = windowConfigToWindowCriteria(this.window);
const {
filterOn: {
post: {
subreddits: {
include = [],
exclude = []
} = {},
} = {},
} = {}
} = strongWindow;
// typeof x === string -- a patch for now...technically this is all it supports but eventually will need to be able to do any SubredditState
if (include.length > 0 && !include.some(x => x.name !== undefined && x.name.toLocaleLowerCase() === item.subreddit.display_name.toLocaleLowerCase())) {
shouldIncludeSelf = false;
} else if (exclude.length > 0 && exclude.some(x => x.name !== undefined && x.name.toLocaleLowerCase() === item.subreddit.display_name.toLocaleLowerCase())) {
shouldIncludeSelf = false;
}
if(strongWindow.fetch === undefined && this.lookAt !== undefined) {
switch(this.lookAt) {
@@ -159,25 +138,10 @@ export class RecentActivityRule extends Rule {
}
}
activities = await this.resources.getAuthorActivities(item.author, strongWindow);
switch (strongWindow.fetch) {
case 'comment':
if (shouldIncludeSelf && item instanceof Comment && !activities.some(x => x.name === item.name)) {
activities.unshift(item);
}
break;
case 'submission':
if (shouldIncludeSelf && item instanceof Submission && !activities.some(x => x.name === item.name)) {
activities.unshift(item);
}
break;
default:
if (shouldIncludeSelf && !activities.some(x => x.name === item.name)) {
activities.unshift(item);
}
break;
}
// ACID is a bitch
// reddit may not return the activity being checked in the author's recent history due to availability/consistency issues or *something*
// so add current activity as a prefetched activity and add it to the returned activities (after it goes through filtering)
activities = await this.resources.getAuthorActivities(item.author, strongWindow, undefined, [item]);
let viableActivity = activities;
// if config does not specify reference then we set the default based on whether the item is a submission or not

View File

@@ -7,6 +7,7 @@ import {
PASS, triggeredIndicator, windowConfigToWindowCriteria
} from "../util";
import {
RegExResultWithTest,
RuleResult,
} from "../Common/interfaces";
import dayjs from 'dayjs';
@@ -14,10 +15,11 @@ import {SimpleError} from "../Utils/Errors";
import {JoinOperands} from "../Common/Infrastructure/Atomic";
import {ActivityWindowConfig} from "../Common/Infrastructure/ActivityWindow";
import {
comparisonTextOp,
comparisonTextOp, GenericComparison,
parseGenericValueComparison,
parseGenericValueOrPercentComparison
} from "../Common/Infrastructure/Comparisons";
import {SnoowrapActivity} from "../Common/Infrastructure/Reddit";
export interface RegexCriteria {
/**
@@ -27,13 +29,23 @@ export interface RegexCriteria {
* */
name?: string
/**
* A valid Regular Expression to test content against
* A valid Regular Expression, or list of expressions, to test content against
*
* If no flags are specified then the **global** flag is used by default
*
* @examples ["/reddit|FoxxMD/ig"]
* */
regex: string,
regex: string | string[],
/**
* Determines if ALL regexes listed are run or if regexes are only run until one is matched.
*
* * `true` => all regexes are always run
* * `false` => regexes are run until one matches
*
* @default false
* */
exhaustive?: boolean
/**
* Which content from an Activity to test the regex against
@@ -157,6 +169,7 @@ export class RegexRule extends Rule {
const {
name = (index + 1),
regex,
exhaustive = false,
testOn: testOnVals = ['title', 'body'],
lookAt = 'all',
matchThreshold = '> 0',
@@ -174,13 +187,7 @@ export class RegexRule extends Rule {
return acc.concat(curr);
}, []);
// check regex
const regexContent = await this.resources.getContent(regex);
const reg = parseStringToRegex(regexContent, 'g');
if(reg === undefined) {
throw new SimpleError(`Value given for regex on Criteria ${name} was not valid: ${regex}`);
}
// ok cool its a valid regex
const regexTests: RegExp[] = await this.convertToRegexArray(name, regex);
const matchComparison = parseGenericValueComparison(matchThreshold);
const activityMatchComparison = activityMatchThreshold === null ? undefined : parseGenericValueOrPercentComparison(activityMatchThreshold);
@@ -198,12 +205,13 @@ export class RegexRule extends Rule {
// first lets see if the activity we are checking satisfies thresholds
// since we may be able to avoid api calls to get history
let actMatches = this.getMatchesFromActivity(item, testOn, reg);
matches = matches.concat(actMatches).slice(0, 100);
matchCount += actMatches.length;
let actMatches = getMatchesFromActivity(item, testOn, regexTests, exhaustive);
const actMatchSummary = regexResultsSummary(actMatches);
matches = matches.concat(actMatchSummary.matches).slice(0, 100);
matchCount += actMatchSummary.matches.length;
activitiesTested++;
const singleMatched = comparisonTextOp(actMatches.length, matchComparison.operator, matchComparison.value);
const singleMatched = comparisonTextOp(actMatchSummary.matches.length, matchComparison.operator, matchComparison.value);
if (singleMatched) {
activitiesMatchedCount++;
}
@@ -233,7 +241,7 @@ export class RegexRule extends Rule {
}
history = await this.resources.getAuthorActivities(item.author, strongWindow);
// remove current activity it exists in history so we don't count it twice
// remove current activity if it exists in history so we don't count it twice
history = history.filter(x => x.id !== item.id);
const historyLength = history.length;
@@ -252,10 +260,12 @@ export class RegexRule extends Rule {
for (const h of history) {
activitiesTested++;
const aMatches = this.getMatchesFromActivity(h, testOn, reg);
matches = matches.concat(aMatches).slice(0, 100);
matchCount += aMatches.length;
const matched = comparisonTextOp(aMatches.length, matchComparison.operator, matchComparison.value);
const aMatches = getMatchesFromActivity(h, testOn, regexTests, exhaustive);
actMatches = actMatches.concat(aMatches);
const actHistoryMatchSummary = regexResultsSummary(aMatches);
matches = matches.concat(actHistoryMatchSummary.matches).slice(0, 100);
matchCount += actHistoryMatchSummary.matches.length;
const matched = comparisonTextOp(actHistoryMatchSummary.matches.length, matchComparison.operator, matchComparison.value);
if (matched) {
activitiesMatchedCount++;
}
@@ -282,10 +292,19 @@ export class RegexRule extends Rule {
humanWindow = '1 Item';
}
// to provide at least one useful regex for this criteria
// use the first regex found by default
let relevantRegex: string = regexTests[0].toString();
// but if more than one regex was listed AND we did have matches
// then use the first regex that actually got a match
if(regexTests.length > 0 && actMatches.length > 0) {
relevantRegex = actMatches[0].test.toString();
}
const critResults = {
criteria: {
name,
regex: regex !== regexContent ? `${regex} from ${regexContent}` : regex,
regex: relevantRegex,
testOn,
matchThreshold,
activityMatchThreshold,
@@ -352,44 +371,117 @@ export class RegexRule extends Rule {
return Promise.resolve([criteriaMet, this.getResult(criteriaMet, {result, data: {results: criteriaResults, matchSample }})]);
}
protected getMatchesFromActivity(a: (Submission | Comment), testOn: string[], reg: RegExp): string[] {
let m: string[] = [];
// determine what content we are testing
let contents: string[] = [];
if (asSubmission(a)) {
for (const l of testOn) {
switch (l) {
case 'title':
contents.push(a.title);
break;
case 'body':
if (a.is_self) {
contents.push(a.selftext);
}
break;
case 'url':
if (isExternalUrlSubmission(a)) {
contents.push(a.url);
}
break;
}
protected async convertToRegexArray(name: string | number, value: string | string[]): Promise<RegExp[]> {
const regexTests: RegExp[] = [];
const regexStringVals = typeof value === 'string' ? [value] : value;
for(const r of regexStringVals) {
// check regex
const regexContent = await this.resources.getContent(r);
const reg = parseStringToRegex(regexContent, 'ig');
if (reg === undefined) {
throw new SimpleError(`Value given for regex on Criteria ${name} was not valid: ${value}`);
}
} else {
contents.push(a.body)
// ok cool its a valid regex
regexTests.push(reg);
}
for (const c of contents) {
const results = parseRegex(reg, c);
if(results !== undefined) {
for(const r of results) {
m.push(r.match);
}
}
}
return m;
return regexTests;
}
}
export const getMatchResultsFromContent = (contents: string[], reg: RegExp): RegExResultWithTest[] => {
let m: RegExResultWithTest[] = [];
for (const c of contents) {
const results = parseRegex(reg, c);
if(results !== undefined) {
for(const r of results) {
m.push({...r, test: reg});
}
}
}
return m;
}
export const regexResultsSummary = (results: RegExResultWithTest[]) => {
const matchResults: ActivityMatchResults = {
matches: [],
matchesByTest: {},
groups: {}
}
for (const r of results) {
if (matchResults.matchesByTest[r.test.toString()] === undefined) {
matchResults.matchesByTest[r.test.toString()] = [];
}
matchResults.matchesByTest[r.test.toString()].push(r.match);
matchResults.matches.push(r.match);
if (r.named !== undefined) {
Object.entries(r.named).forEach(([key, val]) => {
if (matchResults.groups[key] === undefined) {
matchResults.groups[key] = [];
}
matchResults.groups[key].push(val);
});
}
}
return matchResults;
}
export const getMatchesFromActivity = (a: (Submission | Comment), testOn: string[], regexes: RegExp[], exhaustive: boolean): RegExResultWithTest[] => {
// determine what content we are testing
let contents: string[] = getMatchableContent(a, testOn);
let results: RegExResultWithTest[] = [];
for (const reg of regexes) {
const res = getMatchResultsFromContent(contents, reg);
if(res.length > 0) {
results = results.concat(res);
// only continue testing if the user wants to exhaustively check all regexes (to get more matches?)
if(!exhaustive) {
return results;
}
}
}
return results;
}
const getMatchableContent = (a: SnoowrapActivity, testOn: string[]) => {
let contents: string[] = [];
if (asSubmission(a)) {
for (const l of testOn) {
switch (l) {
case 'title':
contents.push(a.title);
break;
case 'body':
if (a.is_self) {
contents.push(a.selftext);
}
break;
case 'url':
if (isExternalUrlSubmission(a)) {
contents.push(a.url);
}
break;
}
}
} else {
contents.push(a.body)
}
return contents;
}
interface RegexMatchComparisonOptions {
matchComparison: GenericComparison
activityMatchComparison?: GenericComparison
totalMatchComparison?: GenericComparison
}
interface ActivityMatchResults {
matches: string[]
matchesByTest: Record<string, string[]>
groups: Record<string, string[]>
}
interface RegexConfig {
/**
* A list of Regular Expressions and conditions under which tested Activity(ies) are matched

View File

@@ -93,8 +93,8 @@ import {EntityRunState} from "../Common/Entities/EntityRunState/EntityRunState";
import {
ActivitySourceValue,
EventRetentionPolicyRange,
Invokee,
PollOn,
Invokee, POLLING_COMMENTS, POLLING_MODQUEUE, POLLING_SUBMISSIONS, POLLING_UNMODERATED,
PollOn, pollOnTypes,
recordOutputTypes,
RunState
} from "../Common/Infrastructure/Atomic";
@@ -635,7 +635,7 @@ export class Manager extends EventEmitter implements RunningStates {
const configBuilder = new ConfigBuilder({logger: this.logger});
const validJson = configBuilder.validateJson(configObj);
const {
polling = [{pollOn: 'unmoderated', limit: DEFAULT_POLLING_LIMIT, interval: DEFAULT_POLLING_INTERVAL}],
polling = [{pollOn: POLLING_SUBMISSIONS, limit: DEFAULT_POLLING_LIMIT, interval: DEFAULT_POLLING_INTERVAL}],
caching,
credentials,
dryRun,
@@ -957,7 +957,7 @@ export class Manager extends EventEmitter implements RunningStates {
await this.resources.setActivityLastSeenDate(item.name);
// if modqueue is running then we know we are checking for new reports every X seconds
if(options.activitySource.identifier === 'modqueue') {
if(options.activitySource.identifier === POLLING_MODQUEUE) {
// if the activity is from modqueue and only has one report then we know that report was just created
if(item.num_reports === 1
// otherwise if it has more than one report AND we have seen it (its only seen if it has already been stored (in below block))
@@ -1325,25 +1325,20 @@ export class Manager extends EventEmitter implements RunningStates {
}
}
isPollingShared(streamName: string): boolean {
isPollingShared(streamName: PollOn): boolean {
const pollOption = this.pollOptions.find(x => x.pollOn === streamName);
return pollOption !== undefined && pollOption.limit === DEFAULT_POLLING_LIMIT && pollOption.interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(streamName as PollOn);
return pollOption !== undefined && pollOption.limit === DEFAULT_POLLING_LIMIT && pollOption.interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(streamName);
}
async buildPolling() {
const sources: PollOn[] = ['unmoderated', 'modqueue', 'newComm', 'newSub'];
const sources = [...pollOnTypes];
const subName = this.subreddit.display_name;
for (const source of sources) {
if (!sources.includes(source)) {
this.logger.error(`'${source}' is not a valid polling source. Valid sources: unmoderated | modqueue | newComm | newSub`);
continue;
}
const pollOpt = this.pollOptions.find(x => x.pollOn.toLowerCase() === source.toLowerCase());
const pollOpt = this.pollOptions.find(x => x.pollOn === source);
if (pollOpt === undefined) {
if(this.sharedStreamCallbacks.has(source)) {
this.logger.debug(`Removing listener for shared polling on ${source.toUpperCase()} because it no longer exists in config`);
@@ -1366,11 +1361,11 @@ export class Manager extends EventEmitter implements RunningStates {
let modStreamType: string | undefined;
switch (source) {
case 'unmoderated':
case POLLING_UNMODERATED:
if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
modStreamType = 'unmoderated';
modStreamType = POLLING_UNMODERATED;
// use default mod stream from resources
stream = this.cacheManager.modStreams.get('unmoderated') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
stream = this.cacheManager.modStreams.get(POLLING_UNMODERATED) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
} else {
stream = new UnmoderatedStream(this.client, {
subreddit: this.subreddit.display_name,
@@ -1380,11 +1375,11 @@ export class Manager extends EventEmitter implements RunningStates {
});
}
break;
case 'modqueue':
case POLLING_MODQUEUE:
if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
modStreamType = 'modqueue';
modStreamType = POLLING_MODQUEUE;
// use default mod stream from resources
stream = this.cacheManager.modStreams.get('modqueue') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
stream = this.cacheManager.modStreams.get(POLLING_MODQUEUE) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
} else {
stream = new ModQueueStream(this.client, {
subreddit: this.subreddit.display_name,
@@ -1394,11 +1389,11 @@ export class Manager extends EventEmitter implements RunningStates {
});
}
break;
case 'newSub':
case POLLING_SUBMISSIONS:
if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
modStreamType = 'newSub';
modStreamType = POLLING_SUBMISSIONS;
// use default mod stream from resources
stream = this.cacheManager.modStreams.get('newSub') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
stream = this.cacheManager.modStreams.get(POLLING_SUBMISSIONS) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
} else {
stream = new SubmissionStream(this.client, {
subreddit: this.subreddit.display_name,
@@ -1408,11 +1403,11 @@ export class Manager extends EventEmitter implements RunningStates {
});
}
break;
case 'newComm':
case POLLING_COMMENTS:
if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) {
modStreamType = 'newComm';
modStreamType = POLLING_COMMENTS;
// use default mod stream from resources
stream = this.cacheManager.modStreams.get('newComm') as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
stream = this.cacheManager.modStreams.get(POLLING_COMMENTS) as SPoll<Snoowrap.Submission | Snoowrap.Comment>;
} else {
stream = new CommentStream(this.client, {
subreddit: this.subreddit.display_name,
@@ -1422,6 +1417,8 @@ export class Manager extends EventEmitter implements RunningStates {
});
}
break;
default:
throw new CMError(`This shouldn't happen! All polling sources are enumerated in switch. Source value: ${source}`)
}
if (stream === undefined) {
@@ -1514,10 +1511,10 @@ export class Manager extends EventEmitter implements RunningStates {
}
noChecksWarning = (source: PollOn) => (listing: any) => {
if (this.commentChecks.length === 0 && ['modqueue', 'newComm'].some(x => x === source)) {
if (this.commentChecks.length === 0 && [POLLING_MODQUEUE, POLLING_COMMENTS].some(x => x === source)) {
this.logger.warn(`Polling '${source.toUpperCase()}' may return Comments but no comments checks were configured.`);
}
if (this.submissionChecks.length === 0 && ['unmoderated', 'modqueue', 'newSub'].some(x => x === source)) {
if (this.submissionChecks.length === 0 && [POLLING_UNMODERATED, POLLING_MODQUEUE, POLLING_SUBMISSIONS].some(x => x === source)) {
this.logger.warn(`Polling '${source.toUpperCase()}' may return Submissions but no submission checks were configured.`);
}
}
@@ -1670,7 +1667,7 @@ export class Manager extends EventEmitter implements RunningStates {
}
this.startedAt = dayjs();
const modQueuePollOpts = this.pollOptions.find(x => x.pollOn === 'modqueue');
const modQueuePollOpts = this.pollOptions.find(x => x.pollOn === POLLING_MODQUEUE);
if(modQueuePollOpts !== undefined) {
this.modqueueInterval = modQueuePollOpts.interval;
}

View File

@@ -1030,13 +1030,13 @@ export class SubredditResources {
}
}
async getAuthorActivities(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing): Promise<SnoowrapActivity[]> {
async getAuthorActivities(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing, prefetchedActivities?: SnoowrapActivity[]): Promise<SnoowrapActivity[]> {
const {post} = await this.getAuthorActivitiesWithFilter(user, options, customListing);
const {post} = await this.getAuthorActivitiesWithFilter(user, options, customListing, prefetchedActivities);
return post;
}
async getAuthorActivitiesWithFilter(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing): Promise<FetchedActivitiesResult> {
async getAuthorActivitiesWithFilter(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing, prefetchedActivities?: SnoowrapActivity[]): Promise<FetchedActivitiesResult> {
let listFuncName: string;
let listFunc: ListingFunc;
@@ -1064,21 +1064,21 @@ export class SubredditResources {
...(cloneDeep(options)),
}
return await this.getActivities(user, criteriaWithDefaults, {func: listFunc, name: listFuncName});
return await this.getActivities(user, criteriaWithDefaults, {func: listFunc, name: listFuncName}, prefetchedActivities);
}
async getAuthorComments(user: RedditUser, options: ActivityWindowCriteria): Promise<Comment[]> {
return await this.getAuthorActivities(user, {...options, fetch: 'comment'}) as unknown as Promise<Comment[]>;
async getAuthorComments(user: RedditUser, options: ActivityWindowCriteria, prefetchedActivities?: SnoowrapActivity[]): Promise<Comment[]> {
return await this.getAuthorActivities(user, {...options, fetch: 'comment'}, undefined, prefetchedActivities) as unknown as Promise<Comment[]>;
}
async getAuthorSubmissions(user: RedditUser, options: ActivityWindowCriteria): Promise<Submission[]> {
async getAuthorSubmissions(user: RedditUser, options: ActivityWindowCriteria, prefetchedActivities?: SnoowrapActivity[]): Promise<Submission[]> {
return await this.getAuthorActivities(user, {
...options,
fetch: 'submission'
}) as unknown as Promise<Submission[]>;
}, undefined,prefetchedActivities) as unknown as Promise<Submission[]>;
}
async getActivities(user: RedditUser, options: ActivityWindowCriteria, listingData: NamedListing): Promise<FetchedActivitiesResult> {
async getActivities(user: RedditUser, options: ActivityWindowCriteria, listingData: NamedListing, prefetchedActivities: SnoowrapActivity[] = []): Promise<FetchedActivitiesResult> {
try {
@@ -1213,12 +1213,24 @@ export class SubredditResources {
}
}
let unFilteredItems: SnoowrapActivity[] | undefined;
let preFilteredPrefetchedActivities = [...prefetchedActivities];
if(preFilteredPrefetchedActivities.length > 0) {
switch(options.fetch) {
// TODO this may not work if using a custom listingFunc that does not include fetch type
case 'comment':
preFilteredPrefetchedActivities = preFilteredPrefetchedActivities.filter(x => asComment(x));
break;
case 'submission':
preFilteredPrefetchedActivities = preFilteredPrefetchedActivities.filter(x => asSubmission(x));
break;
}
preFilteredPrefetchedActivities = await this.filterListingWithHistoryOptions(preFilteredPrefetchedActivities, user, options.filterOn?.pre);
}
let unFilteredItems: SnoowrapActivity[] | undefined = [...preFilteredPrefetchedActivities];
pre = pre.concat(preFilteredPrefetchedActivities);
const { func: listingFunc } = listingData;
let listing = await listingFunc(getAuthorHistoryAPIOptions(options));
let hitEnd = false;
let offset = chunkSize;
@@ -1228,6 +1240,9 @@ export class SubredditResources {
timeOk = false;
let listSlice = listing.slice(offset - chunkSize);
// filter out any from slice that were already included from prefetched list so that prefetched aren't included twice
listSlice = preFilteredPrefetchedActivities.length === 0 ? listSlice : listSlice.filter(x => !preFilteredPrefetchedActivities.some(y => y.name === x.name));
let preListSlice = await this.filterListingWithHistoryOptions(listSlice, user, options.filterOn?.pre);
// its more likely the time criteria is going to be hit before the count criteria
@@ -1502,6 +1517,7 @@ export class SubredditResources {
usernotes,
ruleResults,
actionResults,
author: (val) => this.getAuthor(val)
});
}

View File

@@ -133,6 +133,7 @@ export interface TemplateContext {
ruleResults?: RuleResultEntity[]
actionResults?: ActionResultEntity[]
activity?: SnoowrapActivity
author?: (val: string | RedditUser) => Promise<RedditUser>
[key: string]: any
}
@@ -140,11 +141,25 @@ export const renderContent = async (template: string, data: TemplateContext = {}
const {
usernotes,
ruleResults,
author,
actionResults,
activity,
...restContext
} = data;
let fetchedUser: RedditUser | undefined;
// @ts-ignore
const user = async (): Promise<RedditUser> => {
if(fetchedUser === undefined) {
if(author !== undefined) {
// @ts-ignore
fetchedUser = await author(activity.author);
}
}
// @ts-ignore
return fetchedUser;
}
let view: GenericContentTemplateData = {
botLink: BOT_LINK,
...restContext
@@ -171,10 +186,24 @@ export const renderContent = async (template: string, data: TemplateContext = {}
view.modmailLink = `https://www.reddit.com/message/compose?to=%2Fr%2F${subreddit}&message=${encodeURIComponent(permalink)}`;
const author: any = {
toString: () => getActivityAuthorName(activity.author)
};
if(template.includes('{{item.author.')) {
// @ts-ignore
const auth = await user();
author.age = dayjs.unix(auth.created).fromNow(true);
author.linkKarma = auth.link_karma;
author.commentKarma = auth.comment_karma;
author.totalKarma = auth.comment_karma + auth.link_karma;
author.verified = auth.has_verified_email;
}
const templateData: any = {
kind: activity instanceof Submission ? 'submission' : 'comment',
// @ts-ignore
author: getActivityAuthorName(await activity.author),
author,
votes: activity.score,
age: dayjs.duration(dayjs().diff(dayjs.unix(activity.created))).humanize(),
permalink,

View File

@@ -77,6 +77,7 @@ import {
ImageHashCacheData,
ModUserNoteLabel,
modUserNoteLabels,
PollOn, pollOnTypeMapping, pollOnTypes,
RedditEntity,
RedditEntityType,
RelativeDateTimeMatch,
@@ -3088,3 +3089,12 @@ export const toStrongSharingACLConfig = (data: SharingACLConfig | string[]): Str
exclude: (data.exclude ?? []).map(x => parseStringToRegexOrLiteralSearch(x))
}
}
export const toPollOn = (val: string | PollOn): PollOn => {
const clean = val.toLowerCase().trim();
const pVal = pollOnTypeMapping.get(clean);
if(pVal !== undefined) {
return pVal;
}
throw new SimpleError(`'${val}' is not a valid polling source. Valid sources: ${pollOnTypes.join(' | ')}`);
}