feat(filter): Improve building filters from config

* Fix ignoring of filters when they are plain objects
* Fix default filter merging behavior to account for "new" filter structure (named criteria)
* Add tests for building/merging filters
This commit is contained in:
FoxxMD
2022-10-26 14:43:44 -04:00
parent d318286507
commit 0278a4d673
6 changed files with 450 additions and 73 deletions

View File

@@ -41,6 +41,10 @@ export interface FilterOptionsJson<T> {
}
export const asFilterOptionsJson = <T>(val: any): val is FilterOptionsJson<T> => {
return val !== null && typeof val === 'object' && (val.include !== undefined || val.exclude !== undefined);
}
export interface FilterOptionsConfig<T> extends FilterOptionsJson<T> {
/**
@@ -66,7 +70,7 @@ export interface FilterOptions<T> extends FilterOptionsConfig<T> {
export type MinimalOrFullFilter<T> = MaybeAnonymousCriteria<T>[] | FilterOptions<T>
export type MinimalOrFullMaybeAnonymousFilter<T> = MaybeAnonymousCriteria<T>[] | FilterOptionsConfig<T>
export type MinimalOrFullFilterJson<T> = MaybeAnonymousOrStringCriteria<T>[] | FilterOptionsJson<T>
export type MinimalOrFullFilterJson<T> = MaybeAnonymousOrStringCriteria<T> | MaybeAnonymousOrStringCriteria<T>[] | FilterOptionsJson<T>
export type StructuredFilter<T> = Omit<T, 'authorIs' | 'itemIs'> & {
itemIs?: MinimalOrFullFilter<TypedActivityState>
authorIs?: MinimalOrFullFilter<AuthorCriteria>

View File

@@ -78,6 +78,7 @@ import {
PollOn
} from "./Common/Infrastructure/Atomic";
import {
asFilterOptionsJson,
FilterCriteriaDefaults,
FilterCriteriaDefaultsJson,
MaybeAnonymousOrStringCriteria, MinimalOrFullFilter, MinimalOrFullFilterJson, NamedCriteria
@@ -385,7 +386,7 @@ export class ConfigBuilder {
return await this.hydrateConfig(config, resource);
}
async parseToStructured(hydratedConfig: SubredditConfigHydratedData, resource: SubredditResources, filterCriteriaDefaultsFromBot?: FilterCriteriaDefaults, postCheckBehaviorDefaultsFromBot: PostBehavior = {}): Promise<RunConfigObject[]> {
async parseToStructured(hydratedConfig: SubredditConfigHydratedData, filterCriteriaDefaultsFromBot?: FilterCriteriaDefaults, postCheckBehaviorDefaultsFromBot: PostBehavior = {}): Promise<RunConfigObject[]> {
let namedRules: Map<string, RuleConfigObject> = new Map();
let namedActions: Map<string, ActionConfigObject> = new Map();
const {filterCriteriaDefaults, postCheckBehaviorDefaults} = hydratedConfig;
@@ -548,7 +549,7 @@ const parseFilterJson = <T>(addToFilter: FilterJsonFuncArg<T>) => (val: MinimalO
for (const v of val) {
addToFilter(v);
}
} else {
} else if(asFilterOptionsJson<T>(val)) {
const {include = [], exclude = []} = val;
for (const v of include) {
addToFilter(v);
@@ -566,46 +567,6 @@ export const extractNamedFilters = (config: SubredditConfigHydratedData, namedAu
const parseAuthorIs = parseFilterJson(addToAuthors);
const parseItemIs = parseFilterJson(addToItems);
// const parseAuthorIs = (val: MinimalOrFullFilterJson<AuthorCriteria> | undefined) => {
// if (val === undefined) {
// return;
// }
// if (Array.isArray(val)) {
// for (const v of val) {
// addToAuthors(v);
// }
// } else {
// const {include = [], exclude = []} = val;
// for (const v of include) {
// addToAuthors(v);
// }
// for (const v of exclude) {
// addToAuthors(v);
// }
// }
// }
// const parseItemIs = (val: MinimalOrFullFilterJson<TypedActivityState> | undefined) => {
// if (val === undefined) {
// return;
// }
// if (Array.isArray(val)) {
// for (const v of val) {
// addToItems(v);
// }
// } else {
// const {include = [], exclude = []} = val;
// for (const v of include) {
// addToItems(v);
// }
// for (const v of exclude) {
// addToItems(v);
// }
// }
// }
//const namedRules = new Map();
const {
filterCriteriaDefaults,
runs = []
@@ -681,34 +642,36 @@ export const insertNameFilters = (namedAuthorFilters: Map<string, NamedCriteria<
authorIs: undefined,
itemIs: undefined,
}
if(val.authorIs !== undefined) {
if (val.authorIs !== undefined) {
if (Array.isArray(val.authorIs)) {
runnableOpts.authorIs = val.authorIs.map(x => getNamedAuthorOrReturn(x))
} else {
runnableOpts.authorIs = {};
const {include, exclude} = val.authorIs;
if(include !== undefined) {
} else if (asFilterOptionsJson<AuthorCriteria>(val.authorIs)) {
const {include, exclude, ...rest} = val.authorIs;
runnableOpts.authorIs = {...rest};
if (include !== undefined) {
runnableOpts.authorIs.include = include.map(x => getNamedAuthorOrReturn(x))
}
if(exclude !== undefined) {
} else if (exclude !== undefined) {
runnableOpts.authorIs.exclude = exclude.map(x => getNamedAuthorOrReturn(x))
}
} else {
// assume object is criteria
runnableOpts.authorIs = [getNamedAuthorOrReturn(val.authorIs)];
}
}
if(val.itemIs !== undefined) {
if (val.itemIs !== undefined) {
if (Array.isArray(val.itemIs)) {
runnableOpts.itemIs = val.itemIs.map(x => getNamedItemOrReturn(x))
} else {
runnableOpts.itemIs = {};
const {include, exclude} = val.itemIs;
if(include !== undefined) {
} else if (asFilterOptionsJson<TypedActivityState>(val.itemIs)) {
const {include, exclude, ...rest} = val.itemIs;
runnableOpts.itemIs = {...rest};
if (include !== undefined) {
runnableOpts.itemIs.include = include.map(x => getNamedItemOrReturn(x))
}
if(exclude !== undefined) {
} else if (exclude !== undefined) {
runnableOpts.itemIs.exclude = exclude.map(x => getNamedItemOrReturn(x))
}
} else {
// assume object is criteria
runnableOpts.itemIs = [getNamedItemOrReturn(val.itemIs)];
}
}

View File

@@ -703,7 +703,7 @@ export class Manager extends EventEmitter implements RunningStates {
const hydratedConfig = await configBuilder.hydrateConfig(validJson, this.resources);
this.lastParseConfigHash = objectHash.sha1(hydratedConfig);
const structuredRuns = await configBuilder.parseToStructured(hydratedConfig, this.resources, this.filterCriteriaDefaults, this.postCheckBehaviorDefaults);
const structuredRuns = await configBuilder.parseToStructured(hydratedConfig, this.filterCriteriaDefaults, this.postCheckBehaviorDefaults);
let runs: Run[] = [];

View File

@@ -2475,14 +2475,22 @@ export const mergeFilters = (objectConfig: RunnableBaseJson, filterDefs: FilterC
let derivedAuthorIs: AuthorOptions = buildFilter(authorIsDefault);
if (authorIsBehavior === 'merge') {
derivedAuthorIs = merge.all([authorIs, authorIsDefault], {arrayMerge: removeFromSourceIfKeysExistsInDestination});
derivedAuthorIs = {
excludeCondition: authorIs.excludeCondition ?? derivedAuthorIs.excludeCondition,
include: addNonConflictingCriteria(derivedAuthorIs.include, authorIs.include),
exclude: addNonConflictingCriteria(derivedAuthorIs.exclude, authorIs.exclude),
}
} else if (!filterIsEmpty(authorIs)) {
derivedAuthorIs = authorIs;
}
let derivedItemIs: ItemOptions = buildFilter(itemIsDefault);
if (itemIsBehavior === 'merge') {
derivedItemIs = merge.all([itemIs, itemIsDefault], {arrayMerge: removeFromSourceIfKeysExistsInDestination});
derivedItemIs = {
excludeCondition: itemIs.excludeCondition ?? derivedItemIs.excludeCondition,
include: addNonConflictingCriteria(derivedItemIs.include, itemIs.include),
exclude: addNonConflictingCriteria(derivedItemIs.exclude, itemIs.exclude),
}
} else if (!filterIsEmpty(itemIs)) {
derivedItemIs = itemIs;
}
@@ -2490,6 +2498,23 @@ export const mergeFilters = (objectConfig: RunnableBaseJson, filterDefs: FilterC
return [derivedAuthorIs, derivedItemIs];
}
export const addNonConflictingCriteria = <T>(defaultCriteria: NamedCriteria<T>[] = [], explicitCriteria: NamedCriteria<T>[] = []): NamedCriteria<T>[] => {
if(explicitCriteria.length === 0) {
return defaultCriteria;
}
const allExplicitKeys = Array.from(explicitCriteria.reduce((acc, curr) => {
Object.keys(curr.criteria).forEach(key => acc.add(key));
return acc;
}, new Set()));
const nonConflicting = defaultCriteria.filter(x => {
return intersect(Object.keys(x.criteria), allExplicitKeys).length === 0;
});
if(nonConflicting.length > 0) {
return explicitCriteria.concat(nonConflicting);
}
return explicitCriteria;
}
export const filterIsEmpty = (obj: FilterOptions<any>): boolean => {
return (obj.include === undefined || obj.include.length === 0) && (obj.exclude === undefined || obj.exclude.length === 0);
}

228
tests/config.test.ts Normal file
View File

@@ -0,0 +1,228 @@
import {describe, it} from 'mocha';
import {assert} from 'chai';
import {insertNameFilters} from "../src/ConfigBuilder";
import {
authorAgeDayCrit, authorAgeMonthCrit,
authorFlair1Crit,
fullAuthorAnonymousAll,
fullAuthorAnonymousExclude,
fullAuthorAnonymousInclude,
fullAuthorFullExclude,
fullAuthorFullInclude,
fullItemAnonymousAll,
fullItemAnonymousExclude,
fullItemAnonymousInclude,
fullItemFullAll,
fullItemFullExclude,
fullItemFullInclude,
itemApprovedCrit,
itemRemovedCrit,
maybeAnonymousFullAuthorFilter,
minimalAuthorFilter,
namedAuthorFilter,
namedAuthorFilters,
namedItemFilter,
namedItemFilters
} from "./testFactory";
import {buildFilter, mergeFilters} from "../src/util";
import {FilterOptions, FilterOptionsConfig, NamedCriteria} from "../src/Common/Infrastructure/Filters/FilterShapes";
import {AuthorCriteria} from "../src";
import {filterCriteriaDefault} from "../src/Common/defaults";
const namedFilters = insertNameFilters(namedAuthorFilters, namedItemFilters);
describe('Filter Building', function () {
describe('Convert string or plain objects/arrays to AT LEAST filters with anonymous criteria', function () {
describe('Author Filter', function () {
describe('Anonymous Filters', function () {
it('Accepts plain object', function () {
const filters = namedFilters({authorIs: authorAgeDayCrit()})
assert.deepEqual(filters.authorIs, [{criteria: authorAgeDayCrit()}]);
});
it('Accepts plain array of objects', function () {
const filters = namedFilters({authorIs: [authorAgeDayCrit(), authorFlair1Crit()]})
assert.deepEqual(filters.authorIs, [{criteria: authorAgeDayCrit()}, {criteria: authorFlair1Crit()}]);
});
it('Accepts full anonymous include config and returns full anonymous include config', function () {
const authFull = namedFilters({authorIs: fullAuthorAnonymousInclude()})
assert.deepEqual(authFull.authorIs, fullAuthorFullInclude());
});
it('Accepts full anonymous exclude config and returns full anonymous exclude config', function () {
const authFull = namedFilters({authorIs: fullAuthorAnonymousExclude()})
assert.deepEqual(authFull.authorIs, fullAuthorFullExclude());
});
it('Accepts full anonymous include-exclude config and returns full anonymous include with no exclude', function () {
const authFull = namedFilters({authorIs: fullAuthorAnonymousAll()})
assert.deepEqual(authFull.authorIs, fullAuthorFullInclude());
});
});
describe('Named Filters', function () {
it('Inserts named filter from plain array', function () {
const filters = namedFilters({authorIs: ['test1Author', authorFlair1Crit()]})
assert.deepEqual(filters.authorIs, [namedAuthorFilter(), {criteria: authorFlair1Crit()}]);
});
});
});
describe('Item Filter', function () {
describe('Anonymous Filters', function () {
it('Accepts plain object', function () {
const filters = namedFilters({itemIs: itemRemovedCrit()})
assert.deepEqual(filters.itemIs, [{criteria: itemRemovedCrit()}]);
});
it('Accepts plain array of objects', function () {
const filters = namedFilters({itemIs: [itemRemovedCrit(), itemApprovedCrit()]})
assert.deepEqual(filters.itemIs, [{criteria: itemRemovedCrit()}, {criteria: itemApprovedCrit()}]);
});
it('Accepts full anonymous include config and returns full anonymous include config', function () {
const fullFilter = namedFilters({itemIs: fullItemAnonymousInclude()})
assert.deepEqual(fullFilter.itemIs, fullItemFullInclude());
});
it('Accepts full anonymous exclude config and returns full anonymous exclude config', function () {
const fullFilter = namedFilters({itemIs: fullItemAnonymousExclude()})
assert.deepEqual(fullFilter.itemIs, fullItemFullExclude());
});
it('Accepts full anonymous include-exclude config and returns full anonymous include with no exclude', function () {
const fullFilter = namedFilters({itemIs: fullItemAnonymousAll()})
assert.deepEqual(fullFilter.itemIs, fullItemFullAll());
});
});
describe('Named Filters', function () {
it('Inserts named filter from plain array', function () {
const filters = namedFilters({itemIs: ['test1Item', itemApprovedCrit()]})
assert.deepEqual(filters.itemIs, [namedItemFilter(), {criteria: itemApprovedCrit()}]);
});
});
});
describe('Full Filter', function () {
it('Accepts and returns full filter', function () {
const filters = namedFilters({
itemIs: ['test1Item', itemApprovedCrit()],
authorIs: ['test1Author', authorFlair1Crit()]
})
assert.deepEqual(filters.itemIs, [namedItemFilter(), {criteria: itemApprovedCrit()}]);
assert.deepEqual(filters.authorIs, [namedAuthorFilter(), {criteria: authorFlair1Crit()}]);
});
});
});
describe('Convert hydrated/anonymous criteria to full FilterOptions', function () {
describe('Author Filter', function () {
it('Converts minimal (array) filter into include', function () {
const opts = buildFilter(minimalAuthorFilter());
assert.deepEqual(opts, {
include: (minimalAuthorFilter() as NamedCriteria<AuthorCriteria>[]).map((x) => ({
...x,
name: undefined
})),
excludeCondition: 'OR',
exclude: []
});
});
it('Converts anonymous full filter into FilterOptions', function () {
const opts = buildFilter(maybeAnonymousFullAuthorFilter());
assert.deepEqual(opts, {
include: [
{
criteria: authorAgeDayCrit()
},
{
criteria: authorAgeMonthCrit()
}
].map((x) => ({...x, name: undefined})),
excludeCondition: undefined,
exclude: [
{
criteria: authorAgeDayCrit()
},
{
criteria: authorAgeMonthCrit()
}
].map((x) => ({...x, name: undefined}))
})
});
});
});
describe('Filter merging', function () {
it('Merges (adds) when user-defined filter and defaults filter are present', function () {
const [author] = mergeFilters({
authorIs: {
exclude: [
{
criteria: {age: '> 1 hour'}
}]
}
}, filterCriteriaDefault);
assert.deepEqual(author.exclude, [
{
criteria: {age: '> 1 hour'},
name: undefined
}, {
criteria: {isMod: true},
name: undefined
}]);
});
it('Does not merge when user-defined filter and defaults filter are present with conflicting properties', function () {
const [author] = mergeFilters({
authorIs: {
exclude: [{
criteria: {
age: '> 1 hour',
isMod: true
}
}]
}
}, filterCriteriaDefault);
assert.deepEqual(author.exclude, [{criteria: {age: '> 1 hour', isMod: true}, name: undefined}]);
});
it('User-defined filter replaces defaults filter when replace behavior is set', function () {
const [author] = mergeFilters({
authorIs: {
exclude: [
{
criteria: {age: '> 1 hour'}
}
]
}
}, {
authorIsBehavior: 'replace',
authorIs: {
exclude: [
{
criteria: {name: ['test']}
}]
}
});
assert.deepEqual(author.exclude, [{criteria: {age: '> 1 hour'}, name: undefined}]);
});
it('Ignores mods by default', function () {
const [author] = mergeFilters({}, filterCriteriaDefault);
assert.deepEqual(author.exclude, [
{
criteria: {isMod: true},
name: undefined
}]);
});
});
});

View File

@@ -1,7 +1,7 @@
import {OperatorConfig, OperatorJsonConfig} from "../src/Common/interfaces";
import Snoowrap from "snoowrap";
import Bot from "../src/Bot/index"
import {buildOperatorConfigWithDefaults} from "../src/ConfigBuilder";
import {buildOperatorConfigWithDefaults, insertNameFilters} from "../src/ConfigBuilder";
import {App} from "../src/App";
import {YamlOperatorConfigDocument} from "../src/Common/Config/Operator";
import {NoopLogger} from "../src/Utils/loggerFactory";
@@ -10,6 +10,13 @@ import {Bot as BotEntity} from "../src/Common/Entities/Bot";
import {SubredditResources} from "../src/Subreddit/SubredditResources";
import {Subreddit, Comment, Submission} from 'snoowrap/dist/objects';
import dayjs from 'dayjs';
import {
FilterOptions, MaybeAnonymousCriteria,
MinimalOrFullFilter,
MinimalOrFullFilterJson, MinimalOrFullMaybeAnonymousFilter, NamedCriteria
} from "../src/Common/Infrastructure/Filters/FilterShapes";
import {AuthorCriteria} from "../src";
import {TypedActivityState} from "../src/Common/Infrastructure/Filters/FilterCriteria";
const mockSnoowrap = new Snoowrap({userAgent: 'test', accessToken: 'test'});
@@ -80,10 +87,10 @@ export const getBot = async () => {
await bot.cacheManager.set('test', {
logger: NoopLogger,
caching: {
authorTTL: false,
submissionTTL: false,
commentTTL: false,
provider: 'memory'
authorTTL: false,
submissionTTL: false,
commentTTL: false,
provider: 'memory'
},
subreddit: bot.client.getSubreddit('test'),
client: bot.client,
@@ -145,12 +152,12 @@ export const sampleActivity = {
}, snoowrap, true)
},
commentRemoved: (snoowrap = mockSnoowrap) => {
return new Comment({
can_mod_post: true,
banned_at_utc: dayjs().subtract(10, 'minutes').unix(),
removed: true,
replies: ''
}, snoowrap, true);
return new Comment({
can_mod_post: true,
banned_at_utc: dayjs().subtract(10, 'minutes').unix(),
removed: true,
replies: ''
}, snoowrap, true);
},
submissionDeleted: (snoowrap = mockSnoowrap) => {
return new Submission({
@@ -200,3 +207,153 @@ export const sampleActivity = {
}
}
}
export const authorAgeDayCrit = (): AuthorCriteria => ({
age: '> 1 day'
});
export const authorAgeMonthCrit = (): AuthorCriteria => ({
age: '> 1 month'
});
export const authorFlair1Crit = (): AuthorCriteria => ({
flairText: 'flair 1'
});
export const authorFlair2Crit = (): AuthorCriteria => ({
flairText: 'flair 2'
});
export const fullAuthorFullInclude = (): FilterOptions<AuthorCriteria> => ({
include: [
{
criteria: authorAgeDayCrit()
},
{
criteria: authorFlair1Crit()
}
]
})
export const fullAuthorFullExclude = (): FilterOptions<AuthorCriteria> => ({
exclude: [
{
criteria: authorAgeMonthCrit()
},
{
criteria: authorFlair2Crit()
}
]
})
export const fullAuthorFullAll = (): FilterOptions<AuthorCriteria> => ({
include: fullAuthorFullInclude().include,
})
export const fullAuthorAnonymousInclude = (): MinimalOrFullFilterJson<AuthorCriteria> => ({
include: [
authorAgeDayCrit(),
authorFlair1Crit()
]
})
export const fullAuthorAnonymousExclude = (): MinimalOrFullFilterJson<AuthorCriteria> => ({
exclude: [
authorAgeMonthCrit(),
authorFlair2Crit()
]
});
export const fullAuthorAnonymousAll = (): MinimalOrFullFilterJson<AuthorCriteria> => ({
include: (fullAuthorAnonymousInclude() as FilterOptions<AuthorCriteria>).include,
exclude: (fullAuthorAnonymousExclude() as FilterOptions<AuthorCriteria>).exclude,
})
export const namedAuthorFilter = (): NamedCriteria<AuthorCriteria> => ({
name: 'test1Author',
criteria: authorAgeDayCrit()
});
export const itemRemovedCrit = (): TypedActivityState => ({
removed: false
});
export const itemApprovedCrit = (): TypedActivityState => ({
approved: true
});
export const itemFlairCrit = (): TypedActivityState => ({
link_flair_text: ['test1','test2']
});
export const fullItemFullInclude = (): FilterOptions<TypedActivityState> => ({
include: [
{
criteria: itemRemovedCrit()
},
{
criteria: itemApprovedCrit()
}
]
})
export const fullItemFullExclude = (): FilterOptions<TypedActivityState> => ({
exclude: [
{
criteria: itemRemovedCrit()
},
{
criteria: itemFlairCrit()
}
]
})
export const fullItemFullAll = (): FilterOptions<TypedActivityState> => ({
include: fullItemFullInclude().include,
})
export const fullItemAnonymousInclude = (): MinimalOrFullFilterJson<TypedActivityState> => ({
include: [
itemRemovedCrit(),
itemApprovedCrit()
]
})
export const fullItemAnonymousExclude = (): MinimalOrFullFilterJson<TypedActivityState> => ({
exclude: [
itemRemovedCrit(),
itemFlairCrit()
]
});
export const fullItemAnonymousAll = (): MinimalOrFullFilterJson<TypedActivityState> => ({
include: (fullItemAnonymousInclude() as FilterOptions<TypedActivityState>).include,
exclude: (fullItemAnonymousExclude() as FilterOptions<TypedActivityState>).exclude,
})
export const namedItemFilter = (): NamedCriteria<TypedActivityState> => ({
name: 'test1Item',
criteria: itemRemovedCrit()
});
export const namedAuthorFilters = new Map([['test1author', namedAuthorFilter()]]);
export const namedItemFilters = new Map([['test1item', namedItemFilter()]]);
export const minimalAuthorFilter = (): MinimalOrFullMaybeAnonymousFilter<AuthorCriteria> => ([
{
criteria: authorAgeDayCrit()
},
{
criteria: authorAgeMonthCrit()
}
]);
export const maybeAnonymousFullAuthorFilter = (): MinimalOrFullMaybeAnonymousFilter<AuthorCriteria> => ({
include: [
{
criteria: authorAgeDayCrit()
},
{
criteria: authorAgeMonthCrit()
}
],
exclude: [
{
criteria: authorAgeDayCrit()
},
{
criteria: authorAgeMonthCrit()
}
]
});