Support logical operations in filters

package.json

@@ -9,7 +9,7 @@
   "scripts": {
     "start": "NODE_ENV=production node dist/server.js",
     "build": "rimraf dist && tsc -b && copyfiles \"src/**/*.*\" -e \"src/**/*.ts\" -u 1 dist",
-    "dev": "LOG_LEVEL=trace ts-node-dev src/server.ts --clear --watch \"src/**/*.ts\" --rs --transpile-only | pino-colada",
+    "dev": "LOG_LEVEL=trace ts-node-dev --files src/server.ts --clear --watch \"src/**/*.ts\" --rs --transpile-only | pino-colada",
     "cli": "ts-node --script-mode --transpile-only src/cli/index.ts"
   },
   "repository": {

src/database/index.ts

@@ -1,13 +1,10 @@
 import knex from 'knex';
-import logger from '../logger';
 import dotenv from 'dotenv';
 
 import SchemaInspector from '../knex-schema-inspector/lib/index';
 
 dotenv.config();
 
-const log = logger.child({ module: 'sql' });
-
 const database = knex({
 	client: process.env.DB_CLIENT,
 	connection: {
@@ -27,8 +24,6 @@ const database = knex({
 	},
 });
 
-database.on('query', (data) => log.trace(data.sql));
-
 export const schemaInspector = SchemaInspector(database);
 
 export default database;

src/database/run-ast.ts

@@ -1,7 +1,8 @@
 import { AST, NestedCollectionAST } from '../types/ast';
 import { uniq } from 'lodash';
 import database, { schemaInspector } from './index';
-import { Query } from '../types/query';
+import { Filter, Query } from '../types';
+import { QueryBuilder } from 'knex';
 
 export default async function runAST(ast: AST, query = ast.query) {
 	const toplevelFields: string[] = [];
@@ -23,33 +24,10 @@ export default async function runAST(ast: AST, query = ast.query) {
 		nestedCollections.push(child);
 	}
 
-	const dbQuery = database.select(toplevelFields).from(ast.name);
+	let dbQuery = database.select(toplevelFields).from(ast.name);
 
 	if (query.filter) {
-		query.filter.forEach((filter) => {
-			if (filter.operator === 'in') {
-				let value = filter.value;
-				if (typeof value === 'string') value = value.split(',');
-
-				dbQuery.whereIn(filter.column, value as string[]);
-			}
-
-			if (filter.operator === 'eq') {
-				dbQuery.where({ [filter.column]: filter.value });
-			}
-
-			if (filter.operator === 'neq') {
-				dbQuery.whereNot({ [filter.column]: filter.value });
-			}
-
-			if (filter.operator === 'null') {
-				dbQuery.whereNull(filter.column);
-			}
-
-			if (filter.operator === 'nnull') {
-				dbQuery.whereNotNull(filter.column);
-			}
-		});
+		applyFilter(dbQuery, query.filter);
 	}
 
 	if (query.sort) {
@@ -100,30 +78,24 @@ export default async function runAST(ast: AST, query = ast.query) {
 		if (m2o) {
 			batchQuery = {
 				...batch.query,
-				filter: [
-					...(batch.query.filter || []),
-					{
-						column: 'id',
-						operator: 'in',
-						// filter removes null / undefined
-						value: uniq(results.map((res) => res[batch.relation.field_many])).filter(
+				filter: {
+					...(batch.query.filter || {}),
+					[batch.relation.primary_one]: {
+						_in: uniq(results.map((res) => res[batch.relation.field_many])).filter(
 							(id) => id
 						),
 					},
-				],
+				},
 			};
 		} else {
 			batchQuery = {
 				...batch.query,
-				filter: [
-					...(batch.query.filter || []),
-					{
-						column: batch.relation.field_many,
-						operator: 'in',
-						// filter removes null / undefined
-						value: uniq(results.map((res) => res[batch.parentKey])).filter((id) => id),
+				filter: {
+					...(batch.query.filter || {}),
+					[batch.relation.field_many]: {
+						_in: uniq(results.map((res) => res[batch.parentKey])).filter((id) => id),
 					},
-				],
+				},
 			};
 		}
 
@@ -166,3 +138,50 @@ function isM2O(child: NestedCollectionAST) {
 		child.relation.collection_one === child.name && child.relation.field_many === child.fieldKey
 	);
 }
+
+function applyFilter(dbQuery: QueryBuilder, filter: Filter) {
+	for (const [key, value] of Object.entries(filter)) {
+		if (key.startsWith('_') === false) {
+			let operator = Object.keys(value)[0];
+			operator = operator.slice(1);
+			operator = operator.toLowerCase();
+
+			const compareValue = Object.values(value)[0];
+
+			if (operator === 'eq') {
+				dbQuery.where({ [key]: compareValue });
+			}
+
+			if (operator === 'neq') {
+				dbQuery.whereNot({ [key]: compareValue });
+			}
+
+			if (operator === 'in') {
+				let value = compareValue;
+				if (typeof value === 'string') value = value.split(',');
+
+				dbQuery.whereIn(key, value as string[]);
+			}
+
+			if (operator === 'null') {
+				dbQuery.whereNull(key);
+			}
+
+			if (operator === 'nnull') {
+				dbQuery.whereNotNull(key);
+			}
+		}
+
+		if (key === '_or') {
+			value.forEach((subFilter: Record<string, any>) => {
+				dbQuery.orWhere((subQuery) => applyFilter(subQuery, subFilter));
+			});
+		}
+
+		if (key === '_and') {
+			value.forEach((subFilter: Record<string, any>) => {
+				dbQuery.andWhere((subQuery) => applyFilter(subQuery, subFilter));
+			});
+		}
+	}
+}

src/middleware/example.json

@@ -0,0 +1,16 @@
+{
+    "_or": [
+        {
+            "_and": [
+                { "email": { "_eq": "rijk@rngr.org" }},
+                { "first_name": { "_eq": "Rijk" }}
+            ]
+        },
+        {
+            "_and": [
+                { "email": { "_eq": "ben@rngr.org" } },
+                { "first_name": { "_eq": "Ben" } }
+            ]
+        }
+    ]
+}

src/middleware/sanitize-query.ts

@@ -4,8 +4,9 @@
  */
 
 import { RequestHandler } from 'express';
-import { Query, Sort, Filter, FilterOperator } from '../types/query';
+import { Query, Sort, Filter } from '../types/query';
 import { Meta } from '../types/meta';
+import logger from '../logger';
 
 const sanitizeQuery: RequestHandler = (req, res, next) => {
 	if (!req.query) return;
@@ -84,14 +85,20 @@ function sanitizeSort(rawSort: any) {
 }
 
 function sanitizeFilter(rawFilter: any) {
-	const filters: Filter[] = [];
+	let filters: Filter = rawFilter;
 
-	Object.keys(rawFilter).forEach((column) => {
-		Object.keys(rawFilter[column]).forEach((operator: FilterOperator) => {
-			const value = rawFilter[column][operator];
-			filters.push({ column, operator, value });
-		});
-	});
+	if (typeof rawFilter === 'string') {
+		try {
+			filters = JSON.parse(rawFilter);
+		} catch {
+			logger.warn('Invalid value passed for filter query parameter.');
+		}
+	}
+
+	/**
+	 * @todo
+	 * validate filter syntax?
+	 */
 
 	return filters;
 }

src/services/items.ts

@@ -98,14 +98,12 @@ export const readItem = async <T = any>(
 
 	query = {
 		...query,
-		filter: [
-			...(query.filter || []),
-			{
-				column: primaryKeyField,
-				operator: 'eq',
-				value: pk,
+		filter: {
+			...(query.filter || {}),
+			[primaryKeyField]: {
+				_eq: pk,
 			},
-		],
+		},
 	};
 
 	const ast = await getAST(collection, query);

src/types/query.ts

@@ -3,7 +3,7 @@ import { Meta } from './meta';
 export type Query = {
 	fields?: string[];
 	sort?: Sort[];
-	filter?: Filter[];
+	filter?: Filter;
 	limit?: number;
 	offset?: number;
 	page?: number;
@@ -18,9 +18,7 @@ export type Sort = {
 };
 
 export type Filter = {
-	column: string;
-	operator: FilterOperator;
-	value: null | string | number | (string | number)[];
+	[keyOrOperator: string]: Filter | any;
 };
 
 export type FilterOperator = 'eq' | 'neq' | 'in' | 'nin' | 'null' | 'nnull';