From 300951e0786c37ec40fbbaf520f68dcf891e0cf8 Mon Sep 17 00:00:00 2001 From: rijkvanzanten Date: Mon, 13 Jul 2020 14:55:07 -0400 Subject: [PATCH] Support logical operations in filters --- package.json | 2 +- src/database/index.ts | 5 -- src/database/run-ast.ts | 103 ++++++++++++++++++------------- src/middleware/example.json | 16 +++++ src/middleware/sanitize-query.ts | 23 ++++--- src/services/items.ts | 12 ++-- src/types/query.ts | 6 +- 7 files changed, 100 insertions(+), 67 deletions(-) create mode 100644 src/middleware/example.json diff --git a/package.json b/package.json index fe35b6c97a..d68ade057c 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "scripts": { "start": "NODE_ENV=production node dist/server.js", "build": "rimraf dist && tsc -b && copyfiles \"src/**/*.*\" -e \"src/**/*.ts\" -u 1 dist", - "dev": "LOG_LEVEL=trace ts-node-dev src/server.ts --clear --watch \"src/**/*.ts\" --rs --transpile-only | pino-colada", + "dev": "LOG_LEVEL=trace ts-node-dev --files src/server.ts --clear --watch \"src/**/*.ts\" --rs --transpile-only | pino-colada", "cli": "ts-node --script-mode --transpile-only src/cli/index.ts" }, "repository": { diff --git a/src/database/index.ts b/src/database/index.ts index cc41e511ef..e8994942e3 100644 --- a/src/database/index.ts +++ b/src/database/index.ts @@ -1,13 +1,10 @@ import knex from 'knex'; -import logger from '../logger'; import dotenv from 'dotenv'; import SchemaInspector from '../knex-schema-inspector/lib/index'; dotenv.config(); -const log = logger.child({ module: 'sql' }); - const database = knex({ client: process.env.DB_CLIENT, connection: { @@ -27,8 +24,6 @@ const database = knex({ }, }); -database.on('query', (data) => log.trace(data.sql)); - export const schemaInspector = SchemaInspector(database); export default database; diff --git a/src/database/run-ast.ts b/src/database/run-ast.ts index e45359f901..cc6cdf8d13 100644 --- a/src/database/run-ast.ts +++ b/src/database/run-ast.ts @@ -1,7 +1,8 @@ import { AST, NestedCollectionAST } from '../types/ast'; import { uniq } from 'lodash'; import database, { schemaInspector } from './index'; -import { Query } from '../types/query'; +import { Filter, Query } from '../types'; +import { QueryBuilder } from 'knex'; export default async function runAST(ast: AST, query = ast.query) { const toplevelFields: string[] = []; @@ -23,33 +24,10 @@ export default async function runAST(ast: AST, query = ast.query) { nestedCollections.push(child); } - const dbQuery = database.select(toplevelFields).from(ast.name); + let dbQuery = database.select(toplevelFields).from(ast.name); if (query.filter) { - query.filter.forEach((filter) => { - if (filter.operator === 'in') { - let value = filter.value; - if (typeof value === 'string') value = value.split(','); - - dbQuery.whereIn(filter.column, value as string[]); - } - - if (filter.operator === 'eq') { - dbQuery.where({ [filter.column]: filter.value }); - } - - if (filter.operator === 'neq') { - dbQuery.whereNot({ [filter.column]: filter.value }); - } - - if (filter.operator === 'null') { - dbQuery.whereNull(filter.column); - } - - if (filter.operator === 'nnull') { - dbQuery.whereNotNull(filter.column); - } - }); + applyFilter(dbQuery, query.filter); } if (query.sort) { @@ -100,30 +78,24 @@ export default async function runAST(ast: AST, query = ast.query) { if (m2o) { batchQuery = { ...batch.query, - filter: [ - ...(batch.query.filter || []), - { - column: 'id', - operator: 'in', - // filter removes null / undefined - value: uniq(results.map((res) => res[batch.relation.field_many])).filter( + filter: { + ...(batch.query.filter || {}), + [batch.relation.primary_one]: { + _in: uniq(results.map((res) => res[batch.relation.field_many])).filter( (id) => id ), }, - ], + }, }; } else { batchQuery = { ...batch.query, - filter: [ - ...(batch.query.filter || []), - { - column: batch.relation.field_many, - operator: 'in', - // filter removes null / undefined - value: uniq(results.map((res) => res[batch.parentKey])).filter((id) => id), + filter: { + ...(batch.query.filter || {}), + [batch.relation.field_many]: { + _in: uniq(results.map((res) => res[batch.parentKey])).filter((id) => id), }, - ], + }, }; } @@ -166,3 +138,50 @@ function isM2O(child: NestedCollectionAST) { child.relation.collection_one === child.name && child.relation.field_many === child.fieldKey ); } + +function applyFilter(dbQuery: QueryBuilder, filter: Filter) { + for (const [key, value] of Object.entries(filter)) { + if (key.startsWith('_') === false) { + let operator = Object.keys(value)[0]; + operator = operator.slice(1); + operator = operator.toLowerCase(); + + const compareValue = Object.values(value)[0]; + + if (operator === 'eq') { + dbQuery.where({ [key]: compareValue }); + } + + if (operator === 'neq') { + dbQuery.whereNot({ [key]: compareValue }); + } + + if (operator === 'in') { + let value = compareValue; + if (typeof value === 'string') value = value.split(','); + + dbQuery.whereIn(key, value as string[]); + } + + if (operator === 'null') { + dbQuery.whereNull(key); + } + + if (operator === 'nnull') { + dbQuery.whereNotNull(key); + } + } + + if (key === '_or') { + value.forEach((subFilter: Record) => { + dbQuery.orWhere((subQuery) => applyFilter(subQuery, subFilter)); + }); + } + + if (key === '_and') { + value.forEach((subFilter: Record) => { + dbQuery.andWhere((subQuery) => applyFilter(subQuery, subFilter)); + }); + } + } +} diff --git a/src/middleware/example.json b/src/middleware/example.json new file mode 100644 index 0000000000..b709d68972 --- /dev/null +++ b/src/middleware/example.json @@ -0,0 +1,16 @@ +{ + "_or": [ + { + "_and": [ + { "email": { "_eq": "rijk@rngr.org" }}, + { "first_name": { "_eq": "Rijk" }} + ] + }, + { + "_and": [ + { "email": { "_eq": "ben@rngr.org" } }, + { "first_name": { "_eq": "Ben" } } + ] + } + ] +} diff --git a/src/middleware/sanitize-query.ts b/src/middleware/sanitize-query.ts index eae5a7dc4a..e49c57f20b 100644 --- a/src/middleware/sanitize-query.ts +++ b/src/middleware/sanitize-query.ts @@ -4,8 +4,9 @@ */ import { RequestHandler } from 'express'; -import { Query, Sort, Filter, FilterOperator } from '../types/query'; +import { Query, Sort, Filter } from '../types/query'; import { Meta } from '../types/meta'; +import logger from '../logger'; const sanitizeQuery: RequestHandler = (req, res, next) => { if (!req.query) return; @@ -84,14 +85,20 @@ function sanitizeSort(rawSort: any) { } function sanitizeFilter(rawFilter: any) { - const filters: Filter[] = []; + let filters: Filter = rawFilter; - Object.keys(rawFilter).forEach((column) => { - Object.keys(rawFilter[column]).forEach((operator: FilterOperator) => { - const value = rawFilter[column][operator]; - filters.push({ column, operator, value }); - }); - }); + if (typeof rawFilter === 'string') { + try { + filters = JSON.parse(rawFilter); + } catch { + logger.warn('Invalid value passed for filter query parameter.'); + } + } + + /** + * @todo + * validate filter syntax? + */ return filters; } diff --git a/src/services/items.ts b/src/services/items.ts index 74f6ea7f70..7ab26ba46c 100644 --- a/src/services/items.ts +++ b/src/services/items.ts @@ -98,14 +98,12 @@ export const readItem = async ( query = { ...query, - filter: [ - ...(query.filter || []), - { - column: primaryKeyField, - operator: 'eq', - value: pk, + filter: { + ...(query.filter || {}), + [primaryKeyField]: { + _eq: pk, }, - ], + }, }; const ast = await getAST(collection, query); diff --git a/src/types/query.ts b/src/types/query.ts index 992acdd140..079046e557 100644 --- a/src/types/query.ts +++ b/src/types/query.ts @@ -3,7 +3,7 @@ import { Meta } from './meta'; export type Query = { fields?: string[]; sort?: Sort[]; - filter?: Filter[]; + filter?: Filter; limit?: number; offset?: number; page?: number; @@ -18,9 +18,7 @@ export type Sort = { }; export type Filter = { - column: string; - operator: FilterOperator; - value: null | string | number | (string | number)[]; + [keyOrOperator: string]: Filter | any; }; export type FilterOperator = 'eq' | 'neq' | 'in' | 'nin' | 'null' | 'nnull';