From febdd0adbdcb639f72706c3a830786b125d4a672 Mon Sep 17 00:00:00 2001 From: rijkvanzanten Date: Tue, 20 Oct 2020 11:02:02 -0400 Subject: [PATCH 1/3] Attempt at resolving joins in nested where queries --- api/src/services/meta.ts | 2 +- api/src/utils/apply-query.ts | 299 ++++++++++++++++++----------------- 2 files changed, 157 insertions(+), 144 deletions(-) diff --git a/api/src/services/meta.ts b/api/src/services/meta.ts index 9764dcdcee..fad8c127b8 100644 --- a/api/src/services/meta.ts +++ b/api/src/services/meta.ts @@ -40,7 +40,7 @@ export class MetaService { const dbQuery = database(collection).count('*', { as: 'count' }); if (query.filter) { - applyFilter(dbQuery, query.filter, collection); + await applyFilter(dbQuery, query.filter, collection); } const records = await dbQuery; diff --git a/api/src/utils/apply-query.ts b/api/src/utils/apply-query.ts index d0ead782c3..77bd8adece 100644 --- a/api/src/utils/apply-query.ts +++ b/api/src/utils/apply-query.ts @@ -46,116 +46,179 @@ export default async function applyQuery(collection: string, dbQuery: QueryBuild } } -export async function applyFilter(dbQuery: QueryBuilder, filter: Filter, collection: string) { - for (const [key, value] of Object.entries(filter)) { - if (key === '_or') { - value.forEach((subFilter: Record) => { - dbQuery.orWhere((subQuery) => applyFilter(subQuery, subFilter, collection)); - }); +export async function applyFilter(rootQuery: QueryBuilder, rootFilter: Filter, collection: string) { + const relations = await database.select('*').from('directus_relations'); - continue; + parseLevel(rootQuery, rootFilter, collection); + + function parseLevel(dbQuery: QueryBuilder, filter: Filter, collection: string) { + for (const [key, value] of Object.entries(filter)) { + if (key === '_or') { + dbQuery.orWhere((subQuery) => { + value.forEach((subFilter: Record) => { + parseLevel(subQuery, subFilter, collection); + }); + }); + + continue; + } + + if (key === '_and') { + dbQuery.andWhere((subQuery) => { + value.forEach((subFilter: Record) => { + parseLevel(subQuery, subFilter, collection); + }); + }); + + continue; + } + + const filterPath = getFilterPath(key, value); + const { operator: filterOperator, value: filterValue } = getOperation(key, value); + + const column = + filterPath.length > 1 + ? applyJoins(filterPath, collection) + : `${collection}.${filterPath[0]}`; + + applyFilterToQuery(column, filterOperator, filterValue); } - if (key === '_and') { - value.forEach((subFilter: Record) => { - dbQuery.andWhere((subQuery) => applyFilter(subQuery, subFilter, collection)); - }); + function applyFilterToQuery(key: string, operator: string, compareValue: any) { + if (operator === '_eq') { + dbQuery.where({ [key]: compareValue }); + } - continue; + if (operator === '_neq') { + dbQuery.whereNot({ [key]: compareValue }); + } + + if (operator === '_contains') { + dbQuery.where(key, 'like', `%${compareValue}%`); + } + + if (operator === '_ncontains') { + dbQuery.where(key, 'like', `%${compareValue}%`); + } + + if (operator === '_gt') { + dbQuery.where(key, '>', compareValue); + } + + if (operator === '_gte') { + dbQuery.where(key, '>=', compareValue); + } + + if (operator === '_lt') { + dbQuery.where(key, '<', compareValue); + } + + if (operator === '_lte') { + dbQuery.where(key, '<=', compareValue); + } + + if (operator === '_in') { + let value = compareValue; + if (typeof value === 'string') value = value.split(','); + + dbQuery.whereIn(key, value as string[]); + } + + if (operator === '_nin') { + let value = compareValue; + if (typeof value === 'string') value = value.split(','); + + dbQuery.whereNotIn(key, value as string[]); + } + + if (operator === '_null') { + dbQuery.whereNull(key); + } + + if (operator === '_nnull') { + dbQuery.whereNotNull(key); + } + + if (operator === '_empty') { + dbQuery.andWhere((query) => { + query.whereNull(key); + query.orWhere(key, '=', ''); + }); + } + + if (operator === '_nempty') { + dbQuery.andWhere((query) => { + query.whereNotNull(key); + query.orWhere(key, '!=', ''); + }); + } + + if (operator === '_between') { + let value = compareValue; + if (typeof value === 'string') value = value.split(','); + + dbQuery.whereBetween(key, value); + } + + if (operator === '_nbetween') { + let value = compareValue; + if (typeof value === 'string') value = value.split(','); + + dbQuery.whereNotBetween(key, value); + } } - - const filterPath = getFilterPath(key, value); - const { operator: filterOperator, value: filterValue } = getOperation(key, value); - - const column = - filterPath.length > 1 - ? await applyJoins(dbQuery, filterPath, collection) - : `${collection}.${filterPath[0]}`; - - applyFilterToQuery(column, filterOperator, filterValue); } - function applyFilterToQuery(key: string, operator: string, compareValue: any) { - if (operator === '_eq') { - dbQuery.where({ [key]: compareValue }); - } + function applyJoins(path: string[], collection: string) { + path = clone(path); - if (operator === '_neq') { - dbQuery.whereNot({ [key]: compareValue }); - } + let keyName = ''; - if (operator === '_contains') { - dbQuery.where(key, 'like', `%${compareValue}%`); - } + addJoins(path); - if (operator === '_ncontains') { - dbQuery.where(key, 'like', `%${compareValue}%`); - } + return keyName; - if (operator === '_gt') { - dbQuery.where(key, '>', compareValue); - } - - if (operator === '_gte') { - dbQuery.where(key, '>=', compareValue); - } - - if (operator === '_lt') { - dbQuery.where(key, '<', compareValue); - } - - if (operator === '_lte') { - dbQuery.where(key, '<=', compareValue); - } - - if (operator === '_in') { - let value = compareValue; - if (typeof value === 'string') value = value.split(','); - - dbQuery.whereIn(key, value as string[]); - } - - if (operator === '_nin') { - let value = compareValue; - if (typeof value === 'string') value = value.split(','); - - dbQuery.whereNotIn(key, value as string[]); - } - - if (operator === '_null') { - dbQuery.whereNull(key); - } - - if (operator === '_nnull') { - dbQuery.whereNotNull(key); - } - - if (operator === '_empty') { - dbQuery.andWhere((query) => { - query.whereNull(key); - query.orWhere(key, '=', ''); + function addJoins(pathParts: string[], parentCollection: string = collection) { + const relation = relations.find((relation) => { + return ( + (relation.many_collection === parentCollection && + relation.many_field === pathParts[0]) || + (relation.one_collection === parentCollection && + relation.one_field === pathParts[0]) + ); }); - } - if (operator === '_nempty') { - dbQuery.andWhere((query) => { - query.whereNotNull(key); - query.orWhere(key, '!=', ''); - }); - } + if (!relation) return; - if (operator === '_between') { - let value = compareValue; - if (typeof value === 'string') value = value.split(','); + const isM2O = + relation.many_collection === parentCollection && + relation.many_field === pathParts[0]; - dbQuery.whereBetween(key, value); - } + if (isM2O) { + rootQuery.leftJoin( + relation.one_collection!, + `${parentCollection}.${relation.many_field}`, + `${relation.one_collection}.${relation.one_primary}` + ); + } else { + rootQuery.leftJoin( + relation.many_collection, + `${parentCollection}.${relation.one_primary}`, + `${relation.many_collection}.${relation.many_field}` + ); + } - if (operator === '_nbetween') { - let value = compareValue; - if (typeof value === 'string') value = value.split(','); + pathParts.shift(); - dbQuery.whereNotBetween(key, value); + const parent = isM2O ? relation.one_collection! : relation.many_collection; + + if (pathParts.length === 1) { + keyName = `${parent}.${pathParts[0]}`; + } + + if (pathParts.length) { + addJoins(pathParts, parent); + } } } } @@ -175,53 +238,3 @@ function getOperation(key: string, value: Record): { operator: stri return { operator: key as string, value }; return getOperation(Object.keys(value)[0], Object.values(value)[0]); } - -async function applyJoins(dbQuery: QueryBuilder, path: string[], collection: string) { - path = clone(path); - - let keyName = ''; - - await addJoins(path); - - return keyName; - - async function addJoins(pathParts: string[], parentCollection: string = collection) { - const relation = await database - .select('*') - .from('directus_relations') - .where({ one_collection: parentCollection, one_field: pathParts[0] }) - .orWhere({ many_collection: parentCollection, many_field: pathParts[0] }) - .first(); - - if (!relation) return; - - const isM2O = - relation.many_collection === parentCollection && relation.many_field === pathParts[0]; - - if (isM2O) { - dbQuery.leftJoin( - relation.one_collection, - `${parentCollection}.${relation.many_field}`, - `${relation.one_collection}.${relation.one_primary}` - ); - } else { - dbQuery.leftJoin( - relation.many_collection, - `${relation.one_collection}.${relation.one_primary}`, - `${relation.many_collection}.${relation.many_field}` - ); - } - - pathParts.shift(); - - const parent = isM2O ? relation.one_collection : relation.many_collection; - - if (pathParts.length === 1) { - keyName = `${parent}.${pathParts[0]}`; - } - - if (pathParts.length) { - await addJoins(pathParts, parent); - } - } -} From 8ecb8da3abe04b4ed08e2415a97a203c2aaf41ad Mon Sep 17 00:00:00 2001 From: rijkvanzanten Date: Mon, 26 Oct 2020 17:30:27 +0100 Subject: [PATCH 2/3] Default to _eq for filter --- api/src/utils/apply-query.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/api/src/utils/apply-query.ts b/api/src/utils/apply-query.ts index 77bd8adece..c5d1b0b5ab 100644 --- a/api/src/utils/apply-query.ts +++ b/api/src/utils/apply-query.ts @@ -1,7 +1,7 @@ import { QueryBuilder } from 'knex'; import { Query, Filter } from '../types'; import database, { schemaInspector } from '../database'; -import { clone } from 'lodash'; +import { clone, isPlainObject } from 'lodash'; export default async function applyQuery(collection: string, dbQuery: QueryBuilder, query: Query) { if (query.filter) { @@ -226,7 +226,11 @@ export async function applyFilter(rootQuery: QueryBuilder, rootFilter: Filter, c function getFilterPath(key: string, value: Record) { const path = [key]; - if (Object.keys(value)[0].startsWith('_') === false) { + if (Object.keys(value)[0].startsWith('_') === true) { + return path; + } + + if (isPlainObject(value)) { path.push(...getFilterPath(Object.keys(value)[0], Object.values(value)[0])); } @@ -234,7 +238,11 @@ function getFilterPath(key: string, value: Record) { } function getOperation(key: string, value: Record): { operator: string; value: any } { - if (key.startsWith('_') && key !== '_and' && key !== '_or') + if (key.startsWith('_') && key !== '_and' && key !== '_or') { return { operator: key as string, value }; + } else if (isPlainObject(value) === false) { + return { operator: '_eq', value }; + } + return getOperation(Object.keys(value)[0], Object.values(value)[0]); } From c49e1b02796011994864036ed5487c7e16e87519 Mon Sep 17 00:00:00 2001 From: rijkvanzanten Date: Mon, 26 Oct 2020 18:33:38 +0100 Subject: [PATCH 3/3] Extract adding joins to separate function --- api/src/utils/apply-query.ts | 171 ++++++++++++++++++++++++----------- 1 file changed, 118 insertions(+), 53 deletions(-) diff --git a/api/src/utils/apply-query.ts b/api/src/utils/apply-query.ts index c5d1b0b5ab..13f3f6ad93 100644 --- a/api/src/utils/apply-query.ts +++ b/api/src/utils/apply-query.ts @@ -49,14 +49,16 @@ export default async function applyQuery(collection: string, dbQuery: QueryBuild export async function applyFilter(rootQuery: QueryBuilder, rootFilter: Filter, collection: string) { const relations = await database.select('*').from('directus_relations'); - parseLevel(rootQuery, rootFilter, collection); + addWhereClauses(rootQuery, rootFilter, collection); + addJoins(rootQuery, rootFilter, collection); - function parseLevel(dbQuery: QueryBuilder, filter: Filter, collection: string) { + function addWhereClauses(dbQuery: QueryBuilder, filter: Filter, collection: string) { for (const [key, value] of Object.entries(filter)) { if (key === '_or') { + /** @NOTE these callback functions aren't called until Knex runs the query */ dbQuery.orWhere((subQuery) => { value.forEach((subFilter: Record) => { - parseLevel(subQuery, subFilter, collection); + addWhereClauses(subQuery, subFilter, collection); }); }); @@ -64,9 +66,10 @@ export async function applyFilter(rootQuery: QueryBuilder, rootFilter: Filter, c } if (key === '_and') { + /** @NOTE these callback functions aren't called until Knex runs the query */ dbQuery.andWhere((subQuery) => { value.forEach((subFilter: Record) => { - parseLevel(subQuery, subFilter, collection); + addWhereClauses(subQuery, subFilter, collection); }); }); @@ -76,12 +79,12 @@ export async function applyFilter(rootQuery: QueryBuilder, rootFilter: Filter, c const filterPath = getFilterPath(key, value); const { operator: filterOperator, value: filterValue } = getOperation(key, value); - const column = - filterPath.length > 1 - ? applyJoins(filterPath, collection) - : `${collection}.${filterPath[0]}`; - - applyFilterToQuery(column, filterOperator, filterValue); + if (filterPath.length > 1) { + const columnName = getWhereColumn(filterPath, collection); + applyFilterToQuery(columnName, filterOperator, filterValue); + } else { + applyFilterToQuery(`${collection}.${filterPath[0]}`, filterOperator, filterValue); + } } function applyFilterToQuery(key: string, operator: string, compareValue: any) { @@ -167,57 +170,119 @@ export async function applyFilter(rootQuery: QueryBuilder, rootFilter: Filter, c dbQuery.whereNotBetween(key, value); } } + + function getWhereColumn(path: string[], collection: string) { + path = clone(path); + + let columnName = ''; + + followRelation(path); + + return columnName; + + function followRelation(pathParts: string[], parentCollection: string = collection) { + const relation = relations.find((relation) => { + return ( + (relation.many_collection === parentCollection && + relation.many_field === pathParts[0]) || + (relation.one_collection === parentCollection && + relation.one_field === pathParts[0]) + ); + }); + + if (!relation) return; + + const isM2O = + relation.many_collection === parentCollection && + relation.many_field === pathParts[0]; + + pathParts.shift(); + + const parent = isM2O ? relation.one_collection! : relation.many_collection; + + if (pathParts.length === 1) { + columnName = `${parent}.${pathParts[0]}`; + } + + if (pathParts.length) { + followRelation(pathParts, parent); + } + } + } } - function applyJoins(path: string[], collection: string) { - path = clone(path); + /** + * @NOTE Yes this is very similar in structure and functionality as the other loop. However, + * due to the order of execution that Knex has in the nested andWhere / orWhere structures, + * joins that are added in there aren't added in time + */ + function addJoins(dbQuery: QueryBuilder, filter: Filter, collection: string) { + for (const [key, value] of Object.entries(filter)) { + if (key === '_or') { + value.forEach((subFilter: Record) => { + addJoins(dbQuery, subFilter, collection); + }); - let keyName = ''; - - addJoins(path); - - return keyName; - - function addJoins(pathParts: string[], parentCollection: string = collection) { - const relation = relations.find((relation) => { - return ( - (relation.many_collection === parentCollection && - relation.many_field === pathParts[0]) || - (relation.one_collection === parentCollection && - relation.one_field === pathParts[0]) - ); - }); - - if (!relation) return; - - const isM2O = - relation.many_collection === parentCollection && - relation.many_field === pathParts[0]; - - if (isM2O) { - rootQuery.leftJoin( - relation.one_collection!, - `${parentCollection}.${relation.many_field}`, - `${relation.one_collection}.${relation.one_primary}` - ); - } else { - rootQuery.leftJoin( - relation.many_collection, - `${parentCollection}.${relation.one_primary}`, - `${relation.many_collection}.${relation.many_field}` - ); + continue; } - pathParts.shift(); + if (key === '_and') { + value.forEach((subFilter: Record) => { + addJoins(dbQuery, subFilter, collection); + }); - const parent = isM2O ? relation.one_collection! : relation.many_collection; - - if (pathParts.length === 1) { - keyName = `${parent}.${pathParts[0]}`; + continue; } - if (pathParts.length) { - addJoins(pathParts, parent); + const filterPath = getFilterPath(key, value); + + if (filterPath.length > 1) { + addJoin(filterPath, collection); + } + } + + function addJoin(path: string[], collection: string) { + path = clone(path); + + followRelation(path); + + function followRelation(pathParts: string[], parentCollection: string = collection) { + const relation = relations.find((relation) => { + return ( + (relation.many_collection === parentCollection && + relation.many_field === pathParts[0]) || + (relation.one_collection === parentCollection && + relation.one_field === pathParts[0]) + ); + }); + + if (!relation) return; + + const isM2O = + relation.many_collection === parentCollection && + relation.many_field === pathParts[0]; + + if (isM2O) { + dbQuery.leftJoin( + relation.one_collection!, + `${parentCollection}.${relation.many_field}`, + `${relation.one_collection}.${relation.one_primary}` + ); + } else { + dbQuery.leftJoin( + relation.many_collection, + `${parentCollection}.${relation.one_primary}`, + `${relation.many_collection}.${relation.many_field}` + ); + } + + pathParts.shift(); + + const parent = isM2O ? relation.one_collection! : relation.many_collection; + + if (pathParts.length) { + followRelation(pathParts, parent); + } } } }