diff --git a/.changeset/green-boxes-hope.md b/.changeset/green-boxes-hope.md new file mode 100644 index 0000000000..abbdce0b8a --- /dev/null +++ b/.changeset/green-boxes-hope.md @@ -0,0 +1,5 @@ +--- +"@directus/api": patch +--- + +Optimised countDistinct aggregation on primary key field diff --git a/api/src/utils/apply-query.ts b/api/src/utils/apply-query.ts index 5c39c3a744..cbf171b992 100644 --- a/api/src/utils/apply-query.ts +++ b/api/src/utils/apply-query.ts @@ -38,6 +38,7 @@ export default function applyQuery( options?: { aliasMap?: AliasMap; isInnerQuery?: boolean; hasMultiRelationalSort?: boolean | undefined } ) { const aliasMap: AliasMap = options?.aliasMap ?? Object.create(null); + let hasJoins = false; let hasMultiRelationalFilter = false; applyLimit(knex, dbQuery, query.limit); @@ -51,7 +52,11 @@ export default function applyQuery( } if (query.sort && !options?.isInnerQuery && !options?.hasMultiRelationalSort) { - applySort(knex, schema, dbQuery, query.sort, collection, aliasMap); + const sortResult = applySort(knex, schema, dbQuery, query.sort, collection, aliasMap); + + if (!hasJoins) { + hasJoins = sortResult.hasJoins; + } } if (query.search) { @@ -62,22 +67,21 @@ export default function applyQuery( dbQuery.groupBy(query.group.map((column) => getColumn(knex, collection, column, false, schema))); } - if (query.aggregate) { - applyAggregate(dbQuery, query.aggregate, collection); - } - if (query.filter) { - hasMultiRelationalFilter = applyFilter( - knex, - schema, - dbQuery, - query.filter, - collection, - aliasMap - ).hasMultiRelationalFilter; + const filterResult = applyFilter(knex, schema, dbQuery, query.filter, collection, aliasMap); + + if (!hasJoins) { + hasJoins = filterResult.hasJoins; + } + + hasMultiRelationalFilter = filterResult.hasMultiRelationalFilter; } - return { query: dbQuery, hasMultiRelationalFilter }; + if (query.aggregate) { + applyAggregate(schema, dbQuery, query.aggregate, collection, hasJoins); + } + + return { query: dbQuery, hasJoins, hasMultiRelationalFilter }; } /** @@ -129,13 +133,14 @@ type AddJoinProps = { knex: Knex; }; -function addJoin({ path, collection, aliasMap, rootQuery, schema, relations, knex }: AddJoinProps): boolean { +function addJoin({ path, collection, aliasMap, rootQuery, schema, relations, knex }: AddJoinProps) { let hasMultiRelational = false; + let isJoinAdded = false; path = clone(path); followRelation(path); - return hasMultiRelational; + return { hasMultiRelational, isJoinAdded }; function followRelation(pathParts: string[], parentCollection: string = collection, parentFields?: string) { /** @@ -168,6 +173,8 @@ function addJoin({ path, collection, aliasMap, rootQuery, schema, relations, kne ); aliasMap[aliasKey]!.collection = relation.related_collection!; + + isJoinAdded = true; } else if (relationType === 'a2o') { const pathScope = pathParts[0]!.split(':')[1]; @@ -191,6 +198,8 @@ function addJoin({ path, collection, aliasMap, rootQuery, schema, relations, kne }); aliasMap[aliasKey]!.collection = pathScope; + + isJoinAdded = true; } else if (relationType === 'o2a') { rootQuery.leftJoin({ [alias]: relation.collection }, (joinClause) => { joinClause @@ -208,6 +217,7 @@ function addJoin({ path, collection, aliasMap, rootQuery, schema, relations, kne aliasMap[aliasKey]!.collection = relation.collection; hasMultiRelational = true; + isJoinAdded = true; } else if (relationType === 'o2m') { rootQuery.leftJoin( { [alias]: relation.collection }, @@ -218,6 +228,7 @@ function addJoin({ path, collection, aliasMap, rootQuery, schema, relations, kne aliasMap[aliasKey]!.collection = relation.collection; hasMultiRelational = true; + isJoinAdded = true; } } @@ -257,6 +268,7 @@ export function applySort( returnRecords = false ) { const relations: Relation[] = schema.relations; + let hasJoins = false; let hasMultiRelationalSort = false; const sortRecords = rootSort.map((sortField) => { @@ -283,7 +295,7 @@ export function applySort( } } - const hasMultiRelational = addJoin({ + const { hasMultiRelational, isJoinAdded } = addJoin({ path: column, collection, aliasMap, @@ -303,6 +315,10 @@ export function applySort( const [alias, field] = columnPath.split('.'); + if (!hasJoins) { + hasJoins = isJoinAdded; + } + if (!hasMultiRelationalSort) { hasMultiRelationalSort = hasMultiRelational; } @@ -313,14 +329,14 @@ export function applySort( }; }); - if (returnRecords) return { sortRecords, hasMultiRelationalSort }; + if (returnRecords) return { sortRecords, hasJoins, hasMultiRelationalSort }; // Clears the order if any, eg: from MSSQL offset rootQuery.clear('order'); rootQuery.orderBy(sortRecords); - return undefined; + return { hasJoins, hasMultiRelationalSort }; } export function applyLimit(knex: Knex, rootQuery: Knex.QueryBuilder, limit: any) { @@ -345,12 +361,13 @@ export function applyFilter( ) { const helpers = getHelpers(knex); const relations: Relation[] = schema.relations; + let hasJoins = false; let hasMultiRelationalFilter = false; addJoins(rootQuery, rootFilter, collection); addWhereClauses(knex, rootQuery, rootFilter, collection); - return { query: rootQuery, hasMultiRelationalFilter }; + return { query: rootQuery, hasJoins, hasMultiRelationalFilter }; function addJoins(dbQuery: Knex.QueryBuilder, filter: Filter, collection: string) { for (const [key, value] of Object.entries(filter)) { @@ -374,7 +391,7 @@ export function applyFilter( filterPath.length > 1 || (!(key.includes('(') && key.includes(')')) && schema.collections[collection]?.fields[key]?.type === 'alias') ) { - const hasMultiRelational = addJoin({ + const { hasMultiRelational, isJoinAdded } = addJoin({ path: filterPath, collection, knex, @@ -384,6 +401,10 @@ export function applyFilter( aliasMap, }); + if (!hasJoins) { + hasJoins = isJoinAdded; + } + if (!hasMultiRelationalFilter) { hasMultiRelationalFilter = hasMultiRelational; } @@ -785,7 +806,13 @@ function validateNumber(value: string, parsed: number) { return String(parsed) === value; } -export function applyAggregate(dbQuery: Knex.QueryBuilder, aggregate: Aggregate, collection: string): void { +export function applyAggregate( + schema: SchemaOverview, + dbQuery: Knex.QueryBuilder, + aggregate: Aggregate, + collection: string, + hasJoins: boolean +): void { for (const [operation, fields] of Object.entries(aggregate)) { if (!fields) continue; @@ -811,7 +838,12 @@ export function applyAggregate(dbQuery: Knex.QueryBuilder, aggregate: Aggregate, } if (operation === 'countDistinct') { - dbQuery.countDistinct(`${collection}.${field}`, { as: `countDistinct->${field}` }); + if (!hasJoins && schema.collections[collection]?.primary === field) { + // Optimize to count as primary keys are unique + dbQuery.count(`${collection}.${field}`, { as: `countDistinct->${field}` }); + } else { + dbQuery.countDistinct(`${collection}.${field}`, { as: `countDistinct->${field}` }); + } } if (operation === 'sum') {