Optimise countDistinct aggregation on primary key field (#18717)

* Optimise countDistinct aggregation on primary key field

* Create green-boxes-hope.md

.changeset/green-boxes-hope.md

@@ -0,0 +1,5 @@
+---
+"@directus/api": patch
+---
+
+Optimised countDistinct aggregation on primary key field

api/src/utils/apply-query.ts

@@ -38,6 +38,7 @@ export default function applyQuery(
 	options?: { aliasMap?: AliasMap; isInnerQuery?: boolean; hasMultiRelationalSort?: boolean | undefined }
 ) {
 	const aliasMap: AliasMap = options?.aliasMap ?? Object.create(null);
+	let hasJoins = false;
 	let hasMultiRelationalFilter = false;
 
 	applyLimit(knex, dbQuery, query.limit);
@@ -51,7 +52,11 @@ export default function applyQuery(
 	}
 
 	if (query.sort && !options?.isInnerQuery && !options?.hasMultiRelationalSort) {
-		applySort(knex, schema, dbQuery, query.sort, collection, aliasMap);
+		const sortResult = applySort(knex, schema, dbQuery, query.sort, collection, aliasMap);
+
+		if (!hasJoins) {
+			hasJoins = sortResult.hasJoins;
+		}
 	}
 
 	if (query.search) {
@@ -62,22 +67,21 @@ export default function applyQuery(
 		dbQuery.groupBy(query.group.map((column) => getColumn(knex, collection, column, false, schema)));
 	}
 
-	if (query.aggregate) {
-		applyAggregate(dbQuery, query.aggregate, collection);
-	}
-
 	if (query.filter) {
-		hasMultiRelationalFilter = applyFilter(
-			knex,
-			schema,
-			dbQuery,
-			query.filter,
-			collection,
-			aliasMap
-		).hasMultiRelationalFilter;
+		const filterResult = applyFilter(knex, schema, dbQuery, query.filter, collection, aliasMap);
+
+		if (!hasJoins) {
+			hasJoins = filterResult.hasJoins;
+		}
+
+		hasMultiRelationalFilter = filterResult.hasMultiRelationalFilter;
 	}
 
-	return { query: dbQuery, hasMultiRelationalFilter };
+	if (query.aggregate) {
+		applyAggregate(schema, dbQuery, query.aggregate, collection, hasJoins);
+	}
+
+	return { query: dbQuery, hasJoins, hasMultiRelationalFilter };
 }
 
 /**
@@ -129,13 +133,14 @@ type AddJoinProps = {
 	knex: Knex;
 };
 
-function addJoin({ path, collection, aliasMap, rootQuery, schema, relations, knex }: AddJoinProps): boolean {
+function addJoin({ path, collection, aliasMap, rootQuery, schema, relations, knex }: AddJoinProps) {
 	let hasMultiRelational = false;
+	let isJoinAdded = false;
 
 	path = clone(path);
 	followRelation(path);
 
-	return hasMultiRelational;
+	return { hasMultiRelational, isJoinAdded };
 
 	function followRelation(pathParts: string[], parentCollection: string = collection, parentFields?: string) {
 		/**
@@ -168,6 +173,8 @@ function addJoin({ path, collection, aliasMap, rootQuery, schema, relations, kne
 				);
 
 				aliasMap[aliasKey]!.collection = relation.related_collection!;
+
+				isJoinAdded = true;
 			} else if (relationType === 'a2o') {
 				const pathScope = pathParts[0]!.split(':')[1];
 
@@ -191,6 +198,8 @@ function addJoin({ path, collection, aliasMap, rootQuery, schema, relations, kne
 				});
 
 				aliasMap[aliasKey]!.collection = pathScope;
+
+				isJoinAdded = true;
 			} else if (relationType === 'o2a') {
 				rootQuery.leftJoin({ [alias]: relation.collection }, (joinClause) => {
 					joinClause
@@ -208,6 +217,7 @@ function addJoin({ path, collection, aliasMap, rootQuery, schema, relations, kne
 				aliasMap[aliasKey]!.collection = relation.collection;
 
 				hasMultiRelational = true;
+				isJoinAdded = true;
 			} else if (relationType === 'o2m') {
 				rootQuery.leftJoin(
 					{ [alias]: relation.collection },
@@ -218,6 +228,7 @@ function addJoin({ path, collection, aliasMap, rootQuery, schema, relations, kne
 				aliasMap[aliasKey]!.collection = relation.collection;
 
 				hasMultiRelational = true;
+				isJoinAdded = true;
 			}
 		}
 
@@ -257,6 +268,7 @@ export function applySort(
 	returnRecords = false
 ) {
 	const relations: Relation[] = schema.relations;
+	let hasJoins = false;
 	let hasMultiRelationalSort = false;
 
 	const sortRecords = rootSort.map((sortField) => {
@@ -283,7 +295,7 @@ export function applySort(
 			}
 		}
 
-		const hasMultiRelational = addJoin({
+		const { hasMultiRelational, isJoinAdded } = addJoin({
 			path: column,
 			collection,
 			aliasMap,
@@ -303,6 +315,10 @@ export function applySort(
 
 		const [alias, field] = columnPath.split('.');
 
+		if (!hasJoins) {
+			hasJoins = isJoinAdded;
+		}
+
 		if (!hasMultiRelationalSort) {
 			hasMultiRelationalSort = hasMultiRelational;
 		}
@@ -313,14 +329,14 @@ export function applySort(
 		};
 	});
 
-	if (returnRecords) return { sortRecords, hasMultiRelationalSort };
+	if (returnRecords) return { sortRecords, hasJoins, hasMultiRelationalSort };
 
 	// Clears the order if any, eg: from MSSQL offset
 	rootQuery.clear('order');
 
 	rootQuery.orderBy(sortRecords);
 
-	return undefined;
+	return { hasJoins, hasMultiRelationalSort };
 }
 
 export function applyLimit(knex: Knex, rootQuery: Knex.QueryBuilder, limit: any) {
@@ -345,12 +361,13 @@ export function applyFilter(
 ) {
 	const helpers = getHelpers(knex);
 	const relations: Relation[] = schema.relations;
+	let hasJoins = false;
 	let hasMultiRelationalFilter = false;
 
 	addJoins(rootQuery, rootFilter, collection);
 	addWhereClauses(knex, rootQuery, rootFilter, collection);
 
-	return { query: rootQuery, hasMultiRelationalFilter };
+	return { query: rootQuery, hasJoins, hasMultiRelationalFilter };
 
 	function addJoins(dbQuery: Knex.QueryBuilder, filter: Filter, collection: string) {
 		for (const [key, value] of Object.entries(filter)) {
@@ -374,7 +391,7 @@ export function applyFilter(
 				filterPath.length > 1 ||
 				(!(key.includes('(') && key.includes(')')) && schema.collections[collection]?.fields[key]?.type === 'alias')
 			) {
-				const hasMultiRelational = addJoin({
+				const { hasMultiRelational, isJoinAdded } = addJoin({
 					path: filterPath,
 					collection,
 					knex,
@@ -384,6 +401,10 @@ export function applyFilter(
 					aliasMap,
 				});
 
+				if (!hasJoins) {
+					hasJoins = isJoinAdded;
+				}
+
 				if (!hasMultiRelationalFilter) {
 					hasMultiRelationalFilter = hasMultiRelational;
 				}
@@ -785,7 +806,13 @@ function validateNumber(value: string, parsed: number) {
 	return String(parsed) === value;
 }
 
-export function applyAggregate(dbQuery: Knex.QueryBuilder, aggregate: Aggregate, collection: string): void {
+export function applyAggregate(
+	schema: SchemaOverview,
+	dbQuery: Knex.QueryBuilder,
+	aggregate: Aggregate,
+	collection: string,
+	hasJoins: boolean
+): void {
 	for (const [operation, fields] of Object.entries(aggregate)) {
 		if (!fields) continue;
 
@@ -811,7 +838,12 @@ export function applyAggregate(dbQuery: Knex.QueryBuilder, aggregate: Aggregate,
 			}
 
 			if (operation === 'countDistinct') {
-				dbQuery.countDistinct(`${collection}.${field}`, { as: `countDistinct->${field}` });
+				if (!hasJoins && schema.collections[collection]?.primary === field) {
+					// Optimize to count as primary keys are unique
+					dbQuery.count(`${collection}.${field}`, { as: `countDistinct->${field}` });
+				} else {
+					dbQuery.countDistinct(`${collection}.${field}`, { as: `countDistinct->${field}` });
+				}
 			}
 
 			if (operation === 'sum') {