Add functions support to the app + add count function (#12488)

* Rename date functions to fn, add json_array_length for pg

* Add json count to mssql

* Add json array count support to other vendors

* Add UI for selecting API functions

* Make it not break

* Render functions in filter preview better

* Include functions in field altering

* Add schema access to database helper

* Allow filtering against o2m/m2m/m2a count

* Add data function execution helpers in utils

* Fix type issue

* Inject function results in validate step

* Render field keys with function names translated

* Allow selecting nested/functions in field validation step

* Make sure number comparisons are treated as numbers

* Add check if instanceof date when casting to a Number

* Prevent selecting foreign keys for junction sort (#12463)

* [SDK] Add further request options to `items` functions (#12503)

* add possibility to set further options to the request

* fix options type

* add typings to interface

* add test if headers are passed thourght

* create reusable options param

* set higher priority to options param

* Small stylistic cleanup

Co-authored-by: Azri Kahar <42867097+azrikahar@users.noreply.github.com>
Co-authored-by: ian <licitdev@gmail.com>
Co-authored-by: Jürg Hunziker <juerg.hunziker@gmail.com>

api/src/database/helpers/date/dialects/default.ts

@@ -0,0 +1,3 @@
+import { DateHelper } from '../types';
+
+export class DateHelperDefault extends DateHelper {}

api/src/database/helpers/date/dialects/sqlite.ts

@@ -1,39 +1,6 @@
 import { DateHelper } from '../types';
-import { Knex } from 'knex';
 
 export class DateHelperSQLite extends DateHelper {
-	year(table: string, column: string): Knex.Raw {
-		return this.knex.raw("strftime('%Y', ??.?? / 1000, 'unixepoch')", [table, column]);
-	}
-
-	month(table: string, column: string): Knex.Raw {
-		return this.knex.raw("strftime('%m', ??.?? / 1000, 'unixepoch')", [table, column]);
-	}
-
-	week(table: string, column: string): Knex.Raw {
-		return this.knex.raw("strftime('%W', ??.?? / 1000, 'unixepoch')", [table, column]);
-	}
-
-	day(table: string, column: string): Knex.Raw {
-		return this.knex.raw("strftime('%d', ??.?? / 1000, 'unixepoch')", [table, column]);
-	}
-
-	weekday(table: string, column: string): Knex.Raw {
-		return this.knex.raw("strftime('%w', ??.?? / 1000, 'unixepoch')", [table, column]);
-	}
-
-	hour(table: string, column: string): Knex.Raw {
-		return this.knex.raw("strftime('%H', ??.?? / 1000, 'unixepoch')", [table, column]);
-	}
-
-	minute(table: string, column: string): Knex.Raw {
-		return this.knex.raw("strftime('%M', ??.?? / 1000, 'unixepoch')", [table, column]);
-	}
-
-	second(table: string, column: string): Knex.Raw {
-		return this.knex.raw("strftime('%S', ??.?? / 1000, 'unixepoch')", [table, column]);
-	}
-
 	parse(date: string): string {
 		const newDate = new Date(date);
 		return (newDate.getTime() - newDate.getTimezoneOffset() * 60 * 1000).toString();

api/src/database/helpers/date/index.ts

@@ -1,7 +1,7 @@
-export { DateHelperPostgres as postgres } from './dialects/postgres';
-export { DateHelperPostgres as redshift } from './dialects/postgres';
-export { DateHelperPostgres as cockroachdb } from './dialects/postgres';
-export { DateHelperOracle as oracle } from './dialects/oracle';
+export { DateHelperDefault as postgres } from './dialects/default';
+export { DateHelperDefault as redshift } from './dialects/default';
+export { DateHelperDefault as cockroachdb } from './dialects/default';
+export { DateHelperDefault as oracle } from './dialects/default';
+export { DateHelperDefault as mysql } from './dialects/default';
+export { DateHelperDefault as mssql } from './dialects/default';
 export { DateHelperSQLite as sqlite } from './dialects/sqlite';
-export { DateHelperMySQL as mysql } from './dialects/mysql';
-export { DateHelperMSSQL as mssql } from './dialects/mssql';

api/src/database/helpers/date/types.ts

@@ -1,15 +1,6 @@
 import { DatabaseHelper } from '../types';
-import { Knex } from 'knex';
 
 export abstract class DateHelper extends DatabaseHelper {
-	abstract year(table: string, column: string): Knex.Raw;
-	abstract month(table: string, column: string): Knex.Raw;
-	abstract week(table: string, column: string): Knex.Raw;
-	abstract day(table: string, column: string): Knex.Raw;
-	abstract weekday(table: string, column: string): Knex.Raw;
-	abstract hour(table: string, column: string): Knex.Raw;
-	abstract minute(table: string, column: string): Knex.Raw;
-	abstract second(table: string, column: string): Knex.Raw;
 	parse(date: string): string {
 		return date;
 	}

api/src/database/helpers/fn/dialects/mssql.ts

@@ -1,7 +1,7 @@
-import { DateHelper } from '../types';
+import { FnHelper } from '../types';
 import { Knex } from 'knex';
 
-export class DateHelperMSSQL extends DateHelper {
+export class FnHelperMSSQL extends FnHelper {
 	year(table: string, column: string): Knex.Raw {
 		return this.knex.raw('DATEPART(year, ??.??)', [table, column]);
 	}
@@ -33,4 +33,18 @@ export class DateHelperMSSQL extends DateHelper {
 	second(table: string, column: string): Knex.Raw {
 		return this.knex.raw('DATEPART(second, ??.??)', [table, column]);
 	}
+
+	count(table: string, column: string): Knex.Raw<any> {
+		const type = this.schema.collections?.[table]?.fields?.[column]?.type ?? 'unknown';
+
+		if (type === 'json') {
+			return this.knex.raw(`(SELECT COUNT(*) FROM OPENJSON(??.??, '$'))`, [table, column]);
+		}
+
+		if (type === 'alias') {
+			return this._relationalCount(table, column);
+		}
+
+		throw new Error(`Couldn't extract type from ${table}.${column}`);
+	}
 }

api/src/database/helpers/fn/dialects/mysql.ts

@@ -1,7 +1,7 @@
-import { DateHelper } from '../types';
+import { FnHelper } from '../types';
 import { Knex } from 'knex';
 
-export class DateHelperMySQL extends DateHelper {
+export class FnHelperMySQL extends FnHelper {
 	year(table: string, column: string): Knex.Raw {
 		return this.knex.raw('YEAR(??.??)', [table, column]);
 	}
@@ -33,4 +33,18 @@ export class DateHelperMySQL extends DateHelper {
 	second(table: string, column: string): Knex.Raw {
 		return this.knex.raw('SECOND(??.??)', [table, column]);
 	}
+
+	count(table: string, column: string): Knex.Raw {
+		const type = this.schema.collections?.[table]?.fields?.[column]?.type ?? 'unknown';
+
+		if (type === 'json') {
+			return this.knex.raw('JSON_LENGTH(??.??)', [table, column]);
+		}
+
+		if (type === 'alias') {
+			return this._relationalCount(table, column);
+		}
+
+		throw new Error(`Couldn't extract type from ${table}.${column}`);
+	}
 }

api/src/database/helpers/fn/dialects/oracle.ts

@@ -1,7 +1,7 @@
-import { DateHelper } from '../types';
+import { FnHelper } from '../types';
 import { Knex } from 'knex';
 
-export class DateHelperOracle extends DateHelper {
+export class FnHelperOracle extends FnHelper {
 	year(table: string, column: string): Knex.Raw {
 		return this.knex.raw("TO_CHAR(??.??, 'IYYY')", [table, column]);
 	}
@@ -33,4 +33,18 @@ export class DateHelperOracle extends DateHelper {
 	second(table: string, column: string): Knex.Raw {
 		return this.knex.raw("TO_CHAR(??.??, 'SS')", [table, column]);
 	}
+
+	count(table: string, column: string): Knex.Raw<any> {
+		const type = this.schema.collections?.[table]?.fields?.[column]?.type ?? 'unknown';
+
+		if (type === 'json') {
+			return this.knex.raw("json_value(??.??, '$.size()')", [table, column]);
+		}
+
+		if (type === 'alias') {
+			return this._relationalCount(table, column);
+		}
+
+		throw new Error(`Couldn't extract type from ${table}.${column}`);
+	}
 }

api/src/database/helpers/fn/dialects/postgres.ts

@@ -1,7 +1,7 @@
-import { DateHelper } from '../types';
+import { FnHelper } from '../types';
 import { Knex } from 'knex';
 
-export class DateHelperPostgres extends DateHelper {
+export class FnHelperPostgres extends FnHelper {
 	year(table: string, column: string): Knex.Raw {
 		return this.knex.raw('EXTRACT(YEAR FROM ??.??)', [table, column]);
 	}
@@ -33,4 +33,18 @@ export class DateHelperPostgres extends DateHelper {
 	second(table: string, column: string): Knex.Raw {
 		return this.knex.raw('EXTRACT(SECOND FROM ??.??)', [table, column]);
 	}
+
+	count(table: string, column: string): Knex.Raw {
+		const type = this.schema.collections?.[table]?.fields?.[column]?.type ?? 'unknown';
+
+		if (type === 'json') {
+			return this.knex.raw('json_array_length(??.??)', [table, column]);
+		}
+
+		if (type === 'alias') {
+			return this._relationalCount(table, column);
+		}
+
+		throw new Error(`Couldn't extract type from ${table}.${column}`);
+	}
 }

api/src/database/helpers/fn/dialects/sqlite.ts

@@ -0,0 +1,50 @@
+import { FnHelper } from '../types';
+import { Knex } from 'knex';
+
+export class FnHelperSQLite extends FnHelper {
+	year(table: string, column: string): Knex.Raw {
+		return this.knex.raw("strftime('%Y', ??.?? / 1000, 'unixepoch')", [table, column]);
+	}
+
+	month(table: string, column: string): Knex.Raw {
+		return this.knex.raw("strftime('%m', ??.?? / 1000, 'unixepoch')", [table, column]);
+	}
+
+	week(table: string, column: string): Knex.Raw {
+		return this.knex.raw("strftime('%W', ??.?? / 1000, 'unixepoch')", [table, column]);
+	}
+
+	day(table: string, column: string): Knex.Raw {
+		return this.knex.raw("strftime('%d', ??.?? / 1000, 'unixepoch')", [table, column]);
+	}
+
+	weekday(table: string, column: string): Knex.Raw {
+		return this.knex.raw("strftime('%w', ??.?? / 1000, 'unixepoch')", [table, column]);
+	}
+
+	hour(table: string, column: string): Knex.Raw {
+		return this.knex.raw("strftime('%H', ??.?? / 1000, 'unixepoch')", [table, column]);
+	}
+
+	minute(table: string, column: string): Knex.Raw {
+		return this.knex.raw("strftime('%M', ??.?? / 1000, 'unixepoch')", [table, column]);
+	}
+
+	second(table: string, column: string): Knex.Raw {
+		return this.knex.raw("strftime('%S', ??.?? / 1000, 'unixepoch')", [table, column]);
+	}
+
+	count(table: string, column: string): Knex.Raw<any> {
+		const type = this.schema.collections?.[table]?.fields?.[column]?.type ?? 'unknown';
+
+		if (type === 'json') {
+			return this.knex.raw(`json_array_length(??.??, '$')`, [table, column]);
+		}
+
+		if (type === 'alias') {
+			return this._relationalCount(table, column);
+		}
+
+		throw new Error(`Couldn't extract type from ${table}.${column}`);
+	}
+}

api/src/database/helpers/fn/index.ts

@@ -0,0 +1,7 @@
+export { FnHelperPostgres as postgres } from './dialects/postgres';
+export { FnHelperPostgres as redshift } from './dialects/postgres';
+export { FnHelperPostgres as cockroachdb } from './dialects/postgres';
+export { FnHelperOracle as oracle } from './dialects/oracle';
+export { FnHelperSQLite as sqlite } from './dialects/sqlite';
+export { FnHelperMySQL as mysql } from './dialects/mysql';
+export { FnHelperMSSQL as mssql } from './dialects/mssql';

api/src/database/helpers/fn/types.ts

@@ -0,0 +1,42 @@
+import { SchemaOverview } from '@directus/shared/types';
+import { Knex } from 'knex';
+import { DatabaseHelper } from '../types';
+
+export abstract class FnHelper extends DatabaseHelper {
+	constructor(protected knex: Knex, protected schema: SchemaOverview) {
+		super(knex);
+		this.schema = schema;
+	}
+
+	abstract year(table: string, column: string): Knex.Raw;
+	abstract month(table: string, column: string): Knex.Raw;
+	abstract week(table: string, column: string): Knex.Raw;
+	abstract day(table: string, column: string): Knex.Raw;
+	abstract weekday(table: string, column: string): Knex.Raw;
+	abstract hour(table: string, column: string): Knex.Raw;
+	abstract minute(table: string, column: string): Knex.Raw;
+	abstract second(table: string, column: string): Knex.Raw;
+	abstract count(table: string, column: string): Knex.Raw;
+
+	protected _relationalCount(table: string, column: string): Knex.Raw {
+		const relation = this.schema.relations.find(
+			(relation) => relation.related_collection === table && relation?.meta?.one_field === column
+		);
+
+		const currentPrimary = this.schema.collections[table].primary;
+
+		if (!relation) {
+			throw new Error(`Field ${table}.${column} isn't a nested relational collection`);
+		}
+
+		return this.knex.raw(
+			'(' +
+				this.knex
+					.count('*')
+					.from(relation.collection)
+					.where(relation.field, '=', this.knex.raw(`??.??`, [table, currentPrimary]))
+					.toQuery() +
+				')'
+		);
+	}
+}

api/src/database/helpers/index.ts

@@ -1,12 +1,15 @@
-import { getDatabaseClient } from '..';
+import { SchemaOverview } from '@directus/shared/types';
 import { Knex } from 'knex';
+import { getDatabaseClient } from '..';
 
 import * as dateHelpers from './date';
+import * as fnHelpers from './fn';
 import * as geometryHelpers from './geometry';
 import * as schemaHelpers from './schema';
 
 export function getHelpers(database: Knex) {
 	const client = getDatabaseClient(database);
+
 	return {
 		date: new dateHelpers[client](database),
 		st: new geometryHelpers[client](database),
@@ -14,4 +17,9 @@ export function getHelpers(database: Knex) {
 	};
 }
 
+export function getFunctions(database: Knex, schema: SchemaOverview) {
+	const client = getDatabaseClient(database);
+	return new fnHelpers[client](database, schema);
+}
+
 export type Helpers = ReturnType<typeof getHelpers>;

api/src/database/run-ast.ts

@@ -225,7 +225,7 @@ function getColumnPreprocessor(knex: Knex, schema: SchemaOverview, table: string
 			return helpers.st.asText(table, field.field);
 		}
 
-		return getColumn(knex, table, fieldNode.name, alias);
+		return getColumn(knex, table, fieldNode.name, alias, schema);
 	};
 }