Update geometric types and patch new field flow. (#9397)

* Update geometric types and patch new field flow.

* Add migration

* Fixed migrations

* Also fixed migrations

* Update migration ID

* Cleanup type selector a bit

* Add missing fallback interface/display for new types

Co-authored-by: rijkvanzanten <rijkvanzanten@me.com>

api/src/database/helpers/geometry.ts

@@ -26,7 +26,7 @@ export function getGeometryHelper(database?: Knex): KnexSpatial {
 	return geometryHelper!;
 }
 
-class KnexSpatial {
+abstract class KnexSpatial {
 	constructor(protected knex: Knex) {}
 	supported(): boolean | Promise<boolean> {
 		return true;
@@ -38,12 +38,15 @@ class KnexSpatial {
 		return expression.wrap('NOT ', '');
 	}
 	createColumn(table: Knex.CreateTableBuilder, field: RawField | Field) {
-		const type = field.schema?.geometry_type ?? 'geometry';
+		const type = field.type.split('.')[1] ?? 'geometry';
 		return table.specificType(field.field, type);
 	}
 	asText(table: string, column: string): Knex.Raw {
 		return this.knex.raw('st_astext(??.??) as ??', [table, column, column]);
 	}
+	asGeoJSON?(table: string, column: string): Knex.Raw {
+		return this.knex.raw('st_asgeojson(??.??) as ??', [table, column, column]);
+	}
 	fromText(text: string): Knex.Raw {
 		return this.knex.raw('st_geomfromtext(?, 4326)', text);
 	}
@@ -80,6 +83,9 @@ class KnexSpatial_SQLite extends KnexSpatial {
 		const res = await this.knex.select('name').from('pragma_function_list').where({ name: 'spatialite_version' });
 		return res.length > 0;
 	}
+	asGeoJSON(table: string, column: string): Knex.Raw {
+		return this.knex.raw('asgeojson(??.??) as ??', [table, column, column]);
+	}
 }
 class KnexSpatial_PG extends KnexSpatial {
 	async supported() {
@@ -87,7 +93,7 @@ class KnexSpatial_PG extends KnexSpatial {
 		return res.length > 0;
 	}
 	createColumn(table: Knex.CreateTableBuilder, field: RawField | Field) {
-		const type = field.schema?.geometry_type ?? 'geometry';
+		const type = field.type.split('.')[1] ?? 'geometry';
 		return table.specificType(field.field, `geometry(${type})`);
 	}
 	_intersects_bbox(key: string, geojson: GeoJSONGeometry): Knex.Raw {
@@ -107,8 +113,9 @@ class KnexSpatial_MySQL extends KnexSpatial {
 
 class KnexSpatial_Redshift extends KnexSpatial {
 	createColumn(table: Knex.CreateTableBuilder, field: RawField | Field) {
-		const type = field.schema?.geometry_type ?? 'geometry';
-		if (type !== 'geometry') field.meta!.special![1] = type;
+		if (field.type.split('.')[1]) {
+			field.meta!.special = [field.type];
+		}
 		return table.specificType(field.field, 'geometry');
 	}
 }
@@ -121,8 +128,9 @@ class KnexSpatial_MSSQL extends KnexSpatial {
 		return expression.wrap(``, ` = 0`);
 	}
 	createColumn(table: Knex.CreateTableBuilder, field: RawField | Field) {
-		const type = field.schema?.geometry_type ?? 'geometry';
-		if (type !== 'geometry') field.meta!.special![1] = type;
+		if (field.type.split('.')[1]) {
+			field.meta!.special = [field.type];
+		}
 		return table.specificType(field.field, 'geometry');
 	}
 	asText(table: string, column: string): Knex.Raw {
@@ -142,6 +150,7 @@ class KnexSpatial_MSSQL extends KnexSpatial {
 	collect(table: string, column: string): Knex.Raw {
 		return this.knex.raw('geometry::CollectionAggregate(??.??).STAsText()', [table, column]);
 	}
+	asGeoJSON: undefined;
 }
 
 class KnexSpatial_Oracle extends KnexSpatial {
@@ -152,13 +161,17 @@ class KnexSpatial_Oracle extends KnexSpatial {
 		return expression.wrap(``, ` = 'FALSE'`);
 	}
 	createColumn(table: Knex.CreateTableBuilder, field: RawField | Field) {
-		const type = field.schema?.geometry_type ?? 'geometry';
-		if (type !== 'geometry') field.meta!.special![1] = type;
+		if (field.type.split('.')[1]) {
+			field.meta!.special = [field.type];
+		}
 		return table.specificType(field.field, 'sdo_geometry');
 	}
 	asText(table: string, column: string): Knex.Raw {
 		return this.knex.raw('sdo_util.to_wktgeometry(??.??) as ??', [table, column, column]);
 	}
+	asGeoJSON(table: string, column: string): Knex.Raw {
+		return this.knex.raw('sdo_util.to_geojson(??.??) as ??', [table, column, column]);
+	}
 	fromText(text: string): Knex.Raw {
 		return this.knex.raw('sdo_geometry(?, 4326)', text);
 	}

api/src/database/migrations/20211103B-update-special-geometry.ts

@@ -0,0 +1,23 @@
+import { Knex } from 'knex';
+
+export async function up(knex: Knex): Promise<void> {
+	await knex('directus_fields')
+		.update({ special: knex.raw(`REPLACE(special, 'geometry,', 'geometry.')`) })
+		.where('special', 'like', '%geometry,Point%')
+		.orWhere('special', 'like', '%geometry,LineString%')
+		.orWhere('special', 'like', '%geometry,Polygon%')
+		.orWhere('special', 'like', '%geometry,MultiPoint%')
+		.orWhere('special', 'like', '%geometry,MultiLineString%')
+		.orWhere('special', 'like', '%geometry,MultiPolygon%');
+}
+
+export async function down(knex: Knex): Promise<void> {
+	await knex('directus_fields')
+		.update({ special: knex.raw(`REPLACE(special, 'geometry.', 'geometry,')`) })
+		.where('special', 'like', '%geometry.Point%')
+		.orWhere('special', 'like', '%geometry.LineString%')
+		.orWhere('special', 'like', '%geometry.Polygon%')
+		.orWhere('special', 'like', '%geometry.MultiPoint%')
+		.orWhere('special', 'like', '%geometry.MultiLineString%')
+		.orWhere('special', 'like', '%geometry.MultiPolygon%');
+}

api/src/database/run-ast.ts

@@ -10,7 +10,6 @@ import { stripFunction } from '../utils/strip-function';
 import { toArray } from '@directus/shared/utils';
 import { Query } from '@directus/shared/types';
 import getDatabase from './index';
-import { isNativeGeometry } from '../utils/geometry';
 import { getGeometryHelper } from '../database/helpers/geometry';
 
 type RunASTOptions = {
@@ -192,7 +191,7 @@ function getColumnPreprocessor(knex: Knex, schema: SchemaOverview, table: string
 			alias = fieldNode.fieldKey;
 		}
 
-		if (isNativeGeometry(field)) {
+		if (field.type.startsWith('geometry')) {
 			return helper.asText(table, field.field);
 		}
 

api/src/database/seeds/run.ts

@@ -56,10 +56,11 @@ export default async function runSeed(database: Knex): Promise<void> {
 					column = tableBuilder.string(columnName);
 				} else if (columnInfo.type === 'hash') {
 					column = tableBuilder.string(columnName, 255);
-				} else if (columnInfo.type === 'geometry') {
+				} else if (columnInfo.type?.startsWith('geometry')) {
 					const helper = getGeometryHelper();
-					column = helper.createColumn(tableBuilder, { field: columnName } as Field);
+					column = helper.createColumn(tableBuilder, { field: columnName, type: columnInfo.type } as Field);
 				} else {
+					// @ts-ignore
 					column = tableBuilder[columnInfo.type!](columnName);
 				}
 

api/src/services/fields.ts

@@ -86,13 +86,13 @@ export class FieldsService {
 				return field.field === column.name && field.collection === column.table;
 			});
 
-			const { type, ...info } = getLocalType(column, field);
+			const type = getLocalType(column, field);
 
 			const data = {
 				collection: column.table,
 				field: column.name,
 				type: type,
-				schema: { ...column, ...info },
+				schema: column,
 				meta: field || null,
 			};
 
@@ -124,7 +124,7 @@ export class FieldsService {
 		});
 
 		const aliasFieldsAsField = aliasFields.map((field) => {
-			const { type } = getLocalType(undefined, field);
+			const type = getLocalType(undefined, field);
 
 			const data = {
 				collection: field.collection,
@@ -205,14 +205,14 @@ export class FieldsService {
 			// Do nothing
 		}
 
-		const { type = 'alias', ...info } = getLocalType(column, fieldInfo);
+		const type = getLocalType(column, fieldInfo);
 
 		const data = {
 			collection,
 			field,
 			type,
 			meta: fieldInfo || null,
-			schema: type === 'alias' ? null : { ...column, ...info },
+			schema: type === 'alias' ? null : column,
 		};
 
 		return data;
@@ -467,10 +467,11 @@ export class FieldsService {
 			column = table.dateTime(field.field, { useTz: false });
 		} else if (field.type === 'timestamp') {
 			column = table.timestamp(field.field, { useTz: true });
-		} else if (field.type === 'geometry') {
+		} else if (field.type.startsWith('geometry')) {
 			const helper = getGeometryHelper();
 			column = helper.createColumn(table, field);
 		} else {
+			// @ts-ignore
 			column = table[field.type](field.field);
 		}
 

api/src/services/payload.ts

@@ -10,7 +10,6 @@ import { Accountability, Query } from '@directus/shared/types';
 import { toArray } from '@directus/shared/utils';
 import { ItemsService } from './items';
 import { unflatten } from 'flat';
-import { isNativeGeometry } from '../utils/geometry';
 import { getGeometryHelper } from '../database/helpers/geometry';
 import { parse as wktToGeoJSON } from 'wellknown';
 import { generateHash } from '../utils/generate-hash';
@@ -230,13 +229,11 @@ export class PayloadService {
 
 		const process =
 			action == 'read'
-				? (value: any) => {
-						if (typeof value === 'string') return wktToGeoJSON(value);
-				  }
+				? (value: any) => (typeof value === 'string' ? wktToGeoJSON(value) : value)
 				: (value: any) => helper.fromGeoJSON(typeof value == 'string' ? JSON.parse(value) : value);
 
 		const fieldsInCollection = Object.entries(this.schema.collections[this.collection].fields);
-		const geometryColumns = fieldsInCollection.filter(([_, field]) => isNativeGeometry(field));
+		const geometryColumns = fieldsInCollection.filter(([_, field]) => field.type.startsWith('geometry'));
 
 		for (const [name] of geometryColumns) {
 			for (const payload of payloads) {

api/src/services/specifications.ts

@@ -551,7 +551,25 @@ class OASSpecsService implements SpecificationSubService {
 			format: 'uuid',
 		},
 		geometry: {
-			type: 'string',
+			type: 'object',
+		},
+		'geometry.Point': {
+			type: 'object',
+		},
+		'geometry.LineString': {
+			type: 'object',
+		},
+		'geometry.Polygon': {
+			type: 'object',
+		},
+		'geometry.MultiPoint': {
+			type: 'object',
+		},
+		'geometry.MultiLineString': {
+			type: 'object',
+		},
+		'geometry.MultiPolygon': {
+			type: 'object',
 		},
 	};
 }

api/src/utils/geometry.ts

@@ -1,18 +0,0 @@
-import { FieldOverview } from '../types';
-const dbGeometricTypes = new Set([
-	'point',
-	'polygon',
-	'linestring',
-	'multipoint',
-	'multipolygon',
-	'multilinestring',
-	'geometry',
-	'geometrycollection',
-	'sdo_geometry',
-	'user-defined',
-]);
-
-export function isNativeGeometry(field: FieldOverview): boolean {
-	const { type, dbType } = field;
-	return type == 'geometry' && dbGeometricTypes.has(dbType!.toLowerCase());
-}

api/src/utils/get-default-value.ts

@@ -7,7 +7,7 @@ import env from '../env';
 export default function getDefaultValue(
 	column: SchemaOverview[string]['columns'][string] | Column
 ): string | boolean | number | Record<string, any> | any[] | null {
-	const { type } = getLocalType(column);
+	const type = getLocalType(column);
 
 	let defaultValue = column.default_value ?? null;
 	if (defaultValue === null) return null;

api/src/utils/get-local-type.ts

@@ -1,168 +1,143 @@
 import { SchemaOverview } from '@directus/schema/dist/types/overview';
 import { Column } from 'knex-schema-inspector/dist/types/column';
 import { FieldMeta, Type } from '@directus/shared/types';
+import { getDatabaseClient } from '../database';
 import getDatabase from '../database';
 
-type LocalTypeEntry = {
-	type: Type | 'unknown';
-	geometry_type?: 'Point' | 'LineString' | 'Polygon' | 'MultiPoint' | 'MultiLineString' | 'MultiPolygon';
-};
-
-const localTypeMap: Record<string, LocalTypeEntry> = {
+const localTypeMap: Record<string, Type | 'unknown'> = {
 	// Shared
-	boolean: { type: 'boolean' },
-	tinyint: { type: 'integer' },
-	smallint: { type: 'integer' },
-	mediumint: { type: 'integer' },
-	int: { type: 'integer' },
-	integer: { type: 'integer' },
-	serial: { type: 'integer' },
-	bigint: { type: 'bigInteger' },
-	bigserial: { type: 'bigInteger' },
-	clob: { type: 'text' },
-	tinytext: { type: 'text' },
-	mediumtext: { type: 'text' },
-	longtext: { type: 'text' },
-	text: { type: 'text' },
-	varchar: { type: 'string' },
-	longvarchar: { type: 'string' },
-	varchar2: { type: 'string' },
-	nvarchar: { type: 'string' },
-	image: { type: 'binary' },
-	ntext: { type: 'text' },
-	char: { type: 'string' },
-	date: { type: 'date' },
-	datetime: { type: 'dateTime' },
-	dateTime: { type: 'dateTime' },
-	timestamp: { type: 'timestamp' },
-	time: { type: 'time' },
-	float: { type: 'float' },
-	double: { type: 'float' },
-	'double precision': { type: 'float' },
-	real: { type: 'float' },
-	decimal: { type: 'decimal' },
-	numeric: { type: 'integer' },
+	boolean: 'boolean',
+	tinyint: 'integer',
+	smallint: 'integer',
+	mediumint: 'integer',
+	int: 'integer',
+	integer: 'integer',
+	serial: 'integer',
+	bigint: 'bigInteger',
+	bigserial: 'bigInteger',
+	clob: 'text',
+	tinytext: 'text',
+	mediumtext: 'text',
+	longtext: 'text',
+	text: 'text',
+	varchar: 'string',
+	longvarchar: 'string',
+	varchar2: 'string',
+	nvarchar: 'string',
+	image: 'binary',
+	ntext: 'text',
+	char: 'string',
+	date: 'date',
+	datetime: 'dateTime',
+	dateTime: 'dateTime',
+	timestamp: 'timestamp',
+	time: 'time',
+	float: 'float',
+	double: 'float',
+	'double precision': 'float',
+	real: 'float',
+	decimal: 'decimal',
+	numeric: 'integer',
 
 	// Geometries
-	point: { type: 'geometry', geometry_type: 'Point' },
-	linestring: { type: 'geometry', geometry_type: 'LineString' },
-	polygon: { type: 'geometry', geometry_type: 'Polygon' },
-	multipoint: { type: 'geometry', geometry_type: 'MultiPoint' },
-	multilinestring: { type: 'geometry', geometry_type: 'MultiLineString' },
-	multipolygon: { type: 'geometry', geometry_type: 'MultiPolygon' },
-	geometry: { type: 'geometry' },
+	geometry: 'geometry',
+	point: 'geometry.Point',
+	linestring: 'geometry.LineString',
+	polygon: 'geometry.Polygon',
+	multipoint: 'geometry.MultiPoint',
+	multilinestring: 'geometry.MultiLineString',
+	multipolygon: 'geometry.MultiPolygon',
 
 	// MySQL
-	string: { type: 'text' },
-	year: { type: 'integer' },
-	blob: { type: 'binary' },
-	mediumblob: { type: 'binary' },
-	'int unsigned': { type: 'integer' },
+	string: 'text',
+	year: 'integer',
+	blob: 'binary',
+	mediumblob: 'binary',
+	'int unsigned': 'integer',
+	'tinyint(0)': 'boolean',
+	'tinyint(1)': 'boolean',
 
 	// MS SQL
-	bit: { type: 'boolean' },
-	smallmoney: { type: 'float' },
-	money: { type: 'float' },
-	datetimeoffset: { type: 'timestamp' },
-	datetime2: { type: 'dateTime' },
-	smalldatetime: { type: 'dateTime' },
-	nchar: { type: 'text' },
-	binary: { type: 'binary' },
-	varbinary: { type: 'binary' },
-	uniqueidentifier: { type: 'uuid' },
+	bit: 'boolean',
+	smallmoney: 'float',
+	money: 'float',
+	datetimeoffset: 'timestamp',
+	datetime2: 'dateTime',
+	smalldatetime: 'dateTime',
+	nchar: 'text',
+	binary: 'binary',
+	varbinary: 'binary',
+	uniqueidentifier: 'uuid',
 
 	// Postgres
-	json: { type: 'json' },
-	jsonb: { type: 'json' },
-	uuid: { type: 'uuid' },
-	int2: { type: 'integer' },
-	serial4: { type: 'integer' },
-	int4: { type: 'integer' },
-	serial8: { type: 'integer' },
-	int8: { type: 'integer' },
-	bool: { type: 'boolean' },
-	'character varying': { type: 'string' },
-	character: { type: 'string' },
-	interval: { type: 'string' },
-	_varchar: { type: 'string' },
-	bpchar: { type: 'string' },
-	timestamptz: { type: 'timestamp' },
-	'timestamp with time zone': { type: 'timestamp' },
-	'timestamp without time zone': { type: 'dateTime' },
-	timetz: { type: 'time' },
-	'time with time zone': { type: 'time' },
-	'time without time zone': { type: 'time' },
-	float4: { type: 'float' },
-	float8: { type: 'float' },
+	json: 'json',
+	jsonb: 'json',
+	uuid: 'uuid',
+	int2: 'integer',
+	serial4: 'integer',
+	int4: 'integer',
+	serial8: 'integer',
+	int8: 'integer',
+	bool: 'boolean',
+	'character varying': 'string',
+	character: 'string',
+	interval: 'string',
+	_varchar: 'string',
+	bpchar: 'string',
+	timestamptz: 'timestamp',
+	'timestamp with time zone': 'timestamp',
+	'timestamp with local time zone': 'timestamp',
+	'timestamp without time zone': 'dateTime',
+	'timestamp without local time zone': 'dateTime',
+	timetz: 'time',
+	'time with time zone': 'time',
+	'time without time zone': 'time',
+	float4: 'float',
+	float8: 'float',
 
 	// Oracle
-	number: { type: 'integer' },
-	sdo_geometry: { type: 'geometry' },
+	number: 'integer',
+	sdo_geometry: 'geometry',
 
 	// SQLite
-	integerfirst: { type: 'integer' },
+	integerfirst: 'integer',
 };
 
 export default function getLocalType(
 	column?: SchemaOverview[string]['columns'][string] | Column,
 	field?: { special?: FieldMeta['special'] }
-): LocalTypeEntry {
+): Type | 'unknown' {
 	const database = getDatabase();
+	const databaseClient = getDatabaseClient(database);
 
-	const type = (
-		column ? localTypeMap[column.data_type.toLowerCase().split('(')[0]] : { type: 'alias' }
-	) as LocalTypeEntry;
+	const type = column ? localTypeMap[column.data_type.toLowerCase().split('(')[0]] : 'alias';
 
 	const special = field?.special;
 
 	if (special) {
-		if (special.includes('json')) return { type: 'json' };
-		if (special.includes('hash')) return { type: 'hash' };
-		if (special.includes('csv')) return { type: 'csv' };
-		if (special.includes('uuid')) return { type: 'uuid' };
-		if (type?.type == 'geometry' && !type.geometry_type) {
-			type.geometry_type = special[1] as any;
+		if (special.includes('json')) return 'json';
+		if (special.includes('hash')) return 'hash';
+		if (special.includes('csv')) return 'csv';
+		if (special.includes('uuid')) return 'uuid';
+		if (type.startsWith('geometry')) {
+			return (special[0] as Type) || 'geometry';
 		}
 	}
 
-	/** Handle OracleDB timestamp with time zone */
-	if (
-		column &&
-		(database.client.constructor.name === 'Client_Oracledb' || database.client.constructor.name === 'Client_Oracle')
-	) {
-		const type = column.data_type.toLowerCase();
-
-		if (type.startsWith('timestamp')) {
-			if (type.endsWith('with local time zone')) {
-				return { type: 'timestamp' };
-			} else {
-				return { type: 'dateTime' };
-			}
-		}
-	}
 	/** Handle Postgres numeric decimals */
 	if (column && column.data_type === 'numeric' && column.numeric_precision !== null && column.numeric_scale !== null) {
-		return { type: 'decimal' };
+		return 'decimal';
 	}
 
 	/** Handle MS SQL varchar(MAX) (eg TEXT) types */
 	if (column && column.data_type === 'nvarchar' && column.max_length === -1) {
-		return { type: 'text' };
+		return 'text';
 	}
 
 	/** Handle Boolean as TINYINT and edgecase MySQL where it still is just tinyint */
-	if (
-		column &&
-		((database.client.constructor.name === 'Client_MySQL' && column.data_type.toLowerCase() === 'tinyint') ||
-			column.data_type.toLowerCase() === 'tinyint(1)' ||
-			column.data_type.toLowerCase() === 'tinyint(0)')
-	) {
-		return { type: 'boolean' };
+	if (column && databaseClient === 'mysql' && column.data_type.toLowerCase() === 'tinyint') {
+		return 'boolean';
 	}
 
-	if (type) {
-		return type;
-	}
-
-	return { type: 'unknown' };
+	return type ?? 'unknown';
 }

api/src/utils/get-schema.ts

@@ -101,7 +101,7 @@ async function getDatabaseSchema(
 					defaultValue: getDefaultValue(column) ?? null,
 					nullable: column.is_nullable ?? true,
 					generated: column.is_generated ?? false,
-					type: getLocalType(column).type,
+					type: getLocalType(column),
 					dbType: column.data_type,
 					precision: column.numeric_precision || null,
 					scale: column.numeric_scale || null,
@@ -132,7 +132,7 @@ async function getDatabaseSchema(
 		const existing = result.collections[field.collection].fields[field.field];
 		const column = schemaOverview[field.collection].columns[field.field];
 		const special = field.special ? toArray(field.special) : [];
-		const { type = 'alias' } = existing && column ? getLocalType(column, { special }) : {};
+		const type = (existing && getLocalType(column, { special })) || 'alias';
 
 		result.collections[field.collection].fields[field.field] = {
 			field: field.field,