Add support for Geometry type, add Map Layout & Interface (#5684)

* Added map layout

* Cleanup and bug fixes

* Removed package-lock

* Cleanup and fixes

* Small fix

* Added back package-lock

* Saved camera, autofitting option, bug fixes

* Refactor and ui improvements

* Improvements

* Added seled mode

* Removed unused dependency

* Changed selection behaviour, cleanup.

* update import and dependencies

* make custom style into drawer

* remove unused imports

* use lodash functions

* add popups

* allow header to become small

* reorganize settings

* add styling to popup

* change default template

* add projection option

* add basic map interface

* finish simple map

* add mapbox style

* support more mapbox layouts

* add api key option

* add mapbox backgrounds to layout

* warn when no api key is set

* fix for latest version

* Improved map layout and interface, bug fixes, refactoring.

.

.

* Added postgis geometry format, added marker icon shadow

* Made map buttons bigger and their icons thinner. Added transition to header bar.

* Bug fixes and error handling in map interface.

* Moved box-select control out of the map component. Removed material icons sprite and use addImage for marker support.

* Handle MultiGeometry -> Geometry interface error.

* Removed hardcoded styles. Added migrations for basemap column. Lots of refactoring.

Removed hardcoded styles. Added migrations for basemap column. Lots of refactoring.

* Fixed style reloading error. Added translations.

* Moved worker code to lib.

* Removed worker code. Prevent Mapbox from removing access_token from the URL.

* Refactoring.

* Change basemap selection to in-map dropdown for layout and interface.

* Touchscreen selection support and small fixes.

* Small change.

* Fixed unused imports.

* Added support for PostgreSQL identity column

* Renamed migration. Added crs translation.

* Only show fields using the map interface in the map layout.

* Removed logging.

* Reverted Dockerfile change.

* Improved crs support.

* Fixed translations.

* Check for schema identity before updating it.

* Fixed popup not updating on feature hover.

* Added feature hover styling. Fixed layer customization input. Added out of bounds error handling.

* Added geometry type and support for database native geometries.

* Fixed linting.

* Fixed layout.

* Fixed layout.

* Actually fixed linting

* Full support for native geometries
Fixed basemap input
Improved feature popup on hover
Locked interfaced support

* Fixed geometryType option not updating

* Bug fixes in interface

* Fixed crash when empty basemap settings. Fixed fitBounds option not updating.

* Added back storage type option. Improved interface behaviour.

* Dropped wkb because of vendor inconsistency with binary data

* Updated layout to match new geometry type. Fixed geojson payload transform.

* Added missing geometry_format attributes to local types.

* Fixed typos & refactoring

* Removed dependency on proj4

* Fix error when empty map interface options

* Set geometry SRID to 4326 when inserting into the database

* Add support for selectMode

* Fix error on initial source load

* Added geocoder, use GeoJSON for api i/o, removed geometry_format option, refactoring

* Added geometry intersects filter. Created geometry helper class.

* Fix error when null geometryOptions, added mapbox_key setting.

* Moved all geometry parsing/serializing into processGeometries in `payload.ts`. Fixed type errors.

* Migrate to Vue 3

* Use wellknown instead of wkx

* Fixed basemap selection.

* Added available operator for geometry type

* Added nintersects filter, fixed map interface for filter input

* Added intersects_bbox filter & bug fixes.

* Fixed icons rendering

* Fixed cursor icon in select mode

* Added geometry aggregate function

* Fixed geometry processing bug when imported from relational field.

* Fixed error with geocoder instanciation

* Removed @types/maplibre-gl dependency

* Removed fitViewToData options

* Merge remote-tracking branch 'upstream/main' into map-layout

* Fixed style and geometryType in map interface options

* Fixed style change on map interface.

* Improved fitViewToData behaviour

* Fixed type imports and previous merge conflict

* Fixed linting

* Added available operators

* Fix and merge migrations

* Remove outdated p-queue dep

* Fix get-schema column extract

* Replace pg with postgis for local debugging

* Re-add missing import

* Add mapbox as a basemap when key exists

* Remove unused tz flag

* Process delta in payloadservice

* Set default map, add limit number styling

* Default display template to just PK

* Tweak styling of error dialog

* Fix method usage in helpers

* Move sdo_geo to oracle section

* Remove extensions from ts config exclude

* Move geo types to shared, remove _Geometry

* Remove unused type

* Tiny Tweaks

* Remove fit to bounds option in favor of on

* Validate incoming intersects query

* Deepmap filter values

* Add GraphQL support

* No defaultValue for geometryType

* Resolve c

* Fix translations

Co-authored-by: Nitwel <nitwel@arcor.de>
Co-authored-by: Rijk van Zanten <rijkvanzanten@me.com>

api/src/services/collections.ts

@@ -7,11 +7,11 @@ import { systemCollectionRows } from '../database/system-data/collections';
 import env from '../env';
 import { ForbiddenException, InvalidPayloadException } from '../exceptions';
 import logger from '../logger';
-import { FieldsService, RawField } from '../services/fields';
+import { FieldsService } from '../services/fields';
 import { ItemsService, MutationOptions } from '../services/items';
 import Keyv from 'keyv';
 import { AbstractServiceOptions, Collection, CollectionMeta, SchemaOverview } from '../types';
-import { Accountability, FieldMeta } from '@directus/shared/types';
+import { Accountability, FieldMeta, RawField } from '@directus/shared/types';
 
 export type RawCollection = {
 	collection: string;

api/src/services/fields.ts

@@ -13,16 +13,14 @@ import { ItemsService } from '../services/items';
 import { PayloadService } from '../services/payload';
 import { AbstractServiceOptions, SchemaOverview } from '../types';
 import { Accountability } from '@directus/shared/types';
-import { Field, FieldMeta, Type } from '@directus/shared/types';
+import { Field, FieldMeta, RawField, Type } from '@directus/shared/types';
 import getDefaultValue from '../utils/get-default-value';
 import getLocalType from '../utils/get-local-type';
 import { toArray } from '@directus/shared/utils';
 import { isEqual, isNil } from 'lodash';
 import { RelationsService } from './relations';
+import { getGeometryHelper } from '../database/helpers/geometry';
 import Keyv from 'keyv';
-import { DeepPartial } from '@directus/shared/types';
-
-export type RawField = DeepPartial<Field> & { field: string; type: Type };
 
 export class FieldsService {
 	knex: Knex;
@@ -77,25 +75,22 @@ export class FieldsService {
 			fields.push(...systemFieldRows);
 		}
 
-		let columns = await this.schemaInspector.columnInfo(collection);
-
-		columns = columns.map((column) => {
-			return {
-				...column,
-				default_value: getDefaultValue(column),
-			};
-		});
+		const columns = (await this.schemaInspector.columnInfo(collection)).map((column) => ({
+			...column,
+			default_value: getDefaultValue(column),
+		}));
 
 		const columnsWithSystem = columns.map((column) => {
 			const field = fields.find((field) => {
 				return field.field === column.name && field.collection === column.table;
 			});
 
+			const { type = 'alias', ...info } = column ? getLocalType(column, field) : {};
 			const data = {
 				collection: column.table,
 				field: column.name,
-				type: column ? getLocalType(column, field) : 'alias',
-				schema: column,
+				type: type,
+				schema: { ...column, ...info },
 				meta: field || null,
 			};
 
@@ -202,12 +197,13 @@ export class FieldsService {
 			// Do nothing
 		}
 
+		const { type = 'alias', ...info } = column ? getLocalType(column, fieldInfo) : {};
 		const data = {
 			collection,
 			field,
-			type: column ? getLocalType(column, fieldInfo) : 'alias',
+			type,
 			meta: fieldInfo || null,
-			schema: column || null,
+			schema: type == 'alias' ? null : { ...column, ...info },
 		};
 
 		return data;
@@ -458,6 +454,9 @@ export class FieldsService {
 			column = table.dateTime(field.field, { useTz: false });
 		} else if (field.type === 'timestamp') {
 			column = table.timestamp(field.field, { useTz: true });
+		} else if (field.type === 'geometry') {
+			const helper = getGeometryHelper();
+			column = helper.createColumn(table, field);
 		} else {
 			column = table[field.type](field.field);
 		}

api/src/services/graphql.ts

@@ -1,4 +1,5 @@
 import argon2 from 'argon2';
+import { validateQuery } from '../utils/validate-query';
 import {
 	ArgumentNode,
 	BooleanValueNode,
@@ -92,6 +93,12 @@ const GraphQLVoid = new GraphQLScalarType({
 	},
 });
 
+export const GraphQLGeoJSON = new GraphQLScalarType({
+	...GraphQLJSON,
+	name: 'GraphQLGeoJSON',
+	description: 'GeoJSON value',
+});
+
 export const GraphQLDate = new GraphQLScalarType({
 	...GraphQLString,
 	name: 'Date',
@@ -536,6 +543,30 @@ export class GraphQLService {
 				},
 			});
 
+			const GeometryFilterOperators = schemaComposer.createInputTC({
+				name: 'geometry_filter_operators',
+				fields: {
+					_eq: {
+						type: GraphQLGeoJSON,
+					},
+					_neq: {
+						type: GraphQLGeoJSON,
+					},
+					_intersects: {
+						type: GraphQLGeoJSON,
+					},
+					_nintersects: {
+						type: GraphQLGeoJSON,
+					},
+					_intersects_bbox: {
+						type: GraphQLGeoJSON,
+					},
+					_nintersects_bbox: {
+						type: GraphQLGeoJSON,
+					},
+				},
+			});
+
 			for (const collection of Object.values(schema.read.collections)) {
 				if (Object.keys(collection.fields).length === 0) continue;
 				if (SYSTEM_DENY_LIST.includes(collection.collection)) continue;
@@ -557,6 +588,9 @@ export class GraphQLService {
 							case GraphQLDate:
 								filterOperatorType = DateFilterOperators;
 								break;
+							case GraphQLGeoJSON:
+								filterOperatorType = GeometryFilterOperators;
+								break;
 							default:
 								filterOperatorType = StringFilterOperators;
 						}
@@ -1088,6 +1122,8 @@ export class GraphQLService {
 
 		query.fields = parseFields(selections);
 
+		validateQuery(query);
+
 		return query;
 	}
 

api/src/services/items.ts

@@ -172,8 +172,8 @@ export class ItemsService<Item extends AnyItem = AnyItem> implements AbstractSer
 						activity: activityID,
 						collection: this.collection,
 						item: primaryKey,
-						data: JSON.stringify(payload),
-						delta: JSON.stringify(payload),
+						data: await payloadService.prepareDelta(payload),
+						delta: await payloadService.prepareDelta(payload),
 					};
 
 					const revisionID = (await trx.insert(revisionRecord).into('directus_revisions').returning('id'))[0] as number;
@@ -480,14 +480,23 @@ export class ItemsService<Item extends AnyItem = AnyItem> implements AbstractSer
 
 					const snapshots = await itemsService.readMany(keys);
 
-					const revisionRecords = activityPrimaryKeys.map((key, index) => ({
-						activity: key,
-						collection: this.collection,
-						item: keys[index],
-						data:
-							snapshots && Array.isArray(snapshots) ? JSON.stringify(snapshots?.[index]) : JSON.stringify(snapshots),
-						delta: JSON.stringify(payloadWithTypeCasting),
-					}));
+					const revisionRecords: {
+						activity: PrimaryKey;
+						collection: string;
+						item: PrimaryKey;
+						data: string;
+						delta: string;
+					}[] = [];
+
+					for (let i = 0; i < activityPrimaryKeys.length; i++) {
+						revisionRecords.push({
+							activity: activityPrimaryKeys[i],
+							collection: this.collection,
+							item: keys[i],
+							data: snapshots && Array.isArray(snapshots) ? JSON.stringify(snapshots[i]) : JSON.stringify(snapshots),
+							delta: await payloadService.prepareDelta(payloadWithTypeCasting),
+						});
+					}
 
 					for (let i = 0; i < revisionRecords.length; i++) {
 						const revisionID = (

api/src/services/payload.ts

@@ -10,6 +10,9 @@ import { AbstractServiceOptions, Item, PrimaryKey, Query, SchemaOverview, Altera
 import { Accountability } from '@directus/shared/types';
 import { toArray } from '@directus/shared/utils';
 import { ItemsService } from './items';
+import { isNativeGeometry } from '../utils/geometry';
+import { getGeometryHelper } from '../database/helpers/geometry';
+import { parse as wktToGeoJSON } from 'wellknown';
 
 type Action = 'create' | 'read' | 'update';
 
@@ -19,6 +22,7 @@ type Transformers = {
 		value: any;
 		payload: Partial<Item>;
 		accountability: Accountability | null;
+		specials: string[];
 	}) => Promise<any>;
 };
 
@@ -148,13 +152,16 @@ export class PayloadService {
 			})
 		);
 
-		await this.processDates(processedPayload, action);
+		this.processGeometries(processedPayload, action);
+		this.processDates(processedPayload, action);
 
 		if (['create', 'update'].includes(action)) {
 			processedPayload.forEach((record) => {
 				for (const [key, value] of Object.entries(record)) {
-					if (Array.isArray(value) || (typeof value === 'object' && value instanceof Date !== true && value !== null)) {
-						record[key] = JSON.stringify(value);
+					if (Array.isArray(value) || (typeof value === 'object' && !(value instanceof Date) && value !== null)) {
+						if (!value.isRawInstance) {
+							record[key] = JSON.stringify(value);
+						}
 					}
 				}
 			});
@@ -185,6 +192,7 @@ export class PayloadService {
 					value,
 					payload,
 					accountability,
+					specials: fieldSpecials,
 				});
 			}
 		}
@@ -192,14 +200,40 @@ export class PayloadService {
 		return value;
 	}
 
+	/**
+	 * Native geometries are stored in custom binary format. We need to insert them with
+	 * the function st_geomfromtext. For this to work, that function call must not be
+	 * escaped. It's therefore placed as a Knex.Raw object in the payload. Thus the need
+	 * to check if the value is a raw instance before stringifying it in the next step.
+	 */
+	processGeometries<T extends Partial<Record<string, any>>[]>(payloads: T, action: Action): T {
+		const helper = getGeometryHelper();
+
+		const process =
+			action == 'read'
+				? (value: any) => {
+						if (typeof value === 'string') return wktToGeoJSON(value);
+				  }
+				: (value: any) => helper.fromGeoJSON(typeof value == 'string' ? JSON.parse(value) : value);
+
+		const fieldsInCollection = Object.entries(this.schema.collections[this.collection].fields);
+		const geometryColumns = fieldsInCollection.filter(([_, field]) => isNativeGeometry(field));
+
+		for (const [name] of geometryColumns) {
+			for (const payload of payloads) {
+				if (payload[name]) {
+					payload[name] = process(payload[name]);
+				}
+			}
+		}
+
+		return payloads;
+	}
 	/**
 	 * Knex returns `datetime` and `date` columns as Date.. This is wrong for date / datetime, as those
 	 * shouldn't return with time / timezone info respectively
 	 */
-	async processDates(
-		payloads: Partial<Record<string, any>>[],
-		action: Action
-	): Promise<Partial<Record<string, any>>[]> {
+	processDates(payloads: Partial<Record<string, any>>[], action: Action): Partial<Record<string, any>>[] {
 		const fieldsInCollection = Object.entries(this.schema.collections[this.collection].fields);
 
 		const dateColumns = fieldsInCollection.filter(([_name, field]) =>
@@ -638,4 +672,22 @@ export class PayloadService {
 
 		return { revisions };
 	}
+
+	/**
+	 * Transforms the input partial payload to match the output structure, to have consistency
+	 * between delta and data
+	 */
+	async prepareDelta(data: Partial<Item>): Promise<string> {
+		let payload = cloneDeep(data);
+
+		for (const key in payload) {
+			if (payload[key]?.isRawInstance) {
+				payload[key] = payload[key].bindings[0];
+			}
+		}
+
+		payload = await this.processValues('read', payload);
+
+		return JSON.stringify(payload);
+	}
 }

api/src/services/specifications.ts

@@ -526,6 +526,9 @@ class OASSpecsService implements SpecificationSubService {
 			type: 'string',
 			format: 'uuid',
 		},
+		geometry: {
+			type: 'string',
+		},
 	};
 }