Rework nested ast typings to nodes

2026-02-03 16:04:55 -05:00 · 2020-10-08 13:56:19 -04:00
parent c69210f33f
commit 5b91bc3577
4 changed files with 210 additions and 108 deletions
--- a/api/src/database/run-ast.ts
+++ b/api/src/database/run-ast.ts
@@ -1,4 +1,4 @@
-import { AST, NestedCollectionAST } from '../types/ast';
+import { AST, NestedCollectionNode } from '../types/ast';
 import { clone, cloneDeep, uniq, pick } from 'lodash';
 import database from './index';
 import SchemaInspector from 'knex-schema-inspector';
@@ -13,14 +13,20 @@ type RunASTOptions = {
 	child?: boolean;
 };

-export default async function runAST(originalAST: AST, options?: RunASTOptions): Promise<null | Item | Item[]> {
+export default async function runAST(
+	originalAST: AST | NestedCollectionNode,
+	options?: RunASTOptions
+): Promise<null | Item | Item[]> {
 	const ast = cloneDeep(originalAST);

 	const query = options?.query || ast.query;
 	const knex = options?.knex || database;

 	// Retrieve the database columns to select in the current AST
-	const { columnsToSelect, primaryKeyField, nestedCollectionASTs } = await parseCurrentLevel(ast, knex);
+	const { columnsToSelect, primaryKeyField, nestedCollectionNodes } = await parseCurrentLevel(
+		ast,
+		knex
+	);

 	// The actual knex query builder instance. This is a promise that resolves with the raw items from the db
 	const dbQuery = await getDBQuery(knex, ast.name, columnsToSelect, query, primaryKeyField);
@@ -36,25 +42,25 @@ export default async function runAST(originalAST: AST, options?: RunASTOptions):
 	if (!items || items.length === 0) return items;

 	// Apply the `_in` filters to the nested collection batches
-	const nestedASTs = applyParentFilters(nestedCollectionASTs, items);
+	const nestedNodes = applyParentFilters(nestedCollectionNodes, items);

-	for (const nestedAST of nestedASTs) {
+	for (const nestedNode of nestedNodes) {
 		let tempLimit: number | null = null;

 		// Nested o2m-items are fetched from the db in a single query. This means that we're fetching
 		// all nested items for all parent items at once. Because of this, we can't limit that query
 		// to the "standard" item limit. Instead of _n_ nested items per parent item, it would mean
 		// that there's _n_ items, which are then divided on the parent items. (no good)
-		if (isO2M(nestedAST) && typeof nestedAST.query.limit === 'number') {
-			tempLimit = nestedAST.query.limit;
-			nestedAST.query.limit = -1;
+		if (isO2M(nestedNode) && typeof nestedNode.query.limit === 'number') {
+			tempLimit = nestedNode.query.limit;
+			nestedNode.query.limit = -1;
 		}

-		let nestedItems = await runAST(nestedAST, { knex, child: true });
+		let nestedItems = await runAST(nestedNode, { knex, child: true });

 		if (nestedItems) {
 			// Merge all fetched nested records with the parent items
-			items = mergeWithParentItems(nestedItems, items, nestedAST, tempLimit);
+			items = mergeWithParentItems(nestedItems, items, nestedNode, tempLimit);
 		}
 	}

@@ -69,7 +75,7 @@ export default async function runAST(originalAST: AST, options?: RunASTOptions):
 	return items;
 }

-async function parseCurrentLevel(ast: AST, knex: Knex) {
+async function parseCurrentLevel(ast: AST | NestedCollectionNode, knex: Knex) {
 	const schemaInspector = SchemaInspector(knex);

 	const primaryKeyField = await schemaInspector.primary(ast.name);
@@ -79,7 +85,7 @@ async function parseCurrentLevel(ast: AST, knex: Knex) {
 	);

 	const columnsToSelect: string[] = [];
-	const nestedCollectionASTs: NestedCollectionAST[] = [];
+	const nestedCollectionNodes: NestedCollectionNode[] = [];

 	for (const child of ast.children) {
 		if (child.type === 'field') {
@@ -98,7 +104,7 @@ async function parseCurrentLevel(ast: AST, knex: Knex) {
 			columnsToSelect.push(child.relation.many_field);
 		}

-		nestedCollectionASTs.push(child);
+		nestedCollectionNodes.push(child);
 	}

 	/** Always fetch primary key in case there's a nested relation that needs it */
@@ -106,10 +112,16 @@ async function parseCurrentLevel(ast: AST, knex: Knex) {
 		columnsToSelect.push(primaryKeyField);
 	}

-	return { columnsToSelect, nestedCollectionASTs, primaryKeyField };
+	return { columnsToSelect, nestedCollectionNodes, primaryKeyField };
 }

-async function getDBQuery(knex: Knex, table: string, columns: string[], query: Query, primaryKeyField: string): Promise<QueryBuilder> {
+async function getDBQuery(
+	knex: Knex,
+	table: string,
+	columns: string[],
+	query: Query,
+	primaryKeyField: string
+): Promise<QueryBuilder> {
 	let dbQuery = knex.select(columns.map((column) => `${table}.${column}`)).from(table);

 	const queryCopy = clone(query);
@@ -127,92 +139,114 @@ async function getDBQuery(knex: Knex, table: string, columns: string[], query: Q
 	return dbQuery;
 }

-function applyParentFilters(nestedCollectionASTs: NestedCollectionAST[], parentItem: Item | Item[]) {
+function applyParentFilters(
+	nestedCollectionNodes: NestedCollectionNode[],
+	parentItem: Item | Item[]
+) {
 	const parentItems = Array.isArray(parentItem) ? parentItem : [parentItem];

-	for (const nestedAST of nestedCollectionASTs) {
-		if (!nestedAST.relation) continue;
+	for (const nestedNode of nestedCollectionNodes) {
+		if (!nestedNode.relation) continue;

-		if (isM2O(nestedAST)) {
-			nestedAST.query = {
-				...nestedAST.query,
+		if (isM2O(nestedNode)) {
+			nestedNode.query = {
+				...nestedNode.query,
 				filter: {
-					...(nestedAST.query.filter || {}),
-					[nestedAST.relation.one_primary]: {
-						_in: uniq(parentItems.map((res) => res[nestedAST.relation.many_field])).filter(
-							(id) => id
-						),
-					}
-				}
-			}
+					...(nestedNode.query.filter || {}),
+					[nestedNode.relation.one_primary]: {
+						_in: uniq(
+							parentItems.map((res) => res[nestedNode.relation.many_field])
+						).filter((id) => id),
+					},
+				},
+			};
 		} else {
-			const relatedM2OisFetched = !!nestedAST.children.find((child) => {
-				return child.type === 'field' && child.name === nestedAST.relation.many_field
+			const relatedM2OisFetched = !!nestedNode.children.find((child) => {
+				return child.type === 'field' && child.name === nestedNode.relation.many_field;
 			});

 			if (relatedM2OisFetched === false) {
-				nestedAST.children.push({ type: 'field', name: nestedAST.relation.many_field });
+				nestedNode.children.push({ type: 'field', name: nestedNode.relation.many_field });
 			}

-			nestedAST.query = {
-				...nestedAST.query,
+			nestedNode.query = {
+				...nestedNode.query,
 				filter: {
-					...(nestedAST.query.filter || {}),
-					[nestedAST.relation.many_field]: {
-						_in: uniq(parentItems.map((res) => res[nestedAST.parentKey])).filter((id) => id),
-					}
-				}
-			}
+					...(nestedNode.query.filter || {}),
+					[nestedNode.relation.many_field]: {
+						_in: uniq(parentItems.map((res) => res[nestedNode.parentKey])).filter(
+							(id) => id
+						),
+					},
+				},
+			};
 		}
 	}

-	return nestedCollectionASTs;
+	return nestedCollectionNodes;
 }

-function mergeWithParentItems(nestedItem: Item | Item[], parentItem: Item | Item[], nestedAST: NestedCollectionAST, o2mLimit?: number | null) {
+function mergeWithParentItems(
+	nestedItem: Item | Item[],
+	parentItem: Item | Item[],
+	nestedNode: NestedCollectionNode,
+	o2mLimit?: number | null
+) {
 	const nestedItems = Array.isArray(nestedItem) ? nestedItem : [nestedItem];
 	const parentItems = clone(Array.isArray(parentItem) ? parentItem : [parentItem]);

-	if (isM2O(nestedAST)) {
+	if (isM2O(nestedNode)) {
 		for (const parentItem of parentItems) {
 			const itemChild = nestedItems.find((nestedItem) => {
-				return nestedItem[nestedAST.relation.one_primary] === parentItem[nestedAST.fieldKey];
+				return (
+					nestedItem[nestedNode.relation.one_primary] === parentItem[nestedNode.fieldKey]
+				);
 			});

-			parentItem[nestedAST.fieldKey] = itemChild || null;
+			parentItem[nestedNode.fieldKey] = itemChild || null;
 		}
 	} else {
 		for (const parentItem of parentItems) {
 			let itemChildren = nestedItems.filter((nestedItem) => {
 				if (nestedItem === null) return false;
-				if (Array.isArray(nestedItem[nestedAST.relation.many_field])) return true;
+				if (Array.isArray(nestedItem[nestedNode.relation.many_field])) return true;

 				return (
-					nestedItem[nestedAST.relation.many_field] === parentItem[nestedAST.relation.one_primary] ||
-					nestedItem[nestedAST.relation.many_field]?.[nestedAST.relation.many_primary] === parentItem[nestedAST.relation.one_primary]
+					nestedItem[nestedNode.relation.many_field] ===
+						parentItem[nestedNode.relation.one_primary] ||
+					nestedItem[nestedNode.relation.many_field]?.[
+						nestedNode.relation.many_primary
+					] === parentItem[nestedNode.relation.one_primary]
 				);
 			});

 			// We re-apply the requested limit here. This forces the _n_ nested items per parent concept
 			if (o2mLimit !== null) {
 				itemChildren = itemChildren.slice(0, o2mLimit);
-				nestedAST.query.limit = o2mLimit;
+				nestedNode.query.limit = o2mLimit;
 			}

-			parentItem[nestedAST.fieldKey] = itemChildren.length > 0 ? itemChildren : null;
+			parentItem[nestedNode.fieldKey] = itemChildren.length > 0 ? itemChildren : null;
 		}
 	}

 	return Array.isArray(parentItem) ? parentItems : parentItems[0];
 }

-function removeTemporaryFields(rawItem: Item | Item[], ast: AST | NestedCollectionAST): Item | Item[] {
+function removeTemporaryFields(
+	rawItem: Item | Item[],
+	ast: AST | NestedCollectionNode
+): Item | Item[] {
 	const rawItems: Item[] = Array.isArray(rawItem) ? rawItem : [rawItem];

 	const items: Item[] = [];

-	const fields = ast.children.filter((child) => child.type === 'field').map((child) => child.name);
-	const nestedCollections = ast.children.filter((child) => child.type === 'collection') as NestedCollectionAST[];
+	const fields = ast.children
+		.filter((child) => child.type === 'field')
+		.map((child) => child.name);
+	const nestedCollections = ast.children.filter(
+		(child) => child.type !== 'field'
+	) as NestedCollectionNode[];

 	for (const rawItem of rawItems) {
 		if (rawItem === null) return rawItem;
@@ -220,7 +254,10 @@ function removeTemporaryFields(rawItem: Item | Item[], ast: AST | NestedCollecti

 		for (const nestedCollection of nestedCollections) {
 			if (item[nestedCollection.fieldKey] !== null) {
-				item[nestedCollection.fieldKey] = removeTemporaryFields(rawItem[nestedCollection.fieldKey], nestedCollection);
+				item[nestedCollection.fieldKey] = removeTemporaryFields(
+					rawItem[nestedCollection.fieldKey],
+					nestedCollection
+				);
 			}
 		}

@@ -230,12 +267,12 @@ function removeTemporaryFields(rawItem: Item | Item[], ast: AST | NestedCollecti
 	return Array.isArray(rawItem) ? items : items[0];
 }

-function isM2O(child: NestedCollectionAST) {
+function isM2O(child: NestedCollectionNode) {
 	return (
 		child.relation.one_collection === child.name && child.relation.many_field === child.fieldKey
 	);
 }

-function isO2M(child: NestedCollectionAST) {
+function isO2M(child: NestedCollectionNode) {
 	return isM2O(child) === false;
 }
--- a/api/src/services/authorization.ts
+++ b/api/src/services/authorization.ts
@@ -3,8 +3,8 @@ import {
 	Accountability,
 	AbstractServiceOptions,
 	AST,
-	NestedCollectionAST,
-	FieldAST,
+	NestedCollectionNode,
+	FieldNode,
 	Query,
 	Permission,
 	PermissionsAction,
@@ -74,30 +74,28 @@ export class AuthorizationService {
 		 * Traverses the AST and returns an array of all collections that are being fetched
 		 */
 		function getCollectionsFromAST(
-			ast: AST | NestedCollectionAST
+			ast: AST | NestedCollectionNode
 		): { collection: string; field: string }[] {
 			const collections = [];

-			if (ast.type === 'collection') {
+			if (ast.type !== 'root') {
 				collections.push({
 					collection: ast.name,
-					field: (ast as NestedCollectionAST).fieldKey
-						? (ast as NestedCollectionAST).fieldKey
-						: null,
+					field: ast.fieldKey || null,
 				});
 			}

-			for (const subAST of ast.children) {
-				if (subAST.type === 'collection') {
-					collections.push(...getCollectionsFromAST(subAST));
+			for (const nestedNode of ast.children) {
+				if (nestedNode.type !== 'field') {
+					collections.push(...getCollectionsFromAST(nestedNode));
 				}
 			}

 			return collections as { collection: string; field: string }[];
 		}

-		function validateFields(ast: AST | NestedCollectionAST) {
-			if (ast.type === 'collection') {
+		function validateFields(ast: AST | NestedCollectionNode | FieldNode) {
+			if (ast.type !== 'field') {
 				const collection = ast.name;

 				// We check the availability of the permissions in the step before this is run
@@ -108,7 +106,7 @@ export class AuthorizationService {
 				const allowedFields = permissions.fields?.split(',') || [];

 				for (const childAST of ast.children) {
-					if (childAST.type === 'collection') {
+					if (childAST.type !== 'field') {
 						validateFields(childAST);
 						continue;
 					}
@@ -127,10 +125,10 @@ export class AuthorizationService {
 		}

 		function applyFilters(
-			ast: AST | NestedCollectionAST | FieldAST,
+			ast: AST | NestedCollectionNode | FieldNode,
 			accountability: Accountability | null
-		): AST | NestedCollectionAST | FieldAST {
-			if (ast.type === 'collection') {
+		): AST | NestedCollectionNode | FieldNode {
+			if (ast.type !== 'field') {
 				const collection = ast.name;

 				// We check the availability of the permissions in the step before this is run
@@ -164,8 +162,8 @@ export class AuthorizationService {
 				}

 				ast.children = ast.children.map((child) => applyFilters(child, accountability)) as (
-					| NestedCollectionAST
-					| FieldAST
+					| NestedCollectionNode
+					| FieldNode
 				)[];
 			}

@@ -198,7 +196,17 @@ export class AuthorizationService {
 		let permission: Permission | undefined;

 		if (this.accountability?.admin === true) {
-			permission = { id: 0, role: this.accountability?.role, collection, action, permissions: {}, validation: {}, limit: null, fields: '*', presets: {}, }
+			permission = {
+				id: 0,
+				role: this.accountability?.role,
+				collection,
+				action,
+				permissions: {},
+				validation: {},
+				limit: null,
+				fields: '*',
+				presets: {},
+			};
 		} else {
 			permission = await this.knex
 				.select<Permission>('*')
@@ -238,10 +246,23 @@ export class AuthorizationService {
 		let requiredColumns: string[] = [];

 		for (const column of columns) {
-			const field = await this.knex.select<{ special: string }>('special').from('directus_fields').where({ collection, field: column.name }).first();
+			const field = await this.knex
+				.select<{ special: string }>('special')
+				.from('directus_fields')
+				.where({ collection, field: column.name })
+				.first();
 			const specials = (field?.special || '').split(',');
-			const hasGenerateSpecial = ['uuid', 'date-created', 'role-created', 'user-created'].some((name) => specials.includes(name));
-			const isRequired = column.is_nullable === false && column.has_auto_increment === false && column.default_value === null && hasGenerateSpecial === false;
+			const hasGenerateSpecial = [
+				'uuid',
+				'date-created',
+				'role-created',
+				'user-created',
+			].some((name) => specials.includes(name));
+			const isRequired =
+				column.is_nullable === false &&
+				column.has_auto_increment === false &&
+				column.default_value === null &&
+				hasGenerateSpecial === false;

 			if (isRequired) {
 				requiredColumns.push(column.name);
@@ -250,23 +271,20 @@ export class AuthorizationService {

 		if (requiredColumns.length > 0) {
 			permission.validation = {
-				_and: [
-					permission.validation,
-					{}
-				]
-			}
+				_and: [permission.validation, {}],
+			};

 			if (action === 'create') {
 				for (const name of requiredColumns) {
 					permission.validation._and[1][name] = {
-						_required: true
-					}
+						_required: true,
+					};
 				}
 			} else {
 				for (const name of requiredColumns) {
 					permission.validation._and[1][name] = {
-						_nnull: true
-					}
+						_nnull: true,
+					};
 				}
 			}
 		}
@@ -282,7 +300,10 @@ export class AuthorizationService {
 		}
 	}

-	validateJoi(validation: Record<string, any>, payloads: Partial<Record<string, any>>[]): FailedValidationException[] {
+	validateJoi(
+		validation: Record<string, any>,
+		payloads: Partial<Record<string, any>>[]
+	): FailedValidationException[] {
 		const errors: FailedValidationException[] = [];

 		/**
@@ -291,13 +312,21 @@ export class AuthorizationService {

 		if (Object.keys(validation)[0] === '_and') {
 			const subValidation = Object.values(validation)[0];
-			const nestedErrors = flatten<FailedValidationException>(subValidation.map((subObj: Record<string, any>) => this.validateJoi(subObj, payloads))).filter((err?: FailedValidationException) => err);
+			const nestedErrors = flatten<FailedValidationException>(
+				subValidation.map((subObj: Record<string, any>) =>
+					this.validateJoi(subObj, payloads)
+				)
+			).filter((err?: FailedValidationException) => err);
 			errors.push(...nestedErrors);
 		}

 		if (Object.keys(validation)[0] === '_or') {
 			const subValidation = Object.values(validation)[0];
-			const nestedErrors = flatten<FailedValidationException>(subValidation.map((subObj: Record<string, any>) => this.validateJoi(subObj, payloads)));
+			const nestedErrors = flatten<FailedValidationException>(
+				subValidation.map((subObj: Record<string, any>) =>
+					this.validateJoi(subObj, payloads)
+				)
+			);
 			const allErrored = nestedErrors.every((err?: FailedValidationException) => err);

 			if (allErrored) {
@@ -311,7 +340,9 @@ export class AuthorizationService {
 			const { error } = schema.validate(payload, { abortEarly: false });

 			if (error) {
-				errors.push(...error.details.map((details) => new FailedValidationException(details)));
+				errors.push(
+					...error.details.map((details) => new FailedValidationException(details))
+				);
 			}
 		}

--- a/api/src/types/ast.ts
+++ b/api/src/types/ast.ts
@@ -1,24 +1,36 @@
 import { Query } from './query';
 import { Relation } from './relation';

-export type NestedCollectionAST = {
-	type: 'collection';
+export type M2ONode = {
+	type: 'm2o';
 	name: string;
-	children: (NestedCollectionAST | FieldAST)[];
+	children: (NestedCollectionNode | FieldNode)[];
 	query: Query;
 	fieldKey: string;
 	relation: Relation;
 	parentKey: string;
 };

-export type FieldAST = {
+export type O2MNode = {
+	type: 'o2m';
+	name: string;
+	children: (NestedCollectionNode | FieldNode)[];
+	query: Query;
+	fieldKey: string;
+	relation: Relation;
+	parentKey: string;
+};
+
+export type NestedCollectionNode = M2ONode | O2MNode;
+
+export type FieldNode = {
 	type: 'field';
 	name: string;
 };

 export type AST = {
-	type: 'collection';
+	type: 'root';
 	name: string;
-	children: (NestedCollectionAST | FieldAST)[];
+	children: (NestedCollectionNode | FieldNode)[];
 	query: Query;
 };
--- a/api/src/utils/get-ast-from-query.ts
+++ b/api/src/utils/get-ast-from-query.ts
@@ -4,8 +4,8 @@

 import {
 	AST,
-	NestedCollectionAST,
-	FieldAST,
+	FieldNode,
+	NestedCollectionNode,
 	Query,
 	Relation,
 	PermissionsAction,
@@ -49,7 +49,7 @@ export default async function getASTFromQuery(
 			: null;

 	const ast: AST = {
-		type: 'collection',
+		type: 'root',
 		name: collection,
 		query: query,
 		children: [],
@@ -62,7 +62,9 @@ export default async function getASTFromQuery(
 	delete query.fields;
 	delete query.deep;

-	ast.children = (await parseFields(collection, fields, deep)).filter(filterEmptyChildCollections);
+	ast.children = (await parseFields(collection, fields, deep)).filter(
+		filterEmptyChildCollections
+	);

 	return ast;

@@ -122,12 +124,16 @@ export default async function getASTFromQuery(
 		return fields;
 	}

-	async function parseFields(parentCollection: string, fields: string[], deep?: Record<string, Query>) {
+	async function parseFields(
+		parentCollection: string,
+		fields: string[],
+		deep?: Record<string, Query>
+	) {
 		fields = convertWildcards(parentCollection, fields);

 		if (!fields) return [];

-		const children: (NestedCollectionAST | FieldAST)[] = [];
+		const children: (NestedCollectionNode | FieldNode)[] = [];

 		const relationalStructure: Record<string, string[]> = {};

@@ -155,8 +161,10 @@ export default async function getASTFromQuery(

 			if (!relation) continue;

-			const child: NestedCollectionAST = {
-				type: 'collection',
+			const relationType = getRelationType(relatedCollection, relationalField, relation);
+
+			const child: NestedCollectionNode = {
+				type: relationType,
 				name: relatedCollection,
 				fieldKey: relationalField,
 				parentKey: await schemaInspector.primary(parentCollection),
@@ -198,8 +206,22 @@ export default async function getASTFromQuery(
 		}
 	}

-	function filterEmptyChildCollections(childAST: FieldAST | NestedCollectionAST) {
-		if (childAST.type === 'collection' && childAST.children.length === 0) return false;
-		return true;
+	function filterEmptyChildCollections(childNode: FieldNode | NestedCollectionNode) {
+		if (childNode.type === 'field') return true;
+		if (childNode.children.length > 0) return true;
+		return false;
+	}
+
+	function getRelationType(
+		relatedCollection: string,
+		relationalField: string,
+		relation: Relation
+	): 'o2m' | 'm2o' {
+		if (
+			relation.one_collection === relatedCollection &&
+			relation.many_field === relationalField
+		)
+			return 'm2o';
+		return 'o2m';
 	}
 }