From d302c6c263ad057bf65ef73ba6f56ec269e93918 Mon Sep 17 00:00:00 2001 From: Rijk van Zanten Date: Sat, 14 Aug 2021 20:33:55 +0200 Subject: [PATCH] Add support for aliasing fields (#7419) * Don't double split csv values * Still join them on create tho * Add support for `alias` query param * Support aliases in wildcards --- api/src/database/run-ast.ts | 87 ++++++++++++++++------- api/src/services/payload.ts | 3 +- api/src/types/ast.ts | 1 + api/src/types/query.ts | 1 + api/src/utils/get-ast-from-query.ts | 104 ++++++++++++++++++++-------- api/src/utils/get-column.ts | 12 ++-- api/src/utils/sanitize-query.ts | 18 +++++ api/src/utils/validate-query.ts | 25 +++++++ 8 files changed, 188 insertions(+), 63 deletions(-) diff --git a/api/src/database/run-ast.ts b/api/src/database/run-ast.ts index f08661eeac..d252db026c 100644 --- a/api/src/database/run-ast.ts +++ b/api/src/database/run-ast.ts @@ -2,7 +2,7 @@ import { Knex } from 'knex'; import { clone, cloneDeep, pick, uniq } from 'lodash'; import { PayloadService } from '../services/payload'; import { Item, Query, SchemaOverview } from '../types'; -import { AST, FieldNode, NestedCollectionNode } from '../types/ast'; +import { AST, FieldNode, NestedCollectionNode, M2ONode } from '../types/ast'; import { applyFunctionToColumnName } from '../utils/apply-function-to-column-name'; import applyQuery from '../utils/apply-query'; import { getColumn } from '../utils/get-column'; @@ -60,14 +60,15 @@ export default async function runAST( async function run(collection: string, children: (NestedCollectionNode | FieldNode)[], query: Query) { // Retrieve the database columns to select in the current AST - const { columnsToSelect, primaryKeyField, nestedCollectionNodes } = await parseCurrentLevel( + const { fieldNodes, primaryKeyField, nestedCollectionNodes } = await parseCurrentLevel( schema, collection, - children + children, + query ); // The actual knex query builder instance. This is a promise that resolves with the raw items from the db - const dbQuery = await getDBQuery(schema, knex, collection, columnsToSelect, query, options?.nested); + const dbQuery = await getDBQuery(schema, knex, collection, fieldNodes, query, options?.nested); const rawItems: Item | Item[] = await dbQuery; @@ -106,7 +107,8 @@ export default async function runAST( async function parseCurrentLevel( schema: SchemaOverview, collection: string, - children: (NestedCollectionNode | FieldNode)[] + children: (NestedCollectionNode | FieldNode)[], + query: Query ) { const primaryKeyField = schema.collections[collection].primary; const columnsInCollection = Object.keys(schema.collections[collection].fields); @@ -117,8 +119,17 @@ async function parseCurrentLevel( for (const child of children) { if (child.type === 'field') { const fieldKey = stripFunction(child.name); + if (columnsInCollection.includes(fieldKey) || fieldKey === '*') { columnsToSelectInternal.push(child.name); // maintain original name here (includes functions) + + if (query.alias) { + columnsToSelectInternal.push( + ...Object.entries(query.alias) + .filter(([_key, value]) => value === child.name) + .map(([key]) => key) + ); + } } continue; @@ -127,7 +138,7 @@ async function parseCurrentLevel( if (!child.relation) continue; if (child.type === 'm2o') { - columnsToSelectInternal.push(child.relation.field); + columnsToSelectInternal.push(child.fieldKey); } if (child.type === 'm2a') { @@ -138,30 +149,49 @@ async function parseCurrentLevel( nestedCollectionNodes.push(child); } - /** - * Always fetch primary key in case there's a nested relation that needs it + const isAggregate = (query.aggregate && Object.keys(query.aggregate).length > 0) ?? false; + + /** Always fetch primary key in case there's a nested relation that needs it. Aggregate payloads + * can't have nested relational fields */ - const childrenContainRelational = children.some((child) => child.type !== 'field'); - if (childrenContainRelational && columnsToSelectInternal.includes(primaryKeyField) === false) { + if (isAggregate === false && columnsToSelectInternal.includes(primaryKeyField) === false) { columnsToSelectInternal.push(primaryKeyField); } /** Make sure select list has unique values */ const columnsToSelect = [...new Set(columnsToSelectInternal)]; - return { columnsToSelect, nestedCollectionNodes, primaryKeyField }; + const fieldNodes = columnsToSelect.map( + (column: string) => + children.find((childNode) => childNode.fieldKey === column) ?? { type: 'field', name: column, fieldKey: column } + ) as FieldNode[]; + + return { fieldNodes, nestedCollectionNodes, primaryKeyField }; } function getColumnPreprocessor(knex: Knex, schema: SchemaOverview, table: string) { const helper = getGeometryHelper(); - return function (column: string): Knex.Raw { - const field = schema.collections[table].fields[column]; - if (isNativeGeometry(field)) { - return helper.asText(table, column); + return function (fieldNode: FieldNode | M2ONode): Knex.Raw { + let field; + + if (fieldNode.type === 'field') { + field = schema.collections[table].fields[fieldNode.name]; + } else { + field = schema.collections[fieldNode.relation.collection].fields[fieldNode.relation.field]; } - return getColumn(knex, table, column); + let alias = undefined; + + if (fieldNode.name !== fieldNode.fieldKey) { + alias = fieldNode.fieldKey; + } + + if (isNativeGeometry(field)) { + return helper.asText(table, field.field); + } + + return getColumn(knex, table, field.field, alias); }; } @@ -169,12 +199,12 @@ function getDBQuery( schema: SchemaOverview, knex: Knex, table: string, - columns: string[], + fieldNodes: FieldNode[], query: Query, nested?: boolean ): Knex.QueryBuilder { const preProcess = getColumnPreprocessor(knex, schema, table); - const dbQuery = knex.select(columns.map(preProcess)).from(table); + const dbQuery = knex.select(fieldNodes.map(preProcess)).from(table); const queryCopy = clone(query); queryCopy.limit = typeof queryCopy.limit === 'number' ? queryCopy.limit : 100; @@ -217,11 +247,19 @@ function applyParentFilters( }); if (relatedM2OisFetched === false) { - nestedNode.children.push({ type: 'field', name: nestedNode.relation.field }); + nestedNode.children.push({ + type: 'field', + name: nestedNode.relation.field, + fieldKey: nestedNode.relation.field, + }); } if (nestedNode.relation.meta?.sort_field) { - nestedNode.children.push({ type: 'field', name: nestedNode.relation.meta.sort_field }); + nestedNode.children.push({ + type: 'field', + name: nestedNode.relation.meta.sort_field, + fieldKey: nestedNode.relation.meta.sort_field, + }); } nestedNode.query = { @@ -399,10 +437,9 @@ function removeTemporaryFields( const nestedCollectionNodes: NestedCollectionNode[] = []; for (const child of ast.children) { - if (child.type === 'field') { - fields.push(child.name); - } else { - fields.push(child.fieldKey); + fields.push(child.fieldKey); + + if (child.type !== 'field') { nestedCollectionNodes.push(child); } } @@ -414,7 +451,7 @@ function removeTemporaryFields( if (operation === 'count' && aggregateFields.includes('*')) fields.push('count'); - fields.push(...aggregateFields.map((field) => `${field}_${operation}`)); + fields.push(...aggregateFields.map((field) => `${operation}.${field}`)); } } diff --git a/api/src/services/payload.ts b/api/src/services/payload.ts index 7a5bb20222..1250805ed1 100644 --- a/api/src/services/payload.ts +++ b/api/src/services/payload.ts @@ -113,8 +113,7 @@ export class PayloadService { }, async csv({ action, value }) { if (!value) return; - if (action === 'read') return value.split(','); - + if (action === 'read' && Array.isArray(value) === false) return value.split(','); if (Array.isArray(value)) return value.join(','); return value; }, diff --git a/api/src/types/ast.ts b/api/src/types/ast.ts index 3ab3b302a5..75b32e44dc 100644 --- a/api/src/types/ast.ts +++ b/api/src/types/ast.ts @@ -45,6 +45,7 @@ export type NestedCollectionNode = M2ONode | O2MNode | M2ANode; export type FieldNode = { type: 'field'; name: string; + fieldKey: string; }; export type AST = { diff --git a/api/src/types/query.ts b/api/src/types/query.ts index b7e5e98d34..b046afd5c7 100644 --- a/api/src/types/query.ts +++ b/api/src/types/query.ts @@ -13,6 +13,7 @@ export type Query = { group?: string[]; aggregate?: Aggregate; deep?: Record; + alias?: Record; }; export type Sort = { diff --git a/api/src/utils/get-ast-from-query.ts b/api/src/utils/get-ast-from-query.ts index 27e689bdaf..91be1e5d3f 100644 --- a/api/src/utils/get-ast-from-query.ts +++ b/api/src/utils/get-ast-from-query.ts @@ -3,7 +3,7 @@ */ import { Knex } from 'knex'; -import { cloneDeep, mapKeys, omitBy } from 'lodash'; +import { cloneDeep, mapKeys, omitBy, uniq } from 'lodash'; import { Accountability } from '@directus/shared/types'; import { AST, FieldNode, NestedCollectionNode, PermissionsAction, Query, SchemaOverview } from '../types'; import { getRelationType } from '../utils/get-relation-type'; @@ -65,6 +65,8 @@ export default async function getASTFromQuery( fields = query.group; } + fields = uniq(fields); + const deep = query.deep || {}; // Prevent fields/deep from showing up in the query object in further use @@ -109,34 +111,42 @@ export default async function getASTFromQuery( const relationalStructure: Record = {}; - for (const field of fields) { + for (const fieldKey of fields) { + let name = fieldKey; + + const isAlias = (query.alias && name in query.alias) ?? false; + + if (isAlias) { + name = query.alias![fieldKey]; + } + const isRelational = - field.includes('.') || + name.includes('.') || // We'll always treat top level o2m fields as a related item. This is an alias field, otherwise it won't return // anything !!schema.relations.find( - (relation) => relation.related_collection === parentCollection && relation.meta?.one_field === field + (relation) => relation.related_collection === parentCollection && relation.meta?.one_field === name ); if (isRelational) { // field is relational - const parts = field.split('.'); + const parts = name.split('.'); - let fieldKey = parts[0]; + let rootField = parts[0]; let collectionScope: string | null = null; // m2a related collection scoped field selector `fields=sections.section_id:headings.title` - if (fieldKey.includes(':')) { - const [key, scope] = fieldKey.split(':'); - fieldKey = key; + if (rootField.includes(':')) { + const [key, scope] = rootField.split(':'); + rootField = key; collectionScope = scope; } - if (fieldKey in relationalStructure === false) { + if (rootField in relationalStructure === false) { if (collectionScope) { - relationalStructure[fieldKey] = { [collectionScope]: [] }; + relationalStructure[rootField] = { [collectionScope]: [] }; } else { - relationalStructure[fieldKey] = []; + relationalStructure[rootField] = []; } } @@ -144,30 +154,36 @@ export default async function getASTFromQuery( const childKey = parts.slice(1).join('.'); if (collectionScope) { - if (collectionScope in relationalStructure[fieldKey] === false) { - (relationalStructure[fieldKey] as anyNested)[collectionScope] = []; + if (collectionScope in relationalStructure[rootField] === false) { + (relationalStructure[rootField] as anyNested)[collectionScope] = []; } - (relationalStructure[fieldKey] as anyNested)[collectionScope].push(childKey); + (relationalStructure[rootField] as anyNested)[collectionScope].push(childKey); } else { - (relationalStructure[fieldKey] as string[]).push(childKey); + (relationalStructure[rootField] as string[]).push(childKey); } } } else { - children.push({ type: 'field', name: field }); + children.push({ type: 'field', name, fieldKey }); } } - for (const [relationalField, nestedFields] of Object.entries(relationalStructure)) { - const relatedCollection = getRelatedCollection(parentCollection, relationalField); - const relation = getRelation(parentCollection, relationalField); + for (const [fieldKey, nestedFields] of Object.entries(relationalStructure)) { + let fieldName = fieldKey; + + if (query.alias && fieldKey in query.alias) { + fieldName = query.alias[fieldKey]; + } + + const relatedCollection = getRelatedCollection(parentCollection, fieldName); + const relation = getRelation(parentCollection, fieldName); if (!relation) continue; const relationType = getRelationType({ relation, collection: parentCollection, - field: relationalField, + field: fieldName, }); if (!relationType) continue; @@ -187,7 +203,7 @@ export default async function getASTFromQuery( query: {}, relatedKey: {}, parentKey: schema.collections[parentCollection].primary, - fieldKey: relationalField, + fieldKey: fieldKey, relation: relation, }; @@ -195,10 +211,10 @@ export default async function getASTFromQuery( child.children[relatedCollection] = await parseFields( relatedCollection, Array.isArray(nestedFields) ? nestedFields : (nestedFields as anyNested)[relatedCollection] || ['*'], - deep?.[`${relationalField}:${relatedCollection}`] + deep?.[`${fieldKey}:${relatedCollection}`] ); - child.query[relatedCollection] = getDeepQuery(deep?.[`${relationalField}:${relatedCollection}`] || {}); + child.query[relatedCollection] = getDeepQuery(deep?.[`${fieldKey}:${relatedCollection}`] || {}); child.relatedKey[relatedCollection] = schema.collections[relatedCollection].primary; } @@ -210,12 +226,12 @@ export default async function getASTFromQuery( child = { type: relationType, name: relatedCollection, - fieldKey: relationalField, + fieldKey: fieldKey, parentKey: schema.collections[parentCollection].primary, relatedKey: schema.collections[relatedCollection].primary, relation: relation, - query: getDeepQuery(deep?.[relationalField] || {}), - children: await parseFields(relatedCollection, nestedFields as string[], deep?.[relationalField] || {}), + query: getDeepQuery(deep?.[fieldKey] || {}), + children: await parseFields(relatedCollection, nestedFields as string[], deep?.[fieldKey] || {}), }; if (relationType === 'o2m' && !child!.query.sort) { @@ -230,7 +246,18 @@ export default async function getASTFromQuery( } } - return children; + // Deduplicate any children fields that are included both as a regular field, and as a nested m2o field + const nestedCollectionNodes = children.filter((childNode) => childNode.type !== 'field'); + + return children.filter((childNode) => { + const existsAsNestedRelational = !!nestedCollectionNodes.find( + (nestedCollectionNode) => childNode.fieldKey === nestedCollectionNode.fieldKey + ); + + if (childNode.type === 'field' && existsAsNestedRelational) return false; + + return true; + }); } async function convertWildcards(parentCollection: string, fields: string[]) { @@ -256,12 +283,18 @@ export default async function getASTFromQuery( if (fieldKey.includes('*') === false) continue; if (fieldKey === '*') { + const aliases = Object.keys(query.alias ?? {}); // Set to all fields in collection if (allowedFields.includes('*')) { - fields.splice(index, 1, ...fieldsInCollection); + fields.splice(index, 1, ...fieldsInCollection, ...aliases); } else { // Set to all allowed fields - fields.splice(index, 1, ...allowedFields); + const allowedAliases = aliases.filter((fieldKey) => { + const name = query.alias![fieldKey]; + return allowedFields!.includes(name); + }); + + fields.splice(index, 1, ...allowedFields, ...allowedAliases); } } @@ -283,6 +316,16 @@ export default async function getASTFromQuery( const nonRelationalFields = allowedFields.filter((fieldKey) => relationalFields.includes(fieldKey) === false); + const aliasFields = Object.keys(query.alias ?? {}).map((fieldKey) => { + const name = query.alias![fieldKey]; + + if (relationalFields.includes(name)) { + return `${fieldKey}.${parts.slice(1).join('.')}`; + } + + return fieldKey; + }); + fields.splice( index, 1, @@ -291,6 +334,7 @@ export default async function getASTFromQuery( return `${relationalField}.${parts.slice(1).join('.')}`; }), ...nonRelationalFields, + ...aliasFields, ] ); } diff --git a/api/src/utils/get-column.ts b/api/src/utils/get-column.ts index 6ca51558ab..43ced767d7 100644 --- a/api/src/utils/get-column.ts +++ b/api/src/utils/get-column.ts @@ -28,15 +28,15 @@ export function getColumn( if (functionName in fn) { const result = fn[functionName as keyof typeof fn](table, columnName); - if (alias) { - return knex.raw(result + ' AS ??', [alias]); - } - - return result; + return knex.raw(result + ' AS ??', [alias]); } else { throw new Error(`Invalid function specified "${functionName}"`); } } - return knex.raw('??.??', [table, column]); + if (column !== alias) { + return knex.ref(`${table}.${column}`).as(alias); + } + + return knex.ref(`${table}.${column}`); } diff --git a/api/src/utils/sanitize-query.ts b/api/src/utils/sanitize-query.ts index 3e6d7ebc7c..14d4100780 100644 --- a/api/src/utils/sanitize-query.ts +++ b/api/src/utils/sanitize-query.ts @@ -61,6 +61,10 @@ export function sanitizeQuery(rawQuery: Record, accountability?: Ac query.deep = sanitizeDeep(rawQuery.deep, accountability); } + if (rawQuery.alias) { + query.alias = sanitizeAlias(rawQuery.alias); + } + return query; } @@ -202,3 +206,17 @@ function sanitizeDeep(deep: Record, accountability?: Accountability } } } + +function sanitizeAlias(rawAlias: any) { + let alias: Record = rawAlias; + + if (typeof rawAlias === 'string') { + try { + alias = JSON.parse(rawAlias); + } catch (err) { + logger.warn('Invalid value passed for alias query parameter.'); + } + } + + return alias; +} diff --git a/api/src/utils/validate-query.ts b/api/src/utils/validate-query.ts index ba572388bb..5e33b456be 100644 --- a/api/src/utils/validate-query.ts +++ b/api/src/utils/validate-query.ts @@ -22,6 +22,7 @@ const querySchema = Joi.object({ export: Joi.string().valid('json', 'csv', 'xml'), aggregate: Joi.object(), deep: Joi.object(), + alias: Joi.object(), }).id('query'); export function validateQuery(query: Query): Query { @@ -31,6 +32,10 @@ export function validateQuery(query: Query): Query { validateFilter(query.filter); } + if (query.alias) { + validateAlias(query.alias); + } + if (error) { throw new InvalidQueryException(error.message); } @@ -141,3 +146,23 @@ function validateGeometry(value: any, key: string) { return true; } + +function validateAlias(alias: any) { + if (isPlainObject(alias) === false) { + throw new InvalidQueryException(`"alias" has to be an object`); + } + + for (const [key, value] of Object.entries(alias)) { + if (typeof key !== 'string') { + throw new InvalidQueryException(`"alias" key has to be a string. "${typeof key}" given.`); + } + + if (typeof value !== 'string') { + throw new InvalidQueryException(`"alias" value has to be a string. "${typeof key}" given.`); + } + + if (key.includes('.') || value.includes('.')) { + throw new InvalidQueryException(`"alias" key/value can't contain a period character \`.\``); + } + } +}