From 676d78f6ee279aba7f605daebfe7f535bb43abe8 Mon Sep 17 00:00:00 2001 From: rijkvanzanten Date: Fri, 9 Oct 2020 18:23:34 -0400 Subject: [PATCH] Add m2a data fetching --- api/src/database/run-ast.ts | 265 ++++++++++++++++++++++++------------ 1 file changed, 175 insertions(+), 90 deletions(-) diff --git a/api/src/database/run-ast.ts b/api/src/database/run-ast.ts index bdd041ac81..b0ad1958d8 100644 --- a/api/src/database/run-ast.ts +++ b/api/src/database/run-ast.ts @@ -1,4 +1,4 @@ -import { AST, NestedCollectionNode, FieldNode } from '../types/ast'; +import { AST, NestedCollectionNode, FieldNode, M2ONode, O2MNode } from '../types/ast'; import { clone, cloneDeep, uniq, pick } from 'lodash'; import database from './index'; import SchemaInspector from 'knex-schema-inspector'; @@ -19,83 +19,101 @@ export default async function runAST( ): Promise { const ast = cloneDeep(originalAST); - // @ts-ignore - console.log((ast as AST).children[3]); - - if (ast.type === 'm2a') return null; - - const query = options?.query || ast.query; const knex = options?.knex || database; - // Retrieve the database columns to select in the current AST - const { columnsToSelect, primaryKeyField, nestedCollectionNodes } = await parseCurrentLevel( - ast, - knex - ); + if (ast.type === 'm2a') { + const results: { [collection: string]: null | Item | Item[] } = {}; - // The actual knex query builder instance. This is a promise that resolves with the raw items from the db - const dbQuery = await getDBQuery(knex, ast.name, columnsToSelect, query, primaryKeyField); - - const rawItems: Item | Item[] = await dbQuery; - - if (!rawItems) return null; - - // Run the items through the special transforms - const payloadService = new PayloadService(ast.name, { knex }); - let items = await payloadService.processValues('read', rawItems); - - if (!items || items.length === 0) return items; - - // Apply the `_in` filters to the nested collection batches - const nestedNodes = applyParentFilters(nestedCollectionNodes, items); - - for (const nestedNode of nestedNodes) { - if (nestedNode.type === 'm2a') continue; - let tempLimit: number | null = null; - - // Nested o2m-items are fetched from the db in a single query. This means that we're fetching - // all nested items for all parent items at once. Because of this, we can't limit that query - // to the "standard" item limit. Instead of _n_ nested items per parent item, it would mean - // that there's _n_ items, which are then divided on the parent items. (no good) - if (nestedNode.type === 'o2m' && typeof nestedNode.query.limit === 'number') { - tempLimit = nestedNode.query.limit; - nestedNode.query.limit = -1; + for (const collection of ast.names) { + results[collection] = await run( + collection, + ast.children[collection], + ast.query[collection] + ); } - let nestedItems = await runAST(nestedNode, { knex, child: true }); + return results; + } else { + return await run(ast.name, ast.children, options?.query || ast.query); + } - if (nestedItems) { - // Merge all fetched nested records with the parent items - items = mergeWithParentItems(nestedItems, items, nestedNode, tempLimit); + async function run( + collection: string, + children: (NestedCollectionNode | FieldNode)[], + query: Query + ) { + // Retrieve the database columns to select in the current AST + const { columnsToSelect, primaryKeyField, nestedCollectionNodes } = await parseCurrentLevel( + collection, + children, + knex + ); + + // The actual knex query builder instance. This is a promise that resolves with the raw items from the db + const dbQuery = await getDBQuery(knex, collection, columnsToSelect, query, primaryKeyField); + + const rawItems: Item | Item[] = await dbQuery; + + if (!rawItems) return null; + + // Run the items through the special transforms + const payloadService = new PayloadService(collection, { knex }); + let items: null | Item | Item[] = await payloadService.processValues('read', rawItems); + + if (!items || items.length === 0) return items; + + // Apply the `_in` filters to the nested collection batches + const nestedNodes = applyParentFilters(nestedCollectionNodes, items); + + for (const nestedNode of nestedNodes) { + let tempLimit: number | null = null; + + // Nested o2m-items are fetched from the db in a single query. This means that we're fetching + // all nested items for all parent items at once. Because of this, we can't limit that query + // to the "standard" item limit. Instead of _n_ nested items per parent item, it would mean + // that there's _n_ items, which are then divided on the parent items. (no good) + if (nestedNode.type === 'o2m' && typeof nestedNode.query.limit === 'number') { + tempLimit = nestedNode.query.limit; + nestedNode.query.limit = -1; + } + + let nestedItems = await runAST(nestedNode, { knex, child: true }); + + if (nestedItems) { + // Merge all fetched nested records with the parent items + items = mergeWithParentItems(nestedItems, items, nestedNode, tempLimit); + } } - } - // During the fetching of data, we have to inject a couple of required fields for the child nesting - // to work (primary / foreign keys) even if they're not explicitly requested. After all fetching - // and nesting is done, we parse through the output structure, and filter out all non-requested - // fields - if (options?.child !== true) { - items = removeTemporaryFields(items, originalAST, primaryKeyField); - } + // During the fetching of data, we have to inject a couple of required fields for the child nesting + // to work (primary / foreign keys) even if they're not explicitly requested. After all fetching + // and nesting is done, we parse through the output structure, and filter out all non-requested + // fields + if (options?.child !== true) { + // items = removeTemporaryFields(items, originalAST); + } - return items; + return items; + } } -async function parseCurrentLevel(ast: AST | NestedCollectionNode, knex: Knex) { +async function parseCurrentLevel( + collection: string, + children: (NestedCollectionNode | FieldNode)[], + knex: Knex +) { const schemaInspector = SchemaInspector(knex); - if (ast.type === 'm2a') - return { columnsToSelect: [], nestedCollectionNodes: [], primaryKeyField: 'id' }; - const primaryKeyField = await schemaInspector.primary(ast.name); + const primaryKeyField = await schemaInspector.primary(collection); - const columnsInCollection = (await schemaInspector.columns(ast.name)).map( + const columnsInCollection = (await schemaInspector.columns(collection)).map( ({ column }) => column ); const columnsToSelect: string[] = []; const nestedCollectionNodes: NestedCollectionNode[] = []; - for (const child of ast.children) { + for (const child of children) { if (child.type === 'field') { if (columnsInCollection.includes(child.name) || child.name === '*') { columnsToSelect.push(child.name); @@ -106,7 +124,7 @@ async function parseCurrentLevel(ast: AST | NestedCollectionNode, knex: Knex) { if (!child.relation) continue; - if (child.type === 'm2o') { + if (child.type === 'm2o' || child.type === 'm2a') { columnsToSelect.push(child.relation.many_field); } @@ -153,7 +171,6 @@ function applyParentFilters( for (const nestedNode of nestedCollectionNodes) { if (!nestedNode.relation) continue; - if (nestedNode.type === 'm2a') continue; if (nestedNode.type === 'm2o') { nestedNode.query = { @@ -167,7 +184,7 @@ function applyParentFilters( }, }, }; - } else { + } else if (nestedNode.type === 'o2m') { const relatedM2OisFetched = !!nestedNode.children.find((child) => { return child.type === 'field' && child.name === nestedNode.relation.many_field; }); @@ -187,6 +204,30 @@ function applyParentFilters( }, }, }; + } else if (nestedNode.type === 'm2a') { + const keysPerCollection: { [collection: string]: (string | number)[] } = {}; + + for (const parentItem of parentItems) { + const collection = parentItem[nestedNode.relation.one_collection_field!]; + if (!keysPerCollection[collection]) keysPerCollection[collection] = []; + keysPerCollection[collection].push(parentItem[nestedNode.relation.many_field]); + } + + for (const relatedCollection of nestedNode.names) { + nestedNode.query[relatedCollection] = { + ...nestedNode.query[relatedCollection], + filter: { + _and: [ + nestedNode.query[relatedCollection].filter, + { + [nestedNode.relatedKey[relatedCollection]]: { + _in: uniq(keysPerCollection[relatedCollection]), + }, + }, + ].filter((f) => f), + }, + }; + } } } @@ -212,7 +253,7 @@ function mergeWithParentItems( parentItem[nestedNode.fieldKey] = itemChild || null; } - } else { + } else if (nestedNode.type === 'o2m') { for (const parentItem of parentItems) { let itemChildren = nestedItems.filter((nestedItem) => { if (nestedItem === null) return false; @@ -235,6 +276,21 @@ function mergeWithParentItems( parentItem[nestedNode.fieldKey] = itemChildren.length > 0 ? itemChildren : null; } + } else if (nestedNode.type === 'm2a') { + for (const parentItem of parentItems) { + const relatedCollection = parentItem[nestedNode.relation.one_collection_field!]; + + const itemChild = (nestedItem as Record)[relatedCollection].find( + (nestedItem) => { + return ( + nestedItem[nestedNode.relatedKey[relatedCollection]] === + parentItem[nestedNode.fieldKey] + ); + } + ); + + parentItem[nestedNode.fieldKey] = itemChild || null; + } } return Array.isArray(parentItem) ? parentItems : parentItems[0]; @@ -242,41 +298,70 @@ function mergeWithParentItems( function removeTemporaryFields( rawItem: Item | Item[], - ast: AST | NestedCollectionNode, - primaryKeyField: string -): Item | Item[] { - const rawItems: Item[] = Array.isArray(rawItem) ? rawItem : [rawItem]; - if (ast.type === 'm2a') return {}; - + ast: AST | NestedCollectionNode +): null | Item | Item[] { + const rawItems = cloneDeep(Array.isArray(rawItem) ? rawItem : [rawItem]); const items: Item[] = []; - const fields = ast.children - .filter((child) => child.type === 'field') - .map((child) => (child as FieldNode).name); /** @TODO */ + if (ast.type === 'm2a') { + } else { + const fields: string[] = []; + const nestedCollectionNodes: NestedCollectionNode[] = []; - const nestedCollections = ast.children.filter( - (child) => child.type !== 'field' - ) as NestedCollectionNode[]; - - for (const rawItem of rawItems) { - if (rawItem === null) return rawItem; - - const item = fields.length > 0 ? pick(rawItem, fields) : rawItem[primaryKeyField]; - - for (const nestedCollection of nestedCollections) { - if (nestedCollection.type === 'm2a') continue; - - if (item[nestedCollection.fieldKey] !== null) { - item[nestedCollection.fieldKey] = removeTemporaryFields( - rawItem[nestedCollection.fieldKey], - nestedCollection, - nestedCollection.relatedKey - ); + for (const child of ast.children) { + if (child.type === 'field') { + fields.push(child.name); + } else { + nestedCollectionNodes.push(child); } } - items.push(item); + for (const rawItem of rawItems) { + const item = fields.length > 0 ? pick(rawItem, fields) : rawItem.id; /** @TODO */ + for (const nestedNode of nestedCollectionNodes) { + rawItem[nestedNode.fieldKey] = removeTemporaryFields( + rawItem[nestedNode.fieldKey], + nestedNode + ); + } + } } return Array.isArray(rawItem) ? items : items[0]; + + // const rawItems: Item[] = Array.isArray(rawItem) ? rawItem : [rawItem]; + // let items: Item[] = []; + + // if (ast.type === 'm2a') { + // return rawItems + // } else { + // const fields = ast.children + // .filter((child) => child.type === 'field') + // .map((child) => (child as FieldNode).name); + + // const nestedNodes = ast.children.filter( + // (child) => child.type !== 'field' + // ) as NestedCollectionNode[]; + + // items = rawItems.map((item) => parseItem(item, fields, nestedNodes)); + + // return Array.isArray(rawItem) ? items : items[0]; + // } + + // function parseItem(rawItem: Item, fields: string[], nestedNodes: NestedCollectionNode[]): Item { + // if (rawItem === null) return rawItem; + + // const item = fields.length > 0 ? pick(rawItem, fields) : rawItem.id; /** @TODO */ + + // for (const nestedNode of nestedNodes) { + // if (item[nestedNode.fieldKey] !== null) { + // item[nestedNode.fieldKey] = removeTemporaryFields( + // rawItem[nestedNode.fieldKey], + // nestedNode, + // ); + // } + // } + + // return item; + // } }