Merge branch 'main' into add-modified_on

This commit is contained in:
Rijk van Zanten
2020-10-11 18:53:42 -04:00
committed by GitHub
142 changed files with 31727 additions and 2875 deletions

8021
api/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"name": "directus",
"version": "9.0.0-beta.9",
"version": "9.0.0-beta.10",
"license": "GPL-3.0-only",
"homepage": "https://github.com/directus/next#readme",
"description": "Directus is a real-time API and App dashboard for managing SQL database content.",
@@ -66,7 +66,7 @@
"dependencies": {
"@directus/app": "file:../app",
"@directus/format-title": "^3.2.0",
"@directus/specs": "^9.0.0-beta.8",
"@directus/specs": "file:../packages/spec",
"@slynova/flydrive": "^1.0.2",
"@slynova/flydrive-gcs": "^1.0.2",
"@slynova/flydrive-s3": "^1.0.2",
@@ -127,7 +127,7 @@
"mssql": "^6.2.0",
"mysql": "^2.18.1",
"oracledb": "^5.0.0",
"pg": "^8.3.3",
"pg": "^8.4.0",
"sqlite3": "^5.0.0"
},
"gitHead": "4476da28dbbc2824e680137aa28b2b91b5afabec",

View File

@@ -13,14 +13,20 @@ type RunASTOptions = {
child?: boolean;
};
export default async function runAST(originalAST: AST, options?: RunASTOptions): Promise<null | Item | Item[]> {
export default async function runAST(
originalAST: AST,
options?: RunASTOptions
): Promise<null | Item | Item[]> {
const ast = cloneDeep(originalAST);
const query = options?.query || ast.query;
const knex = options?.knex || database;
// Retrieve the database columns to select in the current AST
const { columnsToSelect, primaryKeyField, nestedCollectionASTs } = await parseCurrentLevel(ast, knex);
const { columnsToSelect, primaryKeyField, nestedCollectionASTs } = await parseCurrentLevel(
ast,
knex
);
// The actual knex query builder instance. This is a promise that resolves with the raw items from the db
const dbQuery = await getDBQuery(knex, ast.name, columnsToSelect, query, primaryKeyField);
@@ -63,7 +69,7 @@ export default async function runAST(originalAST: AST, options?: RunASTOptions):
// and nesting is done, we parse through the output structure, and filter out all non-requested
// fields
if (options?.child !== true) {
items = removeTemporaryFields(items, originalAST);
items = removeTemporaryFields(items, originalAST, primaryKeyField);
}
return items;
@@ -109,7 +115,13 @@ async function parseCurrentLevel(ast: AST, knex: Knex) {
return { columnsToSelect, nestedCollectionASTs, primaryKeyField };
}
async function getDBQuery(knex: Knex, table: string, columns: string[], query: Query, primaryKeyField: string): Promise<QueryBuilder> {
async function getDBQuery(
knex: Knex,
table: string,
columns: string[],
query: Query,
primaryKeyField: string
): Promise<QueryBuilder> {
let dbQuery = knex.select(columns.map((column) => `${table}.${column}`)).from(table);
const queryCopy = clone(query);
@@ -127,7 +139,10 @@ async function getDBQuery(knex: Knex, table: string, columns: string[], query: Q
return dbQuery;
}
function applyParentFilters(nestedCollectionASTs: NestedCollectionAST[], parentItem: Item | Item[]) {
function applyParentFilters(
nestedCollectionASTs: NestedCollectionAST[],
parentItem: Item | Item[]
) {
const parentItems = Array.isArray(parentItem) ? parentItem : [parentItem];
for (const nestedAST of nestedCollectionASTs) {
@@ -139,15 +154,15 @@ function applyParentFilters(nestedCollectionASTs: NestedCollectionAST[], parentI
filter: {
...(nestedAST.query.filter || {}),
[nestedAST.relation.one_primary]: {
_in: uniq(parentItems.map((res) => res[nestedAST.relation.many_field])).filter(
(id) => id
),
}
}
}
_in: uniq(
parentItems.map((res) => res[nestedAST.relation.many_field])
).filter((id) => id),
},
},
};
} else {
const relatedM2OisFetched = !!nestedAST.children.find((child) => {
return child.type === 'field' && child.name === nestedAST.relation.many_field
return child.type === 'field' && child.name === nestedAST.relation.many_field;
});
if (relatedM2OisFetched === false) {
@@ -159,24 +174,33 @@ function applyParentFilters(nestedCollectionASTs: NestedCollectionAST[], parentI
filter: {
...(nestedAST.query.filter || {}),
[nestedAST.relation.many_field]: {
_in: uniq(parentItems.map((res) => res[nestedAST.parentKey])).filter((id) => id),
}
}
}
_in: uniq(parentItems.map((res) => res[nestedAST.parentKey])).filter(
(id) => id
),
},
},
};
}
}
return nestedCollectionASTs;
}
function mergeWithParentItems(nestedItem: Item | Item[], parentItem: Item | Item[], nestedAST: NestedCollectionAST, o2mLimit?: number | null) {
function mergeWithParentItems(
nestedItem: Item | Item[],
parentItem: Item | Item[],
nestedAST: NestedCollectionAST,
o2mLimit?: number | null
) {
const nestedItems = Array.isArray(nestedItem) ? nestedItem : [nestedItem];
const parentItems = clone(Array.isArray(parentItem) ? parentItem : [parentItem]);
if (isM2O(nestedAST)) {
for (const parentItem of parentItems) {
const itemChild = nestedItems.find((nestedItem) => {
return nestedItem[nestedAST.relation.one_primary] === parentItem[nestedAST.fieldKey];
return (
nestedItem[nestedAST.relation.one_primary] === parentItem[nestedAST.fieldKey]
);
});
parentItem[nestedAST.fieldKey] = itemChild || null;
@@ -188,8 +212,10 @@ function mergeWithParentItems(nestedItem: Item | Item[], parentItem: Item | Item
if (Array.isArray(nestedItem[nestedAST.relation.many_field])) return true;
return (
nestedItem[nestedAST.relation.many_field] === parentItem[nestedAST.relation.one_primary] ||
nestedItem[nestedAST.relation.many_field]?.[nestedAST.relation.many_primary] === parentItem[nestedAST.relation.one_primary]
nestedItem[nestedAST.relation.many_field] ===
parentItem[nestedAST.relation.one_primary] ||
nestedItem[nestedAST.relation.many_field]?.[nestedAST.relation.many_primary] ===
parentItem[nestedAST.relation.one_primary]
);
});
@@ -206,21 +232,34 @@ function mergeWithParentItems(nestedItem: Item | Item[], parentItem: Item | Item
return Array.isArray(parentItem) ? parentItems : parentItems[0];
}
function removeTemporaryFields(rawItem: Item | Item[], ast: AST | NestedCollectionAST): Item | Item[] {
function removeTemporaryFields(
rawItem: Item | Item[],
ast: AST | NestedCollectionAST,
primaryKeyField: string
): Item | Item[] {
const rawItems: Item[] = Array.isArray(rawItem) ? rawItem : [rawItem];
const items: Item[] = [];
const fields = ast.children.filter((child) => child.type === 'field').map((child) => child.name);
const nestedCollections = ast.children.filter((child) => child.type === 'collection') as NestedCollectionAST[];
const fields = ast.children
.filter((child) => child.type === 'field')
.map((child) => child.name);
const nestedCollections = ast.children.filter(
(child) => child.type === 'collection'
) as NestedCollectionAST[];
for (const rawItem of rawItems) {
if (rawItem === null) return rawItem;
const item = fields.includes('*') ? rawItem : pick(rawItem, fields);
const item = fields.length > 0 ? pick(rawItem, fields) : rawItem[primaryKeyField];
for (const nestedCollection of nestedCollections) {
if (item[nestedCollection.fieldKey] !== null) {
item[nestedCollection.fieldKey] = removeTemporaryFields(rawItem[nestedCollection.fieldKey], nestedCollection);
item[nestedCollection.fieldKey] = removeTemporaryFields(
rawItem[nestedCollection.fieldKey],
nestedCollection,
nestedCollection.relatedKey
);
}
}

View File

@@ -11,28 +11,42 @@ defaults:
data:
- collection: directus_activity
note: Accountability logs for all events
- collection: directus_collections
icon: list_alt
note: Additional collection configuration and metadata
- collection: directus_fields
icon: input
note: Additional field configuration and metadata
- collection: directus_files
icon: folder
note: Metadata for all managed file assets
- collection: directus_folders
note: Provides virtual directories for files
- collection: directus_permissions
icon: admin_panel_settings
note: Access permissions for each role
- collection: directus_presets
icon: bookmark_border
note: Presets for collection defaults and bookmarks
- collection: directus_relations
icon: merge_type
note: Relationship configuration and metadata
- collection: directus_revisions
note: Data snapshots for all activity
- collection: directus_roles
icon: supervised_user_circle
note: Permission groups for system users
- collection: directus_sessions
note: User session information
- collection: directus_settings
singleton: true
note: Project configuration options
- collection: directus_users
archive_field: status
archive_value: archived
unarchive_value: draft
icon: people_alt
note: System users for the platform
- collection: directus_webhooks
note: Configuration for event-based HTTP requests

View File

@@ -32,6 +32,11 @@ data:
many_primary: id
one_collection: directus_users
one_primary: id
- many_collection: directus_presets
many_field: role
many_primary: id
one_collection: directus_roles
one_primary: id
- many_collection: directus_folders
many_field: parent
many_primary: id

View File

@@ -33,6 +33,7 @@ fields:
special: json
sort: 3
width: full
display: tags
- collection: directus_files
field: location
interface: text-input
@@ -104,4 +105,10 @@ fields:
locked: true
special: date-updated
width: half
display: datetime
display: datetime
- collection: directus_files
field: created_on
display: datetime
- collection: directus_files
field: created_by
display: user

View File

@@ -20,7 +20,7 @@ import logger from '../logger';
import { PayloadService } from './payload';
import { AuthorizationService } from './authorization';
import { pick, clone } from 'lodash';
import { pick, clone, cloneDeep } from 'lodash';
import getDefaultValue from '../utils/get-default-value';
import { InvalidPayloadException } from '../exceptions';
@@ -29,6 +29,7 @@ export class ItemsService implements AbstractService {
knex: Knex;
accountability: Accountability | null;
eventScope: string;
schemaInspector: ReturnType<typeof SchemaInspector>;
constructor(collection: string, options?: AbstractServiceOptions) {
this.collection = collection;
@@ -38,15 +39,16 @@ export class ItemsService implements AbstractService {
? this.collection.substring(9)
: 'items';
this.schemaInspector = SchemaInspector(this.knex);
return this;
}
async create(data: Partial<Item>[]): Promise<PrimaryKey[]>;
async create(data: Partial<Item>): Promise<PrimaryKey>;
async create(data: Partial<Item> | Partial<Item>[]): Promise<PrimaryKey | PrimaryKey[]> {
const schemaInspector = SchemaInspector(this.knex);
const primaryKeyField = await schemaInspector.primary(this.collection);
const columns = await schemaInspector.columns(this.collection);
const primaryKeyField = await this.schemaInspector.primary(this.collection);
const columns = await this.schemaInspector.columns(this.collection);
let payloads = clone(Array.isArray(data) ? data : [data]);
@@ -220,8 +222,7 @@ export class ItemsService implements AbstractService {
action: PermissionsAction = 'read'
): Promise<null | Item | Item[]> {
query = clone(query);
const schemaInspector = SchemaInspector(this.knex);
const primaryKeyField = await schemaInspector.primary(this.collection);
const primaryKeyField = await this.schemaInspector.primary(this.collection);
const keys = Array.isArray(key) ? key : [key];
if (keys.length === 1) {
@@ -263,9 +264,8 @@ export class ItemsService implements AbstractService {
data: Partial<Item> | Partial<Item>[],
key?: PrimaryKey | PrimaryKey[]
): Promise<PrimaryKey | PrimaryKey[]> {
const schemaInspector = SchemaInspector(this.knex);
const primaryKeyField = await schemaInspector.primary(this.collection);
const columns = await schemaInspector.columns(this.collection);
const primaryKeyField = await this.schemaInspector.primary(this.collection);
const columns = await this.schemaInspector.columns(this.collection);
// Updating one or more items to the same payload
if (data && key) {
@@ -421,12 +421,58 @@ export class ItemsService implements AbstractService {
return keys;
}
async updateByQuery(data: Partial<Item>, query: Query): Promise<PrimaryKey[]> {
const primaryKeyField = await this.schemaInspector.primary(this.collection);
const readQuery = cloneDeep(query);
readQuery.fields = [primaryKeyField];
// Not authenticated:
const itemsService = new ItemsService(this.collection);
let itemsToUpdate = await itemsService.readByQuery(readQuery);
itemsToUpdate = Array.isArray(itemsToUpdate) ? itemsToUpdate : [itemsToUpdate];
const keys: PrimaryKey[] = itemsToUpdate.map(
(item: Partial<Item>) => item[primaryKeyField]
);
return await this.update(data, keys);
}
upsert(data: Partial<Item>): Promise<PrimaryKey>;
upsert(data: Partial<Item>[]): Promise<PrimaryKey[]>;
async upsert(data: Partial<Item> | Partial<Item>[]): Promise<PrimaryKey | PrimaryKey[]> {
const primaryKeyField = await this.schemaInspector.primary(this.collection);
const payloads = Array.isArray(data) ? data : [data];
const primaryKeys: PrimaryKey[] = [];
for (const payload of payloads) {
const primaryKey = payload[primaryKeyField];
const exists =
primaryKey &&
!!(await this.knex
.select(primaryKeyField)
.from(this.collection)
.where({ [primaryKeyField]: primaryKey })
.first());
if (exists) {
const keys = await this.update([payload]);
primaryKeys.push(...keys);
} else {
const key = await this.create(payload);
primaryKeys.push(key);
}
}
return Array.isArray(data) ? primaryKeys : primaryKeys[0];
}
delete(key: PrimaryKey): Promise<PrimaryKey>;
delete(keys: PrimaryKey[]): Promise<PrimaryKey[]>;
async delete(key: PrimaryKey | PrimaryKey[]): Promise<PrimaryKey | PrimaryKey[]> {
const keys = (Array.isArray(key) ? key : [key]) as PrimaryKey[];
const schemaInspector = SchemaInspector(this.knex);
const primaryKeyField = await schemaInspector.primary(this.collection);
const primaryKeyField = await this.schemaInspector.primary(this.collection);
if (this.accountability && this.accountability.admin !== true) {
const authorizationService = new AuthorizationService({
@@ -480,15 +526,31 @@ export class ItemsService implements AbstractService {
return key;
}
async deleteByQuery(query: Query): Promise<PrimaryKey[]> {
const primaryKeyField = await this.schemaInspector.primary(this.collection);
const readQuery = cloneDeep(query);
readQuery.fields = [primaryKeyField];
// Not authenticated:
const itemsService = new ItemsService(this.collection);
let itemsToDelete = await itemsService.readByQuery(readQuery);
itemsToDelete = Array.isArray(itemsToDelete) ? itemsToDelete : [itemsToDelete];
const keys: PrimaryKey[] = itemsToDelete.map(
(item: Partial<Item>) => item[primaryKeyField]
);
return await this.delete(keys);
}
async readSingleton(query: Query) {
query = clone(query);
const schemaInspector = SchemaInspector(this.knex);
query.single = true;
const record = (await this.readByQuery(query)) as Item;
if (!record) {
const columns = await schemaInspector.columnInfo(this.collection);
const columns = await this.schemaInspector.columnInfo(this.collection);
const defaults: Record<string, any> = {};
for (const column of columns) {
@@ -502,8 +564,7 @@ export class ItemsService implements AbstractService {
}
async upsertSingleton(data: Partial<Item>) {
const schemaInspector = SchemaInspector(this.knex);
const primaryKeyField = await schemaInspector.primary(this.collection);
const primaryKeyField = await this.schemaInspector.primary(this.collection);
const record = await this.knex
.select(primaryKeyField)

View File

@@ -6,7 +6,7 @@
import argon2 from 'argon2';
import { v4 as uuidv4 } from 'uuid';
import database from '../database';
import { clone, isObject } from 'lodash';
import { clone, isObject, cloneDeep } from 'lodash';
import { Relation, Item, AbstractServiceOptions, Accountability, PrimaryKey } from '../types';
import { ItemsService } from './items';
import { URL } from 'url';
@@ -15,6 +15,7 @@ import env from '../env';
import SchemaInspector from 'knex-schema-inspector';
import getLocalType from '../utils/get-local-type';
import { format, formatISO } from 'date-fns';
import { ForbiddenException } from '../exceptions';
type Action = 'create' | 'read' | 'update';
@@ -313,11 +314,7 @@ export class PayloadService {
const exists = hasPrimaryKey && !!(await itemsService.readByKey(relatedPrimaryKey));
if (exists) {
if (relatedRecord.hasOwnProperty('$delete') && relatedRecord.$delete) {
await itemsService.delete(relatedPrimaryKey);
} else {
await itemsService.update(relatedRecord, relatedPrimaryKey);
}
await itemsService.update(relatedRecord, relatedPrimaryKey);
} else {
relatedPrimaryKey = await itemsService.create(relatedRecord);
}
@@ -353,48 +350,52 @@ export class PayloadService {
});
for (const relation of relationsToProcess) {
const relatedRecords: Partial<Item>[] = payload[relation.one_field].map(
(record: string | number | Partial<Item>) => {
if (typeof record === 'string' || typeof record === 'number') {
record = {
[relation.many_primary]: record,
};
}
return {
...record,
[relation.many_field]: parent || payload[relation.one_primary],
};
}
);
const itemsService = new ItemsService(relation.many_collection, {
accountability: this.accountability,
knex: this.knex,
});
const toBeCreated = relatedRecords.filter(
(record) => record.hasOwnProperty(relation.many_primary) === false
const relatedRecords: Partial<Item>[] = [];
for (const relatedRecord of payload[relation.one_field]) {
let record = cloneDeep(relatedRecord);
if (typeof relatedRecord === 'string' || typeof relatedRecord === 'number') {
const exists = !!(await this.knex
.select(relation.many_primary)
.from(relation.many_collection)
.where({ [relation.many_primary]: record })
.first());
if (exists === false)
throw new ForbiddenException(undefined, {
item: record,
collection: relation.many_collection,
});
record = {
[relation.many_primary]: relatedRecord,
};
}
relatedRecords.push({
...record,
[relation.many_field]: parent || payload[relation.one_primary],
});
}
const primaryKeys = await itemsService.upsert(relatedRecords);
await itemsService.updateByQuery(
{ [relation.many_field]: null },
{
filter: {
[relation.many_primary]: {
_nin: primaryKeys,
},
},
}
);
const toBeUpdated = relatedRecords.filter(
(record) =>
record.hasOwnProperty(relation.many_primary) === true &&
record.hasOwnProperty('$delete') === false
);
const toBeDeleted = relatedRecords
.filter(
(record) =>
record.hasOwnProperty(relation.many_primary) === true &&
record.hasOwnProperty('$delete') &&
record.$delete === true
)
.map((record) => record[relation.many_primary]);
await itemsService.create(toBeCreated);
await itemsService.update(toBeUpdated);
await itemsService.delete(toBeDeleted);
}
}
}

View File

@@ -9,6 +9,7 @@ export type NestedCollectionAST = {
fieldKey: string;
relation: Relation;
parentKey: string;
relatedKey: string;
};
export type FieldAST = {

View File

@@ -62,16 +62,84 @@ export default async function getASTFromQuery(
delete query.fields;
delete query.deep;
ast.children = (await parseFields(collection, fields, deep)).filter(filterEmptyChildCollections);
ast.children = await parseFields(collection, fields, deep);
return ast;
function convertWildcards(parentCollection: string, fields: string[]) {
async function parseFields(
parentCollection: string,
fields: string[],
deep?: Record<string, Query>
) {
fields = await convertWildcards(parentCollection, fields);
if (!fields) return [];
const children: (NestedCollectionAST | FieldAST)[] = [];
const relationalStructure: Record<string, string[]> = {};
for (const field of fields) {
const isRelational =
field.includes('.') ||
!!relations.find(
(relation) =>
(relation.many_collection === parentCollection &&
relation.many_field === field) ||
(relation.one_collection === parentCollection &&
relation.one_field === field)
);
if (isRelational) {
// field is relational
const parts = field.split('.');
if (relationalStructure.hasOwnProperty(parts[0]) === false) {
relationalStructure[parts[0]] = [];
}
if (parts.length > 1) {
relationalStructure[parts[0]].push(parts.slice(1).join('.'));
}
} else {
children.push({ type: 'field', name: field });
}
}
for (const [relationalField, nestedFields] of Object.entries(relationalStructure)) {
const relatedCollection = getRelatedCollection(parentCollection, relationalField);
if (!relatedCollection) continue;
const relation = getRelation(parentCollection, relationalField);
if (!relation) continue;
const child: NestedCollectionAST = {
type: 'collection',
name: relatedCollection,
fieldKey: relationalField,
parentKey: await schemaInspector.primary(parentCollection),
relatedKey: await schemaInspector.primary(relatedCollection),
relation: relation,
query: deep?.[relationalField] || {},
children: await parseFields(relatedCollection, nestedFields),
};
children.push(child);
}
return children;
}
async function convertWildcards(parentCollection: string, fields: string[]) {
const fieldsInCollection = await getFieldsInCollection(parentCollection);
const allowedFields = permissions
? permissions
.find((permission) => parentCollection === permission.collection)
?.fields?.split(',')
: ['*'];
: fieldsInCollection;
if (!allowedFields || allowedFields.length === 0) return [];
@@ -81,8 +149,13 @@ export default async function getASTFromQuery(
if (fieldKey.includes('*') === false) continue;
if (fieldKey === '*') {
if (allowedFields.includes('*')) continue;
fields.splice(index, 1, ...allowedFields);
// Set to all fields in collection
if (allowedFields.includes('*')) {
fields.splice(index, 1, ...fieldsInCollection);
} else {
// Set to all allowed fields
fields.splice(index, 1, ...allowedFields);
}
}
// Swap *.* case for *,<relational-field>.*,<another-relational>.*
@@ -122,57 +195,6 @@ export default async function getASTFromQuery(
return fields;
}
async function parseFields(parentCollection: string, fields: string[], deep?: Record<string, Query>) {
fields = convertWildcards(parentCollection, fields);
if (!fields) return [];
const children: (NestedCollectionAST | FieldAST)[] = [];
const relationalStructure: Record<string, string[]> = {};
for (const field of fields) {
if (field.includes('.') === false) {
children.push({ type: 'field', name: field });
} else {
// field is relational
const parts = field.split('.');
if (relationalStructure.hasOwnProperty(parts[0]) === false) {
relationalStructure[parts[0]] = [];
}
relationalStructure[parts[0]].push(parts.slice(1).join('.'));
}
}
for (const [relationalField, nestedFields] of Object.entries(relationalStructure)) {
const relatedCollection = getRelatedCollection(parentCollection, relationalField);
if (!relatedCollection) continue;
const relation = getRelation(parentCollection, relationalField);
if (!relation) continue;
const child: NestedCollectionAST = {
type: 'collection',
name: relatedCollection,
fieldKey: relationalField,
parentKey: await schemaInspector.primary(parentCollection),
relation: relation,
query: deep?.[relationalField] || {},
children: (await parseFields(relatedCollection, nestedFields)).filter(
filterEmptyChildCollections
),
};
children.push(child);
}
return children;
}
function getRelation(collection: string, field: string) {
const relation = relations.find((relation) => {
return (
@@ -198,8 +220,19 @@ export default async function getASTFromQuery(
}
}
function filterEmptyChildCollections(childAST: FieldAST | NestedCollectionAST) {
if (childAST.type === 'collection' && childAST.children.length === 0) return false;
return true;
async function getFieldsInCollection(collection: string) {
const columns = (await schemaInspector.columns(collection)).map((column) => column.column);
const fields = (
await database.select('field').from('directus_fields').where({ collection })
).map((field) => field.field);
const fieldsInCollection = [
...columns,
...fields.filter((field) => {
return columns.includes(field) === false;
}),
];
return fieldsInCollection;
}
}