Merge pull request #48 from directus/collections

Collections & Fields

.gitignore

@@ -1,3 +1,5 @@
+.DS_Store
+
 # Logs
 logs
 *.log

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "src/knex-schema-inspector"]
+	path = src/knex-schema-inspector
+	url = git@github.com:knex/knex-schema-inspector.git

package-lock.json

@@ -3668,10 +3668,6 @@
         }
       }
     },
-    "knex-schema-inspector": {
-      "version": "github:knex/knex-schema-inspector#b01d5ff067e0f49b9a6b48e830f016a0bf10d315",
-      "from": "github:knex/knex-schema-inspector"
-    },
     "levn": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",

package.json

@@ -84,7 +84,6 @@
     "icc": "^2.0.0",
     "jsonwebtoken": "^8.5.1",
     "knex": "^0.21.1",
-    "knex-schema-inspector": "github:knex/knex-schema-inspector",
     "liquidjs": "^9.12.0",
     "lodash": "^4.17.15",
     "mssql": "^6.2.0",

src/app.ts

@@ -13,8 +13,10 @@ import authenticate from './middleware/authenticate';
 import activityRouter from './routes/activity';
 import assetsRouter from './routes/assets';
 import authRouter from './routes/auth';
+import collectionsRouter from './routes/collections';
 import collectionPresetsRouter from './routes/collection-presets';
 import extensionsRouter from './routes/extensions';
+import fieldsRouter from './routes/fields';
 import filesRouter from './routes/files';
 import foldersRouter from './routes/folders';
 import itemsRouter from './routes/items';
@@ -39,8 +41,10 @@ const app = express()
 	.use('/activity', activityRouter)
 	.use('/assets', assetsRouter)
 	.use('/auth', authRouter)
+	.use('/collections', collectionsRouter)
 	.use('/collection_presets', collectionPresetsRouter)
 	.use('/extensions', extensionsRouter)
+	.use('/fields', fieldsRouter)
 	.use('/files', filesRouter)
 	.use('/folders', foldersRouter)
 	.use('/items', itemsRouter)

src/constants.ts

@@ -1,4 +1,5 @@
 import { Transformation } from './types/assets';
+import { Collection } from './types/collection';
 
 export const SYSTEM_ASSET_WHITELIST: Transformation[] = [
 	{

src/database.ts

@@ -1,6 +1,8 @@
 import knex from 'knex';
 import logger from './logger';
 
+import SchemaInspector from './knex-schema-inspector/lib/index';
+
 const log = logger.child({ module: 'sql' });
 
 const database = knex({
@@ -16,4 +18,6 @@ const database = knex({
 
 database.on('query', (data) => log.trace(data.sql));
 
+export const schemaInspector = SchemaInspector(database);
+
 export default database;

src/exceptions/collection-not-found.ts

@@ -0,0 +1,7 @@
+import { BaseException } from './base';
+
+export class CollectionNotFoundException extends BaseException {
+	constructor(collection: string) {
+		super(`Collection "${collection}" doesn't exist.`, 404, 'COLLECTION_NOT_FOUND');
+	}
+}

src/exceptions/field-not-found.ts

@@ -0,0 +1,11 @@
+import { BaseException } from './base';
+
+export class FieldNotFoundException extends BaseException {
+	constructor(collection: string, field: string) {
+		super(
+			`Field "${field}" in collection "${collection}" doesn't exist.`,
+			404,
+			'FIELD_NOT_FOUND'
+		);
+	}
+}

src/exceptions/index.ts

@@ -1,7 +1,9 @@
 export * from './base';
+export * from './collection-not-found';
+export * from './field-not-found';
 export * from './invalid-credentials';
 export * from './invalid-payload';
 export * from './invalid-query';
 export * from './item-limit';
-export * from './route-not-found';
 export * from './item-not-found';
+export * from './route-not-found';

src/middleware/sanitize-query.ts

@@ -44,7 +44,7 @@ const sanitizeQuery: RequestHandler = (req, res, next) => {
 		query.meta = sanitizeMeta(req.query.meta);
 	}
 
-	res.locals.query = query;
+	req.sanitizedQuery = query;
 	return next();
 };
 

src/middleware/validate-collection.ts

@@ -5,7 +5,7 @@
 import { RequestHandler } from 'express';
 import asyncHandler from 'express-async-handler';
 import database from '../database';
-import { RouteNotFoundException } from '../exceptions';
+import { CollectionNotFoundException } from '../exceptions';
 
 const validateCollection: RequestHandler = asyncHandler(async (req, res, next) => {
 	if (!req.params.collection) return next();
@@ -17,7 +17,7 @@ const validateCollection: RequestHandler = asyncHandler(async (req, res, next) =
 		return next();
 	}
 
-	throw new RouteNotFoundException(req.path);
+	throw new CollectionNotFoundException(req.params.collection);
 });
 
 export default validateCollection;

src/middleware/validate-query.ts

@@ -13,9 +13,9 @@ import { InvalidQueryException } from '../exceptions';
 
 const validateQuery: RequestHandler = asyncHandler(async (req, res, next) => {
 	if (!req.collection) return next();
-	if (!req.query) return next();
+	if (!req.sanitizedQuery) return next();
 
-	const query: Query = req.query;
+	const query: Query = req.sanitizedQuery;
 
 	await Promise.all([
 		validateParams(req.params.collection, query),

src/routes/collections.ts

@@ -0,0 +1,73 @@
+import { Router } from 'express';
+import asyncHandler from 'express-async-handler';
+import sanitizeQuery from '../middleware/sanitize-query';
+import validateQuery from '../middleware/validate-query';
+import * as CollectionsService from '../services/collections';
+import database, { schemaInspector } from '../database';
+import { InvalidPayloadException, CollectionNotFoundException } from '../exceptions';
+import Joi from '@hapi/joi';
+
+const router = Router();
+
+const fieldSchema = Joi.object({
+	field: Joi.string().required(),
+	datatype: Joi.string().required(),
+	note: Joi.string().required(),
+	primary_key: Joi.boolean(),
+	auto_increment: Joi.boolean(),
+});
+
+const collectionSchema = Joi.object({
+	collection: Joi.string().required(),
+	fields: Joi.array().items(fieldSchema).min(1).unique().required(),
+	note: Joi.string(),
+});
+
+router.post(
+	'/',
+	asyncHandler(async (req, res) => {
+		const { error } = collectionSchema.validate(req.body);
+		if (error) throw new InvalidPayloadException(error.message);
+
+		const createdCollection = await CollectionsService.create(req.body);
+		res.json({ data: createdCollection });
+	})
+);
+
+router.get(
+	'/',
+	sanitizeQuery,
+	validateQuery,
+	asyncHandler(async (req, res) => {
+		const data = await CollectionsService.readAll(req.sanitizedQuery);
+		res.json({ data });
+	})
+);
+
+router.get(
+	'/:collection',
+	sanitizeQuery,
+	validateQuery,
+	asyncHandler(async (req, res) => {
+		const exists = await schemaInspector.hasTable(req.params.collection);
+
+		if (exists === false) throw new CollectionNotFoundException(req.params.collection);
+
+		const data = await CollectionsService.readOne(req.params.collection, req.sanitizedQuery);
+		res.json({ data });
+	})
+);
+
+router.delete(
+	'/:collection',
+	asyncHandler(async (req, res) => {
+		if ((await schemaInspector.hasTable(req.params.collection)) === false) {
+			throw new CollectionNotFoundException(req.params.collection);
+		}
+
+		await CollectionsService.deleteCollection(req.params.collection);
+		res.end();
+	})
+);
+
+export default router;

src/routes/fields.ts

@@ -0,0 +1,70 @@
+import { Router } from 'express';
+import asyncHandler from 'express-async-handler';
+import * as FieldsService from '../services/fields';
+import validateCollection from '../middleware/validate-collection';
+import { schemaInspector } from '../database';
+import { FieldNotFoundException, InvalidPayloadException } from '../exceptions';
+import Joi from '@hapi/joi';
+import { Field } from '../types/field';
+
+const router = Router();
+
+router.get(
+	'/',
+	asyncHandler(async (req, res) => {
+		const fields = await FieldsService.readAll();
+		return res.json({ data: fields });
+	})
+);
+
+router.get(
+	'/:collection',
+	validateCollection,
+	asyncHandler(async (req, res) => {
+		const fields = await FieldsService.readAll(req.collection);
+		return res.json({ data: fields });
+	})
+);
+
+router.get(
+	'/:collection/:field',
+	validateCollection,
+	asyncHandler(async (req, res) => {
+		const exists = await schemaInspector.hasColumn(req.collection, req.params.field);
+		if (exists === false) throw new FieldNotFoundException(req.collection, req.params.field);
+
+		const field = await FieldsService.readOne(req.collection, req.params.field);
+		return res.json({ data: field });
+	})
+);
+
+const newFieldSchema = Joi.object({
+	field: Joi.string().required(),
+	database: Joi.object({
+		type: Joi.string().required(),
+	}).required(),
+	system: Joi.object({
+		hidden_browse: Joi.boolean(),
+		/** @todo extract this dynamically from the DB schema */
+	}),
+});
+
+router.post(
+	'/:collection',
+	validateCollection,
+	asyncHandler(async (req, res) => {
+		const { error } = newFieldSchema.validate(req.body);
+
+		if (error) {
+			throw new InvalidPayloadException(error.message);
+		}
+
+		const field: Partial<Field> = req.body;
+
+		const createdField = await FieldsService.createField(req.collection, field);
+
+		res.json({ data: createdField });
+	})
+);
+
+export default router;

src/routes/users.ts

@@ -37,15 +37,6 @@ router.get(
 	asyncHandler(async (req, res) => {
 		const item = await UsersService.readUsers(res.locals.query);
 
-		ActivityService.createActivity({
-			action: ActivityService.Action.UPDATE,
-			collection: req.collection,
-			item: item.id,
-			ip: req.ip,
-			user_agent: req.get('user-agent'),
-			action_by: req.user,
-		});
-
 		return res.json({ data: item });
 	})
 );
@@ -56,8 +47,8 @@ router.get(
 	sanitizeQuery,
 	validateQuery,
 	asyncHandler(async (req, res) => {
-		const record = await UsersService.readUser(req.params.pk, res.locals.query);
-		return res.json({ data: record });
+		const items = await UsersService.readUser(req.params.pk, res.locals.query);
+		return res.json({ data: items });
 	})
 );
 
@@ -65,8 +56,18 @@ router.patch(
 	'/:pk',
 	useCollection('directus_users'),
 	asyncHandler(async (req, res) => {
-		const records = await UsersService.updateUser(req.params.pk, req.body, res.locals.query);
-		return res.json({ data: records });
+		const item = await UsersService.updateUser(req.params.pk, req.body, res.locals.query);
+
+		ActivityService.createActivity({
+			action: ActivityService.Action.UPDATE,
+			collection: req.collection,
+			item: item.id,
+			ip: req.ip,
+			user_agent: req.get('user-agent'),
+			action_by: req.user,
+		});
+
+		return res.json({ data: item });
 	})
 );
 

src/routes/webhooks.ts

@@ -12,7 +12,7 @@ router.post(
 	'/',
 	useCollection('directus_webhooks'),
 	asyncHandler(async (req, res) => {
-		const item = await WebhooksService.createWebhook(req.body, req.query);
+		const item = await WebhooksService.createWebhook(req.body, req.sanitizedQuery);
 
 		ActivityService.createActivity({
 			action: ActivityService.Action.CREATE,
@@ -33,7 +33,7 @@ router.get(
 	sanitizeQuery,
 	validateQuery,
 	asyncHandler(async (req, res) => {
-		const records = await WebhooksService.readWebhooks(req.query);
+		const records = await WebhooksService.readWebhooks(req.sanitizedQuery);
 		return res.json({ data: records });
 	})
 );
@@ -44,7 +44,7 @@ router.get(
 	sanitizeQuery,
 	validateQuery,
 	asyncHandler(async (req, res) => {
-		const record = await WebhooksService.readWebhook(req.params.pk, req.query);
+		const record = await WebhooksService.readWebhook(req.params.pk, req.sanitizedQuery);
 		return res.json({ data: record });
 	})
 );
@@ -53,7 +53,11 @@ router.patch(
 	'/:pk',
 	useCollection('directus_webhooks'),
 	asyncHandler(async (req, res) => {
-		const item = await WebhooksService.updateWebhook(req.params.pk, req.body, req.query);
+		const item = await WebhooksService.updateWebhook(
+			req.params.pk,
+			req.body,
+			req.sanitizedQuery
+		);
 
 		ActivityService.createActivity({
 			action: ActivityService.Action.UPDATE,

src/services/collections.ts

@@ -0,0 +1,117 @@
+import database, { schemaInspector } from '../database';
+import * as ItemsService from '../services/items';
+import { Collection } from '../types/collection';
+import { Query } from '../types/query';
+import { ColumnBuilder } from 'knex';
+
+/** @Todo properly type this */
+export const create = async (payload: any) => {
+	await database.schema.createTable(payload.collection, (table) => {
+		if (payload.note) {
+			table.comment(payload.note);
+		}
+
+		/** @todo move this into fields service */
+
+		payload.fields?.forEach((field: any) => {
+			let column: ColumnBuilder;
+
+			if (field.auto_increment) {
+				column = table.increments(field.field);
+			} else {
+				const datatype = field.length
+					? `${field.datatype}(${field.length})`
+					: field.datatype;
+				column = table.specificType(field.field, datatype);
+
+				// increments() also sets primary key
+				if (field.primary_key) {
+					column.primary();
+				}
+			}
+
+			if (field.note) {
+				column.comment(field.note);
+			}
+		});
+	});
+
+	const collection = await ItemsService.createItem('directus_collections', {
+		collection: payload.collection,
+		hidden: payload.hidden || false,
+		single: payload.single || false,
+		icon: payload.icon || null,
+		note: payload.note || null,
+		translation: payload.translation || null,
+	});
+
+	/**
+	 * @TODO make this flexible and based on payload
+	 */
+	await database('directus_fields').insert(
+		payload.fields.map((field: any) => ({
+			collection: payload.collection,
+			field: field.field,
+			locked: false,
+			required: false,
+			readonly: false,
+			hidden_detail: false,
+			hidden_browse: false,
+		}))
+	);
+
+	return collection;
+};
+
+export const readAll = async (query?: Query) => {
+	const [tables, collections] = await Promise.all([
+		schemaInspector.tableInfo(),
+		ItemsService.readItems<Collection>('directus_collections', query),
+	]);
+
+	const data = tables.map((table) => {
+		const collectionInfo = collections.find((collection) => {
+			return collection.collection === table.name;
+		});
+
+		return {
+			collection: table.name,
+			note: table.comment,
+			hidden: collectionInfo?.hidden || false,
+			single: collectionInfo?.single || false,
+			icon: collectionInfo?.icon || null,
+			translation: collectionInfo?.translation || null,
+		};
+	});
+
+	return data;
+};
+
+export const readOne = async (collection: string, query?: Query) => {
+	const [table, collectionInfo] = await Promise.all([
+		schemaInspector.tableInfo(collection),
+		ItemsService.readItem<Collection>('directus_collections', collection, query),
+	]);
+
+	return {
+		collection: table.name,
+		note: table.comment,
+		hidden: collectionInfo?.hidden || false,
+		single: collectionInfo?.single || false,
+		icon: collectionInfo?.icon || null,
+		translation: collectionInfo?.translation || null,
+	};
+};
+
+export const deleteCollection = async (collection: string) => {
+	await Promise.all([
+		database.schema.dropTable(collection),
+		ItemsService.deleteItem('directus_collections', collection),
+		database.delete().from('directus_fields').where({ collection }),
+		database
+			.delete()
+			.from('directus_relations')
+			.where({ collection_many: collection })
+			.orWhere({ collection_one: collection }),
+	]);
+};

src/services/fields.ts

@@ -0,0 +1,64 @@
+import database, { schemaInspector } from '../database';
+import { Field } from '../types/field';
+
+export const readAll = async (collection?: string) => {
+	const fieldsQuery = database.select('*').from('directus_fields');
+
+	if (collection) {
+		fieldsQuery.where({ collection });
+	}
+
+	const [columns, fields] = await Promise.all([
+		schemaInspector.columnInfo(collection),
+		fieldsQuery,
+	]);
+
+	return columns.map((column) => {
+		const field = fields.find(
+			(field) => field.field === column.name && field.collection === column.table
+		);
+
+		const data = {
+			collection: column.table,
+			field: column.name,
+			database: column,
+			system: field || null,
+		};
+
+		return data;
+	});
+};
+
+export const readOne = async (collection: string, field: string) => {
+	const [column, fieldInfo] = await Promise.all([
+		schemaInspector.columnInfo(collection, field),
+		database.select('*').from('directus_fields').where({ collection, field }).first(),
+	]);
+
+	const data = {
+		collection: column.table,
+		field: column.name,
+		database: column,
+		system: fieldInfo || null,
+	};
+
+	return data;
+};
+
+export const createField = async (collection: string, field: Partial<Field>) => {
+	await database.schema.alterTable('articles', (table) => {
+		table.specificType(field.field, field.database.type);
+		/** @todo add support for other database info (length etc) */
+	});
+
+	if (field.system) {
+		await database('directus_fields').insert({
+			...field.system,
+			collection: collection,
+			field: field.field,
+		});
+	}
+
+	const createdField = await readOne(collection, field.field);
+	return createdField;
+};

src/services/items.ts

@@ -1,4 +1,4 @@
-import database from '../database';
+import database, { schemaInspector } from '../database';
 import { Query } from '../types/query';
 
 export const createItem = async (
@@ -6,11 +6,15 @@ export const createItem = async (
 	data: Record<string, any>,
 	query: Query = {}
 ) => {
-	const result = await database(collection).insert(data).returning('id');
+	const primaryKeyField = await schemaInspector.primary(collection);
+	const result = await database(collection).insert(data).returning(primaryKeyField);
 	return readItem(collection, result[0], query);
 };
 
-export const readItems = async (collection: string, query: Query = {}) => {
+export const readItems = async <T = Record<string, any>>(
+	collection: string,
+	query: Query = {}
+): Promise<T[]> => {
 	const dbQuery = database.select(query?.fields || '*').from(collection);
 
 	if (query.sort) {
@@ -62,12 +66,17 @@ export const readItems = async (collection: string, query: Query = {}) => {
 	return records;
 };
 
-export const readItem = async (collection: string, pk: number | string, query: Query = {}) => {
-	const dbQuery = database.select('*').from(collection).where({ id: pk });
-
-	const records = await dbQuery;
-
-	return records[0];
+export const readItem = async <T = any>(
+	collection: string,
+	pk: number | string,
+	query: Query = {}
+): Promise<T> => {
+	const primaryKeyField = await schemaInspector.primary(collection);
+	return await database
+		.select('*')
+		.from(collection)
+		.where({ [primaryKeyField]: pk })
+		.first();
 };
 
 export const updateItem = async (
@@ -76,10 +85,17 @@ export const updateItem = async (
 	data: Record<string, any>,
 	query: Query = {}
 ) => {
-	const result = await database(collection).update(data).where({ id: pk }).returning('id');
+	const primaryKeyField = await schemaInspector.primary(collection);
+	const result = await database(collection)
+		.update(data)
+		.where({ [primaryKeyField]: pk })
+		.returning('id');
 	return readItem(collection, result[0], query);
 };
 
 export const deleteItem = async (collection: string, pk: number | string) => {
-	return await database(collection).delete().where({ id: pk });
+	const primaryKeyField = await schemaInspector.primary(collection);
+	return await database(collection)
+		.delete()
+		.where({ [primaryKeyField]: pk });
 };

src/services/payload.ts

@@ -1,11 +1,9 @@
 /**
- * # PayloadService
- *
  * Process a given payload for a collection to ensure the special fields (hash, uuid, date etc) are
  * handled correctly.
  */
 
-import { FieldInfo } from '../types/field';
+import { System } from '../types/field';
 import argon2 from 'argon2';
 import { v4 as uuidv4 } from 'uuid';
 import database from '../database';
@@ -25,9 +23,10 @@ export const processValues = async (
 	payload: Record<string, any>
 ) => {
 	const processedPayload = clone(payload);
+
 	const specialFieldsInCollection = await database
 		.select('field', 'special')
-		.from('directus_fields')
+		.from<System>('directus_fields')
 		.where({ collection: collection })
 		.whereNotNull('special');
 
@@ -39,7 +38,7 @@ export const processValues = async (
 };
 
 async function processField(
-	field: FieldInfo,
+	field: Pick<System, 'field' | 'special'>,
 	payload: Record<string, any>,
 	operation: 'create' | 'update'
 ) {

src/services/schema.ts

@@ -1,5 +1,7 @@
 import database from '../database';
 
+/** @TODO replace this with schema inspector */
+
 export const hasCollection = async (collection: string) => {
 	return await database.schema.hasTable(collection);
 };

src/types/collection.ts

@@ -0,0 +1,8 @@
+export type Collection = {
+	collection: string;
+	note: string | null;
+	hidden: boolean;
+	single: boolean;
+	icon: string | null;
+	translation: Record<string, string>;
+};

src/types/express.d.ts

@@ -11,6 +11,7 @@ declare global {
 			user?: string;
 			role?: string;
 			collection?: string;
+			sanitizedQuery?: Record<string, any>;
 		}
 	}
 }

src/types/field.ts

@@ -1,4 +1,6 @@
-export type FieldInfo = {
+import { Column } from '../knex-schema-inspector/lib/types/column';
+
+export type System = {
 	id: number;
 	collection: string;
 	field: string;
@@ -16,3 +18,10 @@ export type FieldInfo = {
 	note: string | null;
 	translation: null;
 };
+
+export type Field = {
+	collection: string;
+	field: string;
+	database: Column;
+	system: System | null;
+};