Add native schema migration capabilities (#7939)

* Add snapshot creation command

* Read and start diffing snapshot

* Add apply snapshot functionality

* Fix cli invocation

* Add log messages

* Fix duplicated if check

* Add (minimal) docs on schema migrations

* Fix missing import

* Update api/src/utils/apply-snapshot.ts

Co-authored-by: Nicola Krumschmidt <nicola.krumschmidt@freenet.de>

* Appease to Nicola's programming professor

Co-authored-by: Nicola Krumschmidt <nicola.krumschmidt@freenet.de>

api/src/cli/commands/schema/apply.ts

@@ -0,0 +1,136 @@
+import chalk from 'chalk';
+import { promises as fs } from 'fs';
+import inquirer from 'inquirer';
+import { load as loadYaml } from 'js-yaml';
+import path from 'path';
+import getDatabase from '../../../database';
+import logger from '../../../logger';
+import { Snapshot } from '../../../types';
+import { getSnapshot } from '../../../utils/get-snapshot';
+import { getSnapshotDiff } from '../../../utils/get-snapshot-diff';
+import { applySnapshot } from '../../../utils/apply-snapshot';
+
+export async function apply(snapshotPath: string, options?: { yes: boolean }): Promise<void> {
+	const filename = path.resolve(process.cwd(), snapshotPath);
+
+	const database = getDatabase();
+
+	let snapshot: Snapshot;
+
+	try {
+		const fileContents = await fs.readFile(filename, 'utf8');
+
+		if (filename.endsWith('.yaml') || filename.endsWith('.yml')) {
+			snapshot = (await loadYaml(fileContents)) as Snapshot;
+		} else {
+			snapshot = JSON.parse(fileContents) as Snapshot;
+		}
+
+		const currentSnapshot = await getSnapshot({ database });
+		const snapshotDiff = getSnapshotDiff(currentSnapshot, snapshot);
+
+		if (
+			snapshotDiff.collections.length === 0 &&
+			snapshotDiff.fields.length === 0 &&
+			snapshotDiff.relations.length === 0
+		) {
+			logger.info('No changes to apply.');
+			database.destroy();
+			process.exit(0);
+		}
+
+		if (options?.yes !== true) {
+			let message = '';
+
+			if (snapshotDiff.collections.length > 0) {
+				message += chalk.black.underline.bold('Collections:');
+
+				for (const { collection, diff } of snapshotDiff.collections) {
+					if (diff[0]?.kind === 'E') {
+						message += `\n  - ${chalk.blue('Update')} ${collection}`;
+
+						for (const change of diff) {
+							if (change.kind === 'E') {
+								const path = change.path!.slice(1).join('.');
+								message += `\n    - Set ${path} to ${change.rhs}`;
+							}
+						}
+					} else if (diff[0]?.kind === 'D') {
+						message += `\n  - ${chalk.red('Delete')} ${collection}`;
+					} else if (diff[0]?.kind === 'N') {
+						message += `\n  - ${chalk.green('Create')} ${collection}`;
+					}
+				}
+			}
+
+			if (snapshotDiff.fields.length > 0) {
+				message += '\n\n' + chalk.black.underline.bold('Fields:');
+
+				for (const { collection, field, diff } of snapshotDiff.fields) {
+					if (diff[0]?.kind === 'E') {
+						message += `\n  - ${chalk.blue('Update')} ${collection}.${field}`;
+
+						for (const change of diff) {
+							if (change.kind === 'E') {
+								const path = change.path!.slice(1).join('.');
+								message += `\n    - Set ${path} to ${change.rhs}`;
+							}
+						}
+					} else if (diff[0]?.kind === 'D') {
+						message += `\n  - ${chalk.red('Delete')} ${collection}.${field}`;
+					} else if (diff[0]?.kind === 'N') {
+						message += `\n  - ${chalk.green('Create')} ${collection}.${field}`;
+					}
+				}
+			}
+
+			if (snapshotDiff.relations.length > 0) {
+				message += '\n\n' + chalk.black.underline.bold('Relations:');
+
+				for (const { collection, field, related_collection, diff } of snapshotDiff.relations) {
+					if (diff[0]?.kind === 'E') {
+						message += `\n  - ${chalk.blue('Update')} ${collection}.${field} -> ${related_collection}`;
+
+						for (const change of diff) {
+							if (change.kind === 'E') {
+								const path = change.path!.slice(1).join('.');
+								message += `\n    - Set ${path} to ${change.rhs}`;
+							}
+						}
+					} else if (diff[0]?.kind === 'D') {
+						message += `\n  - ${chalk.red('Delete')} ${collection}.${field} -> ${related_collection}`;
+					} else if (diff[0]?.kind === 'N') {
+						message += `\n  - ${chalk.green('Create')} ${collection}.${field} -> ${related_collection}`;
+					}
+				}
+			}
+
+			const { proceed } = await inquirer.prompt([
+				{
+					type: 'confirm',
+					name: 'proceed',
+					message:
+						'The following changes will be applied:\n\n' +
+						chalk.black(message) +
+						'\n\n' +
+						'Would you like to continue?',
+				},
+			]);
+
+			if (proceed === false) {
+				process.exit(0);
+			}
+		}
+
+		await applySnapshot(snapshot, { current: currentSnapshot, diff: snapshotDiff, database });
+
+		logger.info(`Snapshot applied successfully`);
+
+		database.destroy();
+		process.exit(0);
+	} catch (err: any) {
+		logger.error(err);
+		database.destroy();
+		process.exit(1);
+	}
+}

api/src/cli/commands/schema/snapshot.ts

@@ -0,0 +1,58 @@
+import getDatabase from '../../../database';
+import logger from '../../../logger';
+import { getSnapshot } from '../../../utils/get-snapshot';
+import { constants as fsConstants, promises as fs } from 'fs';
+import path from 'path';
+import inquirer from 'inquirer';
+import { dump as toYaml } from 'js-yaml';
+
+export async function snapshot(
+	snapshotPath: string,
+	options?: { yes: boolean; format: 'json' | 'yaml' }
+): Promise<void> {
+	const filename = path.resolve(process.cwd(), snapshotPath);
+
+	let snapshotExists: boolean;
+
+	try {
+		await fs.access(filename, fsConstants.F_OK);
+		snapshotExists = true;
+	} catch {
+		snapshotExists = false;
+	}
+
+	if (snapshotExists && options?.yes === false) {
+		const { overwrite } = await inquirer.prompt([
+			{
+				type: 'confirm',
+				name: 'overwrite',
+				message: 'Snapshot already exists. Do you want to overwrite the file?',
+			},
+		]);
+
+		if (overwrite === false) {
+			process.exit(0);
+		}
+	}
+
+	const database = getDatabase();
+
+	const snapshot = await getSnapshot({ database });
+
+	try {
+		if (options?.format === 'yaml') {
+			await fs.writeFile(filename, toYaml(snapshot));
+		} else {
+			await fs.writeFile(filename, JSON.stringify(snapshot));
+		}
+
+		logger.info(`Snapshot saved to ${filename}`);
+
+		database.destroy();
+		process.exit(0);
+	} catch (err: any) {
+		logger.error(err);
+		database.destroy();
+		process.exit(1);
+	}
+}

api/src/cli/index.ts

@@ -1,4 +1,4 @@
-import { Command } from 'commander';
+import { Command, Option } from 'commander';
 import start from '../start';
 import { emitAsyncSafe } from '../emitter';
 import { initializeExtensions, registerExtensionHooks } from '../extensions';
@@ -10,6 +10,8 @@ import init from './commands/init';
 import rolesCreate from './commands/roles/create';
 import usersCreate from './commands/users/create';
 import usersPasswd from './commands/users/passwd';
+import { snapshot } from './commands/schema/snapshot';
+import { apply } from './commands/schema/apply';
 
 const pkg = require('../../package.json');
 
@@ -75,6 +77,23 @@ export async function createCli(): Promise<Command> {
 		.option('--skipAdminInit', 'Skips the creation of the default Admin Role and User')
 		.action(bootstrap);
 
+	const schemaCommands = program.command('schema');
+
+	schemaCommands
+		.command('snapshot')
+		.description('Create a new Schema Snapshot')
+		.option('-y, --yes', `Assume "yes" as answer to all prompts and run non-interactively`, false)
+		.addOption(new Option('--format <format>', 'JSON or YAML format').choices(['json', 'yaml']).default('yaml'))
+		.argument('<path>', 'Path to snapshot file')
+		.action(snapshot);
+
+	schemaCommands
+		.command('apply')
+		.description('Apply a snapshot file to the current database')
+		.option('-y, --yes', `Assume "yes" as answer to all prompts and run non-interactively`)
+		.argument('<path>', 'Path to snapshot file')
+		.action(apply);
+
 	await emitAsyncSafe('cli.init.after', { program });
 
 	return program;

api/src/database/system-data/collections/collections.yaml

@@ -57,3 +57,4 @@ data:
     display_template: '{{ first_name }} {{ last_name }}'
   - collection: directus_webhooks
     note: $t:directus_collection.directus_webhooks
+  - collection: directus_migrations

api/src/database/system-data/fields/migrations.yaml

@@ -0,0 +1,10 @@
+# directus_migrations isn't surfaced in the app, nor accessible from the API
+table: directus_migrations
+
+fields:
+  - collection: directus_migrations
+    field: version
+  - collection: directus_migrations
+    field: name
+  - collection: directus_migrations
+    field: timestamp

api/src/database/system-data/fields/presets.yaml

@@ -33,3 +33,5 @@ fields:
 
   - field: layout
     width: half
+
+  - field: refresh_interval

api/src/database/system-data/relations/relations.yaml

@@ -54,3 +54,21 @@ data:
   - many_collection: directus_settings
     many_field: public_background
     one_collection: directus_files
+  - many_collection: directus_files
+    many_field: modified_by
+    one_collection: directus_users
+  - many_collection: directus_fields
+    many_field: group
+    one_collection: directus_fields
+  - many_collection: directus_permissions
+    many_field: role
+    one_collection: directus_roles
+  - many_collection: directus_revisions
+    many_field: parent
+    one_collection: directus_revisions
+  - many_collection: directus_sessions
+    many_field: user
+    one_collection: directus_users
+  - many_collection: directus_settings
+    many_field: storage_default_folder
+    one_collection: directus_files

api/src/services/collections.ts

@@ -16,7 +16,7 @@ import { Accountability, FieldMeta, RawField } from '@directus/shared/types';
 export type RawCollection = {
 	collection: string;
 	fields?: RawField[];
-	meta?: Partial<CollectionMeta>;
+	meta?: Partial<CollectionMeta> | null;
 };
 
 export class CollectionsService {

api/src/services/fields.ts

@@ -133,7 +133,11 @@ export class FieldsService {
 			return data;
 		}) as Field[];
 
-		const result = [...columnsWithSystem, ...aliasFieldsAsField];
+		const knownCollections = Object.keys(this.schema.collections);
+
+		const result = [...columnsWithSystem, ...aliasFieldsAsField].filter((field) =>
+			knownCollections.includes(field.collection)
+		);
 
 		// Filter the result so we only return the fields you have read access to
 		if (this.accountability && this.accountability.admin !== true) {

api/src/types/index.ts

@@ -15,4 +15,5 @@ export * from './revision';
 export * from './schema';
 export * from './services';
 export * from './sessions';
+export * from './snapshot';
 export * from './webhooks';

api/src/types/snapshot.ts

@@ -0,0 +1,30 @@
+import { Collection } from './collection';
+import { Relation } from './relation';
+import { Field } from '@directus/shared/types';
+import { Diff } from 'deep-diff';
+
+export type Snapshot = {
+	version: number;
+	directus: string;
+	collections: Collection[];
+	fields: Field[];
+	relations: Relation[];
+};
+
+export type SnapshotDiff = {
+	collections: {
+		collection: string;
+		diff: Diff<Collection | undefined>[];
+	}[];
+	fields: {
+		collection: string;
+		field: string;
+		diff: Diff<Field | undefined>[];
+	}[];
+	relations: {
+		collection: string;
+		field: string;
+		related_collection: string | null;
+		diff: Diff<Relation | undefined>[];
+	}[];
+};

api/src/utils/apply-snapshot.ts

@@ -0,0 +1,112 @@
+import { Snapshot, SnapshotDiff, SchemaOverview, Relation } from '../types';
+import { getSnapshot } from './get-snapshot';
+import { getSnapshotDiff } from './get-snapshot-diff';
+import { Knex } from 'knex';
+import getDatabase from '../database';
+import { getSchema } from './get-schema';
+import { CollectionsService, FieldsService, RelationsService } from '../services';
+import { set } from 'lodash';
+import { DiffNew } from 'deep-diff';
+import { Field } from '@directus/shared/types';
+
+export async function applySnapshot(
+	snapshot: Snapshot,
+	options?: { database?: Knex; schema?: SchemaOverview; current?: Snapshot; diff?: SnapshotDiff }
+): Promise<void> {
+	const database = options?.database ?? getDatabase();
+	const schema = options?.schema ?? (await getSchema({ database }));
+
+	const current = options?.current ?? (await getSnapshot({ database, schema }));
+	const snapshotDiff = options?.diff ?? getSnapshotDiff(current, snapshot);
+
+	await database.transaction(async (trx) => {
+		const collectionsService = new CollectionsService({ knex: trx, schema });
+
+		for (const { collection, diff } of snapshotDiff.collections) {
+			if (diff?.[0].kind === 'D') {
+				await collectionsService.deleteOne(collection);
+			}
+
+			if (diff?.[0].kind === 'N' && diff[0].rhs) {
+				// We'll nest the to-be-created fields in the same collection creation, to prevent
+				// creating a collection without a primary key
+				const fields = snapshotDiff.fields
+					.filter((fieldDiff) => fieldDiff.collection === collection)
+					.map((fieldDiff) => (fieldDiff.diff[0] as DiffNew<Field>).rhs);
+
+				await collectionsService.createOne({
+					...diff[0].rhs,
+					fields,
+				});
+
+				// Now that the fields are in for this collection, we can strip them from the field
+				// edits
+				snapshotDiff.fields = snapshotDiff.fields.filter((fieldDiff) => fieldDiff.collection !== collection);
+			}
+
+			if (diff?.[0].kind === 'E') {
+				const updates = diff.reduce((acc, edit) => {
+					if (edit.kind !== 'E') return acc;
+					set(acc, edit.path!, edit.rhs);
+					return acc;
+				}, {});
+
+				await collectionsService.updateOne(collection, updates);
+			}
+		}
+
+		const fieldsService = new FieldsService({ knex: trx, schema: await getSchema({ database: trx }) });
+
+		for (const { collection, field, diff } of snapshotDiff.fields) {
+			if (diff?.[0].kind === 'N') {
+				await fieldsService.createField(collection, (diff[0] as DiffNew<Field>).rhs);
+			}
+
+			if (diff?.[0].kind === 'E') {
+				const updates = diff.reduce((acc, edit) => {
+					if (edit.kind !== 'E') return acc;
+					set(acc, edit.path!, edit.rhs);
+					return acc;
+				}, {});
+
+				await fieldsService.updateField(collection, {
+					field,
+					type: 'unknown', // If the type was updated, the updates spread will overwrite it
+					...updates,
+				});
+			}
+
+			if (diff?.[0].kind === 'D') {
+				await fieldsService.deleteField(collection, field);
+
+				// Field deletion also cleans up the relationship. We should ignore any relationship
+				// changes attached to this now non-existing field
+				snapshotDiff.relations = snapshotDiff.relations.filter(
+					(relation) => (relation.collection === collection && relation.field === field) === false
+				);
+			}
+		}
+
+		const relationsService = new RelationsService({ knex: trx, schema: await getSchema({ database: trx }) });
+
+		for (const { collection, field, diff } of snapshotDiff.relations) {
+			if (diff?.[0].kind === 'N') {
+				await relationsService.createOne((diff[0] as DiffNew<Relation>).rhs);
+			}
+
+			if (diff?.[0].kind === 'E') {
+				const updates = diff.reduce((acc, edit) => {
+					if (edit.kind !== 'E') return acc;
+					set(acc, edit.path!, edit.rhs);
+					return acc;
+				}, {});
+
+				await relationsService.updateOne(collection, field, updates);
+			}
+
+			if (diff?.[0].kind === 'D') {
+				await relationsService.deleteOne(collection, field);
+			}
+		}
+	});
+}

api/src/utils/get-snapshot-diff.ts

@@ -0,0 +1,116 @@
+import { Snapshot, SnapshotDiff } from '../types';
+import { diff } from 'deep-diff';
+import { orderBy } from 'lodash';
+
+export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDiff {
+	const diffedSnapshot: SnapshotDiff = {
+		collections: orderBy(
+			[
+				...current.collections.map((currentCollection) => {
+					const afterCollection = after.collections.find(
+						(afterCollection) => afterCollection.collection === currentCollection.collection
+					);
+
+					return {
+						collection: currentCollection.collection,
+						diff: diff(currentCollection, afterCollection),
+					};
+				}),
+				...after.collections
+					.filter((afterCollection) => {
+						const currentCollection = current.collections.find(
+							(currentCollection) => currentCollection.collection === afterCollection.collection
+						);
+
+						return !!currentCollection === false;
+					})
+					.map((afterCollection) => ({
+						collection: afterCollection.collection,
+						diff: diff(undefined, afterCollection),
+					})),
+			].filter((obj) => Array.isArray(obj.diff)) as SnapshotDiff['collections'],
+			'collection'
+		),
+		fields: orderBy(
+			[
+				...current.fields.map((currentField) => {
+					const afterField = after.fields.find(
+						(afterField) => afterField.collection === currentField.collection && afterField.field === currentField.field
+					);
+
+					return {
+						collection: currentField.collection,
+						field: currentField.field,
+						diff: diff(currentField, afterField),
+					};
+				}),
+				...after.fields
+					.filter((afterField) => {
+						const currentField = current.fields.find(
+							(currentField) =>
+								currentField.collection === afterField.collection && afterField.field === currentField.field
+						);
+
+						return !!currentField === false;
+					})
+					.map((afterField) => ({
+						collection: afterField.collection,
+						field: afterField.field,
+						diff: diff(undefined, afterField),
+					})),
+			].filter((obj) => Array.isArray(obj.diff)) as SnapshotDiff['fields'],
+			['collection']
+		),
+		relations: orderBy(
+			[
+				...current.relations.map((currentRelation) => {
+					const afterRelation = after.relations.find(
+						(afterRelation) =>
+							afterRelation.collection === currentRelation.collection && afterRelation.field === currentRelation.field
+					);
+
+					return {
+						collection: currentRelation.collection,
+						field: currentRelation.field,
+						related_collection: currentRelation.related_collection,
+						diff: diff(currentRelation, afterRelation),
+					};
+				}),
+				...after.relations
+					.filter((afterRelation) => {
+						const currentRelation = current.relations.find(
+							(currentRelation) =>
+								currentRelation.collection === afterRelation.collection && afterRelation.field === currentRelation.field
+						);
+
+						return !!currentRelation === false;
+					})
+					.map((afterRelation) => ({
+						collection: afterRelation.collection,
+						field: afterRelation.field,
+						related_collection: afterRelation.related_collection,
+						diff: diff(undefined, afterRelation),
+					})),
+			].filter((obj) => Array.isArray(obj.diff)) as SnapshotDiff['relations'],
+			['collection']
+		),
+	};
+
+	/**
+	 * When you delete a collection, we don't have to individually drop all the fields/relations as well
+	 */
+
+	const deletedCollections = diffedSnapshot.collections
+		.filter((collection) => collection.diff?.[0].kind === 'D')
+		.map(({ collection }) => collection);
+
+	diffedSnapshot.fields = diffedSnapshot.fields.filter(
+		(field) => deletedCollections.includes(field.collection) === false
+	);
+
+	diffedSnapshot.relations = diffedSnapshot.relations.filter(
+		(relation) => deletedCollections.includes(relation.collection) === false
+	);
+
+	return diffedSnapshot;
+}

api/src/utils/get-snapshot.ts

@@ -0,0 +1,34 @@
+import getDatabase from '../database';
+import { getSchema } from './get-schema';
+import { CollectionsService, FieldsService, RelationsService } from '../services';
+import { version } from '../../package.json';
+import { SchemaOverview, Snapshot } from '../types';
+import { Knex } from 'knex';
+
+export async function getSnapshot(options?: { database?: Knex; schema?: SchemaOverview }): Promise<Snapshot> {
+	const database = options?.database ?? getDatabase();
+	const schema = options?.schema ?? (await getSchema({ database }));
+
+	const collectionsService = new CollectionsService({ knex: database, schema });
+	const fieldsService = new FieldsService({ knex: database, schema });
+	const relationsService = new RelationsService({ knex: database, schema });
+
+	const [collections, fields, relations] = await Promise.all([
+		collectionsService.readByQuery(),
+		fieldsService.readAll(),
+		relationsService.readAll(),
+	]);
+
+	return {
+		version: 1,
+		directus: version,
+		collections: collections.filter((item: any) => excludeSystem(item)),
+		fields: fields.filter((item: any) => excludeSystem(item)),
+		relations: relations.filter((item: any) => excludeSystem(item)),
+	};
+}
+
+function excludeSystem(item: { meta?: { system?: boolean } }) {
+	if (item?.meta?.system === true) return false;
+	return true;
+}