Add native schema migration capabilities (#7939)

* Add snapshot creation command

* Read and start diffing snapshot

* Add apply snapshot functionality

* Fix cli invocation

* Add log messages

* Fix duplicated if check

* Add (minimal) docs on schema migrations

* Fix missing import

* Update api/src/utils/apply-snapshot.ts

Co-authored-by: Nicola Krumschmidt <nicola.krumschmidt@freenet.de>

* Appease to Nicola's programming professor

Co-authored-by: Nicola Krumschmidt <nicola.krumschmidt@freenet.de>

api/src/cli/commands/schema/apply.ts

@@ -0,0 +1,136 @@
+import chalk from 'chalk';
+import { promises as fs } from 'fs';
+import inquirer from 'inquirer';
+import { load as loadYaml } from 'js-yaml';
+import path from 'path';
+import getDatabase from '../../../database';
+import logger from '../../../logger';
+import { Snapshot } from '../../../types';
+import { getSnapshot } from '../../../utils/get-snapshot';
+import { getSnapshotDiff } from '../../../utils/get-snapshot-diff';
+import { applySnapshot } from '../../../utils/apply-snapshot';
+
+export async function apply(snapshotPath: string, options?: { yes: boolean }): Promise<void> {
+	const filename = path.resolve(process.cwd(), snapshotPath);
+
+	const database = getDatabase();
+
+	let snapshot: Snapshot;
+
+	try {
+		const fileContents = await fs.readFile(filename, 'utf8');
+
+		if (filename.endsWith('.yaml') || filename.endsWith('.yml')) {
+			snapshot = (await loadYaml(fileContents)) as Snapshot;
+		} else {
+			snapshot = JSON.parse(fileContents) as Snapshot;
+		}
+
+		const currentSnapshot = await getSnapshot({ database });
+		const snapshotDiff = getSnapshotDiff(currentSnapshot, snapshot);
+
+		if (
+			snapshotDiff.collections.length === 0 &&
+			snapshotDiff.fields.length === 0 &&
+			snapshotDiff.relations.length === 0
+		) {
+			logger.info('No changes to apply.');
+			database.destroy();
+			process.exit(0);
+		}
+
+		if (options?.yes !== true) {
+			let message = '';
+
+			if (snapshotDiff.collections.length > 0) {
+				message += chalk.black.underline.bold('Collections:');
+
+				for (const { collection, diff } of snapshotDiff.collections) {
+					if (diff[0]?.kind === 'E') {
+						message += `\n  - ${chalk.blue('Update')} ${collection}`;
+
+						for (const change of diff) {
+							if (change.kind === 'E') {
+								const path = change.path!.slice(1).join('.');
+								message += `\n    - Set ${path} to ${change.rhs}`;
+							}
+						}
+					} else if (diff[0]?.kind === 'D') {
+						message += `\n  - ${chalk.red('Delete')} ${collection}`;
+					} else if (diff[0]?.kind === 'N') {
+						message += `\n  - ${chalk.green('Create')} ${collection}`;
+					}
+				}
+			}
+
+			if (snapshotDiff.fields.length > 0) {
+				message += '\n\n' + chalk.black.underline.bold('Fields:');
+
+				for (const { collection, field, diff } of snapshotDiff.fields) {
+					if (diff[0]?.kind === 'E') {
+						message += `\n  - ${chalk.blue('Update')} ${collection}.${field}`;
+
+						for (const change of diff) {
+							if (change.kind === 'E') {
+								const path = change.path!.slice(1).join('.');
+								message += `\n    - Set ${path} to ${change.rhs}`;
+							}
+						}
+					} else if (diff[0]?.kind === 'D') {
+						message += `\n  - ${chalk.red('Delete')} ${collection}.${field}`;
+					} else if (diff[0]?.kind === 'N') {
+						message += `\n  - ${chalk.green('Create')} ${collection}.${field}`;
+					}
+				}
+			}
+
+			if (snapshotDiff.relations.length > 0) {
+				message += '\n\n' + chalk.black.underline.bold('Relations:');
+
+				for (const { collection, field, related_collection, diff } of snapshotDiff.relations) {
+					if (diff[0]?.kind === 'E') {
+						message += `\n  - ${chalk.blue('Update')} ${collection}.${field} -> ${related_collection}`;
+
+						for (const change of diff) {
+							if (change.kind === 'E') {
+								const path = change.path!.slice(1).join('.');
+								message += `\n    - Set ${path} to ${change.rhs}`;
+							}
+						}
+					} else if (diff[0]?.kind === 'D') {
+						message += `\n  - ${chalk.red('Delete')} ${collection}.${field} -> ${related_collection}`;
+					} else if (diff[0]?.kind === 'N') {
+						message += `\n  - ${chalk.green('Create')} ${collection}.${field} -> ${related_collection}`;
+					}
+				}
+			}
+
+			const { proceed } = await inquirer.prompt([
+				{
+					type: 'confirm',
+					name: 'proceed',
+					message:
+						'The following changes will be applied:\n\n' +
+						chalk.black(message) +
+						'\n\n' +
+						'Would you like to continue?',
+				},
+			]);
+
+			if (proceed === false) {
+				process.exit(0);
+			}
+		}
+
+		await applySnapshot(snapshot, { current: currentSnapshot, diff: snapshotDiff, database });
+
+		logger.info(`Snapshot applied successfully`);
+
+		database.destroy();
+		process.exit(0);
+	} catch (err: any) {
+		logger.error(err);
+		database.destroy();
+		process.exit(1);
+	}
+}

api/src/cli/commands/schema/snapshot.ts

@@ -0,0 +1,58 @@
+import getDatabase from '../../../database';
+import logger from '../../../logger';
+import { getSnapshot } from '../../../utils/get-snapshot';
+import { constants as fsConstants, promises as fs } from 'fs';
+import path from 'path';
+import inquirer from 'inquirer';
+import { dump as toYaml } from 'js-yaml';
+
+export async function snapshot(
+	snapshotPath: string,
+	options?: { yes: boolean; format: 'json' | 'yaml' }
+): Promise<void> {
+	const filename = path.resolve(process.cwd(), snapshotPath);
+
+	let snapshotExists: boolean;
+
+	try {
+		await fs.access(filename, fsConstants.F_OK);
+		snapshotExists = true;
+	} catch {
+		snapshotExists = false;
+	}
+
+	if (snapshotExists && options?.yes === false) {
+		const { overwrite } = await inquirer.prompt([
+			{
+				type: 'confirm',
+				name: 'overwrite',
+				message: 'Snapshot already exists. Do you want to overwrite the file?',
+			},
+		]);
+
+		if (overwrite === false) {
+			process.exit(0);
+		}
+	}
+
+	const database = getDatabase();
+
+	const snapshot = await getSnapshot({ database });
+
+	try {
+		if (options?.format === 'yaml') {
+			await fs.writeFile(filename, toYaml(snapshot));
+		} else {
+			await fs.writeFile(filename, JSON.stringify(snapshot));
+		}
+
+		logger.info(`Snapshot saved to ${filename}`);
+
+		database.destroy();
+		process.exit(0);
+	} catch (err: any) {
+		logger.error(err);
+		database.destroy();
+		process.exit(1);
+	}
+}

api/src/cli/index.ts

@@ -1,4 +1,4 @@
-import { Command } from 'commander';
+import { Command, Option } from 'commander';
 import start from '../start';
 import { emitAsyncSafe } from '../emitter';
 import { initializeExtensions, registerExtensionHooks } from '../extensions';
@@ -10,6 +10,8 @@ import init from './commands/init';
 import rolesCreate from './commands/roles/create';
 import usersCreate from './commands/users/create';
 import usersPasswd from './commands/users/passwd';
+import { snapshot } from './commands/schema/snapshot';
+import { apply } from './commands/schema/apply';
 
 const pkg = require('../../package.json');
 
@@ -75,6 +77,23 @@ export async function createCli(): Promise<Command> {
 		.option('--skipAdminInit', 'Skips the creation of the default Admin Role and User')
 		.action(bootstrap);
 
+	const schemaCommands = program.command('schema');
+
+	schemaCommands
+		.command('snapshot')
+		.description('Create a new Schema Snapshot')
+		.option('-y, --yes', `Assume "yes" as answer to all prompts and run non-interactively`, false)
+		.addOption(new Option('--format <format>', 'JSON or YAML format').choices(['json', 'yaml']).default('yaml'))
+		.argument('<path>', 'Path to snapshot file')
+		.action(snapshot);
+
+	schemaCommands
+		.command('apply')
+		.description('Apply a snapshot file to the current database')
+		.option('-y, --yes', `Assume "yes" as answer to all prompts and run non-interactively`)
+		.argument('<path>', 'Path to snapshot file')
+		.action(apply);
+
 	await emitAsyncSafe('cli.init.after', { program });
 
 	return program;