End-to-end test setup (#4833)

* First start on test setup

* Setup e2e runner framework

* Use prepublishOnly for docs build

* Setup docker-based e2e test runner

* Spacing

* Setup db-getter + config

* Build image in Dockerode, pull images if needed

* Fix waiting for DB, run test for each running instance

* Get tests running per vendor

* Temp remove oracle

* Close awaiting connections in between

e2e-tests/api/ping.test.ts

@@ -0,0 +1,17 @@
+import request from 'supertest';
+import { getURLsToTest } from '../get-urls-to-test';
+
+describe('/server', () => {
+	it('/ping', () =>
+		Promise.all(
+			getURLsToTest().map((url) =>
+				request(url)
+					.get('/server/ping')
+					.expect('Content-Type', /text\/html/)
+					.expect(200)
+					.then((response) => {
+						expect(response.text).toBe('pong');
+					})
+			)
+		));
+});

e2e-tests/config.ts

@@ -0,0 +1,207 @@
+import Dockerode from 'dockerode';
+import { Knex } from 'knex';
+import { allVendors } from './get-dbs-to-test';
+import { customAlphabet } from 'nanoid';
+
+const generateID = customAlphabet('abcdefghijklmnopqrstuvwxyz', 5);
+
+type Vendor = typeof allVendors[number];
+
+export type Config = {
+	containerConfig: Record<
+		Vendor,
+		(Dockerode.ContainerSpec & Dockerode.ContainerCreateOptions & { name: string }) | false
+	>;
+	knexConfig: Record<Vendor, Knex.Config & { waitTestSQL: string }>;
+	ports: Record<Vendor, number>;
+	names: Record<Vendor, string>;
+	envs: Record<Vendor, string[]>;
+};
+
+export const processID = generateID();
+
+const config: Config = {
+	containerConfig: {
+		postgres: {
+			name: `directus-test-database-postgres-${process.pid}`,
+			Image: 'postgres:12-alpine',
+			Hostname: `directus-test-database-postgres-${process.pid}`,
+			Env: ['POSTGRES_PASSWORD=secret', 'POSTGRES_DB=directus'],
+			HostConfig: {
+				PortBindings: {
+					'5432/tcp': [{ HostPort: '6000' }],
+				},
+			},
+		},
+		mysql: {
+			name: `directus-test-database-mysql-${process.pid}`,
+			Image: 'mysql:8',
+			Cmd: ['--default-authentication-plugin=mysql_native_password'],
+			Hostname: `directus-test-database-mysql-${process.pid}`,
+			Env: ['MYSQL_ROOT_PASSWORD=secret', 'MYSQL_DATABASE=directus'],
+			HostConfig: {
+				PortBindings: {
+					'3306/tcp': [{ HostPort: '6001' }],
+				},
+			},
+		},
+		maria: {
+			name: `directus-test-database-maria-${process.pid}`,
+			Image: 'mariadb:10.5',
+			Hostname: `directus-test-database-maria-${process.pid}`,
+			Env: ['MYSQL_ROOT_PASSWORD=secret', 'MYSQL_DATABASE=directus'],
+			HostConfig: {
+				PortBindings: {
+					'3306/tcp': [{ HostPort: '6002' }],
+				},
+			},
+		},
+		mssql: {
+			name: `directus-test-database-mssql-${process.pid}`,
+			Image: 'mcr.microsoft.com/mssql/server:2019-latest',
+			Hostname: `directus-test-database-mssql-${process.pid}`,
+			Env: ['ACCEPT_EULA=Y', 'SA_PASSWORD=Test@123'],
+			HostConfig: {
+				PortBindings: {
+					'1433/tcp': [{ HostPort: '6003' }],
+				},
+			},
+		},
+		oracle: {
+			name: `directus-test-database-oracle-${process.pid}`,
+			Image: 'quillbuilduser/oracle-18-xe-micro-sq',
+			Hostname: `directus-test-database-oracle-${process.pid}`,
+			Env: [
+				'OPATCH_JRE_MEMORY_OPTIONS=-Xms128m -Xmx256m -XX:PermSize=16m -XX:MaxPermSize=32m -Xss1m',
+				'ORACLE_ALLOW_REMOTE=true',
+			],
+			HostConfig: {
+				PortBindings: {
+					'1521/tcp': [{ HostPort: '6004' }],
+				},
+			},
+		},
+		sqlite3: false,
+	},
+	knexConfig: {
+		postgres: {
+			client: 'pg',
+			connection: {
+				host: 'localhost',
+				port: 6000,
+				user: 'postgres',
+				password: 'secret',
+				database: 'directus',
+			},
+			waitTestSQL: 'SELECT 1',
+		},
+		mysql: {
+			client: 'mysql',
+			connection: {
+				host: 'localhost',
+				port: 6001,
+				user: 'root',
+				password: 'secret',
+				database: 'directus',
+			},
+			waitTestSQL: 'SELECT 1',
+		},
+		maria: {
+			client: 'mysql',
+			connection: {
+				host: 'localhost',
+				port: 6002,
+				user: 'root',
+				password: 'secret',
+				database: 'directus',
+			},
+			waitTestSQL: 'SELECT 1',
+		},
+		mssql: {
+			client: 'mssql',
+			connection: {
+				host: 'localhost',
+				port: 6003,
+				user: 'sa',
+				password: 'Test@123',
+				database: 'model',
+			},
+			waitTestSQL: 'SELECT 1',
+		},
+		oracle: {
+			client: 'oracledb',
+			connection: {
+				user: 'secretsysuser',
+				password: 'secretpassword',
+				connectString: 'localhost:6004/XE',
+			},
+			waitTestSQL: 'SELECT 1 FROM DUAL',
+		},
+		sqlite3: {
+			client: 'sqlite3',
+			connection: {
+				filename: './data.db',
+			},
+			waitTestSQL: 'SELECT 1',
+		},
+	},
+	ports: {
+		postgres: 6100,
+		mysql: 6101,
+		maria: 6102,
+		mssql: 6103,
+		oracle: 6104,
+		sqlite3: 6105,
+	},
+	names: {
+		postgres: 'Postgres',
+		mysql: 'MySQL',
+		maria: 'MariaDB',
+		mssql: 'MS SQL Server',
+		oracle: 'OracleDB',
+		sqlite3: 'SQLite 3',
+	},
+	envs: {
+		postgres: [
+			'DB_CLIENT=pg',
+			`DB_HOST=directus-test-database-postgres-${process.pid}`,
+			'DB_USER=postgres',
+			'DB_PASSWORD=secret',
+			'DB_PORT=5432',
+			'DB_DATABASE=directus',
+		],
+		mysql: [
+			'DB_CLIENT=mysql',
+			`DB_HOST=directus-test-database-mysql-${process.pid}`,
+			'DB_PORT=3306',
+			'DB_USER=root',
+			'DB_PASSWORD=secret',
+			'DB_DATABASE=directus',
+		],
+		maria: [
+			'DB_CLIENT=mysql',
+			`DB_HOST=directus-test-database-maria-${process.pid}`,
+			'DB_PORT=3306',
+			'DB_USER=root',
+			'DB_PASSWORD=secret',
+			'DB_DATABASE=directus',
+		],
+		mssql: [
+			'DB_CLIENT=mssql',
+			`DB_HOST=directus-test-database-mssql-${process.pid}`,
+			'DB_PORT=1433',
+			'DB_USER=sa',
+			'DB_PASSWORD=Test@123',
+			'DB_DATABASE=model',
+		],
+		oracle: [
+			'DB_CLIENT=oracledb',
+			'DB_USER=secretsysuser',
+			'DB_PASSWORD=secretpassword',
+			`DB_CONNECT_STRING=directus-test-database-oracle-${process.pid}:1521/XE`,
+		],
+		sqlite3: ['DB_CLIENT=sqlite3', 'DB_FILENAME=./data.db'],
+	},
+};
+
+export default config;

e2e-tests/get-dbs-to-test.ts

@@ -0,0 +1,24 @@
+/** @TODO once Oracle is officially supported, enable it here */
+export const allVendors = ['mssql', 'mysql', 'postgres', /* 'oracle', */ 'maria', 'sqlite3'];
+
+export function getDBsToTest(): string[] {
+	const testVendors = process.env.TEST_DB || '*';
+
+	let vendors = [];
+
+	if (testVendors === '*') {
+		vendors = allVendors;
+	} else if (testVendors.includes(',')) {
+		vendors = testVendors.split(',').map((v) => v.trim());
+	} else {
+		vendors = [testVendors];
+	}
+
+	for (const vendor of vendors) {
+		if (allVendors.includes(vendor) === false) {
+			throw new Error(`No e2e testing capabilities for vendor "${vendor}".`);
+		}
+	}
+
+	return vendors;
+}

e2e-tests/get-urls-to-test.ts

@@ -0,0 +1,8 @@
+import { getDBsToTest } from './get-dbs-to-test';
+import config from './config';
+
+export function getURLsToTest(): string[] {
+	const dbVendors = getDBsToTest();
+	const urls = dbVendors.map((vendor) => `http://localhost:${config.ports[vendor]!}`);
+	return urls;
+}

e2e-tests/jest.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+	preset: 'ts-jest',
+	verbose: true,
+	globalSetup: './setup/setup.ts',
+	globalTeardown: './setup/teardown.ts',
+};

e2e-tests/readme.md

@@ -0,0 +1,4 @@
+# End-to-End (e2e) Tests
+
+This folder contains the end-to-end tests to go over the whole suite. Unit/integration tests per package can be found in
+the individual packages.

e2e-tests/setup/setup.ts

@@ -0,0 +1,258 @@
+import Dockerode, { ContainerSpec } from 'dockerode';
+import knex, { Knex } from 'knex';
+import { awaitDatabaseConnection, awaitDirectusConnection } from './utils/await-connection';
+import Listr, { ListrTask } from 'listr';
+import { getDBsToTest } from '../get-dbs-to-test';
+import config from '../config';
+import globby from 'globby';
+import path from 'path';
+
+declare module global {
+	let databaseContainers: { vendor: string; container: Dockerode.Container }[];
+	let directusContainers: { vendor: string; container: Dockerode.Container }[];
+	let knexInstances: { vendor: string; knex: Knex }[];
+}
+
+const docker = new Dockerode();
+
+export default async () => {
+	console.log('\n\n');
+
+	console.log(`👮‍♀️ Starting tests!\n`);
+
+	global.databaseContainers = [];
+	global.directusContainers = [];
+	global.knexInstances = [];
+
+	const vendors = getDBsToTest();
+
+	const NODE_VERSION = process.env.TEST_NODE_VERSION || '15-alpine';
+
+	await new Listr([
+		{
+			title: 'Create Directus Docker Image',
+			task: async (_, task) => {
+				const result = await globby(['**/*', '!node_modules', '!**/node_modules', '!**/src'], {
+					cwd: path.resolve(__dirname, '..', '..'),
+				});
+
+				const stream = await docker.buildImage(
+					{
+						context: path.resolve(__dirname, '..', '..'),
+						src: ['Dockerfile', ...result],
+					},
+					{ t: 'directus-test-image', buildargs: { NODE_VERSION } }
+				);
+
+				await new Promise((resolve, reject) => {
+					docker.modem.followProgress(
+						stream,
+						(err: Error, res: any) => {
+							if (err) {
+								reject(err);
+							} else {
+								resolve(res);
+							}
+						},
+						(event: any) => {
+							if (event.stream?.startsWith('Step')) {
+								task.output = event.stream;
+							}
+						}
+					);
+				});
+			},
+		},
+		{
+			title: 'Pulling Required Images',
+			task: () => {
+				return new Listr(
+					vendors
+						.map((vendor) => {
+							return {
+								title: config.names[vendor]!,
+								task: async (_, task) => {
+									const image =
+										config.containerConfig[vendor]! &&
+										(config.containerConfig[vendor]! as Dockerode.ContainerSpec).Image;
+
+									if (!image) return;
+
+									const stream = await docker.pull(image);
+
+									await new Promise((resolve, reject) => {
+										docker.modem.followProgress(
+											stream,
+											(err: Error, res: any) => {
+												if (err) {
+													reject(err);
+												} else {
+													resolve(res);
+												}
+											},
+											(event: any) => {
+												if (event.stream?.startsWith('Step')) {
+													task.output = event.stream;
+												}
+											}
+										);
+									});
+								},
+							} as ListrTask;
+						})
+						.filter((t) => t),
+					{ concurrent: true }
+				);
+			},
+		},
+		{
+			title: 'Create Docker containers',
+			task: () => {
+				return new Listr(
+					vendors.map((vendor) => {
+						return {
+							title: config.names[vendor]!,
+							task: () => {
+								return new Listr([
+									{
+										title: 'Database',
+										task: async () => {
+											if (!config.containerConfig[vendor] || config.containerConfig[vendor] === false) return;
+											const container = await docker.createContainer(config.containerConfig[vendor]! as ContainerSpec);
+											global.databaseContainers.push({
+												vendor,
+												container,
+											});
+										},
+									},
+									{
+										title: 'Directus',
+										task: async () => {
+											const container = await docker.createContainer({
+												name: `directus-test-directus-${vendor}-${process.pid}`,
+												Image: 'directus-test-image',
+												Env: [
+													...config.envs[vendor]!,
+													'ADMIN_EMAIL=admin@example.com',
+													'ADMIN_PASSWORD=password',
+													'KEY=directus-test',
+													'SECRET=directus-test',
+													'TELEMETRY=false',
+												],
+												HostConfig: {
+													Links:
+														vendor === 'sqlite3'
+															? undefined
+															: [
+																	`directus-test-database-${vendor}-${process.pid}:directus-test-database-${vendor}-${process.pid}`,
+															  ],
+													PortBindings: {
+														'8055/tcp': [{ HostPort: String(config.ports[vendor]!) }],
+													},
+												},
+											} as ContainerSpec);
+
+											global.directusContainers.push({
+												vendor,
+												container,
+											});
+										},
+									},
+								]);
+							},
+						};
+					}),
+					{ concurrent: true }
+				);
+			},
+		},
+		{
+			title: 'Start Database Docker containers',
+			task: () => {
+				return new Listr(
+					global.databaseContainers.map(({ vendor, container }) => {
+						return {
+							title: config.names[vendor]!,
+							task: async () => await container.start(),
+						};
+					}),
+					{ concurrent: true }
+				);
+			},
+		},
+		{
+			title: 'Create Knex instances',
+			task: () => {
+				return new Listr(
+					vendors.map((vendor) => {
+						return {
+							title: config.names[vendor]!,
+							task: () => {
+								global.knexInstances.push({ vendor, knex: knex(config.knexConfig[vendor]!) });
+							},
+						};
+					}),
+					{ concurrent: true }
+				);
+			},
+		},
+		{
+			title: 'Wait for databases to be ready',
+			task: async () => {
+				return new Listr(
+					global.knexInstances.map(({ vendor, knex }) => {
+						return {
+							title: config.names[vendor]!,
+							task: async () => await awaitDatabaseConnection(knex, config.knexConfig[vendor]!.waitTestSQL),
+						};
+					}),
+					{ concurrent: true }
+				);
+			},
+		},
+		{
+			title: 'Close Knex instances',
+			task: () => {
+				return new Listr(
+					global.knexInstances.map(({ vendor, knex }) => {
+						return {
+							title: config.names[vendor]!,
+							task: async () => await knex.destroy(),
+						};
+					}),
+					{ concurrent: true }
+				);
+			},
+		},
+		{
+			title: 'Start Directus Docker containers',
+			task: () => {
+				return new Listr(
+					global.directusContainers.map(({ vendor, container }) => {
+						return {
+							title: config.names[vendor]!,
+							task: async () => await container.start(),
+						};
+					}),
+					{ concurrent: true }
+				);
+			},
+		},
+		{
+			title: 'Wait for Directus to be ready',
+			task: async () => {
+				return new Listr(
+					vendors.map((vendor) => {
+						return {
+							title: config.names[vendor]!,
+							task: async () => await awaitDirectusConnection(config.ports[vendor]!),
+						};
+					}),
+					{ concurrent: true }
+				);
+			},
+		},
+	]).run();
+
+	console.log('\n');
+};

e2e-tests/setup/teardown.ts

@@ -0,0 +1,92 @@
+import Listr from 'listr';
+import { Knex } from 'knex';
+import Dockerode from 'dockerode';
+import { getDBsToTest } from '../get-dbs-to-test';
+import config from '../config';
+
+declare module global {
+	let databaseContainers: { vendor: string; container: Dockerode.Container }[];
+	let directusContainers: { vendor: string; container: Dockerode.Container }[];
+	let knexInstances: { vendor: string; knex: Knex }[];
+}
+
+export default async () => {
+	const vendors = getDBsToTest();
+
+	console.log('\n');
+
+	await new Listr([
+		{
+			title: 'Stop Docker containers',
+			task: () => {
+				return new Listr(
+					vendors.map((vendor) => {
+						return {
+							title: config.names[vendor]!,
+							task: () => {
+								return new Listr(
+									[
+										{
+											title: 'Database',
+											task: async () =>
+												await global.databaseContainers
+													.find((containerInfo) => containerInfo.vendor === vendor)
+													?.container.stop(),
+										},
+										{
+											title: 'Directus',
+											task: async () =>
+												await global.directusContainers
+													.find((containerInfo) => containerInfo.vendor === vendor)
+													?.container.stop(),
+										},
+									],
+									{ concurrent: true }
+								);
+							},
+						};
+					}),
+					{ concurrent: true }
+				);
+			},
+		},
+		{
+			title: 'Remove Docker containers',
+			task: () => {
+				return new Listr(
+					vendors.map((vendor) => {
+						return {
+							title: config.names[vendor]!,
+							task: () => {
+								return new Listr(
+									[
+										{
+											title: 'Database',
+											task: async () =>
+												await global.databaseContainers
+													.find((containerInfo) => containerInfo.vendor === vendor)
+													?.container.remove(),
+										},
+										{
+											title: 'Directus',
+											task: async () =>
+												await global.directusContainers
+													.find((containerInfo) => containerInfo.vendor === vendor)
+													?.container.remove(),
+										},
+									],
+									{ concurrent: true }
+								);
+							},
+						};
+					}),
+					{ concurrent: true }
+				);
+			},
+		},
+	]).run();
+
+	console.log('\n');
+
+	console.log(`👮‍♀️ Tests complete!\n`);
+};

e2e-tests/setup/utils/await-connection.ts

@@ -0,0 +1,36 @@
+import { Knex } from 'knex';
+import axios from 'axios';
+
+export async function awaitDatabaseConnection(database: Knex, checkSQL: string, currentAttempt: number = 0) {
+	try {
+		await database.raw(checkSQL);
+	} catch {
+		if (currentAttempt === 10) {
+			throw new Error(`Couldn't connect to DB`);
+		}
+
+		return new Promise((resolve) => {
+			setTimeout(async () => {
+				await awaitDatabaseConnection(database, checkSQL, currentAttempt + 1);
+				resolve(null);
+			}, 5000);
+		});
+	}
+}
+
+export async function awaitDirectusConnection(port: number = 6100, currentAttempt: number = 0) {
+	try {
+		await axios.get(`http://localhost:${port}/server/ping`);
+	} catch {
+		if (currentAttempt === 10) {
+			throw new Error(`Couldn't connect to Directus`);
+		}
+
+		return new Promise((resolve) => {
+			setTimeout(async () => {
+				await awaitDirectusConnection(port, currentAttempt + 1);
+				resolve(null);
+			}, 5000);
+		});
+	}
+}

e2e-tests/start.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+node ./dist/cli/index.js bootstrap;
+node ./dist/start.js;

e2e-tests/tsconfig.json

@@ -0,0 +1,28 @@
+{
+	"compilerOptions": {
+		"target": "ES2018",
+		"lib": ["ES2018"],
+		"module": "CommonJS",
+		"moduleResolution": "node",
+		"declaration": true,
+		"declarationMap": true,
+		"strict": true,
+		"noFallthroughCasesInSwitch": true,
+		"esModuleInterop": true,
+		"noImplicitAny": true,
+		"noImplicitThis": true,
+		"noImplicitReturns": true,
+		"noUnusedLocals": true,
+		"noUncheckedIndexedAccess": true,
+		"noUnusedParameters": true,
+		"alwaysStrict": true,
+		"strictNullChecks": true,
+		"strictFunctionTypes": true,
+		"strictBindCallApply": true,
+		"strictPropertyInitialization": true,
+		"resolveJsonModule": false,
+		"skipLibCheck": true,
+		"forceConsistentCasingInFileNames": true,
+		"allowSyntheticDefaultImports": true
+	}
+}