Start splitting blackbox tests (#21078)

.github/workflows/blackbox.yml

@@ -11,7 +11,7 @@ on:
       - packages/**
       - package.json
       - pnpm-lock.yaml
-      - .github/workflows/blackbox-main.yml
+      - .github/workflows/blackbox.yml
 
 concurrency:
   group: blackbox-${{ github.ref }}
@@ -21,7 +21,24 @@ env:
   NODE_OPTIONS: --max_old_space_size=6144
 
 jobs:
-  test:
+  common:
+    name: Common
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Prepare
+        uses: ./.github/actions/prepare
+
+      - name: Start services
+        run:
+          docker compose -f tests/blackbox/docker-compose.yml up auth-saml redis minio minio-mc -d --quiet-pull --wait
+
+      - name: Run tests
+        run: TEST_DB=sqlite3 pnpm run test:blackbox -- --project common
+
+  db:
     name: ${{ matrix.vendor }}
     runs-on: ${{ matrix.vendor == 'mssql' && 'ubuntu-20.04' || 'ubuntu-latest' }}
     strategy:
@@ -61,4 +78,4 @@ jobs:
           docker compose -f tests/blackbox/docker-compose.yml up auth-saml redis minio minio-mc -d --quiet-pull --wait
 
       - name: Run tests
-        run: TEST_DB=${{ matrix.vendor }} pnpm run test:blackbox
+        run: TEST_DB=${{ matrix.vendor }} pnpm run test:blackbox -- --project db

tests/blackbox/setup/environment.ts

@@ -18,7 +18,7 @@ export default <Environment>{
 			throw 'Missing flow env variables';
 		}
 
-		const testIndex = getReversedTestIndex(testFilePath);
+		const testIndex = getReversedTestIndex(testFilePath, global.__vitest_worker__.ctx.config.name);
 
 		while (testIndex !== 0) {
 			try {

tests/blackbox/setup/sequencer.ts

@@ -6,11 +6,13 @@ import { sequentialTestsList } from './sequential-tests';
 export default class CustomSequencer extends BaseSequencer {
 	override async sort(files: WorkspaceSpec[]) {
 		if (files.length > 1) {
+			const list = sequentialTestsList[files[0]![0].config.name as 'db' | 'common'];
+
 			// If specified, only run these tests sequentially
-			if (sequentialTestsList.only.length > 0) {
+			if (list.only.length > 0) {
 				const onlyTests = [];
 
-				for (const sequentialTest of sequentialTestsList.only) {
+				for (const sequentialTest of list.only) {
 					const testIndex = findIndex(files, ([_, testFile]) => {
 						return testFile.endsWith(sequentialTest);
 					});
@@ -26,7 +28,7 @@ export default class CustomSequencer extends BaseSequencer {
 
 				files = onlyTests;
 			} else {
-				for (const sequentialTest of sequentialTestsList.before.slice().reverse()) {
+				for (const sequentialTest of list.before.slice().reverse()) {
 					const testIndex = findIndex(files, ([_, testFile]) => {
 						return testFile.endsWith(sequentialTest);
 					});
@@ -40,7 +42,7 @@ export default class CustomSequencer extends BaseSequencer {
 					}
 				}
 
-				for (const sequentialTest of sequentialTestsList.after) {
+				for (const sequentialTest of list.after) {
 					const testIndex = findIndex(files, ([_, testFile]) => {
 						return testFile.endsWith(sequentialTest);
 					});

tests/blackbox/setup/sequential-tests.ts

@@ -1,37 +1,47 @@
 // Tests will run sequentially according to this list
-export const sequentialTestsList: SequentialTestsList = {
-	before: [
-		'/common/seed-database.test.ts',
-		'/common/common.test.ts',
-		'/routes/schema/schema.test.ts',
-		'/routes/collections/crud.test.ts',
-		'/routes/fields/change-fields.test.ts',
-	],
-	after: [
-		'/schema/timezone/timezone.test.ts',
-		'/schema/timezone/timezone-changed-node-tz-america.test.ts',
-		'/schema/timezone/timezone-changed-node-tz-asia.test.ts',
-		'/websocket/auth.test.ts',
-		'/websocket/general.test.ts',
-		'/flows/schedule-hook.test.ts',
-		'/logger/redact.test.ts',
-		'/routes/permissions/cache-purge.test.ts',
-		'/routes/flows/webhook.test.ts',
-		'/app/cache.test.ts',
-		'/routes/collections/schema-cache.test.ts',
-		'/routes/assets/concurrency.test.ts',
-	],
-	// If specified, only run these tests sequentially
-	only: [
-		// '/common/seed-database.test.ts',
-		// '/common/common.test.ts',
-	],
+export const sequentialTestsList: Record<'db' | 'common', SequentialTestsList> = {
+	common: {
+		before: ['/common/common.test.ts'],
+		after: [],
+		// If specified, only run these tests sequentially
+		only: [
+			// '/common/common.test.ts',
+		],
+	},
+	db: {
+		before: [
+			'/tests/db/seed-database.test.ts',
+			'/common/common.test.ts',
+			'/tests/db/routes/schema/schema.test.ts',
+			'/tests/db/routes/collections/crud.test.ts',
+			'/tests/db/routes/fields/change-fields.test.ts',
+		],
+		after: [
+			'/tests/db/schema/timezone/timezone.test.ts',
+			'/tests/db/schema/timezone/timezone-changed-node-tz-america.test.ts',
+			'/tests/db/schema/timezone/timezone-changed-node-tz-asia.test.ts',
+			'/tests/db/websocket/auth.test.ts',
+			'/tests/db/websocket/general.test.ts',
+			'/tests/db/routes/permissions/cache-purge.test.ts',
+			'/tests/db/routes/flows/webhook.test.ts',
+			'/tests/db/app/cache.test.ts',
+			'/tests/db/routes/collections/schema-cache.test.ts',
+			'/tests/db/routes/assets/concurrency.test.ts',
+		],
+		// If specified, only run these tests sequentially
+		only: [
+			// '/tests/db/seed-database.test.ts',
+			// '/common/common.test.ts',
+		],
+	},
 };
 
-export function getReversedTestIndex(testFilePath: string) {
-	if (sequentialTestsList.only.length > 0) {
-		for (let index = 0; index < sequentialTestsList.only.length; index++) {
-			const onlyTest = sequentialTestsList.only[index];
+export function getReversedTestIndex(testFilePath: string, project: 'db' | 'common') {
+	const list = sequentialTestsList[project];
+
+	if (list.only.length > 0) {
+		for (let index = 0; index < list.only.length; index++) {
+			const onlyTest = list.only[index];
 
 			if (onlyTest && testFilePath.includes(onlyTest)) {
 				return index;
@@ -39,23 +49,23 @@ export function getReversedTestIndex(testFilePath: string) {
 		}
 	}
 
-	for (let index = 0; index < sequentialTestsList.before.length; index++) {
-		const beforeTest = sequentialTestsList.before[index];
+	for (let index = 0; index < list.before.length; index++) {
+		const beforeTest = list.before[index];
 
 		if (beforeTest && testFilePath.includes(beforeTest)) {
 			return index;
 		}
 	}
 
-	for (let index = 0; index < sequentialTestsList.after.length; index++) {
-		const afterTest = sequentialTestsList.after[index];
+	for (let index = 0; index < list.after.length; index++) {
+		const afterTest = list.after[index];
 
 		if (afterTest && testFilePath.includes(afterTest)) {
-			return 0 - sequentialTestsList.after.length + index;
+			return 0 - list.after.length + index;
 		}
 	}
 
-	return sequentialTestsList.before.length;
+	return list.before.length;
 }
 
 type SequentialTestsList = {

tests/blackbox/tests/db/app/cache.seed.ts

@@ -36,7 +36,7 @@ export const seedDBStructure = () => {
 					type: 'string',
 				});
 
-				// Create seconds collection
+				// Create second collection
 				await CreateCollection(vendor, {
 					collection: collectionIgnored,
 				});

tests/blackbox/tests/db/query/filter/index.ts

@@ -1,7 +1,7 @@
 import vendors from '@common/get-dbs-to-test';
 import type { PrimaryKeyType } from '@common/types';
 import type { ClientFilterOperator } from '@directus/types';
-import * as testsSchema from '@schema/index';
+import * as testsSchema from '../../schema/index';
 import { PrepareRequest, type RequestOptions } from '@utils/prepare-request';
 import { get, set } from 'lodash-es';
 import { describe, expect, it, type SuiteCollector } from 'vitest';

tests/blackbox/tests/db/routes/fields/change-fields.seed.ts

@@ -3,7 +3,7 @@ import vendors from '@common/get-dbs-to-test';
 import { SeedFunctions } from '@common/seed-functions';
 import type { PrimaryKeyType } from '@common/types';
 import { PRIMARY_KEY_TYPES } from '@common/variables';
-import type { CachedTestsSchema, TestsSchema, TestsSchemaVendorValues } from '@query/filter';
+import type { CachedTestsSchema, TestsSchema, TestsSchemaVendorValues } from '../../query/filter';
 import { set } from 'lodash-es';
 import { expect, it } from 'vitest';
 

tests/blackbox/tests/db/routes/fields/change-fields.test.ts

@@ -2,7 +2,7 @@ import { getUrl } from '@common/config';
 import { ClearCaches, CreateField, DeleteField, DisableTestCachingSetup } from '@common/functions';
 import vendors from '@common/get-dbs-to-test';
 import { PRIMARY_KEY_TYPES, USER } from '@common/variables';
-import type { CachedTestsSchema, TestsSchemaVendorValues } from '@query/filter';
+import type { CachedTestsSchema, TestsSchemaVendorValues } from '../../query/filter';
 import request from 'supertest';
 import { beforeAll, describe, expect, it } from 'vitest';
 import { collectionCountries, collectionStates, getTestsSchema, seedDBValues } from './change-fields.seed';

tests/blackbox/tests/db/routes/items/m2a.seed.ts

@@ -3,7 +3,7 @@ import vendors from '@common/get-dbs-to-test';
 import { SeedFunctions } from '@common/seed-functions';
 import type { PrimaryKeyType } from '@common/types';
 import { PRIMARY_KEY_TYPES } from '@common/variables';
-import type { CachedTestsSchema, TestsSchema, TestsSchemaVendorValues } from '@query/filter';
+import type { CachedTestsSchema, TestsSchema, TestsSchemaVendorValues } from '../../query/filter';
 import { set } from 'lodash-es';
 import { expect, it } from 'vitest';
 import {

tests/blackbox/tests/db/routes/items/m2a.test.ts

@@ -5,7 +5,7 @@ import { SeedFunctions } from '@common/seed-functions';
 import { requestGraphQL } from '@common/transport';
 import type { PrimaryKeyType } from '@common/types';
 import { PRIMARY_KEY_TYPES, USER } from '@common/variables';
-import { CheckQueryFilters, type CachedTestsSchema, type TestsSchemaVendorValues } from '@query/filter';
+import { CheckQueryFilters, type CachedTestsSchema, type TestsSchemaVendorValues } from '../../query/filter';
 import { findIndex, without } from 'lodash-es';
 import request from 'supertest';
 import { v4 as uuid } from 'uuid';

tests/blackbox/tests/db/routes/items/m2m.seed.ts

@@ -1,5 +1,5 @@
 import vendors from '@common/get-dbs-to-test';
-import type { CachedTestsSchema, TestsSchema, TestsSchemaVendorValues } from '@query/filter';
+import type { CachedTestsSchema, TestsSchema, TestsSchemaVendorValues } from '../../query/filter';
 import {
 	seedAllFieldTypesStructure,
 	seedAllFieldTypesValues,

tests/blackbox/tests/db/routes/items/m2m.test.ts

@@ -4,7 +4,7 @@ import vendors from '@common/get-dbs-to-test';
 import { requestGraphQL } from '@common/transport';
 import type { PrimaryKeyType } from '@common/types';
 import { PRIMARY_KEY_TYPES, USER } from '@common/variables';
-import { CheckQueryFilters, type CachedTestsSchema, type TestsSchemaVendorValues } from '@query/filter';
+import { CheckQueryFilters, type CachedTestsSchema, type TestsSchemaVendorValues } from '../../query/filter';
 import { findIndex, without } from 'lodash-es';
 import request from 'supertest';
 import { v4 as uuid } from 'uuid';

tests/blackbox/tests/db/routes/items/m2o.seed.ts

@@ -3,7 +3,7 @@ import vendors from '@common/get-dbs-to-test';
 import { SeedFunctions } from '@common/seed-functions';
 import type { PrimaryKeyType } from '@common/types';
 import { PRIMARY_KEY_TYPES } from '@common/variables';
-import type { CachedTestsSchema, TestsSchema, TestsSchemaVendorValues } from '@query/filter';
+import type { CachedTestsSchema, TestsSchema, TestsSchemaVendorValues } from '../../query/filter';
 import { set } from 'lodash-es';
 import { expect, it } from 'vitest';
 import { getTestsAllTypesSchema, seedAllFieldTypesStructure, seedAllFieldTypesValues } from './seed-all-field-types';

tests/blackbox/tests/db/routes/items/m2o.test.ts

@@ -4,7 +4,7 @@ import vendors from '@common/get-dbs-to-test';
 import { createWebSocketConn, createWebSocketGql, requestGraphQL } from '@common/transport';
 import type { PrimaryKeyType } from '@common/types';
 import { PRIMARY_KEY_TYPES, USER } from '@common/variables';
-import { CheckQueryFilters, type CachedTestsSchema, type TestsSchemaVendorValues } from '@query/filter';
+import { CheckQueryFilters, type CachedTestsSchema, type TestsSchemaVendorValues } from '../../query/filter';
 import { without } from 'lodash-es';
 import request from 'supertest';
 import { v4 as uuid } from 'uuid';

tests/blackbox/tests/db/routes/items/o2m.seed.ts

@@ -3,7 +3,7 @@ import vendors from '@common/get-dbs-to-test';
 import { SeedFunctions } from '@common/seed-functions';
 import type { PrimaryKeyType } from '@common/types';
 import { PRIMARY_KEY_TYPES } from '@common/variables';
-import type { CachedTestsSchema, TestsSchema, TestsSchemaVendorValues } from '@query/filter';
+import type { CachedTestsSchema, TestsSchema, TestsSchemaVendorValues } from '../../query/filter';
 import { set } from 'lodash-es';
 import { expect, it } from 'vitest';
 import {

tests/blackbox/tests/db/routes/items/o2m.test.ts

@@ -4,7 +4,7 @@ import vendors from '@common/get-dbs-to-test';
 import { createWebSocketConn, createWebSocketGql, requestGraphQL } from '@common/transport';
 import type { PrimaryKeyType } from '@common/types';
 import { PRIMARY_KEY_TYPES, USER } from '@common/variables';
-import { CheckQueryFilters, type CachedTestsSchema, type TestsSchemaVendorValues } from '@query/filter';
+import { CheckQueryFilters, type CachedTestsSchema, type TestsSchemaVendorValues } from '../../query/filter';
 import { findIndex, without } from 'lodash-es';
 import request from 'supertest';
 import { v4 as uuid } from 'uuid';

tests/blackbox/tests/db/routes/items/seed-all-field-types.ts

@@ -2,7 +2,7 @@ import { CreateField, CreateItem, ReadItem } from '@common/functions';
 import type { Vendor } from '@common/get-dbs-to-test';
 import { SeedFunctions } from '@common/seed-functions';
 import type { PrimaryKeyType } from '@common/types';
-import type { TestsFieldSchema } from '@query/filter';
+import type { TestsFieldSchema } from '../../query/filter';
 import { expect } from 'vitest';
 
 const valuesQuantity = 2;

tests/blackbox/tests/db/routes/items/seed-relational-fields.ts

@@ -4,7 +4,7 @@ import type { Vendor } from '@common/get-dbs-to-test';
 import { SeedFunctions } from '@common/seed-functions';
 import type { PrimaryKeyType } from '@common/types';
 import { USER } from '@common/variables';
-import type { TestsFieldSchema } from '@query/filter';
+import type { TestsFieldSchema } from '../../query/filter';
 import request from 'supertest';
 import { expect } from 'vitest';
 

tests/blackbox/tests/db/routes/schema/schema.test.ts

@@ -8,7 +8,7 @@ import { cloneDeep } from 'lodash-es';
 import request from 'supertest';
 import { v4 as uuid } from 'uuid';
 import { describe, expect, it } from 'vitest';
-import { version as currentDirectusVersion } from '../../../../api/package.json';
+import { version as currentDirectusVersion } from '../../../../../../api/package.json';
 import {
 	collectionAll,
 	collectionM2A,

tests/blackbox/tests/db/schema/alias/index.ts

@@ -1,6 +1,6 @@
 import type { ClientFilterOperator } from '@directus/types';
 import { getFilterOperatorsForType } from '@directus/utils';
-import type { FilterEmptyValidator, FilterValidator } from '@query/filter';
+import type { FilterEmptyValidator, FilterValidator } from '../../query/filter';
 import type { GeneratedFilter } from '..';
 
 export const type = 'alias';

tests/blackbox/tests/db/schema/big-integer/index.ts

@@ -1,6 +1,6 @@
 import type { ClientFilterOperator } from '@directus/types';
 import { getFilterOperatorsForType } from '@directus/utils';
-import type { FilterEmptyValidator, FilterValidator } from '@query/filter';
+import type { FilterEmptyValidator, FilterValidator } from '../../query/filter';
 import type { GeneratedFilter } from '..';
 
 export const type = 'bigInteger';

tests/blackbox/tests/db/schema/boolean/index.ts

@@ -1,6 +1,6 @@
 import type { ClientFilterOperator } from '@directus/types';
 import { getFilterOperatorsForType } from '@directus/utils';
-import type { FilterEmptyValidator, FilterValidator } from '@query/filter';
+import type { FilterEmptyValidator, FilterValidator } from '../../query/filter';
 import type { GeneratedFilter } from '..';
 
 export const type = 'boolean';

tests/blackbox/tests/db/schema/date-time/index.ts

@@ -1,6 +1,6 @@
 import type { ClientFilterOperator } from '@directus/types';
 import { getFilterOperatorsForType } from '@directus/utils';
-import type { FilterEmptyValidator, FilterValidator } from '@query/filter';
+import type { FilterEmptyValidator, FilterValidator } from '../../query/filter';
 import type { GeneratedFilter } from '..';
 
 export const type = 'dateTime';

tests/blackbox/tests/db/schema/float/index.ts

@@ -1,6 +1,6 @@
 import type { ClientFilterOperator } from '@directus/types';
 import { getFilterOperatorsForType } from '@directus/utils';
-import type { FilterEmptyValidator, FilterValidator } from '@query/filter';
+import type { FilterEmptyValidator, FilterValidator } from '../../query/filter';
 import type { GeneratedFilter } from '..';
 
 export const type = 'float';

tests/blackbox/tests/db/schema/geometry/index.ts

@@ -1,5 +1,5 @@
 import type { ClientFilterOperator } from '@directus/types';
-import type { FilterEmptyValidator, FilterValidator } from '@query/filter';
+import type { FilterEmptyValidator, FilterValidator } from '../../query/filter';
 import { getFilterOperatorsForType } from '@directus/utils';
 import type { GeneratedFilter } from '..';
 

tests/blackbox/tests/db/schema/integer/index.ts

@@ -1,6 +1,6 @@
 import type { ClientFilterOperator } from '@directus/types';
 import { getFilterOperatorsForType } from '@directus/utils';
-import type { FilterEmptyValidator, FilterValidator } from '@query/filter';
+import type { FilterEmptyValidator, FilterValidator } from '../../query/filter';
 import type { GeneratedFilter } from '..';
 
 export const type = 'integer';

tests/blackbox/tests/db/schema/json/index.ts

@@ -1,6 +1,6 @@
 import type { ClientFilterOperator } from '@directus/types';
 import { getFilterOperatorsForType } from '@directus/utils';
-import type { FilterEmptyValidator, FilterValidator } from '@query/filter';
+import type { FilterEmptyValidator, FilterValidator } from '../../query/filter';
 import type { GeneratedFilter } from '..';
 
 export const type = 'json';

tests/blackbox/tests/db/schema/string/index.ts

@@ -1,6 +1,6 @@
 import type { ClientFilterOperator } from '@directus/types';
 import { getFilterOperatorsForType } from '@directus/utils';
-import type { FilterEmptyValidator, FilterValidator } from '@query/filter';
+import type { FilterEmptyValidator, FilterValidator } from '../../query/filter';
 import type { GeneratedFilter } from '..';
 
 export const type = 'string';

tests/blackbox/tests/db/schema/uuid/index.ts

@@ -1,6 +1,6 @@
 import type { ClientFilterOperator } from '@directus/types';
 import { getFilterOperatorsForType } from '@directus/utils';
-import type { FilterEmptyValidator, FilterValidator } from '@query/filter';
+import type { FilterEmptyValidator, FilterValidator } from '../../query/filter';
 import type { GeneratedFilter } from '..';
 
 export const type = 'uuid';

tests/blackbox/tests/db/seed-database.test.ts

@@ -1,8 +1,8 @@
 import { globby } from 'globby';
 import { describe, expect, test } from 'vitest';
-import { sequentialTestsList } from '../setup/sequential-tests';
-import { paths } from './config';
-import { ClearCaches, DisableTestCachingSetup } from './functions';
+import { sequentialTestsList } from '../../setup/sequential-tests';
+import { paths } from '../../common/config';
+import { ClearCaches, DisableTestCachingSetup } from '../../common/functions';
 
 describe('Seed Database Structure', async () => {
 	DisableTestCachingSetup();
@@ -15,8 +15,8 @@ describe('Seed Database Structure', async () => {
 		test('No seed files found', () => {
 			expect(true).toBe(true);
 		});
-	} else if (sequentialTestsList.only.length > 0) {
-		const requiredPaths = sequentialTestsList.only.map((testEntry) => {
+	} else if (sequentialTestsList['db'].only.length > 0) {
+		const requiredPaths = sequentialTestsList['db'].only.map((testEntry) => {
 			return testEntry.slice(1).replace('.test.ts', '.seed.ts');
 		});
 
@@ -26,7 +26,7 @@ describe('Seed Database Structure', async () => {
 	}
 
 	for (const path of seeds) {
-		const importedTest = await import(`../${path}`);
+		const importedTest = await import(`../../${path}`);
 
 		if (typeof importedTest.seedDBStructure === 'function') {
 			describe(`Seeding "${path}"`, async () => {

tests/blackbox/tsconfig.json

@@ -5,8 +5,8 @@
 		"moduleResolution": "Bundler",
 		"paths": {
 			"@common/*": ["./common/*"],
-			"@query/*": ["./query/*"],
-			"@schema/*": ["./schema/*"],
+			"@query/*": ["./tests/db/query/*"],
+			"@schema/*": ["./tests/db/schema/*"],
 			"@utils/*": ["./utils/*"]
 		}
 	},

tests/blackbox/vitest.config.ts

@@ -5,7 +5,6 @@ import Sequencer from './setup/sequencer';
 export default defineConfig({
 	plugins: [tsconfigPaths()],
 	test: {
-		globalSetup: './setup/setup.ts',
 		environment: './setup/environment.ts',
 		sequence: {
 			sequencer: Sequencer,

tests/blackbox/vitest.workspace.ts

@@ -0,0 +1,20 @@
+import { defineWorkspace } from 'vitest/config';
+
+export default defineWorkspace([
+	{
+		extends: './vitest.config.ts',
+		test: {
+			name: 'common',
+			include: ['tests/common/**/*.test.ts', 'common/common.test.ts'],
+			globalSetup: './setup/setup.ts',
+		},
+	},
+	{
+		extends: './vitest.config.ts',
+		test: {
+			name: 'db',
+			include: ['tests/db/**/*.test.ts', 'common/common.test.ts'],
+			globalSetup: './setup/setup.ts',
+		},
+	},
+]);