Add support for a package extension bundle type (#15672)

* Add bundle type to constants and types

* Add support for API bundle extensions

* Rename generateExtensionsEntry to generateExtensionsEntrypoint

* Add support for App bundle extensions

* Refactor App extension registration

* Replace extensions inject with useExtensions()

* Replace getInterfaces() with useExtensions()

* Replace getDisplays() with useExtensions()

* Replace getLayouts() with useExtensions()

* Replace getModules() with useExtensions()

* Replace getPanels() with useExtensions()

* Replace getOperations() with useExtensions()

* Add useExtension() composable

* Replace useExtensions() with useExtension() where applicable

* Remove interface getters

* Remove display getters

* Remove layout getters

* Remove module getter

* Remove panel getters

* Remove operation getters

* Rename extension register.ts files to index.ts

* Perform module pre register check in parallel

* Remove Refs from AppExtensionConfigs type

* Remove old extension shims

* Ensure registration of modules is awaited when hydrating

* Add support for scaffolding package extensions

* Add support for building bundle extensions

* Add JsonValue type

* Use json for complex command line flags

* Load internal extensions if custom ones are not available

* Fix extension manifest validation for pack extensions

* Fix tests in shared

* Add SplitEntrypoint type

* Move command specific utils to helpers

* Add SDK version getter

* Move extension dev deps generation to helpers

* Move template path to getter util

* Move template copying to a helper

* Only rename copied template files

* Add directus-extension add command

* Convert provided extension source path to url

* Replace deprecated import.meta.globEager

* Mock URL.createObjectURL to make App unit tests pass

* Update rollup-plugin-typescript2

* indentation

* sort vite glob imported modules

* fix unintentional wrong commit

* Simplify app extension import logic

* reinstall @rollup/plugin-virtual

* add test for getInterfaces() expected sort order

Co-authored-by: Rijk van Zanten <rijkvanzanten@me.com>
Co-authored-by: Azri Kahar <42867097+azrikahar@users.noreply.github.com>

packages/shared/src/utils/node/generate-extensions-entry.test.ts

@@ -1,33 +0,0 @@
-import { describe, expect, it } from 'vitest';
-
-import { Extension } from '../../types/extensions';
-import { generateExtensionsEntry } from './generate-extensions-entry';
-
-describe('generateExtensionsEntry', () => {
-	const type = 'panel';
-
-	it('returns an extension entrypoint exporting all extensions with a type that matches the provided type', () => {
-		const mockExtension: Extension[] = [
-			{ path: './extensions', name: 'mockExtension', type: 'panel', entrypoint: 'index.js', local: true },
-		];
-		expect(generateExtensionsEntry(type, mockExtension)).toBe(
-			`import e0 from './extensions/index.js';
-export default [e0];`
-		);
-	});
-
-	it('returns an empty extension entrypoint if there is no extension with the provided type', () => {
-		const mockExtension: Extension[] = [
-			{
-				path: './extensions',
-				name: 'mockExtension',
-				type: 'pack',
-				version: '1.0.0',
-				host: '^9.0.0',
-				children: [],
-				local: false,
-			},
-		];
-		expect(generateExtensionsEntry(type, mockExtension)).toBe(`export default [];`);
-	});
-});

packages/shared/src/utils/node/generate-extensions-entry.ts

@@ -1,23 +0,0 @@
-import path from 'path';
-import { HYBRID_EXTENSION_TYPES } from '../../constants';
-import { AppExtension, AppExtensionType, Extension, HybridExtension, HybridExtensionType } from '../../types';
-import { isTypeIn } from '../array-helpers';
-import { pathToRelativeUrl } from './path-to-relative-url';
-
-export function generateExtensionsEntry(type: AppExtensionType | HybridExtensionType, extensions: Extension[]): string {
-	const filteredExtensions = extensions.filter(
-		(extension): extension is AppExtension | HybridExtension => extension.type === type
-	);
-
-	return `${filteredExtensions
-		.map(
-			(extension, i) =>
-				`import e${i} from './${pathToRelativeUrl(
-					path.resolve(
-						extension.path,
-						isTypeIn(extension, HYBRID_EXTENSION_TYPES) ? extension.entrypoint.app : extension.entrypoint
-					)
-				)}';\n`
-		)
-		.join('')}export default [${filteredExtensions.map((_, i) => `e${i}`).join(',')}];`;
-}

packages/shared/src/utils/node/generate-extensions-entrypoint.test.ts

@@ -0,0 +1,129 @@
+import { describe, expect, it } from 'vitest';
+
+import { Extension } from '../../types/extensions';
+import { generateExtensionsEntrypoint } from './generate-extensions-entrypoint';
+
+describe('generateExtensionsEntrypoint', () => {
+	it('returns an empty extension entrypoint if there is no App, Hybrid or Bundle extension', () => {
+		const mockExtensions: Extension[] = [
+			{
+				path: './extensions/pack',
+				name: 'mock-pack-extension',
+				type: 'pack',
+				version: '1.0.0',
+				host: '^9.0.0',
+				children: [],
+				local: false,
+			},
+		];
+
+		expect(generateExtensionsEntrypoint(mockExtensions)).toBe(
+			`export const interfaces = [];export const displays = [];export const layouts = [];export const modules = [];export const panels = [];export const operations = [];`
+		);
+	});
+
+	it('returns an extension entrypoint exporting a single App extension', () => {
+		const mockExtensions: Extension[] = [
+			{ path: './extensions/panel', name: 'mock-panel-extension', type: 'panel', entrypoint: 'index.js', local: true },
+		];
+
+		expect(generateExtensionsEntrypoint(mockExtensions)).toBe(
+			`import panel0 from './extensions/panel/index.js';export const interfaces = [];export const displays = [];export const layouts = [];export const modules = [];export const panels = [panel0];export const operations = [];`
+		);
+	});
+
+	it('returns an extension entrypoint exporting a single Hybrid extension', () => {
+		const mockExtensions: Extension[] = [
+			{
+				path: './extensions/operation',
+				name: 'mock-operation-extension',
+				type: 'operation',
+				entrypoint: { app: 'app.js', api: 'api.js' },
+				local: true,
+			},
+		];
+
+		expect(generateExtensionsEntrypoint(mockExtensions)).toBe(
+			`import operation0 from './extensions/operation/app.js';export const interfaces = [];export const displays = [];export const layouts = [];export const modules = [];export const panels = [];export const operations = [operation0];`
+		);
+	});
+
+	it('returns an extension entrypoint exporting from a single Bundle extension', () => {
+		const mockExtensions: Extension[] = [
+			{
+				path: './extensions/bundle',
+				name: 'mock-bundle-extension',
+				version: '1.0.0',
+				type: 'bundle',
+				entrypoint: { app: 'app.js', api: 'api.js' },
+				entries: [
+					{ type: 'interface', name: 'mock-bundle-interface' },
+					{ type: 'operation', name: 'mock-bundle-operation' },
+					{ type: 'hook', name: 'mock-bundle-hook' },
+				],
+				host: '^9.0.0',
+				local: false,
+			},
+		];
+
+		expect(generateExtensionsEntrypoint(mockExtensions)).toBe(
+			`import {interfaces as interfaceBundle0,operations as operationBundle0} from './extensions/bundle/app.js';export const interfaces = [...interfaceBundle0];export const displays = [];export const layouts = [];export const modules = [];export const panels = [];export const operations = [...operationBundle0];`
+		);
+	});
+
+	it('returns an extension entrypoint exporting multiple extensions', () => {
+		const mockExtensions: Extension[] = [
+			{
+				path: './extensions/pack',
+				name: 'mock-pack-extension',
+				type: 'pack',
+				version: '1.0.0',
+				host: '^9.0.0',
+				children: [],
+				local: false,
+			},
+			{
+				path: './extensions/display',
+				name: 'mock-display-extension',
+				type: 'display',
+				entrypoint: 'index.js',
+				local: true,
+			},
+			{
+				path: './extensions/operation',
+				name: 'mock-operation-extension',
+				type: 'operation',
+				entrypoint: { app: 'app.js', api: 'api.js' },
+				local: true,
+			},
+			{
+				path: './extensions/bundle',
+				name: 'mock-bundle0-extension',
+				version: '1.0.0',
+				type: 'bundle',
+				entrypoint: { app: 'app.js', api: 'api.js' },
+				entries: [
+					{ type: 'layout', name: 'mock-bundle-layout' },
+					{ type: 'operation', name: 'mock-bundle-operation' },
+					{ type: 'hook', name: 'mock-bundle-hook' },
+				],
+				host: '^9.0.0',
+				local: false,
+			},
+			{
+				path: './extensions/bundle-no-app',
+				name: 'mock-bundle-no-app-extension',
+				version: '1.0.0',
+				type: 'bundle',
+				entrypoint: { app: 'app.js', api: 'api.js' },
+				entries: [{ type: 'endpoint', name: 'mock-bundle-no-app-endpoint' }],
+				host: '^9.0.0',
+				local: false,
+			},
+		];
+
+		expect(generateExtensionsEntrypoint(mockExtensions)).toBe(
+			`import display0 from './extensions/display/index.js';import operation0 from './extensions/operation/app.js';import {layouts as layoutBundle0,operations as operationBundle0} from './extensions/bundle/app.js';export const interfaces = [];export const displays = [display0];export const layouts = [...layoutBundle0];export const modules = [];export const panels = [];export const operations = [operation0,...operationBundle0];`
+		);
+	});
+});

packages/shared/src/utils/node/generate-extensions-entrypoint.ts

@@ -0,0 +1,55 @@
+import path from 'path';
+import { APP_OR_HYBRID_EXTENSION_TYPES, HYBRID_EXTENSION_TYPES } from '../../constants';
+import { AppExtension, BundleExtension, Extension, HybridExtension } from '../../types';
+import { isIn, isTypeIn } from '../array-helpers';
+import { pluralize } from '../pluralize';
+import { pathToRelativeUrl } from './path-to-relative-url';
+
+export function generateExtensionsEntrypoint(extensions: Extension[]): string {
+	const appOrHybridExtensions = extensions.filter((extension): extension is AppExtension | HybridExtension =>
+		isIn(extension.type, APP_OR_HYBRID_EXTENSION_TYPES)
+	);
+
+	const bundleExtensions = extensions.filter(
+		(extension): extension is BundleExtension =>
+			extension.type === 'bundle' && extension.entries.some((entry) => isIn(entry.type, APP_OR_HYBRID_EXTENSION_TYPES))
+	);
+
+	const appOrHybridExtensionImports = APP_OR_HYBRID_EXTENSION_TYPES.flatMap((type) =>
+		appOrHybridExtensions
+			.filter((extension) => extension.type === type)
+			.map(
+				(extension, i) =>
+					`import ${type}${i} from './${pathToRelativeUrl(
+						path.resolve(
+							extension.path,
+							isTypeIn(extension, HYBRID_EXTENSION_TYPES) ? extension.entrypoint.app : extension.entrypoint
+						)
+					)}';`
+			)
+	);
+
+	const bundleExtensionImports = bundleExtensions.map(
+		(extension, i) =>
+			`import {${APP_OR_HYBRID_EXTENSION_TYPES.filter((type) => extension.entries.some((entry) => entry.type === type))
+				.map((type) => `${pluralize(type)} as ${type}Bundle${i}`)
+				.join(',')}} from './${pathToRelativeUrl(path.resolve(extension.path, extension.entrypoint.app))}';`
+	);
+
+	const extensionExports = APP_OR_HYBRID_EXTENSION_TYPES.map(
+		(type) =>
+			`export const ${pluralize(type)} = [${appOrHybridExtensions
+				.filter((extension) => extension.type === type)
+				.map((_, i) => `${type}${i}`)
+				.concat(
+					bundleExtensions
+						.map((extension, i) =>
+							extension.entries.some((entry) => entry.type === type) ? `...${type}Bundle${i}` : null
+						)
+						.filter((e): e is string => e !== null)
+				)
+				.join(',')}];`
+	);
+
+	return `${appOrHybridExtensionImports.join('')}${bundleExtensionImports.join('')}${extensionExports.join('')}`;
+}

packages/shared/src/utils/node/get-extensions.ts

@@ -9,12 +9,7 @@ import {
 } from '../../types';
 import { resolvePackage } from './resolve-package';
 import { listFolders } from './list-folders';
-import {
-	EXTENSION_NAME_REGEX,
-	EXTENSION_PKG_KEY,
-	HYBRID_EXTENSION_TYPES,
-	PACKAGE_EXTENSION_TYPES,
-} from '../../constants';
+import { EXTENSION_NAME_REGEX, EXTENSION_PKG_KEY, HYBRID_EXTENSION_TYPES } from '../../constants';
 import { pluralize } from '../pluralize';
 import { validateExtensionManifest } from '../validate-extension-manifest';
 import { isIn, isTypeIn } from '../array-helpers';
@@ -37,7 +32,7 @@ async function resolvePackageExtensions(
 		const extensionOptions = extensionManifest[EXTENSION_PKG_KEY];
 
 		if (isIn(extensionOptions.type, types)) {
-			if (isTypeIn(extensionOptions, PACKAGE_EXTENSION_TYPES)) {
+			if (extensionOptions.type === 'pack') {
 				const extensionChildren = Object.keys(extensionManifest.dependencies ?? {}).filter((dep) =>
 					EXTENSION_NAME_REGEX.test(dep)
 				);
@@ -54,6 +49,20 @@ async function resolvePackageExtensions(
 
 				extensions.push(extension);
 				extensions.push(...(await resolvePackageExtensions(extension.children || [], extension.path, types)));
+			} else if (extensionOptions.type === 'bundle') {
+				extensions.push({
+					path: extensionPath,
+					name: extensionName,
+					version: extensionManifest.version,
+					type: extensionOptions.type,
+					entrypoint: {
+						app: extensionOptions.path.app,
+						api: extensionOptions.path.api,
+					},
+					entries: extensionOptions.entries,
+					host: extensionOptions.host,
+					local: false,
+				});
 			} else if (isTypeIn(extensionOptions, HYBRID_EXTENSION_TYPES)) {
 				extensions.push({
 					path: extensionPath,

packages/shared/src/utils/node/index.ts

@@ -1,5 +1,5 @@
 export * from './ensure-extension-dirs';
-export * from './generate-extensions-entry';
+export * from './generate-extensions-entrypoint';
 export * from './get-extensions';
 export * from './list-folders';
 export * from './path-to-relative-url';

packages/shared/src/utils/validate-extension-manifest.test.ts

@@ -1,18 +1,22 @@
 import { describe, expect, it } from 'vitest';
-import { validateExtensionManifest } from './validate-extension-manifest';
+import { validateExtensionManifest, validateExtensionOptionsBundleEntry } from './validate-extension-manifest';
 
-describe('', () => {
+describe('validateExtensionManifest', () => {
 	it('returns false when passed item has no name or version', () => {
-		expect(validateExtensionManifest({})).toBe(false);
+		const mockExtension = {};
+
+		expect(validateExtensionManifest(mockExtension)).toBe(false);
 	});
 
 	it('returns false when passed item has no EXTENSION_PKG_KEY', () => {
 		const mockExtension = { name: 'test', version: '0.1' };
+
 		expect(validateExtensionManifest(mockExtension)).toBe(false);
 	});
 
 	it('returns false when passed item has no type', () => {
 		const mockExtension = { name: 'test', version: '0.1', 'directus:extension': {} };
+
 		expect(validateExtensionManifest(mockExtension)).toBe(false);
 	});
 
@@ -22,42 +26,47 @@ describe('', () => {
 			version: '0.1',
 			'directus:extension': { type: 'not_extension_type' },
 		};
+
 		expect(validateExtensionManifest(mockExtension)).toBe(false);
 	});
 
-	it('returns false when passed item has a package type and has no host', () => {
+	it('returns false when passed item has no host', () => {
 		const mockExtension = {
 			name: 'test',
 			version: '0.1',
 			'directus:extension': { type: 'pack' },
 		};
+
 		expect(validateExtensionManifest(mockExtension)).toBe(false);
 	});
 
-	it('returns false when passed item has a hybrid type and has no path, source, host or they have the wrong format', () => {
+	it('returns false when passed item has a type of bundle and has no path, entries or they have the wrong format', () => {
 		const mockExtension = {
 			name: 'test',
 			version: '0.1',
-			'directus:extension': { type: 'operation' },
+			'directus:extension': { type: 'bundle', host: '^9.0.0' },
 		};
+
 		expect(validateExtensionManifest(mockExtension)).toBe(false);
 	});
 
-	it('returns false when passed item has an App or API type and has no path, source or host', () => {
+	it('returns false when passed item has a Hybrid type and has no path, source or they have the wrong format', () => {
 		const mockExtension = {
 			name: 'test',
 			version: '0.1',
-			'directus:extension': { type: 'interface' },
+			'directus:extension': { type: 'operation', host: '^9.0.0' },
 		};
+
 		expect(validateExtensionManifest(mockExtension)).toBe(false);
 	});
 
-	it('returns false when passed item has a type other than pack and has no path, source or host', () => {
+	it('returns false when passed item has an App or API type and has no path or source', () => {
 		const mockExtension = {
 			name: 'test',
 			version: '0.1',
-			'directus:extension': { type: 'interface' },
+			'directus:extension': { type: 'interface', host: '^9.0.0' },
 		};
+
 		expect(validateExtensionManifest(mockExtension)).toBe(false);
 	});
 
@@ -67,10 +76,11 @@ describe('', () => {
 			version: '0.1',
 			'directus:extension': { type: 'interface', path: './dist/index.js', source: './src/index.js', host: '^9.0.0' },
 		};
+
 		expect(validateExtensionManifest(mockExtension)).toBe(true);
 	});
 
-	it('returns true when passed a valid ExtensionManifest with a hybrid type', () => {
+	it('returns true when passed a valid ExtensionManifest with a Hybrid type', () => {
 		const mockExtension = {
 			name: 'test',
 			version: '0.1',
@@ -81,15 +91,84 @@ describe('', () => {
 				host: '^9.0.0',
 			},
 		};
+
 		expect(validateExtensionManifest(mockExtension)).toBe(true);
 	});
 
-	it('returns true when passed a valid ExtensionManifest with a package type', () => {
+	it('returns true when passed a valid ExtensionManifest with a type of bundle', () => {
+		const mockExtension = {
+			name: 'test',
+			version: '0.1',
+			'directus:extension': {
+				type: 'bundle',
+				path: { app: './dist/app.js', api: './dist/api.js' },
+				entries: [
+					{ type: 'interface', name: 'test-interface', source: './src/test-interface/index.js' },
+					{ type: 'hook', name: 'test-hook', source: './src/test-hook/index.js' },
+					{
+						type: 'operation',
+						name: 'test-operation',
+						source: { app: './src/test-operation/app.js', api: './src/test-operation/api.js' },
+					},
+				],
+				host: '^9.0.0',
+			},
+		};
+
+		expect(validateExtensionManifest(mockExtension)).toBe(true);
+	});
+
+	it('returns true when passed a valid ExtensionManifest with a Package type', () => {
 		const mockExtension = {
 			name: 'test',
 			version: '0.1',
 			'directus:extension': { type: 'pack', host: '^9.0.0' },
 		};
+
 		expect(validateExtensionManifest(mockExtension)).toBe(true);
 	});
 });
+
+describe('validateExtensionOptionsBundleEntry', () => {
+	it('returns false when passed item has no type', () => {
+		const mockEntry = {};
+
+		expect(validateExtensionOptionsBundleEntry(mockEntry)).toBe(false);
+	});
+
+	it('returns false when passed item has an invalid type', () => {
+		const mockEntry = { type: 'bundle' };
+
+		expect(validateExtensionOptionsBundleEntry(mockEntry)).toBe(false);
+	});
+
+	it('returns false when passed item has no name', () => {
+		const mockEntry = { type: 'interface' };
+
+		expect(validateExtensionOptionsBundleEntry(mockEntry)).toBe(false);
+	});
+
+	it('returns false when passed item has a Hybrid type and has no source or it has the wrong format', () => {
+		const mockEntry = { type: 'operation', name: 'test', source: './src/index.js' };
+
+		expect(validateExtensionOptionsBundleEntry(mockEntry)).toBe(false);
+	});
+
+	it('returns false when passed item has an App or API type and has no source', () => {
+		const mockEntry = { type: 'interface', name: 'test' };
+
+		expect(validateExtensionOptionsBundleEntry(mockEntry)).toBe(false);
+	});
+
+	it('returns true when passed a valid ExtensionOptionsBundleEntry with an App or API type', () => {
+		const mockEntry = { type: 'interface', name: 'test', source: './src/index.js' };
+
+		expect(validateExtensionOptionsBundleEntry(mockEntry)).toBe(true);
+	});
+
+	it('returns true when passed a valid ExtensionOptionsBundleEntry with a Hybrid type', () => {
+		const mockEntry = { type: 'operation', name: 'test', source: { app: './src/app.js', api: './src/api.js' } };
+
+		expect(validateExtensionOptionsBundleEntry(mockEntry)).toBe(true);
+	});
+});

packages/shared/src/utils/validate-extension-manifest.ts

@@ -1,10 +1,16 @@
 import {
 	EXTENSION_PACKAGE_TYPES,
 	EXTENSION_PKG_KEY,
+	EXTENSION_TYPES,
 	HYBRID_EXTENSION_TYPES,
 	PACKAGE_EXTENSION_TYPES,
 } from '../constants';
-import { ExtensionManifest, ExtensionManifestRaw } from '../types';
+import {
+	ExtensionManifest,
+	ExtensionManifestRaw,
+	ExtensionOptionsBundleEntry,
+	ExtensionOptionsBundleEntryRaw,
+} from '../types';
 import { isIn } from './array-helpers';
 
 export function validateExtensionManifest(
@@ -16,38 +22,66 @@ export function validateExtensionManifest(
 
 	const extensionOptions = extensionManifest[EXTENSION_PKG_KEY];
 
-	if (!extensionOptions) {
-		return false;
-	}
-
-	if (!extensionOptions.type) {
-		return false;
-	}
-
-	if (!isIn(extensionOptions.type, EXTENSION_PACKAGE_TYPES)) {
+	if (
+		!extensionOptions ||
+		!extensionOptions.type ||
+		!isIn(extensionOptions.type, EXTENSION_PACKAGE_TYPES) ||
+		!extensionOptions.host
+	) {
 		return false;
 	}
 
 	if (isIn(extensionOptions.type, PACKAGE_EXTENSION_TYPES)) {
-		if (!extensionOptions.host) {
-			return false;
+		if (extensionOptions.type === 'bundle') {
+			if (
+				!extensionOptions.path ||
+				typeof extensionOptions.path === 'string' ||
+				!extensionOptions.path.app ||
+				!extensionOptions.path.api ||
+				!extensionOptions.entries ||
+				!Array.isArray(extensionOptions.entries) ||
+				!extensionOptions.entries.every((entry) => validateExtensionOptionsBundleEntry(entry))
+			) {
+				return false;
+			}
 		}
-	} else if (isIn(extensionOptions.type, HYBRID_EXTENSION_TYPES)) {
-		if (
-			!extensionOptions.path ||
-			!extensionOptions.source ||
-			typeof extensionOptions.path === 'string' ||
-			typeof extensionOptions.source === 'string' ||
-			!extensionOptions.path.app ||
-			!extensionOptions.path.api ||
-			!extensionOptions.source.app ||
-			!extensionOptions.source.api ||
-			!extensionOptions.host
-		) {
+	} else {
+		if (isIn(extensionOptions.type, HYBRID_EXTENSION_TYPES)) {
+			if (
+				!extensionOptions.path ||
+				!extensionOptions.source ||
+				typeof extensionOptions.path === 'string' ||
+				typeof extensionOptions.source === 'string' ||
+				!extensionOptions.path.app ||
+				!extensionOptions.path.api ||
+				!extensionOptions.source.app ||
+				!extensionOptions.source.api
+			) {
+				return false;
+			}
+		} else {
+			if (!extensionOptions.path || !extensionOptions.source) {
+				return false;
+			}
+		}
+	}
+
+	return true;
+}
+
+export function validateExtensionOptionsBundleEntry(
+	entry: ExtensionOptionsBundleEntryRaw
+): entry is ExtensionOptionsBundleEntry {
+	if (!entry.type || !isIn(entry.type, EXTENSION_TYPES) || !entry.name) {
+		return false;
+	}
+
+	if (isIn(entry.type, HYBRID_EXTENSION_TYPES)) {
+		if (!entry.source || typeof entry.source === 'string' || !entry.source.app || !entry.source.api) {
 			return false;
 		}
 	} else {
-		if (!extensionOptions.path || !extensionOptions.source || !extensionOptions.host) {
+		if (!entry.source) {
 			return false;
 		}
 	}