Fix extensions (#6377)

* Add support for npm extensions

* Allow extensions to import vue from the main app

* Bundle app extensions on server startup

* Fix return type of useLayoutState

* Add shared package

* Add extension-sdk package

* Add type declaration files to allow deep import of shared package

* Add extension loading to shared

* Refactor extension loading to use shared package

* Remove app bundle newline replacement

* Fix extension loading in development

* Rename extension entrypoints

* Update extension build instructions

* Remove vite auto-replacement workaround

* Update package-lock.json

* Remove newline from generated extension entrypoint

* Update package-lock.json

* Build shared package as cjs and esm

* Move useLayoutState composable to shared

* Reverse vite base env check

* Share useLayoutState composable through extension-sdk

* Update layout docs

* Update package versions

* Small cleanup

* Fix layout docs

* Fix imports

* Add nickrum to codeowners

* Fix typo

* Add 'em to vite config too

* Fix email

Co-authored-by: rijkvanzanten <rijkvanzanten@me.com>

api/package.json

@@ -76,8 +76,11 @@
 		"@directus/drive-s3": "9.0.0-rc.80",
 		"@directus/format-title": "9.0.0-rc.80",
 		"@directus/schema": "9.0.0-rc.80",
+    "@directus/shared": "9.0.0-rc.80",
 		"@directus/specs": "9.0.0-rc.80",
 		"@godaddy/terminus": "^4.9.0",
+		"@rollup/plugin-alias": "^3.1.2",
+		"@rollup/plugin-virtual": "^2.0.3",
 		"argon2": "^0.28.1",
 		"async": "^3.2.0",
 		"async-mutex": "^0.3.1",
@@ -132,6 +135,7 @@
 		"qs": "^6.9.4",
 		"rate-limiter-flexible": "^2.2.2",
 		"resolve-cwd": "^3.0.0",
+		"rollup": "^2.52.1",
 		"sharp": "^0.28.3",
 		"stream-json": "^1.7.1",
 		"uuid": "^8.3.2",

api/src/app.ts

@@ -56,7 +56,7 @@ export default async function createApp(): Promise<express.Application> {
 
 	await initializeExtensions();
 
-	await registerExtensionHooks();
+	registerExtensionHooks();
 
 	const app = express();
 
@@ -170,7 +170,7 @@ export default async function createApp(): Promise<express.Application> {
 
 	// Register custom hooks / endpoints
 	await emitAsyncSafe('routes.custom.init.before', { app });
-	await registerExtensionEndpoints(customRouter);
+	registerExtensionEndpoints(customRouter);
 	await emitAsyncSafe('routes.custom.init.after', { app });
 
 	app.use(notFoundHandler);

api/src/constants.ts

@@ -1,4 +1,4 @@
-import { Transformation } from './types/assets';
+import { Transformation } from './types';
 
 export const SYSTEM_ASSET_ALLOW_LIST: Transformation[] = [
 	{

api/src/controllers/extensions.ts

@@ -1,28 +1,24 @@
-import express, { Router } from 'express';
-import env from '../env';
-import { RouteNotFoundException } from '../exceptions';
-import { listExtensions } from '../extensions';
-import { respond } from '../middleware/respond';
+import { Router } from 'express';
 import asyncHandler from '../utils/async-handler';
+import { RouteNotFoundException } from '../exceptions';
+import { listExtensions, getAppExtensionSource } from '../extensions';
+import { respond } from '../middleware/respond';
+import { depluralize } from '@directus/shared/utils';
+import { AppExtensionType, Plural } from '@directus/shared/types';
+import { APP_EXTENSION_TYPES } from '@directus/shared/constants';
 
 const router = Router();
 
-const extensionsPath = env.EXTENSIONS_PATH as string;
-
-const appExtensions = ['interfaces', 'layouts', 'displays', 'modules'];
-
 router.get(
-	['/:type', '/:type/*'],
+	'/:type',
 	asyncHandler(async (req, res, next) => {
-		if (appExtensions.includes(req.params.type) === false) {
+		const type = depluralize(req.params.type as Plural<AppExtensionType>);
+
+		if (APP_EXTENSION_TYPES.includes(type) === false) {
 			throw new RouteNotFoundException(req.path);
 		}
 
-		return next();
-	}),
-	express.static(extensionsPath),
-	asyncHandler(async (req, res, next) => {
-		const extensions = await listExtensions(req.params.type);
+		const extensions = listExtensions(type);
 
 		res.locals.payload = {
 			data: extensions,
@@ -33,4 +29,23 @@ router.get(
 	respond
 );
 
+router.get(
+	'/:type/index.js',
+	asyncHandler(async (req, res) => {
+		const type = depluralize(req.params.type as Plural<AppExtensionType>);
+
+		if (APP_EXTENSION_TYPES.includes(type) === false) {
+			throw new RouteNotFoundException(req.path);
+		}
+
+		const extensionSource = getAppExtensionSource(type);
+		if (extensionSource === undefined) {
+			throw new RouteNotFoundException(req.path);
+		}
+
+		res.setHeader('Content-Type', 'application/javascript; charset=UTF-8');
+		res.end(extensionSource);
+	})
+);
+
 export default router;

api/src/extensions.ts

@@ -1,90 +1,138 @@
 import express, { Router } from 'express';
-import { ensureDir } from 'fs-extra';
 import path from 'path';
+import { AppExtensionType, Extension, ExtensionType } from '@directus/shared/types';
+import {
+	generateExtensionsEntry,
+	getLocalExtensions,
+	getPackageExtensions,
+	pluralize,
+	resolvePackage,
+} from '@directus/shared/utils';
+import { APP_EXTENSION_TYPES, EXTENSION_TYPES, SHARED_DEPS } from '@directus/shared/constants';
 import getDatabase from './database';
 import emitter from './emitter';
 import env from './env';
 import * as exceptions from './exceptions';
-import { ServiceUnavailableException } from './exceptions';
 import logger from './logger';
-import * as services from './services';
-import { EndpointRegisterFunction, HookRegisterFunction } from './types';
+import { HookRegisterFunction, EndpointRegisterFunction } from './types';
+import fse from 'fs-extra';
 import { getSchema } from './utils/get-schema';
-import listFolders from './utils/list-folders';
+
+import * as services from './services';
 import { schedule, validate } from 'node-cron';
+import { rollup } from 'rollup';
+// @TODO Remove this once a new version of @rollup/plugin-virtual has been released
+// @ts-expect-error
+import virtual from '@rollup/plugin-virtual';
+import alias from '@rollup/plugin-alias';
 
-export async function ensureFoldersExist(): Promise<void> {
-	const folders = ['endpoints', 'hooks', 'interfaces', 'modules', 'layouts', 'displays'];
+let extensions: Extension[] = [];
+let extensionBundles: Partial<Record<AppExtensionType, string>> = {};
 
-	for (const folder of folders) {
-		const folderPath = path.resolve(env.EXTENSIONS_PATH, folder);
+export async function initializeExtensions(): Promise<void> {
+	await ensureDirsExist();
+	extensions = await getExtensions();
+	extensionBundles = await generateExtensionBundles();
+
+	logger.info(`Loaded extensions: ${listExtensions().join(', ')}`);
+}
+
+export function listExtensions(type?: ExtensionType): string[] {
+	if (type === undefined) {
+		return extensions.map((extension) => extension.name);
+	} else {
+		return extensions.filter((extension) => extension.type === type).map((extension) => extension.name);
+	}
+}
+
+export function getAppExtensionSource(type: AppExtensionType): string | undefined {
+	return extensionBundles[type];
+}
+
+export function registerExtensionEndpoints(router: Router): void {
+	const endpoints = extensions.filter((extension) => extension.type === 'endpoint');
+	registerEndpoints(endpoints, router);
+}
+
+export function registerExtensionHooks(): void {
+	const hooks = extensions.filter((extension) => extension.type === 'hook');
+	registerHooks(hooks);
+}
+
+async function getExtensions(): Promise<Extension[]> {
+	const packageExtensions = await getPackageExtensions('.');
+	const localExtensions = await getLocalExtensions(env.EXTENSIONS_PATH);
+
+	return [...packageExtensions, ...localExtensions];
+}
+
+async function generateExtensionBundles() {
+	const sharedDepsMapping = await getSharedDepsMapping(SHARED_DEPS);
+	const internalImports = Object.entries(sharedDepsMapping).map(([name, path]) => ({
+		find: name,
+		replacement: path,
+	}));
+
+	const bundles: Partial<Record<AppExtensionType, string>> = {};
+
+	for (const extensionType of APP_EXTENSION_TYPES) {
+		const entry = generateExtensionsEntry(extensionType, extensions);
+
+		const bundle = await rollup({
+			input: 'entry',
+			external: SHARED_DEPS,
+			plugins: [virtual({ entry }), alias({ entries: internalImports })],
+		});
+		const { output } = await bundle.generate({ format: 'es' });
+
+		bundles[extensionType] = output[0].code;
+
+		await bundle.close();
+	}
+
+	return bundles;
+}
+
+async function ensureDirsExist() {
+	for (const extensionType of EXTENSION_TYPES) {
+		const dirPath = path.resolve(env.EXTENSIONS_PATH, pluralize(extensionType));
 		try {
-			await ensureDir(folderPath);
+			await fse.ensureDir(dirPath);
 		} catch (err) {
 			logger.warn(err);
 		}
 	}
 }
 
-export async function initializeExtensions(): Promise<void> {
-	await ensureFoldersExist();
-}
+async function getSharedDepsMapping(deps: string[]) {
+	const appDir = await fse.readdir(path.join(resolvePackage('@directus/app'), 'dist'));
 
-export async function listExtensions(type: string): Promise<string[]> {
-	const extensionsPath = env.EXTENSIONS_PATH as string;
-	const location = path.join(extensionsPath, type);
+	const depsMapping: Record<string, string> = {};
+	for (const dep of deps) {
+		const depName = appDir.find((file) => dep.replace(/\//g, '_') === file.substring(0, file.indexOf('.')));
 
-	try {
-		return await listFolders(location);
-	} catch (err) {
-		if (err.code === 'ENOENT') {
-			throw new ServiceUnavailableException(`Extension folder "extensions/${type}" couldn't be opened`, {
-				service: 'extensions',
-			});
+		if (depName) {
+			depsMapping[dep] = `${env.PUBLIC_URL}/admin/${depName}`;
+		} else {
+			logger.warn(`Couldn't find extension internal dependency "${dep}"`);
 		}
-		throw err;
 	}
+
+	return depsMapping;
 }
 
-export async function registerExtensions(router: Router): Promise<void> {
-	await registerExtensionHooks();
-	await registerExtensionEndpoints(router);
-}
-
-export async function registerExtensionEndpoints(router: Router): Promise<void> {
-	let endpoints: string[] = [];
-	try {
-		endpoints = await listExtensions('endpoints');
-		registerEndpoints(endpoints, router);
-	} catch (err) {
-		logger.warn(err);
-	}
-}
-
-export async function registerExtensionHooks(): Promise<void> {
-	let hooks: string[] = [];
-	try {
-		hooks = await listExtensions('hooks');
-		registerHooks(hooks);
-	} catch (err) {
-		logger.warn(err);
-	}
-}
-
-function registerHooks(hooks: string[]) {
-	const extensionsPath = env.EXTENSIONS_PATH as string;
-
+function registerHooks(hooks: Extension[]) {
 	for (const hook of hooks) {
 		try {
 			registerHook(hook);
 		} catch (error) {
-			logger.warn(`Couldn't register hook "${hook}"`);
+			logger.warn(`Couldn't register hook "${hook.name}"`);
 			logger.warn(error);
 		}
 	}
 
-	function registerHook(hook: string) {
-		const hookPath = path.resolve(extensionsPath, 'hooks', hook, 'index.js');
+	function registerHook(hook: Extension) {
+		const hookPath = path.resolve(hook.path, hook.entrypoint || '');
 		const hookInstance: HookRegisterFunction | { default?: HookRegisterFunction } = require(hookPath);
 
 		let register: HookRegisterFunction = hookInstance as HookRegisterFunction;
@@ -112,20 +160,18 @@ function registerHooks(hooks: string[]) {
 	}
 }
 
-function registerEndpoints(endpoints: string[], router: Router) {
-	const extensionsPath = env.EXTENSIONS_PATH as string;
-
+function registerEndpoints(endpoints: Extension[], router: Router) {
 	for (const endpoint of endpoints) {
 		try {
 			registerEndpoint(endpoint);
 		} catch (error) {
-			logger.warn(`Couldn't register endpoint "${endpoint}"`);
+			logger.warn(`Couldn't register endpoint "${endpoint.name}"`);
 			logger.warn(error);
 		}
 	}
 
-	function registerEndpoint(endpoint: string) {
-		const endpointPath = path.resolve(extensionsPath, 'endpoints', endpoint, 'index.js');
+	function registerEndpoint(endpoint: Extension) {
+		const endpointPath = path.resolve(endpoint.path, endpoint.entrypoint || '');
 		const endpointInstance: EndpointRegisterFunction | { default?: EndpointRegisterFunction } = require(endpointPath);
 
 		let register: EndpointRegisterFunction = endpointInstance as EndpointRegisterFunction;
@@ -136,7 +182,7 @@ function registerEndpoints(endpoints: string[], router: Router) {
 		}
 
 		const scopedRouter = express.Router();
-		router.use(`/${endpoint}/`, scopedRouter);
+		router.use(`/${endpoint.name}/`, scopedRouter);
 
 		register(scopedRouter, { services, exceptions, env, database: getDatabase(), getSchema });
 	}

api/src/services/graphql.ts

@@ -1277,10 +1277,10 @@ export class GraphQLService {
 					},
 				}),
 				resolve: async () => ({
-					interfaces: await listExtensions('interfaces'),
-					displays: await listExtensions('displays'),
-					layouts: await listExtensions('layouts'),
-					modules: await listExtensions('modules'),
+					interfaces: listExtensions('interface'),
+					displays: listExtensions('display'),
+					layouts: listExtensions('layout'),
+					modules: listExtensions('module'),
 				}),
 			},
 			server_specs_oas: {

api/src/utils/list-folders.ts

@@ -1,24 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { promisify } from 'util';
-
-const readdir = promisify(fs.readdir);
-const stat = promisify(fs.stat);
-
-export default async function listFolders(location: string): Promise<string[]> {
-	const fullPath = path.resolve(location);
-	const files = await readdir(fullPath);
-
-	const directories: string[] = [];
-
-	for (const file of files) {
-		const filePath = path.join(fullPath, file);
-		const stats = await stat(filePath);
-
-		if (stats.isDirectory()) {
-			directories.push(file);
-		}
-	}
-
-	return directories;
-}