Merge pull request #28 from directus/files

Files (crud) + assets

example.env

@@ -14,6 +14,23 @@ DB_NAME="directus"
 DB_USER="postgres"
 DB_PASSWORD="psql1234"
 
+####################################################################################################
+# File Storage
+
+STORAGE_LOCATIONS="finder, digitalocean"
+
+STORAGE_FINDER_PUBLIC_URL="http://localhost:3000/uploads"
+STORAGE_FINDER_DRIVER="local"
+STORAGE_FINDER_ROOT="./uploads"
+
+STORAGE_DIGITALOCEAN_PUBLIC_URL="https://cdn.example.com/"
+STORAGE_DIGITALOCEAN_DRIVER="s3"
+STORAGE_DIGITALOCEAN_KEY="abcdef"
+STORAGE_DIGITALOCEAN_SECRET="ghijkl"
+STORAGE_DIGITALOCEAN_ENDPOINT="ams3.digitaloceanspaces.com"
+STORAGE_DIGITALOCEAN_BUCKET="my-files"
+STORAGE_DIGITALOCEAN_REGION="ams3"
+
 ####################################################################################################
 # Auth
 
@@ -25,7 +42,7 @@ SALT_ROUNDS=10
 ####################################################################################################
 # SSO (oAuth) Providers
 
-OAUTH_PROVIDERS="github,facebook"
+OAUTH_PROVIDERS="github, facebook"
 
 OAUTH_GITHUB_KEY="abcdef"
 OAUTH_GITHUB_SECRET="ghijkl"

package.json

@@ -5,7 +5,7 @@
   "main": "dist/server.js",
   "scripts": {
     "start": "NODE_ENV=production node dist/server.js",
-    "build": "rimraf dist && tsc && copyfiles \"src/**/*.*\" -e \"src/**/*.ts\" -u 1 dist",
+    "build": "rimraf dist && tsc -b && copyfiles \"src/**/*.*\" -e \"src/**/*.ts\" -u 1 dist",
     "dev": "LOG_LEVEL=trace ts-node-dev src/server.ts --clear --watch \"src/**/*.ts\" --rs --transpile-only | pino-colada"
   },
   "repository": {
@@ -30,6 +30,7 @@
   "devDependencies": {
     "@types/atob": "^2.1.2",
     "@types/bcrypt": "^3.0.0",
+    "@types/busboy": "^0.2.3",
     "@types/express": "^4.17.6",
     "@types/express-pino-logger": "^4.0.2",
     "@types/express-session": "^1.17.0",
@@ -38,6 +39,7 @@
     "@types/lodash": "^4.14.156",
     "@types/nodemailer": "^6.4.0",
     "@types/pino": "^6.3.0",
+    "@types/sharp": "^0.25.0",
     "@types/uuid": "^8.0.0",
     "copyfiles": "^2.3.0",
     "eslint": "^7.3.1",
@@ -63,9 +65,14 @@
   },
   "dependencies": {
     "@hapi/joi": "^17.1.1",
+    "@slynova/flydrive": "^1.0.1",
+    "@slynova/flydrive-gcs": "^1.0.1",
+    "@slynova/flydrive-s3": "^1.0.1",
     "atob": "^2.1.2",
     "bcrypt": "^5.0.0",
     "body-parser": "^1.19.0",
+    "busboy": "^0.3.1",
+    "camelcase": "^6.0.0",
     "dotenv": "^8.2.0",
     "express": "^4.17.1",
     "express-async-handler": "^1.1.4",
@@ -84,6 +91,7 @@
     "oracledb": "^4.2.0",
     "pg": "^8.2.1",
     "pino": "^6.3.2",
+    "sharp": "^0.25.4",
     "sqlite3": "^4.2.0",
     "ts-node-dev": "^1.0.0-pre.49",
     "uuid": "^8.2.0"

src/app.ts

@@ -11,6 +11,7 @@ import extractToken from './middleware/extract-token';
 import authenticate from './middleware/authenticate';
 
 import activityRouter from './routes/activity';
+import assetsRouter from './routes/assets';
 import authRouter from './routes/auth';
 import collectionPresetsRouter from './routes/collection-presets';
 import extensionsRouter from './routes/extensions';
@@ -35,6 +36,7 @@ const app = express()
 	.use(extractToken)
 	.use(authenticate)
 	.use('/activity', activityRouter)
+	.use('/assets', assetsRouter)
 	.use('/auth', authRouter)
 	.use('/collection_presets', collectionPresetsRouter)
 	.use('/extensions', extensionsRouter)

src/middleware/validate-singleton.ts

@@ -10,7 +10,11 @@ import APIError, { ErrorCode } from '../error';
 const validateCollection: RequestHandler = asyncHandler(async (req, res, next) => {
 	if (!req.collection) return next();
 
-	const collectionInfo = await req.loaders.collections.load(req.collection);
+	const collectionInfo = await database
+		.select('single')
+		.from('directus_collections')
+		.where({ collection: req.collection })
+		.first();
 
 	if (!collectionInfo) return next();
 

src/routes/assets.ts

@@ -0,0 +1,64 @@
+import { Router } from 'express';
+import asyncHandler from 'express-async-handler';
+import storage from '../storage';
+import database from '../database';
+import sharp, { ResizeOptions } from 'sharp';
+
+const router = Router();
+
+router.get(
+	'/:pk',
+	asyncHandler(async (req, res) => {
+		const file = await database
+			.select('type', 'storage', 'filename_disk', 'filename_download')
+			.from('directus_files')
+			.where({ id: req.params.pk })
+			.first();
+
+		res.setHeader('Content-Disposition', 'attachment; filename=' + file.filename_download);
+		res.setHeader('Content-Type', file.type);
+
+		const resizeOptions: ResizeOptions = {};
+
+		if (req.query.w) {
+			resizeOptions.width = Number(req.query.w);
+		}
+
+		if (req.query.h) {
+			resizeOptions.height = Number(req.query.h);
+		}
+
+		if (req.query.f) {
+			resizeOptions.fit = req.query.f as ResizeOptions['fit'];
+		}
+
+		const assetFilename = file.filename_disk + getAssetSuffix(resizeOptions);
+
+		const { exists } = await storage.disk(file.storage).exists(assetFilename);
+
+		if (exists) {
+			return storage.disk(file.storage).getStream(assetFilename).pipe(res);
+		}
+
+		const readStream = storage.disk(file.storage).getStream(file.filename_disk);
+		const transformer = sharp().resize(resizeOptions);
+
+		await storage.disk(file.storage).put(assetFilename, readStream.pipe(transformer));
+
+		return storage.disk(file.storage).getStream(assetFilename).pipe(res);
+	})
+);
+
+export default router;
+
+function getAssetSuffix(resizeOptions: ResizeOptions) {
+	if (Object.keys(resizeOptions).length === 0) return '';
+
+	return (
+		'__' +
+		Object.entries(resizeOptions)
+			.sort((a, b) => (a[0] > b[0] ? 1 : -1))
+			.map((e) => e.join('_'))
+			.join(',')
+	);
+}

src/routes/files.ts

@@ -1,22 +1,74 @@
 import express from 'express';
 import asyncHandler from 'express-async-handler';
+import Busboy from 'busboy';
 import sanitizeQuery from '../middleware/sanitize-query';
 import validateQuery from '../middleware/validate-query';
 import * as FilesService from '../services/files';
+import logger from '../logger';
 
 const router = express.Router();
 
-/** @TODO This needs to support multipart form-data for file uploads */
-// router.post(
-// 	'/',
-// 	asyncHandler(async (req, res) => {
-// 		const records = await FilesService.createFile(
-// 			req.body,
-// 			res.locals.query
-// 		);
-// 		return res.json({ data: records });
-// 	})
-// );
+const multipartHandler = (operation: 'create' | 'update') =>
+	asyncHandler(async (req, res, next) => {
+		const busboy = new Busboy({ headers: req.headers });
+
+		/**
+		 * The order of the fields in multipart/form-data is important. We require that all fields
+		 * are provided _before_ the files. This allows us to set the storage location, and create
+		 * the row in directus_files async during the upload of the actual file.
+		 */
+
+		let disk: string;
+		let payload: Record<string, any> = {};
+
+		busboy.on('field', (fieldname, val) => {
+			if (fieldname === 'storage') {
+				disk = val;
+			}
+
+			payload[fieldname] = val;
+		});
+
+		busboy.on('file', async (fieldname, fileStream, filename, encoding, mimetype) => {
+			if (!disk) {
+				// @todo error
+				return busboy.emit('error', new Error('no storage provided'));
+			}
+
+			payload = {
+				...payload,
+				filename_disk: filename,
+				filename_download: filename,
+				type: mimetype,
+			};
+
+			fileStream.on('end', () => {
+				logger.info(`File ${filename} uploaded to ${disk}.`);
+			});
+
+			try {
+				if (operation === 'create') {
+					await FilesService.createFile(payload, fileStream);
+				} else {
+					await FilesService.updateFile(req.params.pk, payload, fileStream);
+				}
+			} catch (err) {
+				busboy.emit('error', err);
+			}
+		});
+
+		busboy.on('error', (error: Error) => {
+			next(error);
+		});
+
+		busboy.on('finish', () => {
+			res.status(200).end();
+		});
+
+		return req.pipe(busboy);
+	});
+
+router.post('/', multipartHandler('create'));
 
 router.get(
 	'/',
@@ -40,9 +92,14 @@ router.get(
 
 router.patch(
 	'/:pk',
-	asyncHandler(async (req, res) => {
-		const records = await FilesService.updateFile(req.params.pk, req.body, res.locals.query);
-		return res.json({ data: records });
+	asyncHandler(async (req, res, next) => {
+		if (req.is('multipart/form-data')) {
+			await multipartHandler('update')(req, res, next);
+		} else {
+			await FilesService.updateFile(req.params.pk, req.body);
+		}
+
+		return res.status(200).end();
 	})
 );
 

src/services/files.ts

@@ -1,10 +1,22 @@
 import { Query } from '../types/query';
 import * as ItemsService from './items';
+import storage from '../storage';
+import * as PayloadService from './payload';
+import database from '../database';
+import logger from '../logger';
 
-/** @TODO This is a little more involved ofc, circling back later */
-// export const createFile = async (data: Record<string, any>, query: Query) => {
-// 	return await ItemsService.createItem('directus_files', data, query);
-// };
+export const createFile = async (
+	data: Record<string, any>,
+	stream: NodeJS.ReadableStream,
+	query?: Query
+) => {
+	const payload = await PayloadService.processValues('create', 'directus_files', data);
+
+	await ItemsService.createItem('directus_files', payload, query);
+
+	// @todo type of stream in flydrive is wrong: https://github.com/Slynova-Org/flydrive/issues/145
+	await storage.disk(data.storage).put(data.filename_disk, stream as any);
+};
 
 export const readFiles = async (query: Query) => {
 	return await ItemsService.readItems('directus_files', query);
@@ -14,10 +26,40 @@ export const readFile = async (pk: string | number, query: Query) => {
 	return await ItemsService.readItem('directus_files', pk, query);
 };
 
-export const updateFile = async (pk: string | number, data: Record<string, any>, query: Query) => {
-	return await ItemsService.updateItem('directus_files', pk, data, query);
+// @todo Add query support
+export const updateFile = async (
+	pk: string | number,
+	data: Record<string, any>,
+	stream?: NodeJS.ReadableStream,
+	query?: Query
+) => {
+	const payload = await PayloadService.processValues('update', 'directus_files', data);
+	await ItemsService.updateItem('directus_files', pk, payload, query);
+
+	/**
+	 * @TODO
+	 * Handle changes in storage adapter -> going from local to S3 needs to delete from one, upload to the other
+	 */
+
+	if (stream) {
+		const file = await database
+			.select('storage', 'filename_disk')
+			.from('directus_files')
+			.where({ id: pk })
+			.first();
+
+		// @todo type of stream in flydrive is wrong: https://github.com/Slynova-Org/flydrive/issues/145
+		await storage.disk(file.storage).put(file.filename_disk, stream as any);
+	}
 };
 
 export const deleteFile = async (pk: string | number) => {
-	await ItemsService.deleteItem('directus_files', pk);
+	const file = await database
+		.select('storage', 'filename_disk')
+		.from('directus_files')
+		.where({ id: pk })
+		.first();
+	const { wasDeleted } = await storage.disk(file.storage).delete(file.filename_disk);
+	logger.info(`File ${file.filename_download} deleted: ${wasDeleted}`);
+	await database.delete().from('directus_files').where({ id: pk });
 };

src/storage.ts

@@ -0,0 +1,67 @@
+import {
+	StorageManager,
+	LocalFileSystemStorage,
+	StorageManagerConfig,
+	Storage,
+} from '@slynova/flydrive';
+import camelcase from 'camelcase';
+
+import { AmazonWebServicesS3Storage } from '@slynova/flydrive-s3';
+import { GoogleCloudStorage } from '@slynova/flydrive-gcs';
+
+/** @todo dynamically load storage adapters here */
+
+const storage = new StorageManager(getStorageConfig());
+
+registerDrivers(storage);
+
+export default storage;
+
+function getStorageConfig(): StorageManagerConfig {
+	const config: any = { disks: {} };
+
+	for (const [key, value] of Object.entries(process.env)) {
+		if (key.startsWith('STORAGE') === false) continue;
+		if (key === 'STORAGE_LOCATIONS') continue;
+		if (key.endsWith('PUBLIC_URL')) continue;
+
+		const disk = key.split('_')[1].toLowerCase();
+		if (!config.disks[disk]) config.disks[disk] = { config: {} };
+
+		if (key.endsWith('DRIVER')) {
+			config.disks[disk].driver = value;
+			continue;
+		}
+
+		const configKey = camelcase(
+			key.split('_').filter((_, index) => [0, 1].includes(index) === false)
+		);
+		config.disks[disk].config[configKey] = value;
+	}
+
+	return config;
+}
+
+function registerDrivers(storage: StorageManager) {
+	const usedDrivers: string[] = [];
+
+	for (const [key, value] of Object.entries(process.env)) {
+		if ((key.startsWith('STORAGE') && key.endsWith('DRIVER')) === false) continue;
+		if (usedDrivers.includes(value) === false) usedDrivers.push(value);
+	}
+
+	usedDrivers.forEach((driver) => {
+		storage.registerDriver<Storage>(driver, getStorageDriver(driver));
+	});
+}
+
+function getStorageDriver(driver: string) {
+	switch (driver) {
+		case 'local':
+			return LocalFileSystemStorage;
+		case 's3':
+			return AmazonWebServicesS3Storage;
+		case 'gcs':
+			return GoogleCloudStorage;
+	}
+}

src/types/express.d.ts

@@ -2,7 +2,7 @@
  * Custom properties on the req object in express
  */
 
-import createSystemLoaders from '../loaders';
+export {};
 
 declare global {
 	namespace Express {
@@ -11,7 +11,6 @@ declare global {
 			user?: string;
 			role?: string;
 			collection?: string;
-			loaders?: ReturnType<typeof createSystemLoaders>;
 		}
 	}
 }