Support root option in S3/Azure/GCS (#4364)

* Support `root` option in S3

* Ignore GCS key files for local testing

* Default root to empty string

* Add dev watchers to drive-* packages

* Add file rootpath to Azure

api/.gitignore

@@ -9,3 +9,5 @@ debug.db
 test
 dist
 tmp
+keys.json
+

packages/drive-azure/package.json

@@ -18,7 +18,19 @@
 		"directus"
 	],
 	"scripts": {
-		"build": "tsc -b tsconfig.json"
+		"build": "tsc -b tsconfig.json",
+		"dev": "npm-watch build"
+	},
+	"watch": {
+		"build": {
+			"patterns": [
+				"src/*"
+			],
+			"ignore": "dist",
+			"extensions": "ts",
+			"silent": true,
+			"quiet": true
+		}
 	},
 	"author": "Robin Grundvåg <robgru52@gmail.com>",
 	"contributors": [

packages/drive-azure/src/AzureBlobWebServices.ts

@@ -22,6 +22,8 @@ import {
 	ContainerSASPermissions,
 } from '@azure/storage-blob';
 
+import path from 'path';
+
 import { PassThrough, Readable } from 'stream';
 
 function handleError(err: Error, path: string): Error {
@@ -32,6 +34,7 @@ export class AzureBlobWebServicesStorage extends Storage {
 	protected $client: BlobServiceClient;
 	protected $containerClient: ContainerClient;
 	protected $signedCredentials: StorageSharedKeyCredential;
+	protected $root: string;
 
 	constructor(config: AzureBlobWebServicesStorageConfig) {
 		super();
@@ -42,9 +45,20 @@ export class AzureBlobWebServicesStorage extends Storage {
 			this.$signedCredentials
 		);
 		this.$containerClient = this.$client.getContainerClient(config.containerName);
+		this.$root = config.root ?? '';
+	}
+
+	/**
+	 * Prefixes the given filePath with the storage root location
+	 */
+	protected _fullPath(filePath: string) {
+		return path.join(this.$root, filePath);
 	}
 
 	public async copy(src: string, dest: string): Promise<Response> {
+		src = this._fullPath(src);
+		dest = this._fullPath(dest);
+
 		try {
 			const source = this.$containerClient.getBlockBlobClient(src);
 			const target = this.$containerClient.getBlockBlobClient(dest);
@@ -59,6 +73,8 @@ export class AzureBlobWebServicesStorage extends Storage {
 	}
 
 	public async delete(location: string): Promise<DeleteResponse> {
+		location = this._fullPath(location);
+
 		try {
 			const result = await this.$containerClient.getBlockBlobClient(location).deleteIfExists();
 			return { raw: result, wasDeleted: result.succeeded };
@@ -72,6 +88,8 @@ export class AzureBlobWebServicesStorage extends Storage {
 	}
 
 	public async exists(location: string): Promise<ExistsResponse> {
+		location = this._fullPath(location);
+
 		try {
 			const result = await this.$containerClient.getBlockBlobClient(location).exists();
 			return { exists: result, raw: result };
@@ -81,6 +99,8 @@ export class AzureBlobWebServicesStorage extends Storage {
 	}
 
 	public async get(location: string, encoding: BufferEncoding = 'utf-8'): Promise<ContentResponse<string>> {
+		location = this._fullPath(location);
+
 		try {
 			const bufferResult = await this.getBuffer(location);
 			return {
@@ -93,6 +113,8 @@ export class AzureBlobWebServicesStorage extends Storage {
 	}
 
 	public async getBuffer(location: string): Promise<ContentResponse<Buffer>> {
+		location = this._fullPath(location);
+
 		try {
 			const client = this.$containerClient.getBlobClient(location);
 			return { content: await client.downloadToBuffer(), raw: client };
@@ -102,6 +124,8 @@ export class AzureBlobWebServicesStorage extends Storage {
 	}
 
 	public async getSignedUrl(location: string, options: SignedUrlOptions = {}): Promise<SignedUrlResponse> {
+		location = this._fullPath(location);
+
 		const { expiry = 900 } = options;
 
 		try {
@@ -125,6 +149,8 @@ export class AzureBlobWebServicesStorage extends Storage {
 	}
 
 	public async getStat(location: string): Promise<StatResponse> {
+		location = this._fullPath(location);
+
 		try {
 			const props = await this.$containerClient.getBlobClient(location).getProperties();
 			return {
@@ -138,6 +164,8 @@ export class AzureBlobWebServicesStorage extends Storage {
 	}
 
 	public getStream(location: string, range?: Range): NodeJS.ReadableStream {
+		location = this._fullPath(location);
+
 		const intermediateStream = new PassThrough({ highWaterMark: 1 });
 
 		const stream = this.$containerClient
@@ -165,10 +193,15 @@ export class AzureBlobWebServicesStorage extends Storage {
 	}
 
 	public getUrl(location: string): string {
+		location = this._fullPath(location);
+
 		return this.$containerClient.getBlobClient(location).url;
 	}
 
 	public async move(src: string, dest: string): Promise<Response> {
+		src = this._fullPath(src);
+		dest = this._fullPath(dest);
+
 		const source = this.$containerClient.getBlockBlobClient(src);
 		const target = this.$containerClient.getBlockBlobClient(dest);
 
@@ -181,7 +214,10 @@ export class AzureBlobWebServicesStorage extends Storage {
 	}
 
 	public async put(location: string, content: Buffer | NodeJS.ReadableStream | string): Promise<Response> {
+		location = this._fullPath(location);
+
 		const blockBlobClient = this.$containerClient.getBlockBlobClient(location);
+
 		try {
 			if (isReadableStream(content)) {
 				const result = await blockBlobClient.uploadStream(content as Readable);
@@ -215,4 +251,5 @@ export interface AzureBlobWebServicesStorageConfig {
 	containerName: string;
 	accountName: string;
 	accountKey: string;
+	root?: string;
 }

packages/drive-gcs/package.json

@@ -24,7 +24,19 @@
 		"dist"
 	],
 	"scripts": {
-		"build": "tsc -b tsconfig.json"
+		"build": "tsc -b tsconfig.json",
+		"dev": "npm-watch build"
+	},
+	"watch": {
+		"build": {
+			"patterns": [
+				"src/*"
+			],
+			"ignore": "dist",
+			"extensions": "ts",
+			"silent": true,
+			"quiet": true
+		}
 	},
 	"dependencies": {
 		"@google-cloud/storage": "^5.0.0"

packages/drive-gcs/src/GoogleCloudStorage.ts

@@ -26,6 +26,8 @@ import {
 	Range,
 } from '@directus/drive';
 
+import path from 'path';
+
 function handleError(err: Error & { code?: number | string }, path: string): Error {
 	switch (err.code) {
 		case 401:
@@ -45,6 +47,7 @@ export class GoogleCloudStorage extends Storage {
 	protected $config: GoogleCloudStorageConfig;
 	protected $driver: GCSDriver;
 	protected $bucket: Bucket;
+	protected $root: string;
 
 	public constructor(config: GoogleCloudStorageConfig) {
 		super();
@@ -52,10 +55,18 @@ export class GoogleCloudStorage extends Storage {
 		const GCSStorage = require('@google-cloud/storage').Storage;
 		this.$driver = new GCSStorage(config);
 		this.$bucket = this.$driver.bucket(config.bucket);
+		this.$root = config.root ?? '';
 	}
 
-	private _file(path: string): File {
-		return this.$bucket.file(path);
+	/**
+	 * Prefixes the given filePath with the storage root location
+	 */
+	protected _fullPath(filePath: string) {
+		return path.join(this.$root, filePath);
+	}
+
+	private _file(filePath: string): File {
+		return this.$bucket.file(this._fullPath(filePath));
 	}
 
 	/**
@@ -180,7 +191,7 @@ export class GoogleCloudStorage extends Storage {
 	 * status.
 	 */
 	public getUrl(location: string): string {
-		return `https://storage.googleapis.com/${this.$bucket.name}/${location}`;
+		return `https://storage.googleapis.com/${this.$bucket.name}/${this._fullPath(location)}`;
 	}
 
 	/**
@@ -249,4 +260,5 @@ export class GoogleCloudStorage extends Storage {
 
 export interface GoogleCloudStorageConfig extends StorageOptions {
 	bucket: string;
+	root: string;
 }

packages/drive-s3/package.json

@@ -25,7 +25,19 @@
 		"dist"
 	],
 	"scripts": {
-		"build": "tsc -b tsconfig.json"
+		"build": "tsc -b tsconfig.json",
+		"dev": "npm-watch build"
+	},
+	"watch": {
+		"build": {
+			"patterns": [
+				"src/*"
+			],
+			"ignore": "dist",
+			"extensions": "ts",
+			"silent": true,
+			"quiet": true
+		}
 	},
 	"dependencies": {
 		"aws-sdk": "^2.680.0"

packages/drive-s3/src/AmazonWebServicesS3Storage.ts

@@ -15,6 +15,7 @@ import {
 	DeleteResponse,
 	Range,
 } from '@directus/drive';
+import path from 'path';
 
 function handleError(err: Error, path: string, bucket: string): Error {
 	switch (err.name) {
@@ -32,6 +33,7 @@ function handleError(err: Error, path: string, bucket: string): Error {
 export class AmazonWebServicesS3Storage extends Storage {
 	protected $driver: S3;
 	protected $bucket: string;
+	protected $root: string;
 
 	constructor(config: AmazonWebServicesS3StorageConfig) {
 		super();
@@ -45,12 +47,23 @@ export class AmazonWebServicesS3Storage extends Storage {
 		});
 
 		this.$bucket = config.bucket;
+		this.$root = config.root ?? '';
+	}
+
+	/**
+	 * Prefixes the given filePath with the storage root location
+	 */
+	protected _fullPath(filePath: string) {
+		return path.join(this.$root, filePath);
 	}
 
 	/**
 	 * Copy a file to a location.
 	 */
 	public async copy(src: string, dest: string): Promise<Response> {
+		src = this._fullPath(src);
+		dest = this._fullPath(src);
+
 		const params = {
 			Key: dest,
 			Bucket: this.$bucket,
@@ -69,6 +82,8 @@ export class AmazonWebServicesS3Storage extends Storage {
 	 * Delete existing file.
 	 */
 	public async delete(location: string): Promise<DeleteResponse> {
+		location = this._fullPath(location);
+
 		const params = { Key: location, Bucket: this.$bucket };
 
 		try {
@@ -91,6 +106,8 @@ export class AmazonWebServicesS3Storage extends Storage {
 	 * Determines if a file or folder already exists.
 	 */
 	public async exists(location: string): Promise<ExistsResponse> {
+		location = this._fullPath(location);
+
 		const params = { Key: location, Bucket: this.$bucket };
 
 		try {
@@ -109,7 +126,10 @@ export class AmazonWebServicesS3Storage extends Storage {
 	 * Returns the file contents.
 	 */
 	public async get(location: string, encoding: BufferEncoding = 'utf-8'): Promise<ContentResponse<string>> {
+		location = this._fullPath(location);
+
 		const bufferResult = await this.getBuffer(location);
+
 		return {
 			content: bufferResult.content.toString(encoding),
 			raw: bufferResult.raw,
@@ -120,6 +140,8 @@ export class AmazonWebServicesS3Storage extends Storage {
 	 * Returns the file contents as Buffer.
 	 */
 	public async getBuffer(location: string): Promise<ContentResponse<Buffer>> {
+		location = this._fullPath(location);
+
 		const params = { Key: location, Bucket: this.$bucket };
 
 		try {
@@ -138,6 +160,8 @@ export class AmazonWebServicesS3Storage extends Storage {
 	 * Returns signed url for an existing file
 	 */
 	public async getSignedUrl(location: string, options: SignedUrlOptions = {}): Promise<SignedUrlResponse> {
+		location = this._fullPath(location);
+
 		const { expiry = 900 } = options;
 
 		try {
@@ -158,6 +182,8 @@ export class AmazonWebServicesS3Storage extends Storage {
 	 * Returns file's size and modification date.
 	 */
 	public async getStat(location: string): Promise<StatResponse> {
+		location = this._fullPath(location);
+
 		const params = { Key: location, Bucket: this.$bucket };
 
 		try {
@@ -176,11 +202,14 @@ export class AmazonWebServicesS3Storage extends Storage {
 	 * Returns the stream for the given file.
 	 */
 	public getStream(location: string, range?: Range): NodeJS.ReadableStream {
+		location = this._fullPath(location);
+
 		const params: S3.GetObjectRequest = {
 			Key: location,
 			Bucket: this.$bucket,
 			Range: range ? `${range.start}-${range.end || ''}` : undefined,
 		};
+
 		return this.$driver.getObject(params).createReadStream();
 	}
 
@@ -188,6 +217,8 @@ export class AmazonWebServicesS3Storage extends Storage {
 	 * Returns url for a given key.
 	 */
 	public getUrl(location: string): string {
+		location = this._fullPath(location);
+
 		const { href } = this.$driver.endpoint;
 
 		if (href.startsWith('https://s3.amazonaws')) {
@@ -203,6 +234,9 @@ export class AmazonWebServicesS3Storage extends Storage {
 	 * the hood.
 	 */
 	public async move(src: string, dest: string): Promise<Response> {
+		src = this._fullPath(src);
+		dest = this._fullPath(dest);
+
 		await this.copy(src, dest);
 		await this.delete(src);
 		return { raw: undefined };
@@ -213,7 +247,10 @@ export class AmazonWebServicesS3Storage extends Storage {
 	 * This method will create missing directories on the fly.
 	 */
 	public async put(location: string, content: Buffer | NodeJS.ReadableStream | string): Promise<Response> {
+		location = this._fullPath(location);
+
 		const params = { Key: location, Body: content, Bucket: this.$bucket };
+
 		try {
 			const result = await this.$driver.upload(params).promise();
 			return { raw: result };
@@ -258,4 +295,5 @@ export interface AmazonWebServicesS3StorageConfig extends ClientConfiguration {
 	key: string;
 	secret: string;
 	bucket: string;
+	root?: string;
 }