Cleanup request handler

Squashed commit of the following:

commit 90368698c8
Author: rijkvanzanten <rijkvanzanten@me.com>
Date:   Mon Feb 13 15:49:12 2023 -0500

    Cleanup

commit 61514f4509
Author: rijkvanzanten <rijkvanzanten@me.com>
Date:   Mon Feb 13 15:44:15 2023 -0500

    Rename to index

commit 38fe6b84fa
Author: rijkvanzanten <rijkvanzanten@me.com>
Date:   Mon Feb 13 15:41:23 2023 -0500

    Test coverage 100%

commit f2e36db95e
Author: rijkvanzanten <rijkvanzanten@me.com>
Date:   Mon Feb 13 15:22:45 2023 -0500

    Split up handler from validator

commit 800ac1968a
Author: Pascal Jufer <pascal-jufer@bluewin.ch>
Date:   Mon Feb 13 20:44:48 2023 +0100

    Use shared axios instance with URL check for outgoing requests

api/src/env.ts

@@ -257,7 +257,7 @@ const defaults: Record<string, any> = {
 	IP_TRUST_PROXY: true,
 	IP_CUSTOM_HEADER: false,
 
-	IMPORT_IP_DENY_LIST: '0.0.0.0',
+	IMPORT_IP_DENY_LIST: ['0.0.0.0', '169.254.169.254'],
 
 	SERVE_APP: true,
 

api/src/operations/request/index.test.ts

@@ -2,13 +2,14 @@ import { afterEach, expect, test, vi } from 'vitest';
 
 const axiosDefault = vi.fn();
 
-vi.mock('axios', () => ({
-	default: axiosDefault.mockResolvedValue({
-		status: 200,
-		statusText: 'OK',
-		headers: {},
-		data: {},
-	}),
+vi.mock('../../request', () => ({
+	getAxios: () =>
+		axiosDefault.mockResolvedValue({
+			status: 200,
+			statusText: 'OK',
+			headers: {},
+			data: {},
+		}),
 }));
 
 const url = '/';

api/src/operations/request/index.ts

@@ -1,5 +1,6 @@
 import { defineOperationApi, parseJSON } from '@directus/shared/utils';
 import encodeUrl from 'encodeurl';
+import { getAxios } from '../../request/index';
 
 type Options = {
 	url: string;
@@ -12,8 +13,6 @@ export default defineOperationApi<Options>({
 	id: 'request',
 
 	handler: async ({ url, method, body, headers }) => {
-		const axios = (await import('axios')).default;
-
 		const customHeaders =
 			headers?.reduce((acc, { header, value }) => {
 				acc[header] = value;
@@ -24,6 +23,7 @@ export default defineOperationApi<Options>({
 			customHeaders['Content-Type'] = 'application/json';
 		}
 
+		const axios = await getAxios();
 		const result = await axios({
 			url: encodeUrl(url),
 			method,

api/src/request/index.test.ts

@@ -0,0 +1,31 @@
+import { test, vi, afterEach, beforeEach, expect } from 'vitest';
+import { getAxios, _cache } from './index';
+import axios from 'axios';
+import type { AxiosInstance } from 'axios';
+
+vi.mock('axios');
+
+let mockAxiosInstance: AxiosInstance;
+
+beforeEach(() => {
+	mockAxiosInstance = {
+		interceptors: {
+			response: {
+				use: vi.fn(),
+			},
+		},
+	} as unknown as AxiosInstance;
+
+	vi.mocked(axios.create).mockReturnValue(mockAxiosInstance);
+});
+
+afterEach(() => {
+	vi.resetAllMocks();
+	_cache.axiosInstance = null;
+});
+
+test('Creates and returns new axios instance if cache is empty', async () => {
+	const instance = await getAxios();
+	expect(axios.create).toHaveBeenCalled();
+	expect(instance).toBe(mockAxiosInstance);
+});

api/src/request/index.ts

@@ -0,0 +1,16 @@
+import type { AxiosInstance } from 'axios';
+import { responseInterceptor } from './response-interceptor';
+
+export const _cache: { axiosInstance: AxiosInstance | null } = {
+	axiosInstance: null,
+};
+
+export async function getAxios() {
+	if (!_cache.axiosInstance) {
+		const axios = (await import('axios')).default;
+		_cache.axiosInstance = axios.create();
+		_cache.axiosInstance.interceptors.response.use(responseInterceptor);
+	}
+
+	return _cache.axiosInstance;
+}

api/src/request/response-interceptor.test.ts

@@ -0,0 +1,44 @@
+import { randIp, randUrl } from '@ngneat/falso';
+import type { AxiosResponse } from 'axios';
+import { afterEach, beforeEach, expect, test, vi } from 'vitest';
+import { responseInterceptor } from './response-interceptor';
+import { validateIP } from './validate-ip';
+
+vi.mock('./validate-ip');
+
+let sample: {
+	remoteAddress: string;
+	url: string;
+};
+
+let sampleResponseConfig: AxiosResponse<any, any>;
+
+beforeEach(() => {
+	sample = {
+		remoteAddress: randIp(),
+		url: randUrl(),
+	};
+
+	sampleResponseConfig = {
+		request: {
+			socket: {
+				remoteAddress: sample.remoteAddress,
+			},
+			url: sample.url,
+		},
+	} as AxiosResponse<any, any>;
+});
+
+afterEach(() => {
+	vi.resetAllMocks();
+});
+
+test(`Calls validateIP with IP/url from axios request config`, async () => {
+	await responseInterceptor(sampleResponseConfig);
+	expect(validateIP).toHaveBeenCalledWith(sample.remoteAddress, sample.url);
+});
+
+test(`Returns passed in config as-is`, async () => {
+	const config = await responseInterceptor(sampleResponseConfig);
+	expect(config).toBe(sampleResponseConfig);
+});

api/src/request/response-interceptor.ts

@@ -0,0 +1,7 @@
+import type { AxiosResponse } from 'axios';
+import { validateIP } from './validate-ip';
+
+export const responseInterceptor = async (config: AxiosResponse<any, any>) => {
+	await validateIP(config.request.socket.remoteAddress, config.request.url);
+	return config;
+};

api/src/request/validate-ip.test.ts

@@ -0,0 +1,81 @@
+import { randIp, randUrl } from '@ngneat/falso';
+import os from 'node:os';
+import { afterEach, beforeEach, expect, test, vi } from 'vitest';
+import { getEnv } from '../env';
+import { validateIP } from './validate-ip';
+
+vi.mock('../env');
+vi.mock('node:os');
+
+let sample: {
+	ip: string;
+	url: string;
+};
+
+beforeEach(() => {
+	sample = {
+		ip: randIp(),
+		url: randUrl(),
+	};
+});
+
+afterEach(() => {
+	vi.resetAllMocks();
+});
+
+test(`Does nothing if IP is valid`, async () => {
+	vi.mocked(getEnv).mockReturnValue({ IMPORT_IP_DENY_LIST: [] });
+	await validateIP(sample.ip, sample.url);
+});
+
+test(`Throws error if passed IP is denylisted`, async () => {
+	vi.mocked(getEnv).mockReturnValue({ IMPORT_IP_DENY_LIST: [sample.ip] });
+
+	try {
+		await validateIP(sample.ip, sample.url);
+	} catch (err: any) {
+		expect(err).toBeInstanceOf(Error);
+		expect(err.message).toBe(`Requested URL "${sample.url}" resolves to a denied IP address`);
+	}
+});
+
+test(`Checks against IPs of local networkInterfaces if IP deny list contains 0.0.0.0`, async () => {
+	vi.mocked(getEnv).mockReturnValue({ IMPORT_IP_DENY_LIST: ['0.0.0.0'] });
+	vi.mocked(os.networkInterfaces).mockReturnValue({});
+	await validateIP(sample.ip, sample.url);
+	expect(os.networkInterfaces).toHaveBeenCalledOnce();
+});
+
+test(`Throws error if IP address matches resolved localhost IP`, async () => {
+	vi.mocked(getEnv).mockReturnValue({ IMPORT_IP_DENY_LIST: ['0.0.0.0'] });
+	vi.mocked(os.networkInterfaces).mockReturnValue({
+		fa0: undefined,
+		lo0: [
+			{
+				address: '127.0.0.1',
+				netmask: '255.0.0.0',
+				family: 'IPv4',
+				mac: '00:00:00:00:00:00',
+				internal: true,
+				cidr: '127.0.0.1/8',
+			},
+		],
+		en0: [
+			{
+				address: sample.ip,
+				netmask: '255.0.0.0',
+				family: 'IPv4',
+				mac: '00:00:00:00:00:00',
+				internal: true,
+				cidr: '127.0.0.1/8',
+			},
+		],
+	});
+
+	try {
+		await validateIP(sample.ip, sample.url);
+	} catch (err: any) {
+		expect(err).toBeInstanceOf(Error);
+		expect(err.message).toBe(`Requested URL "${sample.url}" resolves to a denied IP address`);
+	}
+});

api/src/request/validate-ip.ts

@@ -0,0 +1,24 @@
+import os from 'node:os';
+import { getEnv } from '../env';
+
+export const validateIP = async (ip: string, url: string) => {
+	const env = getEnv();
+
+	if (env.IMPORT_IP_DENY_LIST.includes(ip)) {
+		throw new Error(`Requested URL "${url}" resolves to a denied IP address`);
+	}
+
+	if (env.IMPORT_IP_DENY_LIST.includes('0.0.0.0')) {
+		const networkInterfaces = os.networkInterfaces();
+
+		for (const networkInfo of Object.values(networkInterfaces)) {
+			if (!networkInfo) continue;
+
+			for (const info of networkInfo) {
+				if (info.address === ip) {
+					throw new Error(`Requested URL "${url}" resolves to a denied IP address`);
+				}
+			}
+		}
+	}
+};

api/src/services/files.ts

@@ -1,22 +1,19 @@
 import { toArray } from '@directus/shared/utils';
-import { lookup } from 'dns';
 import encodeURL from 'encodeurl';
 import exif from 'exif-reader';
 import { parse as parseIcc } from 'icc';
 import { clone, pick } from 'lodash';
 import { extension } from 'mime-types';
-import net from 'net';
 import type { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
-import os from 'os';
 import path from 'path';
 import sharp from 'sharp';
-import url, { URL } from 'url';
-import { promisify } from 'util';
+import url from 'url';
 import emitter from '../emitter';
 import env from '../env';
 import { ForbiddenException, InvalidPayloadException, ServiceUnavailableException } from '../exceptions';
 import logger from '../logger';
+import { getAxios } from '../request/index';
 import { getStorage } from '../storage';
 import { AbstractServiceOptions, File, Metadata, MutationOptions, PrimaryKey } from '../types';
 import { parseIptc, parseXmp } from '../utils/parse-image-metadata';
@@ -25,8 +22,6 @@ import { ItemsService } from './items';
 // @ts-ignore
 import formatTitle from '@directus/format-title';
 
-const lookupDNS = promisify(lookup);
-
 export class FilesService extends ItemsService {
 	constructor(options: AbstractServiceOptions) {
 		super('directus_files', options);
@@ -224,8 +219,6 @@ export class FilesService extends ItemsService {
 	 * Import a single file from an external URL
 	 */
 	async importOne(importURL: string, body: Partial<File>): Promise<PrimaryKey> {
-		const axios = (await import('axios')).default;
-
 		const fileCreatePermissions = this.accountability?.permissions?.find(
 			(permission) => permission.collection === 'directus_files' && permission.action === 'create'
 		);
@@ -234,62 +227,15 @@ export class FilesService extends ItemsService {
 			throw new ForbiddenException();
 		}
 
-		let resolvedUrl;
-
-		try {
-			resolvedUrl = new URL(importURL);
-		} catch (err: any) {
-			logger.warn(err, `Requested URL ${importURL} isn't a valid URL`);
-			throw new ServiceUnavailableException(`Couldn't fetch file from url "${importURL}"`, {
-				service: 'external-file',
-			});
-		}
-
-		let ip = resolvedUrl.hostname;
-
-		if (net.isIP(ip) === 0) {
-			try {
-				ip = (await lookupDNS(ip)).address;
-			} catch (err: any) {
-				logger.warn(err, `Couldn't lookup the DNS for url ${importURL}`);
-				throw new ServiceUnavailableException(`Couldn't fetch file from url "${importURL}"`, {
-					service: 'external-file',
-				});
-			}
-		}
-
-		if (env.IMPORT_IP_DENY_LIST.includes('0.0.0.0')) {
-			const networkInterfaces = os.networkInterfaces();
-
-			for (const networkInfo of Object.values(networkInterfaces)) {
-				if (!networkInfo) continue;
-
-				for (const info of networkInfo) {
-					if (info.address === ip) {
-						logger.warn(`Requested URL ${importURL} resolves to localhost.`);
-						throw new ServiceUnavailableException(`Couldn't fetch file from url "${importURL}"`, {
-							service: 'external-file',
-						});
-					}
-				}
-			}
-		}
-
-		if (env.IMPORT_IP_DENY_LIST.includes(ip)) {
-			logger.warn(`Requested URL ${importURL} resolves to a denied IP address.`);
-			throw new ServiceUnavailableException(`Couldn't fetch file from url "${importURL}"`, {
-				service: 'external-file',
-			});
-		}
-
 		let fileResponse;
 
 		try {
+			const axios = await getAxios();
 			fileResponse = await axios.get<Readable>(encodeURL(importURL), {
 				responseType: 'stream',
 			});
 		} catch (err: any) {
-			logger.warn(err, `Couldn't fetch file from url "${importURL}"`);
+			logger.warn(err, `Couldn't fetch file from URL "${importURL}"`);
 			throw new ServiceUnavailableException(`Couldn't fetch file from url "${importURL}"`, {
 				service: 'external-file',
 			});

api/src/webhooks.ts

@@ -1,11 +1,12 @@
+import { ActionHandler } from '@directus/shared/types';
 import getDatabase from './database';
 import emitter from './emitter';
 import logger from './logger';
-import { Webhook, WebhookHeader } from './types';
-import { WebhooksService } from './services';
-import { getSchema } from './utils/get-schema';
-import { ActionHandler } from '@directus/shared/types';
 import { getMessenger } from './messenger';
+import { getAxios } from './request/index';
+import { WebhooksService } from './services';
+import { Webhook, WebhookHeader } from './types';
+import { getSchema } from './utils/get-schema';
 import { JobQueue } from './utils/job-queue';
 
 let registered: { event: string; handler: ActionHandler }[] = [];
@@ -55,9 +56,8 @@ export function unregister(): void {
 
 function createHandler(webhook: Webhook, event: string): ActionHandler {
 	return async (meta, context) => {
-		const axios = (await import('axios')).default;
-
 		if (webhook.collections.includes(meta.collection) === false) return;
+		const axios = await getAxios();
 
 		const webhookPayload = {
 			event,