Merge branch 'main' into aggregation

api/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "directus",
-	"version": "9.0.0-rc.75",
+	"version": "9.0.0-rc.76",
 	"license": "GPL-3.0-only",
 	"homepage": "https://github.com/directus/directus#readme",
 	"description": "Directus is a real-time API and App dashboard for managing SQL database content.",
@@ -66,14 +66,14 @@
 		"example.env"
 	],
 	"dependencies": {
-		"@directus/app": "9.0.0-rc.75",
-		"@directus/drive": "9.0.0-rc.75",
-		"@directus/drive-azure": "9.0.0-rc.75",
-		"@directus/drive-gcs": "9.0.0-rc.75",
-		"@directus/drive-s3": "9.0.0-rc.75",
-		"@directus/format-title": "9.0.0-rc.75",
-		"@directus/schema": "9.0.0-rc.75",
-		"@directus/specs": "9.0.0-rc.75",
+		"@directus/app": "9.0.0-rc.76",
+		"@directus/drive": "9.0.0-rc.76",
+		"@directus/drive-azure": "9.0.0-rc.76",
+		"@directus/drive-gcs": "9.0.0-rc.76",
+		"@directus/drive-s3": "9.0.0-rc.76",
+		"@directus/format-title": "9.0.0-rc.76",
+		"@directus/schema": "9.0.0-rc.76",
+		"@directus/specs": "9.0.0-rc.76",
 		"@godaddy/terminus": "^4.9.0",
 		"argon2": "^0.28.1",
 		"async": "^3.2.0",
@@ -102,7 +102,7 @@
 		"graphql": "^15.5.0",
 		"graphql-compose": "^9.0.1",
 		"icc": "^2.0.0",
-		"inquirer": "^8.1.0",
+		"inquirer": "^8.1.1",
 		"joi": "^17.3.0",
 		"js-yaml": "^4.1.0",
 		"js2xmlparser": "^4.0.1",
@@ -125,6 +125,7 @@
 		"otplib": "^12.0.1",
 		"pino": "^6.11.3",
 		"pino-colada": "^2.1.0",
+		"prettier": "^2.3.1",
 		"qs": "^6.9.4",
 		"rate-limiter-flexible": "^2.2.2",
 		"resolve-cwd": "^3.0.0",
@@ -138,7 +139,7 @@
 		"connect-memcached": "^1.0.0",
 		"connect-redis": "^6.0.0",
 		"connect-session-knex": "^2.1.0",
-		"ioredis": "^4.27.2",
+		"ioredis": "^4.27.6",
 		"keyv-memcache": "^1.2.5",
 		"memcached": "^2.2.2",
 		"mysql": "^2.18.1",

api/src/cli/commands/count/index.ts

@@ -1,3 +1,5 @@
+/* eslint-disable no-console */
+
 import getDatabase from '../../../database';
 
 export default async function count(collection: string): Promise<void> {

api/src/cli/commands/database/install.ts

@@ -1,3 +1,5 @@
+/* eslint-disable no-console */
+
 import runMigrations from '../../../database/migrations/run';
 import installSeeds from '../../../database/seeds/run';
 import getDatabase from '../../../database';

api/src/cli/commands/database/migrate.ts

@@ -1,3 +1,5 @@
+/* eslint-disable no-console */
+
 import run from '../../../database/migrations/run';
 import getDatabase from '../../../database';
 

api/src/cli/commands/init/index.ts

@@ -1,3 +1,5 @@
+/* eslint-disable no-console */
+
 import argon2 from 'argon2';
 import chalk from 'chalk';
 import execa from 'execa';

api/src/cli/commands/roles/create.ts

@@ -1,3 +1,5 @@
+/* eslint-disable no-console */
+
 import { getSchema } from '../../../utils/get-schema';
 import { RolesService } from '../../../services';
 import getDatabase from '../../../database';

api/src/cli/commands/users/create.ts

@@ -1,3 +1,5 @@
+/* eslint-disable no-console */
+
 import { getSchema } from '../../../utils/get-schema';
 import { UsersService } from '../../../services';
 import getDatabase from '../../../database';

api/src/cli/commands/users/passwd.ts

@@ -1,3 +1,5 @@
+/* eslint-disable no-console */
+
 import argon2 from 'argon2';
 import { getSchema } from '../../../utils/get-schema';
 import { UsersService } from '../../../services';

api/src/cli/index.ts

@@ -1,5 +1,7 @@
 #!/usr/bin/env node
 
+/* eslint-disable no-console */
+
 import program from 'commander';
 import start from '../start';
 import bootstrap from './commands/bootstrap';

api/src/database/index.ts

@@ -28,7 +28,11 @@ export default function getDatabase(): Knex {
 	if (env.DB_CLIENT && env.DB_CLIENT === 'sqlite3') {
 		requiredEnvVars.push('DB_FILENAME');
 	} else if (env.DB_CLIENT && env.DB_CLIENT === 'oracledb') {
-		requiredEnvVars.push('DB_USER', 'DB_PASSWORD', 'DB_CONNECT_STRING');
+		if (!env.DB_CONNECT_STRING) {
+			requiredEnvVars.push('DB_HOST', 'DB_PORT', 'DB_DATABASE', 'DB_USER', 'DB_PASSWORD');
+		} else {
+			requiredEnvVars.push('DB_USER', 'DB_PASSWORD', 'DB_CONNECT_STRING');
+		}
 	} else {
 		if (env.DB_CLIENT === 'pg') {
 			if (!env.DB_CONNECTION_STRING) {

api/src/database/migrations/20201105B-change-webhook-url-type.ts

@@ -1,4 +1,6 @@
 import { Knex } from 'knex';
+// @ts-ignore
+import Client_Oracledb from 'knex/lib/dialects/oracledb';
 import env from '../../env';
 
 async function oracleAlterUrl(knex: Knex, type: string): Promise<void> {
@@ -10,7 +12,7 @@ async function oracleAlterUrl(knex: Knex, type: string): Promise<void> {
 }
 
 export async function up(knex: Knex): Promise<void> {
-	if (env.DB_CLIENT === 'oracledb') {
+	if (knex.client instanceof Client_Oracledb) {
 		await oracleAlterUrl(knex, 'CLOB');
 		return;
 	}

api/src/database/migrations/20210312A-webhooks-collections-text.ts

@@ -1,4 +1,6 @@
 import { Knex } from 'knex';
+// @ts-ignore
+import Client_Oracledb from 'knex/lib/dialects/oracledb';
 import env from '../../env';
 
 async function oracleAlterCollections(knex: Knex, type: string): Promise<void> {
@@ -10,7 +12,7 @@ async function oracleAlterCollections(knex: Knex, type: string): Promise<void> {
 }
 
 export async function up(knex: Knex): Promise<void> {
-	if (env.DB_CLIENT === 'oracledb') {
+	if (knex.client instanceof Client_Oracledb) {
 		await oracleAlterCollections(knex, 'CLOB');
 		return;
 	}

api/src/database/migrations/run.ts

@@ -1,3 +1,5 @@
+/* eslint-disable no-console */
+
 import formatTitle from '@directus/format-title';
 import fse from 'fs-extra';
 import { Knex } from 'knex';

api/src/env.ts

@@ -50,6 +50,7 @@ const defaults: Record<string, any> = {
 	CACHE_TTL: '10m',
 	CACHE_NAMESPACE: 'system-cache',
 	CACHE_AUTO_PURGE: false,
+	CACHE_CONTROL_S_MAXAGE: '0',
 
 	OAUTH_PROVIDERS: '',
 

api/src/middleware/cache.ts

@@ -2,6 +2,7 @@ import { RequestHandler } from 'express';
 import cache from '../cache';
 import env from '../env';
 import asyncHandler from '../utils/async-handler';
+import { getCacheControlHeader } from '../utils/get-cache-headers';
 import { getCacheKey } from '../utils/get-cache-key';
 
 const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next) => {
@@ -17,18 +18,11 @@ const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next)
 	const cachedData = await cache.get(key);
 
 	if (cachedData) {
-		// Set cache-control header, but only for the public role
-		if (env.CACHE_AUTO_PURGE !== true && !!req.accountability?.role === false) {
-			const expiresAt = await cache.get(`${key}__expires_at`);
-			const maxAge = `max-age=${expiresAt - Date.now()}`;
-			res.setHeader('Cache-Control', `public, ${maxAge}`);
-		} else {
-			// This indicates that the browser/proxy is allowed to cache, but has to revalidate with
-			// the server before use. At this point, we don't include Last-Modified, so it'll always
-			// recreate the local cache. This does NOT mean that cache is disabled all together, as
-			// Directus is still pulling the value from it's internal cache.
-			res.setHeader('Cache-Control', 'no-cache');
-		}
+		const cacheExpiryDate = (await cache.get(`${key}__expires_at`)) as number | null;
+		const cacheTTL = cacheExpiryDate ? cacheExpiryDate - Date.now() : null;
+
+		res.setHeader('Cache-Control', getCacheControlHeader(req, cacheTTL));
+		res.setHeader('Vary', 'Origin, Cache-Control');
 
 		return res.json(cachedData);
 	} else {

api/src/middleware/respond.ts

@@ -7,6 +7,7 @@ import env from '../env';
 import asyncHandler from '../utils/async-handler';
 import { getCacheKey } from '../utils/get-cache-key';
 import { parse as toXML } from 'js2xmlparser';
+import { getCacheControlHeader } from '../utils/get-cache-headers';
 
 export const respond: RequestHandler = asyncHandler(async (req, res) => {
 	if (
@@ -19,20 +20,12 @@ export const respond: RequestHandler = asyncHandler(async (req, res) => {
 		const key = getCacheKey(req);
 		await cache.set(key, res.locals.payload, ms(env.CACHE_TTL as string));
 		await cache.set(`${key}__expires_at`, Date.now() + ms(env.CACHE_TTL as string));
-
-		const noCacheRequested =
-			req.headers['cache-control']?.includes('no-cache') || req.headers['Cache-Control']?.includes('no-cache');
-
-		// Set cache-control header
-		if (env.CACHE_AUTO_PURGE !== true && noCacheRequested === false) {
-			const maxAge = `max-age=${ms(env.CACHE_TTL as string)}`;
-			const access = !!req.accountability?.role === false ? 'public' : 'private';
-			res.setHeader('Cache-Control', `${access}, ${maxAge}`);
-		}
-
-		if (noCacheRequested) {
-			res.setHeader('Cache-Control', 'no-cache');
-		}
+		res.setHeader('Cache-Control', getCacheControlHeader(req, ms(env.CACHE_TTL as string)));
+		res.setHeader('Vary', 'Origin, Cache-Control');
+	} else {
+		// Don't cache anything by default
+		res.setHeader('Cache-Control', 'no-cache');
+		res.setHeader('Vary', 'Origin, Cache-Control');
 	}
 
 	if (req.sanitizedQuery.export) {

api/src/services/mail/index.ts

@@ -9,6 +9,7 @@ import logger from '../../logger';
 import { AbstractServiceOptions, Accountability, SchemaOverview } from '../../types';
 import getMailer from '../../mailer';
 import { Transporter, SendMailOptions } from 'nodemailer';
+import prettier from 'prettier';
 
 const liquidEngine = new Liquid({
 	root: [path.resolve(env.EXTENSIONS_PATH, 'templates'), path.resolve(__dirname, 'templates')],
@@ -61,6 +62,11 @@ export class MailService {
 			html = await this.renderTemplate(template.name, templateData);
 		}
 
+		if (typeof html === 'string') {
+			// Some email clients start acting funky when line length exceeds 75 characters. See #6074
+			html = prettier.format(html as string, { parser: 'html', printWidth: 70, tabWidth: 0 });
+		}
+
 		try {
 			await this.mailer.sendMail({ ...emailOptions, from, html });
 		} catch (error) {

api/src/utils/adjust-date.ts

@@ -24,7 +24,7 @@ import { clone } from 'lodash';
  *
  * The conversion is lifted straight from `ms`.
  */
-export function adjustDate(date: Date, adjustment: string) {
+export function adjustDate(date: Date, adjustment: string): Date | undefined {
 	date = clone(date);
 
 	const subtract = adjustment.startsWith('-');

api/src/utils/get-cache-headers.ts

@@ -0,0 +1,42 @@
+import env from '../env';
+import { Request } from 'express';
+
+/**
+ * Returns the Cache-Control header for the current request
+ *
+ * @param req Express request object
+ * @param ttl TTL of the cache in ms
+ */
+export function getCacheControlHeader(req: Request, ttl: number | null): string {
+	// When the resource / current request isn't cached
+	if (ttl === null) return 'no-cache';
+
+	// When the API cache can invalidate at any moment
+	if (env.CACHE_AUTO_PURGE === true) return 'no-cache';
+
+	const noCacheRequested =
+		req.headers['cache-control']?.includes('no-cache') || req.headers['Cache-Control']?.includes('no-cache');
+
+	// When the user explicitly asked to skip the cache
+	if (noCacheRequested) return 'no-cache';
+
+	// Cache control header uses seconds for everything
+	const ttlSeconds = Math.round(ttl / 1000);
+
+	const access = !!req.accountability?.role === false ? 'public' : 'private';
+
+	let headerValue = `${access}, max-age=${ttlSeconds}`;
+
+	// When the s-maxage flag should be included
+	if (env.CACHE_CONTROL_S_MAXAGE !== false) {
+		// Default to regular max-age flag when true
+		if (env.CACHE_CONTROL_S_MAXAGE === true) {
+			headerValue += `, s-maxage=${ttlSeconds}`;
+		} else {
+			// Set to custom value
+			headerValue += `, s-maxage=${env.CACHE_CONTROL_S_MAXAGE}`;
+		}
+	}
+
+	return headerValue;
+}

api/src/utils/is-url-allowed.ts

@@ -5,8 +5,6 @@ import logger from '../logger';
  * Check if url matches allow list either exactly or by domain+path
  */
 export default function isUrlAllowed(url: string, allowList: string | string[]): boolean {
-	console.log(url, allowList);
-
 	const urlAllowList = toArray(allowList);
 
 	if (urlAllowList.includes(url)) return true;