Add schema caching (#6456)

* Rework cache handler to be function export

* Add default schema caching

* Add schema cache

* Auto purge schema cache on schema change from api

* Only set last_access value on login

* Add note on schema cache setting

api/src/cache.ts

@@ -6,31 +6,39 @@ import { getConfigFromEnv } from './utils/get-config-from-env';
 import { validateEnv } from './utils/validate-env';
 
 let cache: Keyv | null = null;
+let schemaCache: Keyv | null = null;
 
-if (env.CACHE_ENABLED === true) {
-	validateEnv(['CACHE_NAMESPACE', 'CACHE_TTL', 'CACHE_STORE']);
-	cache = getKeyvInstance();
-	cache.on('error', (err) => logger.error(err));
+export function getCache() {
+	if (env.CACHE_ENABLED === true && cache === null) {
+		validateEnv(['CACHE_NAMESPACE', 'CACHE_TTL', 'CACHE_STORE']);
+		cache = getKeyvInstance(ms(env.CACHE_TTL as string));
+		cache.on('error', (err) => logger.error(err));
+	}
+
+	if (env.CACHE_SCHEMA !== false && schemaCache === null) {
+		schemaCache = getKeyvInstance(typeof env.CACHE_SCHEMA === 'string' ? ms(env.CACHE_SCHEMA) : undefined);
+		schemaCache.on('error', (err) => logger.error(err));
+	}
+
+	return { cache, schemaCache };
 }
 
-export default cache;
-
-function getKeyvInstance() {
+function getKeyvInstance(ttl: number | undefined) {
 	switch (env.CACHE_STORE) {
 		case 'redis':
-			return new Keyv(getConfig('redis'));
+			return new Keyv(getConfig('redis', ttl));
 		case 'memcache':
-			return new Keyv(getConfig('memcache'));
+			return new Keyv(getConfig('memcache', ttl));
 		case 'memory':
 		default:
-			return new Keyv(getConfig());
+			return new Keyv(getConfig('memory', ttl));
 	}
 }
 
-function getConfig(store: 'memory' | 'redis' | 'memcache' = 'memory'): Options<any> {
+function getConfig(store: 'memory' | 'redis' | 'memcache' = 'memory', ttl: number | undefined): Options<any> {
 	const config: Options<any> = {
 		namespace: env.CACHE_NAMESPACE,
-		ttl: ms(env.CACHE_TTL as string),
+		ttl,
 	};
 
 	if (store === 'redis') {

api/src/env.ts

@@ -51,6 +51,7 @@ const defaults: Record<string, any> = {
 	CACHE_NAMESPACE: 'system-cache',
 	CACHE_AUTO_PURGE: false,
 	CACHE_CONTROL_S_MAXAGE: '0',
+	CACHE_SCHEMA: true,
 
 	OAUTH_PROVIDERS: '',
 

api/src/middleware/authenticate.ts

@@ -78,10 +78,6 @@ const authenticate: RequestHandler = asyncHandler(async (req, res, next) => {
 		req.accountability.app = user.app_access === true || user.app_access == 1;
 	}
 
-	if (req.accountability?.user) {
-		await database('directus_users').update({ last_access: new Date() }).where({ id: req.accountability.user });
-	}
-
 	return next();
 });
 

api/src/middleware/cache.ts

@@ -1,11 +1,13 @@
 import { RequestHandler } from 'express';
-import cache from '../cache';
+import { getCache } from '../cache';
 import env from '../env';
 import asyncHandler from '../utils/async-handler';
 import { getCacheControlHeader } from '../utils/get-cache-headers';
 import { getCacheKey } from '../utils/get-cache-key';
 
 const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next) => {
+	const { cache } = getCache();
+
 	if (req.method.toLowerCase() !== 'get') return next();
 	if (env.CACHE_ENABLED !== true) return next();
 	if (!cache) return next();

api/src/middleware/respond.ts

@@ -2,7 +2,7 @@ import { RequestHandler } from 'express';
 import { Transform, transforms } from 'json2csv';
 import ms from 'ms';
 import { PassThrough } from 'stream';
-import cache from '../cache';
+import { getCache } from '../cache';
 import env from '../env';
 import asyncHandler from '../utils/async-handler';
 import { getCacheKey } from '../utils/get-cache-key';
@@ -10,6 +10,8 @@ import { parse as toXML } from 'js2xmlparser';
 import { getCacheControlHeader } from '../utils/get-cache-headers';
 
 export const respond: RequestHandler = asyncHandler(async (req, res) => {
+	const { cache } = getCache();
+
 	if (
 		req.method.toLowerCase() === 'get' &&
 		env.CACHE_ENABLED === true &&

api/src/services/authentication.ts

@@ -185,6 +185,8 @@ export class AuthenticationService {
 			});
 		}
 
+		await this.knex('directus_users').update({ last_access: new Date() }).where({ id: user.id });
+
 		emitStatus('success');
 
 		if (allowedAttempts !== null) {
@@ -230,6 +232,8 @@ export class AuthenticationService {
 			.update({ token: newRefreshToken, expires: refreshTokenExpiration })
 			.where({ token: refreshToken });
 
+		await this.knex('directus_users').update({ last_access: new Date() }).where({ id: record.id });
+
 		return {
 			accessToken,
 			refreshToken: newRefreshToken,

api/src/services/collections.ts

@@ -1,6 +1,6 @@
 import SchemaInspector from '@directus/schema';
 import { Knex } from 'knex';
-import cache from '../cache';
+import { getCache } from '../cache';
 import { ALIAS_TYPES } from '../constants';
 import getDatabase, { getSchemaInspector } from '../database';
 import { systemCollectionRows } from '../database/system-data/collections';
@@ -9,6 +9,7 @@ import { ForbiddenException, InvalidPayloadException } from '../exceptions';
 import logger from '../logger';
 import { FieldsService, RawField } from '../services/fields';
 import { ItemsService, MutationOptions } from '../services/items';
+import Keyv from 'keyv';
 import {
 	AbstractServiceOptions,
 	Accountability,
@@ -29,12 +30,18 @@ export class CollectionsService {
 	accountability: Accountability | null;
 	schemaInspector: ReturnType<typeof SchemaInspector>;
 	schema: SchemaOverview;
+	cache: Keyv<any> | null;
+	schemaCache: Keyv<any> | null;
 
 	constructor(options: AbstractServiceOptions) {
 		this.knex = options.knex || getDatabase();
 		this.accountability = options.accountability || null;
 		this.schemaInspector = options.knex ? SchemaInspector(options.knex) : getSchemaInspector();
 		this.schema = options.schema;
+
+		const { cache, schemaCache } = getCache();
+		this.cache = cache;
+		this.schemaCache = schemaCache;
 	}
 
 	/**
@@ -128,8 +135,12 @@ export class CollectionsService {
 			return payload.collection;
 		});
 
-		if (cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
+			await this.cache.clear();
+		}
+
+		if (this.schemaCache) {
+			await this.schemaCache.clear();
 		}
 
 		return payload.collection;
@@ -156,8 +167,12 @@ export class CollectionsService {
 			return collectionNames;
 		});
 
-		if (cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
+			await this.cache.clear();
+		}
+
+		if (this.schemaCache) {
+			await this.schemaCache.clear();
 		}
 
 		return collections;
@@ -416,8 +431,12 @@ export class CollectionsService {
 			await trx.schema.dropTable(collectionKey);
 		});
 
-		if (cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
+			await this.cache.clear();
+		}
+
+		if (this.schemaCache) {
+			await this.schemaCache.clear();
 		}
 
 		return collectionKey;
@@ -443,8 +462,12 @@ export class CollectionsService {
 			}
 		});
 
-		if (cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
+			await this.cache.clear();
+		}
+
+		if (this.schemaCache) {
+			await this.schemaCache.clear();
 		}
 
 		return collectionKeys;

api/src/services/fields.ts

@@ -1,7 +1,7 @@
 import SchemaInspector from '@directus/schema';
 import { Knex } from 'knex';
 import { Column } from 'knex-schema-inspector/dist/types/column';
-import cache from '../cache';
+import { getCache } from '../cache';
 import { ALIAS_TYPES } from '../constants';
 import getDatabase, { getSchemaInspector } from '../database';
 import { systemFieldRows } from '../database/system-data/fields/';
@@ -18,6 +18,7 @@ import getLocalType from '../utils/get-local-type';
 import { toArray } from '../utils/to-array';
 import { isEqual } from 'lodash';
 import { RelationsService } from './relations';
+import Keyv from 'keyv';
 
 export type RawField = DeepPartial<Field> & { field: string; type: typeof types[number] };
 
@@ -28,6 +29,8 @@ export class FieldsService {
 	payloadService: PayloadService;
 	schemaInspector: ReturnType<typeof SchemaInspector>;
 	schema: SchemaOverview;
+	cache: Keyv<any> | null;
+	schemaCache: Keyv<any> | null;
 
 	constructor(options: AbstractServiceOptions) {
 		this.knex = options.knex || getDatabase();
@@ -36,6 +39,10 @@ export class FieldsService {
 		this.itemsService = new ItemsService('directus_fields', options);
 		this.payloadService = new PayloadService('directus_fields', options);
 		this.schema = options.schema;
+
+		const { cache, schemaCache } = getCache();
+		this.cache = cache;
+		this.schemaCache = schemaCache;
 	}
 
 	private get hasReadAccess() {
@@ -244,8 +251,12 @@ export class FieldsService {
 			}
 		});
 
-		if (cache && env.CACHE_AUTO_PURGE) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE) {
+			await this.cache.clear();
+		}
+
+		if (this.schemaCache) {
+			await this.schemaCache.clear();
 		}
 	}
 
@@ -291,8 +302,12 @@ export class FieldsService {
 			}
 		}
 
-		if (cache && env.CACHE_AUTO_PURGE) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE) {
+			await this.cache.clear();
+		}
+
+		if (this.schemaCache) {
+			await this.schemaCache.clear();
 		}
 
 		return field.field;
@@ -396,8 +411,12 @@ export class FieldsService {
 			}
 		});
 
-		if (cache && env.CACHE_AUTO_PURGE) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE) {
+			await this.cache.clear();
+		}
+
+		if (this.schemaCache) {
+			await this.schemaCache.clear();
 		}
 
 		emitAsyncSafe(`fields.delete`, {

api/src/services/files.ts

@@ -7,7 +7,6 @@ import { extension } from 'mime-types';
 import path from 'path';
 import sharp from 'sharp';
 import url from 'url';
-import cache from '../cache';
 import { emitAsyncSafe } from '../emitter';
 import env from '../env';
 import { ForbiddenException, ServiceUnavailableException } from '../exceptions';
@@ -121,8 +120,8 @@ export class FilesService extends ItemsService {
 
 		await sudoService.updateOne(primaryKey, payload, { emitEvents: false });
 
-		if (cache && env.CACHE_AUTO_PURGE) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE) {
+			await this.cache.clear();
 		}
 
 		emitAsyncSafe(`files.upload`, {
@@ -208,8 +207,8 @@ export class FilesService extends ItemsService {
 			}
 		}
 
-		if (cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
+			await this.cache.clear();
 		}
 
 		return keys;

api/src/services/items.ts

@@ -1,6 +1,7 @@
 import { Knex } from 'knex';
 import { clone, cloneDeep, merge, pick, without } from 'lodash';
-import cache from '../cache';
+import { getCache } from '../cache';
+import Keyv from 'keyv';
 import getDatabase from '../database';
 import runAST from '../database/run-ast';
 import emitter, { emitAsyncSafe } from '../emitter';
@@ -52,6 +53,7 @@ export class ItemsService<Item extends AnyItem = AnyItem> implements AbstractSer
 	accountability: Accountability | null;
 	eventScope: string;
 	schema: SchemaOverview;
+	cache: Keyv<any> | null;
 
 	constructor(collection: string, options: AbstractServiceOptions) {
 		this.collection = collection;
@@ -59,6 +61,7 @@ export class ItemsService<Item extends AnyItem = AnyItem> implements AbstractSer
 		this.accountability = options.accountability || null;
 		this.eventScope = this.collection.startsWith('directus_') ? this.collection.substring(9) : 'items';
 		this.schema = options.schema;
+		this.cache = getCache().cache;
 
 		return this;
 	}
@@ -208,8 +211,8 @@ export class ItemsService<Item extends AnyItem = AnyItem> implements AbstractSer
 			});
 		}
 
-		if (cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
+			await this.cache.clear();
 		}
 
 		return primaryKey;
@@ -236,8 +239,8 @@ export class ItemsService<Item extends AnyItem = AnyItem> implements AbstractSer
 			return primaryKeys;
 		});
 
-		if (cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
+			await this.cache.clear();
 		}
 
 		return primaryKeys;
@@ -524,8 +527,8 @@ export class ItemsService<Item extends AnyItem = AnyItem> implements AbstractSer
 			}
 		});
 
-		if (cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
+			await this.cache.clear();
 		}
 
 		if (opts?.emitEvents !== false) {
@@ -589,8 +592,8 @@ export class ItemsService<Item extends AnyItem = AnyItem> implements AbstractSer
 			return primaryKeys;
 		});
 
-		if (cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
+			await this.cache.clear();
 		}
 
 		return primaryKeys;
@@ -673,8 +676,8 @@ export class ItemsService<Item extends AnyItem = AnyItem> implements AbstractSer
 			}
 		});
 
-		if (cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE && opts?.autoPurgeCache !== false) {
+			await this.cache.clear();
 		}
 
 		if (opts?.emitEvents !== false) {

api/src/services/relations.ts

@@ -9,6 +9,8 @@ import SchemaInspector from '@directus/schema';
 import { ForeignKey } from 'knex-schema-inspector/dist/types/foreign-key';
 import getDatabase, { getSchemaInspector } from '../database';
 import { getDefaultIndexName } from '../utils/get-default-index-name';
+import { getCache } from '../cache';
+import Keyv from 'keyv';
 
 export class RelationsService {
 	knex: Knex;
@@ -17,6 +19,7 @@ export class RelationsService {
 	accountability: Accountability | null;
 	schema: SchemaOverview;
 	relationsItemService: ItemsService<RelationMeta>;
+	schemaCache: Keyv<any> | null;
 
 	constructor(options: AbstractServiceOptions) {
 		this.knex = options.knex || getDatabase();
@@ -31,6 +34,8 @@ export class RelationsService {
 			// allowed to extract the relations regardless of permissions to directus_relations. This
 			// happens in `filterForbidden` down below
 		});
+
+		this.schemaCache = getCache().schemaCache;
 	}
 
 	async readAll(collection?: string, opts?: QueryOptions): Promise<Relation[]> {
@@ -183,6 +188,10 @@ export class RelationsService {
 
 			await relationsItemService.createOne(metaRow);
 		});
+
+		if (this.schemaCache) {
+			await this.schemaCache.clear();
+		}
 	}
 
 	/**
@@ -259,6 +268,10 @@ export class RelationsService {
 				}
 			}
 		});
+
+		if (this.schemaCache) {
+			await this.schemaCache.clear();
+		}
 	}
 
 	/**
@@ -296,6 +309,10 @@ export class RelationsService {
 				await trx('directus_relations').delete().where({ many_collection: collection, many_field: field });
 			}
 		});
+
+		if (this.schemaCache) {
+			await this.schemaCache.clear();
+		}
 	}
 
 	/**

api/src/services/server.ts

@@ -6,7 +6,7 @@ import os from 'os';
 import { performance } from 'perf_hooks';
 // @ts-ignore
 import { version } from '../../package.json';
-import cache from '../cache';
+import { getCache } from '../cache';
 import getDatabase, { hasDatabaseConnection } from '../database';
 import env from '../env';
 import logger from '../logger';
@@ -189,6 +189,8 @@ export class ServerService {
 				return {};
 			}
 
+			const { cache } = getCache();
+
 			const checks: Record<string, HealthCheck[]> = {
 				'cache:responseTime': [
 					{

api/src/services/users.ts

@@ -2,7 +2,6 @@ import argon2 from 'argon2';
 import jwt from 'jsonwebtoken';
 import { Knex } from 'knex';
 import { clone } from 'lodash';
-import cache from '../cache';
 import getDatabase from '../database';
 import env from '../env';
 import {
@@ -287,8 +286,8 @@ export class UsersService extends ItemsService {
 
 		await this.knex('directus_users').update({ password: passwordHashed, status: 'active' }).where({ id: user.id });
 
-		if (cache && env.CACHE_AUTO_PURGE) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE) {
+			await this.cache.clear();
 		}
 	}
 
@@ -343,8 +342,8 @@ export class UsersService extends ItemsService {
 
 		await this.knex('directus_users').update({ password: passwordHashed, status: 'active' }).where({ id: user.id });
 
-		if (cache && env.CACHE_AUTO_PURGE) {
-			await cache.clear();
+		if (this.cache && env.CACHE_AUTO_PURGE) {
+			await this.cache.clear();
 		}
 	}
 

api/src/utils/get-schema.ts

@@ -12,20 +12,32 @@ import getDefaultValue from './get-default-value';
 import getLocalType from './get-local-type';
 import { mergePermissions } from './merge-permissions';
 import getDatabase from '../database';
+import { getCache } from '../cache';
+import env from '../env';
+import ms from 'ms';
 
 export async function getSchema(options?: {
 	accountability?: Accountability;
 	database?: Knex;
 }): Promise<SchemaOverview> {
-	// Allows for use in the CLI
 	const database = options?.database || getDatabase();
 	const schemaInspector = SchemaInspector(database);
+	const { schemaCache } = getCache();
 
-	const result: SchemaOverview = {
-		collections: {},
-		relations: [],
-		permissions: [],
-	};
+	let result: SchemaOverview;
+
+	if (env.CACHE_SCHEMA !== false && schemaCache) {
+		const cachedSchema = (await schemaCache.get('schema')) as SchemaOverview;
+
+		if (cachedSchema) {
+			result = cachedSchema;
+		} else {
+			result = await getDatabaseSchema(database, schemaInspector);
+			await schemaCache.set('schema', result, typeof env.CACHE_SCHEMA === 'string' ? ms(env.CACHE_SCHEMA) : undefined);
+		}
+	} else {
+		result = await getDatabaseSchema(database, schemaInspector);
+	}
 
 	let permissions: Permission[] = [];
 
@@ -65,6 +77,19 @@ export async function getSchema(options?: {
 
 	result.permissions = permissions;
 
+	return result;
+}
+
+async function getDatabaseSchema(
+	database: Knex,
+	schemaInspector: ReturnType<typeof SchemaInspector>
+): Promise<SchemaOverview> {
+	const result: SchemaOverview = {
+		collections: {},
+		relations: [],
+		permissions: [],
+	};
+
 	const schemaOverview = await schemaInspector.overview();
 
 	const collections = [

docs/reference/environment-variables.md

@@ -125,14 +125,17 @@ needs, you can extend the above environment variables to configure any of
 
 ## Cache
 
-| Variable                 | Description                                                                            | Default Value    |
-| ------------------------ | -------------------------------------------------------------------------------------- | ---------------- |
-| `CACHE_ENABLED`          | Whether or not caching is enabled.                                                     | `false`          |
-| `CACHE_TTL`              | How long the cache is persisted.                                                       | `30m`            |
-| `CACHE_CONTROL_S_MAXAGE` | Whether to not to add the s-maxage expiration flag. Set to a number for a custom value | `0`              |
-| `CACHE_AUTO_PURGE`       | Automatically purge the cache on `create`/`update`/`delete` actions.                   | `false`          |
-| `CACHE_NAMESPACE`        | How to scope the cache data.                                                           | `directus-cache` |
-| `CACHE_STORE`            | Where to store the cache data. Either `memory`, `redis`, or `memcache`.                | `memory`         |
+| Variable                      | Description                                                                                  | Default Value    |
+| ----------------------------- | -------------------------------------------------------------------------------------------- | ---------------- |
+| `CACHE_ENABLED`               | Whether or not caching is enabled.                                                           | `false`          |
+| `CACHE_TTL`                   | How long the cache is persisted.                                                             | `30m`            |
+| `CACHE_CONTROL_S_MAXAGE`      | Whether to not to add the s-maxage expiration flag. Set to a number for a custom value       | `0`              |
+| `CACHE_AUTO_PURGE`            | Automatically purge the cache on `create`/`update`/`delete` actions.                         | `false`          |
+| `CACHE_SCHEMA` <sup>[1]</sup> | Whether or not the database schema is cached. One of `false`, `true`, or a string time value | `true`           |
+| `CACHE_NAMESPACE`             | How to scope the cache data.                                                                 | `directus-cache` |
+| `CACHE_STORE`                 | Where to store the cache data. Either `memory`, `redis`, or `memcache`.                      | `memory`         |
+
+<sup>[1]</sup> `CACHE_SCHEMA` ignores the `CACHE_ENABLED` value
 
 Based on the `CACHE_STORE` used, you must also provide the following configurations: