Add auto-purge option

And add cache-control header when auto purge is disabled

Fixes #3425

api/example.env

@@ -53,6 +53,9 @@ CACHE_ENABLED=true
 CACHE_TTL="30m"
 CACHE_NAMESPACE="directus-cache"
 CACHE_STORE=memory # memory | redis | memcache
+CACHE_AUTO_PURGE=true
+
+ASSETS_CACHE_CONTROL="max-age=604800"
 
 # CACHE_REDIS="redis://:authpassword@127.0.0.1:6380/4"
 # --OR--

api/src/env.ts

@@ -33,6 +33,7 @@ const defaults: Record<string, any> = {
 	CACHE_STORE: 'memory',
 	CACHE_TTL: '30m',
 	CACHE_NAMESPACE: 'system-cache',
+	CACHE_AUTO_PURGE: false,
 
 	OAUTH_PROVIDERS: '',
 

api/src/middleware/cache.ts

@@ -14,6 +14,14 @@ const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next)
 	const cachedData = await cache.get(key);
 
 	if (cachedData) {
+		// Set cache-control header
+		if (env.CACHE_AUTO_PURGE !== true) {
+			const expiresAt = await cache.get(`${key}__expires_at`);
+			const maxAge = `max-age="${expiresAt - Date.now()}"`;
+			const access = !!req.accountability?.role === false ? 'public' : 'private';
+			res.setHeader('Cache-Control', `${access}, ${maxAge}`);
+		}
+
 		return res.json(cachedData);
 	} else {
 		return next();

api/src/middleware/respond.ts

@@ -5,11 +5,20 @@ import { getCacheKey } from '../utils/get-cache-key';
 import cache from '../cache';
 import { Transform, transforms } from 'json2csv';
 import { PassThrough } from 'stream';
+import ms from 'ms';
 
 export const respond: RequestHandler = asyncHandler(async (req, res) => {
 	if (req.method.toLowerCase() === 'get' && env.CACHE_ENABLED === true && cache && !req.sanitizedQuery.export) {
 		const key = getCacheKey(req);
-		await cache.set(key, res.locals.payload);
+		await cache.set(key, res.locals.payload, ms(env.CACHE_TTL as string));
+		await cache.set(`${key}__expires_at`, Date.now() + ms(env.CACHE_TTL as string));
+
+		// Set cache-control header
+		if (env.CACHE_AUTO_PURGE !== true) {
+			const maxAge = `max-age="${ms(env.CACHE_TTL as string)}"`;
+			const access = !!req.accountability?.role === false ? 'public' : 'private';
+			res.setHeader('Cache-Control', `${access}, ${maxAge}`);
+		}
 	}
 
 	if (req.sanitizedQuery.export) {

api/src/services/collections.ts

@@ -7,6 +7,7 @@ import { ItemsService } from '../services/items';
 import cache from '../cache';
 import { toArray } from '../utils/to-array';
 import { systemCollectionRows } from '../database/system-data/collections';
+import env from '../env';
 
 export class CollectionsService {
 	knex: Knex;
@@ -93,7 +94,7 @@ export class CollectionsService {
 			}
 		});
 
-		if (cache) {
+		if (cache && env.CACHE_AUTO_PURGE) {
 			await cache.clear();
 		}
 
@@ -245,7 +246,7 @@ export class CollectionsService {
 
 		await collectionItemsService.update(collectionUpdates);
 
-		if (cache) {
+		if (cache && env.CACHE_AUTO_PURGE) {
 			await cache.clear();
 		}
 
@@ -316,7 +317,7 @@ export class CollectionsService {
 			await this.knex.schema.dropTable(collectionKey);
 		}
 
-		if (cache) {
+		if (cache && env.CACHE_AUTO_PURGE) {
 			await cache.clear();
 		}
 

api/src/services/fields.ts

@@ -12,6 +12,7 @@ import getDefaultValue from '../utils/get-default-value';
 import cache from '../cache';
 import SchemaInspector from '@directus/schema';
 import { toArray } from '../utils/to-array';
+import env from '../env';
 
 import { systemFieldRows } from '../database/system-data/fields/';
 
@@ -227,7 +228,7 @@ export class FieldsService {
 			});
 		}
 
-		if (cache) {
+		if (cache && env.CACHE_AUTO_PURGE) {
 			await cache.clear();
 		}
 	}
@@ -269,7 +270,7 @@ export class FieldsService {
 			}
 		}
 
-		if (cache) {
+		if (cache && env.CACHE_AUTO_PURGE) {
 			await cache.clear();
 		}
 
@@ -311,7 +312,7 @@ export class FieldsService {
 			}
 		}
 
-		if (cache) {
+		if (cache && env.CACHE_AUTO_PURGE) {
 			await cache.clear();
 		}
 	}

api/src/services/files.ts

@@ -11,6 +11,7 @@ import { ForbiddenException } from '../exceptions';
 import { toArray } from '../utils/to-array';
 import { extension } from 'mime-types';
 import path from 'path';
+import env from '../env';
 
 export class FilesService extends ItemsService {
 	constructor(options: AbstractServiceOptions) {
@@ -86,7 +87,7 @@ export class FilesService extends ItemsService {
 		});
 		await sudoService.update(payload, primaryKey);
 
-		if (cache) {
+		if (cache && env.CACHE_AUTO_PURGE) {
 			await cache.clear();
 		}
 
@@ -116,7 +117,7 @@ export class FilesService extends ItemsService {
 
 		await super.delete(keys);
 
-		if (cache) {
+		if (cache && env.CACHE_AUTO_PURGE) {
 			await cache.clear();
 		}
 

api/src/services/items.ts

@@ -17,6 +17,7 @@ import cache from '../cache';
 import emitter from '../emitter';
 import logger from '../logger';
 import { toArray } from '../utils/to-array';
+import env from '../env';
 
 import { PayloadService } from './payload';
 import { AuthorizationService } from './authorization';
@@ -154,7 +155,7 @@ export class ItemsService<Item extends AnyItem = AnyItem> implements AbstractSer
 				await trx.insert(revisionRecords).into('directus_revisions');
 			}
 
-			if (cache) {
+			if (cache && env.CACHE_AUTO_PURGE) {
 				await cache.clear();
 			}
 
@@ -349,7 +350,7 @@ export class ItemsService<Item extends AnyItem = AnyItem> implements AbstractSer
 				}
 			});
 
-			if (cache) {
+			if (cache && env.CACHE_AUTO_PURGE) {
 				await cache.clear();
 			}
 
@@ -486,7 +487,7 @@ export class ItemsService<Item extends AnyItem = AnyItem> implements AbstractSer
 			}
 		});
 
-		if (cache) {
+		if (cache && env.CACHE_AUTO_PURGE) {
 			await cache.clear();
 		}
 

api/src/services/users.ts

@@ -46,7 +46,7 @@ export class UsersService extends ItemsService {
 			}
 		}
 
-		if (cache) {
+		if (cache && env.CACHE_AUTO_PURGE) {
 			await cache.clear();
 		}
 
@@ -110,7 +110,7 @@ export class UsersService extends ItemsService {
 
 		await this.knex('directus_users').update({ password: passwordHashed, status: 'active' }).where({ id: user.id });
 
-		if (cache) {
+		if (cache && env.CACHE_AUTO_PURGE) {
 			await cache.clear();
 		}
 	}
@@ -144,7 +144,7 @@ export class UsersService extends ItemsService {
 
 		await this.knex('directus_users').update({ password: passwordHashed, status: 'active' }).where({ id: user.id });
 
-		if (cache) {
+		if (cache && env.CACHE_AUTO_PURGE) {
 			await cache.clear();
 		}
 	}

docs/reference/environment-variables.md

@@ -205,6 +205,12 @@ the following configurations.<br>**Default: `memory`**
 - **Memcache**
   - **`CACHE_MEMCACHE`** — Location of your memcache instance
 
+### `CACHE_AUTO_PURGE`
+
+Controls whether or not the cache will be auto-purged on create/update/delete actions within the system. Enabling this
+feature means that the API will remain real-time, while caching subsequent read calls when no changes have happened.
+**Note**: enabling auto-purge will remove the `Cache-Control` header, as the cache can be invalidated at any point.
+
 ### `ASSETS_CACHE_CONTROL`
 
 The value for the `Cache-Control` header for the static assets in the /assets endpoint. Defaults to