Improve cache performance by compressing records (#14833)

* Utils to compress/decompress data
Gzip was chosen because we want smaller data but quick algorithm since this will be ran for every request

* Compress system cache

* Decompress system cache

* Set/Get compressed cache for individual requests

* Switch from gzip to snappy, use json compression too

* Fix cache exp set/get

* Remove unused import

Co-authored-by: rijkvanzanten <rijkvanzanten@me.com>

api/package.json

@@ -155,6 +155,7 @@
 		"rollup": "^2.75.6",
 		"sanitize-html": "^2.7.0",
 		"sharp": "^0.30.6",
+		"snappy": "^7.1.1",
 		"stream-json": "^1.7.4",
 		"strip-bom-stream": "^4.0.0",
 		"supertest": "^6.2.3",

api/src/cache.ts

@@ -4,6 +4,7 @@ import env from './env';
 import logger from './logger';
 import { getConfigFromEnv } from './utils/get-config-from-env';
 import { validateEnv } from './utils/validate-env';
+import { compress, decompress } from './utils/compress';
 
 let cache: Keyv | null = null;
 let systemCache: Keyv | null = null;
@@ -50,10 +51,33 @@ export async function setSystemCache(key: string, value: any, ttl?: number): Pro
 	const { systemCache, lockCache } = getCache();
 
 	if (!(await lockCache.get('system-cache-lock'))) {
-		await systemCache.set(key, value, ttl);
+		await setCacheValue(systemCache, key, value, ttl);
 	}
 }
 
+export async function getSystemCache(key: string): Promise<Record<string, any>> {
+	const { systemCache } = getCache();
+
+	return await getCacheValue(systemCache, key);
+}
+
+export async function setCacheValue(
+	cache: Keyv,
+	key: string,
+	value: Record<string, any> | Record<string, any>[],
+	ttl?: number
+) {
+	const compressed = await compress(value);
+	await cache.set(key, compressed, ttl);
+}
+
+export async function getCacheValue(cache: Keyv, key: string): Promise<any> {
+	const value = await cache.get(key);
+	if (!value) return undefined;
+	const decompressed = await decompress(value);
+	return decompressed;
+}
+
 function getKeyvInstance(ttl: number | undefined, namespaceSuffix?: string): Keyv {
 	switch (env.CACHE_STORE) {
 		case 'redis':

api/src/middleware/cache.ts

@@ -1,5 +1,5 @@
 import { RequestHandler } from 'express';
-import { getCache } from '../cache';
+import { getCache, getCacheValue } from '../cache';
 import env from '../env';
 import asyncHandler from '../utils/async-handler';
 import { getCacheControlHeader } from '../utils/get-cache-headers';
@@ -23,7 +23,7 @@ const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next)
 	let cachedData;
 
 	try {
-		cachedData = await cache.get(key);
+		cachedData = await getCacheValue(cache, key);
 	} catch (err: any) {
 		logger.warn(err, `[cache] Couldn't read key ${key}. ${err.message}`);
 		if (env.CACHE_STATUS_HEADER) res.setHeader(`${env.CACHE_STATUS_HEADER}`, 'MISS');
@@ -34,7 +34,7 @@ const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next)
 		let cacheExpiryDate;
 
 		try {
-			cacheExpiryDate = (await cache.get(`${key}__expires_at`)) as number | null;
+			cacheExpiryDate = (await getCacheValue(cache, `${key}__expires_at`))?.exp;
 		} catch (err: any) {
 			logger.warn(err, `[cache] Couldn't read key ${`${key}__expires_at`}. ${err.message}`);
 			if (env.CACHE_STATUS_HEADER) res.setHeader(`${env.CACHE_STATUS_HEADER}`, 'MISS');

api/src/middleware/respond.ts

@@ -1,6 +1,6 @@
 import { RequestHandler } from 'express';
 import ms from 'ms';
-import { getCache } from '../cache';
+import { getCache, setCacheValue } from '../cache';
 import env from '../env';
 import asyncHandler from '../utils/async-handler';
 import { getCacheKey } from '../utils/get-cache-key';
@@ -33,8 +33,8 @@ export const respond: RequestHandler = asyncHandler(async (req, res) => {
 		const key = getCacheKey(req);
 
 		try {
-			await cache.set(key, res.locals.payload, ms(env.CACHE_TTL as string));
-			await cache.set(`${key}__expires_at`, Date.now() + ms(env.CACHE_TTL as string));
+			await setCacheValue(cache, key, res.locals.payload, ms(env.CACHE_TTL as string));
+			await setCacheValue(cache, `${key}__expires_at`, { exp: Date.now() + ms(env.CACHE_TTL as string) });
 		} catch (err: any) {
 			logger.warn(err, `[cache] Couldn't set key ${key}. ${err}`);
 		}

api/src/utils/compress.ts

@@ -0,0 +1,12 @@
+import { compress as compressSnappy, uncompress as uncompressSnappy } from 'snappy';
+import { compress as compressJSON, decompress as decompressJSON } from '@directus/shared/utils';
+
+export async function compress(raw: Record<string, any> | Record<string, any>[]): Promise<Buffer> {
+	if (!raw) return raw;
+	return await compressSnappy(compressJSON(raw));
+}
+
+export async function decompress(compressed: Buffer): Promise<any> {
+	if (!compressed) return compressed;
+	return decompressJSON((await uncompressSnappy(compressed, { asBuffer: false })) as string);
+}

api/src/utils/get-permissions.ts

@@ -2,7 +2,7 @@ import { Accountability, Permission, SchemaOverview } from '@directus/shared/typ
 import { deepMap, parseFilter, parseJSON, parsePreset } from '@directus/shared/utils';
 import { cloneDeep } from 'lodash';
 import hash from 'object-hash';
-import { getCache, setSystemCache } from '../cache';
+import { getCache, getSystemCache, setSystemCache } from '../cache';
 import getDatabase from '../database';
 import { appAccessMinimalPermissions } from '../database/system-data/app-access-permissions';
 import env from '../env';
@@ -13,7 +13,7 @@ import { mergePermissionsForShare } from './merge-permissions-for-share';
 
 export async function getPermissions(accountability: Accountability, schema: SchemaOverview) {
 	const database = getDatabase();
-	const { systemCache, cache } = getCache();
+	const { cache } = getCache();
 
 	let permissions: Permission[] = [];
 
@@ -21,7 +21,7 @@ export async function getPermissions(accountability: Accountability, schema: Sch
 	const cacheKey = `permissions-${hash({ user, role, app, admin, share_scope })}`;
 
 	if (env.CACHE_PERMISSIONS !== false) {
-		const cachedPermissions = await systemCache.get(cacheKey);
+		const cachedPermissions = await getSystemCache(cacheKey);
 
 		if (cachedPermissions) {
 			if (!cachedPermissions.containDynamicData) {

api/src/utils/get-schema.ts

@@ -3,7 +3,7 @@ import { Accountability, Filter, SchemaOverview } from '@directus/shared/types';
 import { parseJSON, toArray } from '@directus/shared/utils';
 import { Knex } from 'knex';
 import { mapValues } from 'lodash';
-import { getCache, setSystemCache } from '../cache';
+import { getSystemCache, setSystemCache } from '../cache';
 import { ALIAS_TYPES } from '../constants';
 import getDatabase from '../database';
 import { systemCollectionRows } from '../database/system-data/collections';
@@ -20,7 +20,6 @@ export async function getSchema(options?: {
 }): Promise<SchemaOverview> {
 	const database = options?.database || getDatabase();
 	const schemaInspector = SchemaInspector(database);
-	const { systemCache } = getCache();
 
 	let result: SchemaOverview;
 
@@ -28,7 +27,7 @@ export async function getSchema(options?: {
 		let cachedSchema;
 
 		try {
-			cachedSchema = (await systemCache.get('schema')) as SchemaOverview;
+			cachedSchema = (await getSystemCache('schema')) as SchemaOverview;
 		} catch (err: any) {
 			logger.warn(err, `[schema-cache] Couldn't retrieve cache. ${err}`);
 		}

packages/shared/src/utils/index.ts

@@ -1,3 +1,4 @@
+export { compress, decompress } from './compress';
 export * from './abbreviate-number';
 export * from './add-field-flag';
 export * from './adjust-date';

pnpm-lock.yaml

@@ -198,6 +198,7 @@ importers:
       rollup: ^2.75.6
       sanitize-html: ^2.7.0
       sharp: ^0.30.6
+      snappy: ^7.1.1
       sqlite3: ^5.0.8
       stream-json: ^1.7.4
       strip-bom-stream: ^4.0.0
@@ -292,6 +293,7 @@ importers:
       rollup: 2.77.0
       sanitize-html: 2.7.0
       sharp: 0.30.7
+      snappy: 7.1.1
       stream-json: 1.7.4
       strip-bom-stream: 4.0.0
       supertest: 6.2.4
@@ -3225,6 +3227,136 @@ packages:
     engines: { node: '>=6.0.0' }
     dev: true
 
+  /@napi-rs/snappy-android-arm-eabi/7.1.1:
+    resolution:
+      { integrity: sha512-NKd/ztuVEgQaAaNVQ5zZaCB9VV+7+uBXBHqhaE5iSapQhLc41szTlT0s68FCee75OoT3vhqdA6Jp5TrzZ2WOaw== }
+    engines: { node: '>= 10' }
+    cpu: [arm]
+    os: [android]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  /@napi-rs/snappy-android-arm64/7.1.1:
+    resolution:
+      { integrity: sha512-DktruMAO0K0toTnxNHg2GWNIAPJqdvIchCsdsRaKyuEnG101qBg0mYiRCAhxHgbT6RJlOGbUPKkbA9KKRhEJUg== }
+    engines: { node: '>= 10' }
+    cpu: [arm64]
+    os: [android]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  /@napi-rs/snappy-darwin-arm64/7.1.1:
+    resolution:
+      { integrity: sha512-3LZyoAw3Qa5F7sCCTkSkhmGlydwUKU6L3Jl46eKHO2Ctm8Gcjxww6T7MfwlwGZ6JqAM6d1d++WLzUZPCGXVmag== }
+    engines: { node: '>= 10' }
+    cpu: [arm64]
+    os: [darwin]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  /@napi-rs/snappy-darwin-x64/7.1.1:
+    resolution:
+      { integrity: sha512-X1D2F67bQkPwr5iSR29/RnOrLwAkB55YO6t41toABzla3mflLDpzZcakz6FokIukykf7ey31/t73v/4pbgaBkg== }
+    engines: { node: '>= 10' }
+    cpu: [x64]
+    os: [darwin]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  /@napi-rs/snappy-freebsd-x64/7.1.1:
+    resolution:
+      { integrity: sha512-vSeuf+An8jFVHPAn5IbWE9hTGU9PFAaZLj/X7rKTQQtZstnDsHgWe6u4g7FHLuOdwQ8TvhcxAEpNlYIXIk4AJg== }
+    engines: { node: '>= 10' }
+    cpu: [x64]
+    os: [freebsd]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  /@napi-rs/snappy-linux-arm-gnueabihf/7.1.1:
+    resolution:
+      { integrity: sha512-/yyN6QsnOs3D1+jI3SfRX+gtnD86rbixdfmgxv9g40+FrDaDTLAu/3VuZIqH02qqq/xiWbDnkO+42RGxXDzTCw== }
+    engines: { node: '>= 10' }
+    cpu: [arm]
+    os: [linux]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  /@napi-rs/snappy-linux-arm64-gnu/7.1.1:
+    resolution:
+      { integrity: sha512-StEeUCSwUoajgrBtiCQPTkHu+0Q4QlYndghGZNdbN1zJ1ny70YzPpevaFBUyjI/eJ+FN9uICKtwTPtQNSILS5g== }
+    engines: { node: '>= 10' }
+    cpu: [arm64]
+    os: [linux]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  /@napi-rs/snappy-linux-arm64-musl/7.1.1:
+    resolution:
+      { integrity: sha512-jWEBRzj+lswZVYf0b5eY0fjMlBL9L9yqjmTuv2UIMjJNHPuR282LK/s3Fm9sYIXQtKkiCo5JyhmIcoghZ3v0Eg== }
+    engines: { node: '>= 10' }
+    cpu: [arm64]
+    os: [linux]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  /@napi-rs/snappy-linux-x64-gnu/7.1.1:
+    resolution:
+      { integrity: sha512-41DPoAUFAU9VNrj/96qKfStH2Xq88ZYIsSz8BlITDm2ScoeDGOGbmaWguCXU7I+bC2uKWTmUVMXKqk6tVY6LEg== }
+    engines: { node: '>= 10' }
+    cpu: [x64]
+    os: [linux]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  /@napi-rs/snappy-linux-x64-musl/7.1.1:
+    resolution:
+      { integrity: sha512-xR4hzFQqVq6J8Zf6XyUVtFJBaRgDyAQYUoBsCr92tZ7gI/0RlWCV6Q6JMO/wP5CSsvyFJIAtSUXXqlzIpw0GPA== }
+    engines: { node: '>= 10' }
+    cpu: [x64]
+    os: [linux]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  /@napi-rs/snappy-win32-arm64-msvc/7.1.1:
+    resolution:
+      { integrity: sha512-2mHPadctsaYtrfSV5Na8ooTdI5rflPxP1pceY4us6vbjeWrfgB+KQCuEFOHsGXqFNfsi6L9nWH8nB9swnxnSyw== }
+    engines: { node: '>= 10' }
+    cpu: [arm64]
+    os: [win32]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  /@napi-rs/snappy-win32-ia32-msvc/7.1.1:
+    resolution:
+      { integrity: sha512-FOMgs9W71hdgjyl3T9F7b/WEIuoryfgBqsyhtHjAaa/98R0BUHl0bOoHg8ka0b5GgnhLBHkX2Yd6VD+Si9Q2ww== }
+    engines: { node: '>= 10' }
+    cpu: [ia32]
+    os: [win32]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  /@napi-rs/snappy-win32-x64-msvc/7.1.1:
+    resolution:
+      { integrity: sha512-Mu3yELySvzhBcNTVCq+hYxVh+lH3/KjoQ5HIEb3DDPoX0AGRTm3XZa+usq8pFWjl91Cgp9nWK+9lVSkCCIRaKA== }
+    engines: { node: '>= 10' }
+    cpu: [x64]
+    os: [win32]
+    requiresBuild: true
+    dev: false
+    optional: true
+
   /@nodelib/fs.scandir/2.1.5:
     resolution:
       { integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g== }
@@ -13903,6 +14035,26 @@ packages:
     engines: { node: '>= 6.0.0', npm: '>= 3.0.0' }
     optional: true
 
+  /snappy/7.1.1:
+    resolution:
+      { integrity: sha512-mL7GGPJ+WdsaFT5aR/uEqCq8cPg2VbhyifDEP7AeqIVDsAC8LBGYbZP1Qzoa2Ym84OW7JEQXqIpwqFp1EQw5BA== }
+    engines: { node: '>= 10' }
+    optionalDependencies:
+      '@napi-rs/snappy-android-arm-eabi': 7.1.1
+      '@napi-rs/snappy-android-arm64': 7.1.1
+      '@napi-rs/snappy-darwin-arm64': 7.1.1
+      '@napi-rs/snappy-darwin-x64': 7.1.1
+      '@napi-rs/snappy-freebsd-x64': 7.1.1
+      '@napi-rs/snappy-linux-arm-gnueabihf': 7.1.1
+      '@napi-rs/snappy-linux-arm64-gnu': 7.1.1
+      '@napi-rs/snappy-linux-arm64-musl': 7.1.1
+      '@napi-rs/snappy-linux-x64-gnu': 7.1.1
+      '@napi-rs/snappy-linux-x64-musl': 7.1.1
+      '@napi-rs/snappy-win32-arm64-msvc': 7.1.1
+      '@napi-rs/snappy-win32-ia32-msvc': 7.1.1
+      '@napi-rs/snappy-win32-x64-msvc': 7.1.1
+    dev: false
+
   /socket.io-adapter/2.1.0:
     resolution:
       { integrity: sha512-+vDov/aTsLjViYTwS9fPy5pEtTkrbEKsw2M+oVSoFGw6OD1IpvlV1VPhUzNbofCQ8oyMbdYJqDtGdmHQK6TdPg== }
@@ -14997,7 +15149,7 @@ packages:
     dependencies:
       bs-logger: 0.2.6
       fast-json-stable-stringify: 2.1.0
-      jest: 28.1.2
+      jest: 28.1.2_dyhfsldgbafx4up6nvciefunqu
       jest-util: 28.1.3
       json5: 2.2.1
       lodash.memoize: 4.1.2