From cc343fdf9153b1bb02ad3ed9c8dbd51b5cc6171f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Varela?= Date: Thu, 4 Aug 2022 22:35:27 +0100 Subject: [PATCH] Improve cache performance by compressing records (#14833) * Utils to compress/decompress data Gzip was chosen because we want smaller data but quick algorithm since this will be ran for every request * Compress system cache * Decompress system cache * Set/Get compressed cache for individual requests * Switch from gzip to snappy, use json compression too * Fix cache exp set/get * Remove unused import Co-authored-by: rijkvanzanten --- api/package.json | 1 + api/src/cache.ts | 26 ++++- api/src/middleware/cache.ts | 6 +- api/src/middleware/respond.ts | 6 +- api/src/utils/compress.ts | 12 +++ api/src/utils/get-permissions.ts | 6 +- api/src/utils/get-schema.ts | 5 +- packages/shared/src/utils/index.ts | 1 + pnpm-lock.yaml | 154 ++++++++++++++++++++++++++++- 9 files changed, 203 insertions(+), 14 deletions(-) create mode 100644 api/src/utils/compress.ts diff --git a/api/package.json b/api/package.json index 4773f11fef..2fc92edd89 100644 --- a/api/package.json +++ b/api/package.json @@ -155,6 +155,7 @@ "rollup": "^2.75.6", "sanitize-html": "^2.7.0", "sharp": "^0.30.6", + "snappy": "^7.1.1", "stream-json": "^1.7.4", "strip-bom-stream": "^4.0.0", "supertest": "^6.2.3", diff --git a/api/src/cache.ts b/api/src/cache.ts index faa89348c3..9b617c76a8 100644 --- a/api/src/cache.ts +++ b/api/src/cache.ts @@ -4,6 +4,7 @@ import env from './env'; import logger from './logger'; import { getConfigFromEnv } from './utils/get-config-from-env'; import { validateEnv } from './utils/validate-env'; +import { compress, decompress } from './utils/compress'; let cache: Keyv | null = null; let systemCache: Keyv | null = null; @@ -50,10 +51,33 @@ export async function setSystemCache(key: string, value: any, ttl?: number): Pro const { systemCache, lockCache } = getCache(); if (!(await lockCache.get('system-cache-lock'))) { - await systemCache.set(key, value, ttl); + await setCacheValue(systemCache, key, value, ttl); } } +export async function getSystemCache(key: string): Promise> { + const { systemCache } = getCache(); + + return await getCacheValue(systemCache, key); +} + +export async function setCacheValue( + cache: Keyv, + key: string, + value: Record | Record[], + ttl?: number +) { + const compressed = await compress(value); + await cache.set(key, compressed, ttl); +} + +export async function getCacheValue(cache: Keyv, key: string): Promise { + const value = await cache.get(key); + if (!value) return undefined; + const decompressed = await decompress(value); + return decompressed; +} + function getKeyvInstance(ttl: number | undefined, namespaceSuffix?: string): Keyv { switch (env.CACHE_STORE) { case 'redis': diff --git a/api/src/middleware/cache.ts b/api/src/middleware/cache.ts index a80ce272f0..757c59b726 100644 --- a/api/src/middleware/cache.ts +++ b/api/src/middleware/cache.ts @@ -1,5 +1,5 @@ import { RequestHandler } from 'express'; -import { getCache } from '../cache'; +import { getCache, getCacheValue } from '../cache'; import env from '../env'; import asyncHandler from '../utils/async-handler'; import { getCacheControlHeader } from '../utils/get-cache-headers'; @@ -23,7 +23,7 @@ const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next) let cachedData; try { - cachedData = await cache.get(key); + cachedData = await getCacheValue(cache, key); } catch (err: any) { logger.warn(err, `[cache] Couldn't read key ${key}. ${err.message}`); if (env.CACHE_STATUS_HEADER) res.setHeader(`${env.CACHE_STATUS_HEADER}`, 'MISS'); @@ -34,7 +34,7 @@ const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next) let cacheExpiryDate; try { - cacheExpiryDate = (await cache.get(`${key}__expires_at`)) as number | null; + cacheExpiryDate = (await getCacheValue(cache, `${key}__expires_at`))?.exp; } catch (err: any) { logger.warn(err, `[cache] Couldn't read key ${`${key}__expires_at`}. ${err.message}`); if (env.CACHE_STATUS_HEADER) res.setHeader(`${env.CACHE_STATUS_HEADER}`, 'MISS'); diff --git a/api/src/middleware/respond.ts b/api/src/middleware/respond.ts index 855bf81015..7857766720 100644 --- a/api/src/middleware/respond.ts +++ b/api/src/middleware/respond.ts @@ -1,6 +1,6 @@ import { RequestHandler } from 'express'; import ms from 'ms'; -import { getCache } from '../cache'; +import { getCache, setCacheValue } from '../cache'; import env from '../env'; import asyncHandler from '../utils/async-handler'; import { getCacheKey } from '../utils/get-cache-key'; @@ -33,8 +33,8 @@ export const respond: RequestHandler = asyncHandler(async (req, res) => { const key = getCacheKey(req); try { - await cache.set(key, res.locals.payload, ms(env.CACHE_TTL as string)); - await cache.set(`${key}__expires_at`, Date.now() + ms(env.CACHE_TTL as string)); + await setCacheValue(cache, key, res.locals.payload, ms(env.CACHE_TTL as string)); + await setCacheValue(cache, `${key}__expires_at`, { exp: Date.now() + ms(env.CACHE_TTL as string) }); } catch (err: any) { logger.warn(err, `[cache] Couldn't set key ${key}. ${err}`); } diff --git a/api/src/utils/compress.ts b/api/src/utils/compress.ts new file mode 100644 index 0000000000..7683e5dfc9 --- /dev/null +++ b/api/src/utils/compress.ts @@ -0,0 +1,12 @@ +import { compress as compressSnappy, uncompress as uncompressSnappy } from 'snappy'; +import { compress as compressJSON, decompress as decompressJSON } from '@directus/shared/utils'; + +export async function compress(raw: Record | Record[]): Promise { + if (!raw) return raw; + return await compressSnappy(compressJSON(raw)); +} + +export async function decompress(compressed: Buffer): Promise { + if (!compressed) return compressed; + return decompressJSON((await uncompressSnappy(compressed, { asBuffer: false })) as string); +} diff --git a/api/src/utils/get-permissions.ts b/api/src/utils/get-permissions.ts index 5be90eb1ac..f1989d9219 100644 --- a/api/src/utils/get-permissions.ts +++ b/api/src/utils/get-permissions.ts @@ -2,7 +2,7 @@ import { Accountability, Permission, SchemaOverview } from '@directus/shared/typ import { deepMap, parseFilter, parseJSON, parsePreset } from '@directus/shared/utils'; import { cloneDeep } from 'lodash'; import hash from 'object-hash'; -import { getCache, setSystemCache } from '../cache'; +import { getCache, getSystemCache, setSystemCache } from '../cache'; import getDatabase from '../database'; import { appAccessMinimalPermissions } from '../database/system-data/app-access-permissions'; import env from '../env'; @@ -13,7 +13,7 @@ import { mergePermissionsForShare } from './merge-permissions-for-share'; export async function getPermissions(accountability: Accountability, schema: SchemaOverview) { const database = getDatabase(); - const { systemCache, cache } = getCache(); + const { cache } = getCache(); let permissions: Permission[] = []; @@ -21,7 +21,7 @@ export async function getPermissions(accountability: Accountability, schema: Sch const cacheKey = `permissions-${hash({ user, role, app, admin, share_scope })}`; if (env.CACHE_PERMISSIONS !== false) { - const cachedPermissions = await systemCache.get(cacheKey); + const cachedPermissions = await getSystemCache(cacheKey); if (cachedPermissions) { if (!cachedPermissions.containDynamicData) { diff --git a/api/src/utils/get-schema.ts b/api/src/utils/get-schema.ts index bfb78acb8b..4111b040b6 100644 --- a/api/src/utils/get-schema.ts +++ b/api/src/utils/get-schema.ts @@ -3,7 +3,7 @@ import { Accountability, Filter, SchemaOverview } from '@directus/shared/types'; import { parseJSON, toArray } from '@directus/shared/utils'; import { Knex } from 'knex'; import { mapValues } from 'lodash'; -import { getCache, setSystemCache } from '../cache'; +import { getSystemCache, setSystemCache } from '../cache'; import { ALIAS_TYPES } from '../constants'; import getDatabase from '../database'; import { systemCollectionRows } from '../database/system-data/collections'; @@ -20,7 +20,6 @@ export async function getSchema(options?: { }): Promise { const database = options?.database || getDatabase(); const schemaInspector = SchemaInspector(database); - const { systemCache } = getCache(); let result: SchemaOverview; @@ -28,7 +27,7 @@ export async function getSchema(options?: { let cachedSchema; try { - cachedSchema = (await systemCache.get('schema')) as SchemaOverview; + cachedSchema = (await getSystemCache('schema')) as SchemaOverview; } catch (err: any) { logger.warn(err, `[schema-cache] Couldn't retrieve cache. ${err}`); } diff --git a/packages/shared/src/utils/index.ts b/packages/shared/src/utils/index.ts index 1587101652..6d6bec6f80 100644 --- a/packages/shared/src/utils/index.ts +++ b/packages/shared/src/utils/index.ts @@ -1,3 +1,4 @@ +export { compress, decompress } from './compress'; export * from './abbreviate-number'; export * from './add-field-flag'; export * from './adjust-date'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6e4b7fe964..de2d2b9d2a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -198,6 +198,7 @@ importers: rollup: ^2.75.6 sanitize-html: ^2.7.0 sharp: ^0.30.6 + snappy: ^7.1.1 sqlite3: ^5.0.8 stream-json: ^1.7.4 strip-bom-stream: ^4.0.0 @@ -292,6 +293,7 @@ importers: rollup: 2.77.0 sanitize-html: 2.7.0 sharp: 0.30.7 + snappy: 7.1.1 stream-json: 1.7.4 strip-bom-stream: 4.0.0 supertest: 6.2.4 @@ -3225,6 +3227,136 @@ packages: engines: { node: '>=6.0.0' } dev: true + /@napi-rs/snappy-android-arm-eabi/7.1.1: + resolution: + { integrity: sha512-NKd/ztuVEgQaAaNVQ5zZaCB9VV+7+uBXBHqhaE5iSapQhLc41szTlT0s68FCee75OoT3vhqdA6Jp5TrzZ2WOaw== } + engines: { node: '>= 10' } + cpu: [arm] + os: [android] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-android-arm64/7.1.1: + resolution: + { integrity: sha512-DktruMAO0K0toTnxNHg2GWNIAPJqdvIchCsdsRaKyuEnG101qBg0mYiRCAhxHgbT6RJlOGbUPKkbA9KKRhEJUg== } + engines: { node: '>= 10' } + cpu: [arm64] + os: [android] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-darwin-arm64/7.1.1: + resolution: + { integrity: sha512-3LZyoAw3Qa5F7sCCTkSkhmGlydwUKU6L3Jl46eKHO2Ctm8Gcjxww6T7MfwlwGZ6JqAM6d1d++WLzUZPCGXVmag== } + engines: { node: '>= 10' } + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-darwin-x64/7.1.1: + resolution: + { integrity: sha512-X1D2F67bQkPwr5iSR29/RnOrLwAkB55YO6t41toABzla3mflLDpzZcakz6FokIukykf7ey31/t73v/4pbgaBkg== } + engines: { node: '>= 10' } + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-freebsd-x64/7.1.1: + resolution: + { integrity: sha512-vSeuf+An8jFVHPAn5IbWE9hTGU9PFAaZLj/X7rKTQQtZstnDsHgWe6u4g7FHLuOdwQ8TvhcxAEpNlYIXIk4AJg== } + engines: { node: '>= 10' } + cpu: [x64] + os: [freebsd] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-linux-arm-gnueabihf/7.1.1: + resolution: + { integrity: sha512-/yyN6QsnOs3D1+jI3SfRX+gtnD86rbixdfmgxv9g40+FrDaDTLAu/3VuZIqH02qqq/xiWbDnkO+42RGxXDzTCw== } + engines: { node: '>= 10' } + cpu: [arm] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-linux-arm64-gnu/7.1.1: + resolution: + { integrity: sha512-StEeUCSwUoajgrBtiCQPTkHu+0Q4QlYndghGZNdbN1zJ1ny70YzPpevaFBUyjI/eJ+FN9uICKtwTPtQNSILS5g== } + engines: { node: '>= 10' } + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-linux-arm64-musl/7.1.1: + resolution: + { integrity: sha512-jWEBRzj+lswZVYf0b5eY0fjMlBL9L9yqjmTuv2UIMjJNHPuR282LK/s3Fm9sYIXQtKkiCo5JyhmIcoghZ3v0Eg== } + engines: { node: '>= 10' } + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-linux-x64-gnu/7.1.1: + resolution: + { integrity: sha512-41DPoAUFAU9VNrj/96qKfStH2Xq88ZYIsSz8BlITDm2ScoeDGOGbmaWguCXU7I+bC2uKWTmUVMXKqk6tVY6LEg== } + engines: { node: '>= 10' } + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-linux-x64-musl/7.1.1: + resolution: + { integrity: sha512-xR4hzFQqVq6J8Zf6XyUVtFJBaRgDyAQYUoBsCr92tZ7gI/0RlWCV6Q6JMO/wP5CSsvyFJIAtSUXXqlzIpw0GPA== } + engines: { node: '>= 10' } + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-win32-arm64-msvc/7.1.1: + resolution: + { integrity: sha512-2mHPadctsaYtrfSV5Na8ooTdI5rflPxP1pceY4us6vbjeWrfgB+KQCuEFOHsGXqFNfsi6L9nWH8nB9swnxnSyw== } + engines: { node: '>= 10' } + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-win32-ia32-msvc/7.1.1: + resolution: + { integrity: sha512-FOMgs9W71hdgjyl3T9F7b/WEIuoryfgBqsyhtHjAaa/98R0BUHl0bOoHg8ka0b5GgnhLBHkX2Yd6VD+Si9Q2ww== } + engines: { node: '>= 10' } + cpu: [ia32] + os: [win32] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-win32-x64-msvc/7.1.1: + resolution: + { integrity: sha512-Mu3yELySvzhBcNTVCq+hYxVh+lH3/KjoQ5HIEb3DDPoX0AGRTm3XZa+usq8pFWjl91Cgp9nWK+9lVSkCCIRaKA== } + engines: { node: '>= 10' } + cpu: [x64] + os: [win32] + requiresBuild: true + dev: false + optional: true + /@nodelib/fs.scandir/2.1.5: resolution: { integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g== } @@ -13903,6 +14035,26 @@ packages: engines: { node: '>= 6.0.0', npm: '>= 3.0.0' } optional: true + /snappy/7.1.1: + resolution: + { integrity: sha512-mL7GGPJ+WdsaFT5aR/uEqCq8cPg2VbhyifDEP7AeqIVDsAC8LBGYbZP1Qzoa2Ym84OW7JEQXqIpwqFp1EQw5BA== } + engines: { node: '>= 10' } + optionalDependencies: + '@napi-rs/snappy-android-arm-eabi': 7.1.1 + '@napi-rs/snappy-android-arm64': 7.1.1 + '@napi-rs/snappy-darwin-arm64': 7.1.1 + '@napi-rs/snappy-darwin-x64': 7.1.1 + '@napi-rs/snappy-freebsd-x64': 7.1.1 + '@napi-rs/snappy-linux-arm-gnueabihf': 7.1.1 + '@napi-rs/snappy-linux-arm64-gnu': 7.1.1 + '@napi-rs/snappy-linux-arm64-musl': 7.1.1 + '@napi-rs/snappy-linux-x64-gnu': 7.1.1 + '@napi-rs/snappy-linux-x64-musl': 7.1.1 + '@napi-rs/snappy-win32-arm64-msvc': 7.1.1 + '@napi-rs/snappy-win32-ia32-msvc': 7.1.1 + '@napi-rs/snappy-win32-x64-msvc': 7.1.1 + dev: false + /socket.io-adapter/2.1.0: resolution: { integrity: sha512-+vDov/aTsLjViYTwS9fPy5pEtTkrbEKsw2M+oVSoFGw6OD1IpvlV1VPhUzNbofCQ8oyMbdYJqDtGdmHQK6TdPg== } @@ -14997,7 +15149,7 @@ packages: dependencies: bs-logger: 0.2.6 fast-json-stable-stringify: 2.1.0 - jest: 28.1.2 + jest: 28.1.2_dyhfsldgbafx4up6nvciefunqu jest-util: 28.1.3 json5: 2.2.1 lodash.memoize: 4.1.2