From faa71c7595fa656b7dc153267a75278099e63e90 Mon Sep 17 00:00:00 2001
From: Rijk van Zanten <rijkvanzanten@me.com>
Date: Thu, 5 Aug 2021 22:27:10 +0200
Subject: [PATCH] Add cache connection fallbacks (#7226)

---
 api/src/cache.ts              | 10 +++++++---
 api/src/middleware/cache.ts   | 20 ++++++++++++++++++--
 api/src/middleware/respond.ts | 11 +++++++++--
 api/src/utils/get-schema.ts   | 19 +++++++++++++++++--
 4 files changed, 51 insertions(+), 9 deletions(-)

diff --git a/api/src/cache.ts b/api/src/cache.ts
index 423a28b323..f19a92d9e1 100644
--- a/api/src/cache.ts
+++ b/api/src/cache.ts
@@ -12,12 +12,12 @@ export function getCache(): { cache: Keyv | null; schemaCache: Keyv | null } {
 	if (env.CACHE_ENABLED === true && cache === null) {
 		validateEnv(['CACHE_NAMESPACE', 'CACHE_TTL', 'CACHE_STORE']);
 		cache = getKeyvInstance(ms(env.CACHE_TTL as string));
-		cache.on('error', (err) => logger.error(err));
+		cache.on('error', (err) => logger.warn(err, `[cache] ${err}`));
 	}
 
 	if (env.CACHE_SCHEMA !== false && schemaCache === null) {
 		schemaCache = getKeyvInstance(typeof env.CACHE_SCHEMA === 'string' ? ms(env.CACHE_SCHEMA) : undefined);
-		schemaCache.on('error', (err) => logger.error(err));
+		schemaCache.on('error', (err) => logger.warn(err, `[cache] ${err}`));
 	}
 
 	return { cache, schemaCache };
@@ -43,7 +43,11 @@ function getConfig(store: 'memory' | 'redis' | 'memcache' = 'memory', ttl: numbe
 
 	if (store === 'redis') {
 		const KeyvRedis = require('@keyv/redis');
-		config.store = new KeyvRedis(env.CACHE_REDIS || getConfigFromEnv('CACHE_REDIS_'));
+
+		config.store = new KeyvRedis(env.CACHE_REDIS || getConfigFromEnv('CACHE_REDIS_'), {
+			commandTimeout: 500,
+			retryStrategy: false,
+		});
 	}
 
 	if (store === 'memcache') {
diff --git a/api/src/middleware/cache.ts b/api/src/middleware/cache.ts
index ee240ea1d4..747db06885 100644
--- a/api/src/middleware/cache.ts
+++ b/api/src/middleware/cache.ts
@@ -4,6 +4,7 @@ import env from '../env';
 import asyncHandler from '../utils/async-handler';
 import { getCacheControlHeader } from '../utils/get-cache-headers';
 import { getCacheKey } from '../utils/get-cache-key';
+import logger from '../logger';
 
 const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next) => {
 	const { cache } = getCache();
@@ -18,10 +19,25 @@ const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next)
 
 	const key = getCacheKey(req);
 
-	const cachedData = await cache.get(key);
+	let cachedData;
+
+	try {
+		cachedData = await cache.get(key);
+	} catch (err) {
+		logger.warn(err, `[cache] Couldn't read key ${key}. ${err.message}`);
+		return next();
+	}
 
 	if (cachedData) {
-		const cacheExpiryDate = (await cache.get(`${key}__expires_at`)) as number | null;
+		let cacheExpiryDate;
+
+		try {
+			cacheExpiryDate = (await cache.get(`${key}__expires_at`)) as number | null;
+		} catch (err) {
+			logger.warn(err, `[cache] Couldn't read key ${`${key}__expires_at`}. ${err.message}`);
+			return next();
+		}
+
 		const cacheTTL = cacheExpiryDate ? cacheExpiryDate - Date.now() : null;
 
 		res.setHeader('Cache-Control', getCacheControlHeader(req, cacheTTL));
diff --git a/api/src/middleware/respond.ts b/api/src/middleware/respond.ts
index ed4a4032a2..072f3db411 100644
--- a/api/src/middleware/respond.ts
+++ b/api/src/middleware/respond.ts
@@ -8,6 +8,7 @@ import asyncHandler from '../utils/async-handler';
 import { getCacheKey } from '../utils/get-cache-key';
 import { parse as toXML } from 'js2xmlparser';
 import { getCacheControlHeader } from '../utils/get-cache-headers';
+import logger from '../logger';
 
 export const respond: RequestHandler = asyncHandler(async (req, res) => {
 	const { cache } = getCache();
@@ -20,8 +21,14 @@ export const respond: RequestHandler = asyncHandler(async (req, res) => {
 		res.locals.cache !== false
 	) {
 		const key = getCacheKey(req);
-		await cache.set(key, res.locals.payload, ms(env.CACHE_TTL as string));
-		await cache.set(`${key}__expires_at`, Date.now() + ms(env.CACHE_TTL as string));
+
+		try {
+			await cache.set(key, res.locals.payload, ms(env.CACHE_TTL as string));
+			await cache.set(`${key}__expires_at`, Date.now() + ms(env.CACHE_TTL as string));
+		} catch (err) {
+			logger.warn(err, `[cache] Couldn't set key ${key}. ${err}`);
+		}
+
 		res.setHeader('Cache-Control', getCacheControlHeader(req, ms(env.CACHE_TTL as string)));
 		res.setHeader('Vary', 'Origin, Cache-Control');
 	} else {
diff --git a/api/src/utils/get-schema.ts b/api/src/utils/get-schema.ts
index 5697e9dd28..fdbc611589 100644
--- a/api/src/utils/get-schema.ts
+++ b/api/src/utils/get-schema.ts
@@ -28,13 +28,28 @@ export async function getSchema(options?: {
 	let result: SchemaOverview;
 
 	if (env.CACHE_SCHEMA !== false && schemaCache) {
-		const cachedSchema = (await schemaCache.get('schema')) as SchemaOverview;
+		let cachedSchema;
+
+		try {
+			cachedSchema = (await schemaCache.get('schema')) as SchemaOverview;
+		} catch (err) {
+			logger.warn(err, `[schema-cache] Couldn't retrieve cache. ${err}`);
+		}
 
 		if (cachedSchema) {
 			result = cachedSchema;
 		} else {
 			result = await getDatabaseSchema(database, schemaInspector);
-			await schemaCache.set('schema', result, typeof env.CACHE_SCHEMA === 'string' ? ms(env.CACHE_SCHEMA) : undefined);
+
+			try {
+				await schemaCache.set(
+					'schema',
+					result,
+					typeof env.CACHE_SCHEMA === 'string' ? ms(env.CACHE_SCHEMA) : undefined
+				);
+			} catch (err) {
+				logger.warn(err, `[schema-cache] Couldn't save cache. ${err}`);
+			}
 		}
 	} else {
 		result = await getDatabaseSchema(database, schemaInspector);