changes for caching

Most changes should be here, just making sure flush cache is in for each post / delete
This commit is contained in:
kukulaka
2020-08-31 17:16:45 +01:00
parent 226280d6ba
commit 9d6f80149c
16 changed files with 132 additions and 359 deletions

1
api/.gitignore vendored
View File

@@ -1,4 +1,5 @@
node_modules
.env
.cache
.DS_Store
uploads

View File

@@ -120,7 +120,6 @@
"oracledb": "^5.0.0",
"pg": "^8.3.0",
"sqlite3": "^5.0.0",
"rate-limiter-flexible": "^2.1.10",
"redis": "^3.0.2"
},
"devDependencies": {

View File

@@ -10,7 +10,6 @@ import errorHandler from './middleware/error-handler';
import extractToken from './middleware/extract-token';
import authenticate from './middleware/authenticate';
import rateLimiter from './middleware/rate-limiter';
import activityRouter from './controllers/activity';
import assetsRouter from './controllers/assets';
import authRouter from './controllers/auth';
@@ -67,8 +66,6 @@ if (env.NODE_ENV !== 'development') {
});
}
// use the rate limiter - all routes for now
app.use('/auth', authRouter)
.use(authenticate)

View File

@@ -8,15 +8,6 @@
{{ database }}
####################################################################################################
# REDIS Server
{{ redisServer }}
####################################################################################################
# Rate Limiting
{{ rateLimits}}
####################################################################################################
# Caching

View File

@@ -40,7 +40,13 @@ const defaults = {
INMEMEMORY_BLOCK_DURATION: 30,
},
caching: {
CACHE_TYPE: 'redis',
CACHE_ENABLED: true,
CACHE_DRIVER: 'redis',
CACHE_HOST: '127.0.0.1',
CACHE_PORT: '6379',
CACHE_REDIS_PASSWORD: null,
CACHE_TTL: 300,
CACHE_CHECK_LIVE: 300,
},
security: {
KEY: uuidv4(),

View File

@@ -3,19 +3,11 @@ import redis from 'redis';
import asyncHandler from 'express-async-handler';
import sanitizeQuery from '../middleware/sanitize-query';
import useCollection from '../middleware/use-collection';
import cacheMiddleware from '../middleware/cache';
import CacheService from '../services/node-cache';
import checkCacheMiddleware from '../middleware/check-cache';
import setCacheMiddleware from '../middleware/set-cache';
import ActivityService from '../services/activity';
import MetaService from '../services/meta';
import { Action } from '../types';
import env from '../env';
const redisClient = redis.createClient({
enable_offline_queue: false,
host: env.REDIS_HOST,
port: env.REDIS_PORT,
password: env.REDIS_PASSWORD,
});
const router = express.Router();
@@ -23,83 +15,36 @@ router.get(
'/',
useCollection('directus_activity'),
sanitizeQuery,
cacheMiddleware,
checkCacheMiddleware,
asyncHandler(async (req, res) => {
const key = req.url;
const TTL = req.query.TTL;
const TTLnum = Number(TTL);
const dTTL = Number(req.query.dTTL);
const service = new ActivityService({ accountability: req.accountability });
const metaService = new MetaService({ accountability: req.accountability });
const records = await service.readByQuery(req.sanitizedQuery);
const meta = await metaService.getMetaForQuery(req.collection, req.sanitizedQuery);
if (TTL) {
if (env.CACHE_TYPE === 'redis') {
redisClient.setex(
key,
TTLnum,
JSON.stringify({
data: records || null,
meta,
})
);
} else {
const cacheService = new CacheService(TTLnum, dTTL);
cacheService.setCache(
key,
JSON.stringify({
data: records || null,
meta,
})
);
}
}
return res.json({
data: records || null,
meta,
});
})
}),
setCacheMiddleware
);
router.get(
'/:pk',
useCollection('directus_activity'),
sanitizeQuery,
cacheMiddleware,
checkCacheMiddleware,
asyncHandler(async (req, res) => {
const key = req.url;
const TTL = req.query.TTL;
const TTLnum = Number(TTL);
const dTTL = Number(req.query.dTTL);
const service = new ActivityService({ accountability: req.accountability });
const record = await service.readByKey(req.params.pk, req.sanitizedQuery);
if (TTL) {
if (env.CACHE_TYPE === 'redis') {
redisClient.setex(
key,
TTLnum,
JSON.stringify({
data: record || null,
})
);
} else {
const cacheService = new CacheService(TTLnum, dTTL);
cacheService.setCache(
key,
JSON.stringify({
data: record || null,
})
);
}
}
return res.json({
data: record || null,
});
})
}),
setCacheMiddleware
);
router.post(

View File

@@ -2,19 +2,11 @@ import { Router } from 'express';
import redis from 'redis';
import asyncHandler from 'express-async-handler';
import sanitizeQuery from '../middleware/sanitize-query';
import cacheMiddleware from '../middleware/cache';
import checkCacheMiddleware from '../middleware/check-cache';
import setCacheMiddleware from '../middleware/set-cache';
import CollectionsService from '../services/collections';
import CacheService from '../services/node-cache';
import useCollection from '../middleware/use-collection';
import MetaService from '../services/meta';
import env from '../env';
const redisClient = redis.createClient({
enable_offline_queue: false,
host: env.REDIS_HOST,
port: env.REDIS_PORT,
password: env.REDIS_PASSWORD,
});
const router = Router();
@@ -34,56 +26,33 @@ router.post(
router.get(
'/',
useCollection('directus_collections'),
cacheMiddleware,
checkCacheMiddleware,
asyncHandler(async (req, res) => {
const key = req.url;
const TTL = req.query.TTL;
const TTLnum = Number(TTL);
const dTTL = Number(req.query.dTTL);
const collectionsService = new CollectionsService({ accountability: req.accountability });
const metaService = new MetaService({ accountability: req.accountability });
const collections = await collectionsService.readByQuery();
const meta = await metaService.getMetaForQuery(req.collection, {});
if (TTL) {
if (env.CACHE_TYPE === 'redis') {
redisClient.setex(key, TTLnum, JSON.stringify({ data: collections || null, meta }));
} else {
const cacheService = new CacheService(TTLnum, dTTL);
cacheService.setCache(key, JSON.stringify({ data: collections || null, meta }));
}
}
res.json({ data: collections || null, meta });
})
}),
setCacheMiddleware
);
router.get(
'/:collection',
useCollection('directus_collections'),
sanitizeQuery,
cacheMiddleware,
checkCacheMiddleware,
asyncHandler(async (req, res) => {
const key = req.url;
const TTL = req.query.TTL;
const TTLnum = Number(TTL);
const dTTL = Number(req.query.dTTL);
const collectionsService = new CollectionsService({ accountability: req.accountability });
const collectionKey = req.params.collection.includes(',')
? req.params.collection.split(',')
: req.params.collection;
const collection = await collectionsService.readByKey(collectionKey as any);
if (TTL) {
if (env.CACHE_TYPE === 'redis') {
redisClient.setex(key, TTLnum, JSON.stringify({ data: collection || null }));
} else {
const cacheService = new CacheService(TTLnum, dTTL);
cacheService.setCache(key, JSON.stringify({ data: collection || null }));
}
}
res.json({ data: collection || null });
})
}),
setCacheMiddleware
);
router.patch(

View File

@@ -1,6 +1,6 @@
import { Router } from 'express';
import asyncHandler from 'express-async-handler';
import cacheMiddleware from '../middleware/cache';
import checkCacheMiddleware from '../middleware/check-cache';
import * as ExtensionsService from '../services/extensions';
import { RouteNotFoundException } from '../exceptions';
@@ -8,7 +8,7 @@ const router = Router();
router.get(
'/:type',
cacheMiddleware,
checkCacheMiddleware,
asyncHandler(async (req, res) => {
const typeAllowList = ['interfaces', 'layouts', 'displays', 'modules'];

View File

@@ -3,22 +3,14 @@ import asyncHandler from 'express-async-handler';
import redis from 'redis';
import FieldsService from '../services/fields';
import validateCollection from '../middleware/collection-exists';
import CacheService from '../services/node-cache';
import cacheMiddleware from '../middleware/cache';
import checkCacheMiddleware from '../middleware/check-cache';
import setCacheMiddleware from '../middleware/set-cache';
import { schemaInspector } from '../database';
import { FieldNotFoundException, InvalidPayloadException } from '../exceptions';
import Joi from 'joi';
import { Field } from '../types/field';
import useCollection from '../middleware/use-collection';
import { Accountability, types } from '../types';
import env from '../env';
const redisClient = redis.createClient({
enable_offline_queue: false,
host: env.REDIS_HOST,
port: env.REDIS_PORT,
password: env.REDIS_PASSWORD,
});
const router = Router();
@@ -31,79 +23,48 @@ const router = Router();
router.get(
'/',
useCollection('directus_fields'),
cacheMiddleware,
checkCacheMiddleware,
asyncHandler(async (req, res) => {
const key = req.url;
const TTL = req.query.TTL;
const TTLnum = Number(TTL);
const dTTL = Number(req.query.dTTL);
const service = new FieldsService({ accountability: req.accountability });
const fields = await service.readAll();
if (TTL) {
if (env.CACHE_TYPE === 'redis') {
redisClient.setex(key, TTLnum, JSON.stringify({ data: fields || null }));
} else {
const cacheService = new CacheService(TTLnum, dTTL);
cacheService.setCache(key, JSON.stringify({ data: fields || null }));
}
}
return res.json({ data: fields || null });
})
}),
setCacheMiddleware
);
router.get(
'/:collection',
validateCollection,
useCollection('directus_fields'),
cacheMiddleware,
checkCacheMiddleware,
asyncHandler(async (req, res) => {
const key = req.url;
const TTL = req.query.TTL;
const TTLnum = Number(TTL);
const dTTL = Number(req.query.dTTL);
const service = new FieldsService({ accountability: req.accountability });
const fields = await service.readAll(req.params.collection);
if (TTL) {
if (env.CACHE_TYPE === 'redis') {
redisClient.setex(key, TTLnum, JSON.stringify({ data: fields || null }));
} else {
const cacheService = new CacheService(TTLnum, dTTL);
cacheService.setCache(key, JSON.stringify({ data: fields || null }));
}
}
return res.json({ data: fields || null });
})
}),
setCacheMiddleware
);
router.get(
'/:collection/:field',
validateCollection,
useCollection('directus_fields'),
cacheMiddleware,
checkCacheMiddleware,
asyncHandler(async (req, res) => {
const key = req.url;
const TTL = req.query.TTL;
const TTLnum = Number(TTL);
const dTTL = Number(req.query.dTTL);
const service = new FieldsService({ accountability: req.accountability });
const exists = await schemaInspector.hasColumn(req.collection, req.params.field);
if (exists === false) throw new FieldNotFoundException(req.collection, req.params.field);
const field = await service.readOne(req.params.collection, req.params.field);
if (TTL) {
if (env.CACHE_TYPE === 'redis') {
redisClient.setex(key, TTLnum, JSON.stringify({ data: field || null }));
} else {
const cacheService = new CacheService(TTLnum, dTTL);
cacheService.setCache(key, JSON.stringify({ data: field || null }));
}
}
return res.json({ data: field || null });
})
}),
setCacheMiddleware
);
const newFieldSchema = Joi.object({

View File

@@ -2,20 +2,12 @@ import express from 'express';
import redis from 'redis';
import asyncHandler from 'express-async-handler';
import ItemsService from '../services/items';
import cacheMiddleware from '../middleware/cache';
import checkCacheMiddleware from '../middleware/check-cache';
import setCacheMiddleware from '../middleware/set-cache';
import sanitizeQuery from '../middleware/sanitize-query';
import CacheService from '../services/node-cache';
import collectionExists from '../middleware/collection-exists';
import MetaService from '../services/meta';
import { RouteNotFoundException } from '../exceptions';
import env from '../env';
const redisClient = redis.createClient({
enable_offline_queue: false,
host: env.REDIS_HOST,
port: env.REDIS_PORT,
password: env.REDIS_PASSWORD,
});
const router = express.Router();
@@ -40,12 +32,8 @@ router.get(
'/:collection',
collectionExists,
sanitizeQuery,
cacheMiddleware,
checkCacheMiddleware,
asyncHandler(async (req, res) => {
const key = req.url;
const TTL = req.query.TTL;
const TTLnum = Number(TTL);
const dTTL = Number(req.query.dTTL);
const service = new ItemsService(req.collection, { accountability: req.accountability });
const metaService = new MetaService({ accountability: req.accountability });
@@ -54,64 +42,34 @@ router.get(
: await service.readByQuery(req.sanitizedQuery);
const meta = await metaService.getMetaForQuery(req.collection, req.sanitizedQuery);
if (TTL) {
if (env.CACHE_TYPE === 'redis') {
redisClient.setex(
key,
TTLnum,
JSON.stringify({ meta: meta, data: records || null })
);
} else {
const cacheService = new CacheService(TTLnum, dTTL);
cacheService.setCache(key, JSON.stringify({ meta: meta, data: records || null }));
}
}
return res.json({
meta: meta,
data: records || null,
});
})
}),
setCacheMiddleware
);
router.get(
'/:collection/:pk',
collectionExists,
sanitizeQuery,
cacheMiddleware,
checkCacheMiddleware,
asyncHandler(async (req, res) => {
if (req.singleton) {
throw new RouteNotFoundException(req.path);
}
const key = req.url;
const TTL = req.query.TTL;
const TTLnum = Number(TTL);
const dTTL = Number(req.query.dTTL);
const service = new ItemsService(req.collection, { accountability: req.accountability });
const primaryKey = req.params.pk.includes(',') ? req.params.pk.split(',') : req.params.pk;
const result = await service.readByKey(primaryKey as any, req.sanitizedQuery);
if (TTL) {
if (env.CACHE_TYPE === 'redis') {
redisClient.setex(
key,
TTLnum,
JSON.stringify({
data: result || null,
})
);
} else {
const cacheService = new CacheService(TTLnum, dTTL);
cacheService.setCache(
key,
JSON.stringify({
data: result || null,
})
);
}
}
return res.json({
data: result || null,
});
})
}),
setCacheMiddleware
);
router.patch(

View File

@@ -4,19 +4,12 @@ import asyncHandler from 'express-async-handler';
import sanitizeQuery from '../middleware/sanitize-query';
import PermissionsService from '../services/permissions';
import useCollection from '../middleware/use-collection';
import CacheService from '../services/node-cache';
import cacheMiddleware from '../middleware/cache';
import checkCacheMiddleware from '../middleware/check-cache';
import setCacheMiddleware from '../middleware/set-cache';
import MetaService from '../services/meta';
import { InvalidCredentialsException } from '../exceptions';
import env from '../env';
const redisClient = redis.createClient({
enable_offline_queue: false,
host: env.REDIS_HOST,
port: env.REDIS_PORT,
password: env.REDIS_PASSWORD,
});
const router = express.Router();
router.use(useCollection('directus_permissions'));
@@ -35,42 +28,28 @@ router.post(
router.get(
'/',
sanitizeQuery,
cacheMiddleware,
checkCacheMiddleware,
asyncHandler(async (req, res) => {
const key = req.url;
const TTL = req.query.TTL;
const TTLnum = Number(TTL);
const dTTL = Number(req.query.dTTL);
const service = new PermissionsService({ accountability: req.accountability });
const metaService = new MetaService({ accountability: req.accountability });
const item = await service.readByQuery(req.sanitizedQuery);
const meta = await metaService.getMetaForQuery(req.collection, req.sanitizedQuery);
if (TTL) {
if (env.CACHE_TYPE === 'redis') {
redisClient.setex(key, TTLnum, JSON.stringify({ data: item || null, meta }));
} else {
const cacheService = new CacheService(TTLnum, dTTL);
cacheService.setCache(key, JSON.stringify({ data: item || null, meta }));
}
}
return res.json({ data: item || null, meta });
})
}),
setCacheMiddleware
);
router.get(
'/me',
sanitizeQuery,
cacheMiddleware,
checkCacheMiddleware,
asyncHandler(async (req, res) => {
if (!req.accountability?.user || !req.accountability?.role) {
throw new InvalidCredentialsException();
}
const key = req.url;
const TTL = req.query.TTL;
const TTLnum = Number(TTL);
const dTTL = Number(req.query.dTTL);
const service = new PermissionsService();
const query = req.sanitizedQuery || {};
@@ -82,40 +61,23 @@ router.get(
};
const items = await service.readByQuery(req.sanitizedQuery);
if (TTL) {
if (env.CACHE_TYPE === 'redis') {
redisClient.setex(key, TTLnum, JSON.stringify({ data: items || null }));
} else {
const cacheService = new CacheService(TTLnum, dTTL);
cacheService.setCache(key, JSON.stringify({ data: items || null }));
}
}
return res.json({ data: items || null });
})
}),
setCacheMiddleware
);
router.get(
'/:pk',
sanitizeQuery,
cacheMiddleware,
checkCacheMiddleware,
asyncHandler(async (req, res) => {
const key = req.url;
const TTL = req.query.TTL;
const TTLnum = Number(TTL);
const dTTL = Number(req.query.dTTL);
const service = new PermissionsService({ accountability: req.accountability });
const record = await service.readByKey(Number(req.params.pk), req.sanitizedQuery);
if (TTL) {
if (env.CACHE_TYPE === 'redis') {
redisClient.setex(key, TTLnum, JSON.stringify({ data: record || null }));
} else {
const cacheService = new CacheService(TTLnum, dTTL);
cacheService.setCache(key, JSON.stringify({ data: record || null }));
}
}
return res.json({ data: record || null });
})
}),
setCacheMiddleware
);
router.patch(

View File

@@ -19,6 +19,14 @@ const defaults: Record<string, any> = {
CORS_ENABLED: false,
CACHE_ENABLED: true,
CACHE_DRIVER: 'redis',
CACHE_HOST: '127.0.0.1',
CACHE_PORT: '6379',
CACHE_REDIS_PASSWORD: null,
CACHE_TTL: 300,
CACHE_CHECK_LIVE: 300,
OAUTH_PROVIDERS: '',
EXTENSIONS_PATH: './extensions',

View File

@@ -4,36 +4,31 @@
*/
import { RequestHandler } from 'express';
import redis from 'redis';
import NodeCache from 'node-cache';
import asyncHandler from 'express-async-handler';
import CacheService from '../services/node-cache';
import CacheService from '../services/cache';
import { RedisNotFoundException } from '../exceptions';
import { InvalidCacheKeyException } from '../exceptions';
import env from '../env';
const redisClient = redis.createClient({
enable_offline_queue: false,
host: env.REDIS_HOST,
port: env.REDIS_PORT,
password: env.REDIS_PASSWORD,
host: env.CACHE_HOST,
port: env.CACHE_PORT,
password: env.CACHE_REDIS_PASSWORD,
});
const cacheMiddleware: RequestHandler = asyncHandler(async (req, res, next) => {
const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next) => {
// make the key of the cache the URL
// need to check that this will work for all endpoints
// node cache service
// have used query as then can decide whather to use cache or not from api call
if (env.CACHE_ENABLED !== 'true') return next();
if (!req.query.TTL) return next();
if (!req.query.dTTL) return next();
//key needs to have url, query and permissions
const TTLnumber = Number(req.query.TTL);
const dTTL = Number(req.query.dTTL);
const key = req.url;
const key = `${req.url}${req.query}${req.permissions}`;
// we have two options here. Redis or node cache
if (env.CACHE_TYPE === 'redis') {
if (env.CACHE_DRIVER === 'redis') {
if (!redisClient) {
throw new RedisNotFoundException('Redis client does not exist');
}
@@ -49,11 +44,11 @@ const cacheMiddleware: RequestHandler = asyncHandler(async (req, res, next) => {
}
});
} else {
const cacheService = new CacheService(TTLnumber, dTTL);
const cacheService = new CacheService();
res.json(cacheService.getCache(key));
}
return next();
});
export default cacheMiddleware;
export default checkCacheMiddleware;

View File

@@ -1,65 +0,0 @@
/**
* RateLimiter using Redis
* and rate-limiter-flexible
* can extend with further options
* in future
*/
import { RequestHandler } from 'express';
import redis from 'redis';
import asyncHandler from 'express-async-handler';
import { RateLimiterRedis, RateLimiterMemory } from 'rate-limiter-flexible';
import { HitRateLimitException } from '../exceptions';
import { RedisNotFoundException } from '../exceptions';
import env from '../env';
const redisClient = redis.createClient({
enable_offline_queue: false,
host: env.REDIS_HOST,
port: env.REDIS_PORT,
password: env.REDIS_PASSWORD,
});
const rateLimiter: RequestHandler = asyncHandler(async (req, res, next) => {
// options for the rate limiter are set below. Opts can be found
// at https://github.com/animir/node-rate-limiter-flexible/wiki/Options
// more basic for memory store
const opts = {
points: env.CONSUMED_POINTS_LIMIT, // Number of points
duration: env.CONSUMED_RESET_DURATION, // Number of seconds before consumed points are reset.
keyPrefix: 'rlflx', // must be unique for limiters with different purpose
};
let rateLimiterSet = new RateLimiterMemory(opts);
if (env.RATE_LIMIT_TYPE === 'redis') {
const redisOpts = {
...opts,
storeClient: redisClient,
// Custom
execEvenly: env.EXEC_EVENLY, // delay actions after first action - this may need adjusting (leaky bucket)
blockDuration: env.BLOCK_POINT_DURATION, // Do not block if consumed more than points
inmemoryBlockOnConsumed: env.INMEMORY_BLOCK_CONSUMED, // eg if 200 points consumed
inmemoryBlockDuration: env.INMEMEMORY_BLOCK_DURATION, // block for certain amount of seconds
};
rateLimiterSet = new RateLimiterRedis(redisOpts);
// first need to check that redis is running!
if (!redisClient) {
throw new RedisNotFoundException('Redis client does not exist');
}
}
try {
await rateLimiterSet.consume(req.ip);
} catch (rejRes) {
// If there is no error, rateLimiterRedis promise rejected with number of ms before next request allowed
const secs = Math.round(rejRes.msBeforeNext / 1000) || 1;
res.set('Retry-After', String(secs));
throw new HitRateLimitException(`Too many requests, retry after ${secs}.`);
}
return next();
});
export default rateLimiter;

View File

@@ -0,0 +1,44 @@
/**
* Caching using redis
* and node caching
*/
import { RequestHandler } from 'express';
import redis from 'redis';
import NodeCache from 'node-cache';
import asyncHandler from 'express-async-handler';
import CacheService from '../services/cache';
import { RedisNotFoundException } from '../exceptions';
import env from '../env';
const redisClient = redis.createClient({
enable_offline_queue: false,
host: env.CACHE_HOST,
port: env.CACHE_PORT,
password: env.CACHE_REDIS_PASSWORD,
});
const setCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next) => {
// setting the cache
if (env.CACHE_ENABLED !== 'true') return next();
//key needs to have url, query and permissions
const key = `${req.url}${req.query}${req.permissions}`;
// we have two options here. Redis or node cache
if (env.CACHE_DRIVER === 'redis') {
if (!redisClient) {
throw new RedisNotFoundException('Redis client does not exist');
}
redisClient.setex(key, env.CACHE_TTL, JSON.stringify(res.json));
} else {
const cacheService = new CacheService();
cacheService.setCache(key, JSON.stringify(res.json));
}
return next();
});
export default setCacheMiddleware;

View File

@@ -10,16 +10,18 @@
* could put redis cache in here too
*/
import NodeCache from 'node-cache';
import redis from 'redis';
import { InvalidCacheKeyException } from '../exceptions';
import env from '../env';
export default class CacheService {
apiCache: NodeCache;
constructor(stdTTLSecs: number, checkPeriodSecs: number) {
constructor() {
// options found at https://github.com/node-cache/node-cache
this.apiCache = new NodeCache({
stdTTL: stdTTLSecs,
checkperiod: checkPeriodSecs,
stdTTL: env.CACHE_TTL,
checkperiod: env.CACHE_CHECK_LIVE,
useClones: false,
});
}