From 2a1df698181e6ed6f6a7c0df9fa34b6a0c7a8e1f Mon Sep 17 00:00:00 2001 From: Tanya Byrne Date: Mon, 17 Aug 2020 12:56:21 +0100 Subject: [PATCH] #78 rate limiter - all routes --- api/package-lock.json | 95 ++++++++++++++++++++++++++++++ api/package.json | 5 ++ api/src/app.ts | 4 +- api/src/middleware/rate-limiter.ts | 59 +++++++++++++++++++ 4 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 api/src/middleware/rate-limiter.ts diff --git a/api/package-lock.json b/api/package-lock.json index 9098b7b578..2639d07a35 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -677,6 +677,14 @@ "safe-buffer": "*" } }, + "@types/redis": { + "version": "2.8.25", + "resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.25.tgz", + "integrity": "sha512-e5N5Dg712aZ1CPi1Li0XalukPSWd2RTLYzmrMsQ84NkYQ7cqKHC+HroXM1WP65O1zRGfzld72/u9ikumEe+ylA==", + "requires": { + "@types/node": "*" + } + }, "@types/serve-static": { "version": "1.13.5", "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.5.tgz", @@ -1686,6 +1694,11 @@ "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", "integrity": "sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=" }, + "cluster-key-slot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" + }, "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", @@ -2140,6 +2153,11 @@ "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" }, + "denque": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz", + "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==" + }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -2694,6 +2712,11 @@ "pino-http": "^5.1.0" } }, + "express-rate-limit": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-5.1.3.tgz", + "integrity": "sha512-TINcxve5510pXj4n9/1AMupkj3iWxl3JuZaWhCdYDlZeoCPqweGZrxbrlqTCFb1CT5wli7s8e2SH/Qz2c9GorA==" + }, "express-session": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/express-session/-/express-session-1.17.1.tgz", @@ -3838,6 +3861,29 @@ "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==" }, + "ioredis": { + "version": "4.17.3", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.17.3.tgz", + "integrity": "sha512-iRvq4BOYzNFkDnSyhx7cmJNOi1x/HWYe+A4VXHBu4qpwJaGT1Mp+D2bVGJntH9K/Z/GeOM/Nprb8gB3bmitz1Q==", + "requires": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.1.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "redis-commands": "1.5.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.0.1" + }, + "dependencies": { + "redis-commands": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz", + "integrity": "sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg==" + } + } + }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -4469,6 +4515,16 @@ "resolved": "https://registry.npmjs.org/lodash.deburr/-/lodash.deburr-4.1.0.tgz", "integrity": "sha1-3bG7s+8HRYwBd7oH3hRCLLAz/5s=" }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + }, + "lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" + }, "lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", @@ -6093,6 +6149,11 @@ "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, + "rate-limiter-flexible": { + "version": "2.1.10", + "resolved": "https://registry.npmjs.org/rate-limiter-flexible/-/rate-limiter-flexible-2.1.10.tgz", + "integrity": "sha512-Pa+8TPD4xYaiCUB5K4a/+j2FHDUe4HP1g49JmKEmkOkhqPaeVqxJsZuuVaza/svSCOT+V73vtsyBiSFK/e1yXw==" + }, "raw-body": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", @@ -6225,6 +6286,35 @@ } } }, + "redis": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz", + "integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==", + "requires": { + "denque": "^1.4.1", + "redis-commands": "^1.5.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0" + } + }, + "redis-commands": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.6.0.tgz", + "integrity": "sha512-2jnZ0IkjZxvguITjFTrGiLyzQZcTvaw8DAaCXxZq/dsHXz7KfMQ3OUJy7Tz9vnRtZRVz6VRCPDvruvU8Ts44wQ==" + }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "requires": { + "redis-errors": "^1.0.0" + } + }, "regex-not": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", @@ -7032,6 +7122,11 @@ "tweetnacl": "~0.14.0" } }, + "standard-as-callback": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.0.1.tgz", + "integrity": "sha512-NQOxSeB8gOI5WjSaxjBgog2QFw55FV8TkS6Y07BiB3VJ8xNTvUYm0wl0s8ObgQ5NhdpnNfigMIKjgPESzgr4tg==" + }, "static-extend": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", diff --git a/api/package.json b/api/package.json index 37a1b9390b..58abc7d118 100644 --- a/api/package.json +++ b/api/package.json @@ -69,6 +69,7 @@ "@slynova/flydrive": "^1.0.2", "@slynova/flydrive-gcs": "^1.0.2", "@slynova/flydrive-s3": "^1.0.2", + "@types/redis": "^2.8.25", "argon2": "^0.26.2", "atob": "^2.1.2", "axios": "^0.19.2", @@ -85,11 +86,13 @@ "express": "^4.17.1", "express-async-handler": "^1.1.4", "express-pino-logger": "^5.0.0", + "express-rate-limit": "^5.1.3", "express-session": "^1.17.1", "fs-extra": "^9.0.1", "grant": "^5.3.0", "icc": "^2.0.0", "inquirer": "^7.3.3", + "ioredis": "^4.17.3", "joi": "^17.1.1", "js-yaml": "^3.14.0", "jsonwebtoken": "^8.5.1", @@ -104,6 +107,8 @@ "otplib": "^12.0.1", "pino": "^6.4.1", "pino-colada": "^2.1.0", + "rate-limiter-flexible": "^2.1.10", + "redis": "^3.0.2", "resolve-cwd": "^3.0.0", "sharp": "^0.25.4", "uuid": "^8.3.0", diff --git a/api/src/app.ts b/api/src/app.ts index 99bee1d3f9..56738b8b0b 100644 --- a/api/src/app.ts +++ b/api/src/app.ts @@ -10,7 +10,7 @@ import errorHandler from './middleware/error-handler'; import extractToken from './middleware/extract-token'; import authenticate from './middleware/authenticate'; - +import rateLimiter from './middleware/rate-limiter'; import activityRouter from './controllers/activity'; import assetsRouter from './controllers/assets'; import authRouter from './controllers/auth'; @@ -67,6 +67,8 @@ if (env.NODE_ENV !== 'development') { }); } +// use the rate limiter - all routes for now +app.use(rateLimiter); app.use('/auth', authRouter) .use(authenticate) diff --git a/api/src/middleware/rate-limiter.ts b/api/src/middleware/rate-limiter.ts new file mode 100644 index 0000000000..d5f893b04f --- /dev/null +++ b/api/src/middleware/rate-limiter.ts @@ -0,0 +1,59 @@ +/** + * RateLimiter using Redis + * and rate-limiter-flexible + * can extend with further options + * in future + */ +import { RequestHandler } from 'express'; +import redis from 'redis'; +import { RateLimiterRedis } from 'rate-limiter-flexible'; + +const redisClient = redis.createClient({ enable_offline_queue: false }); + +const rateLimiter: RequestHandler = (req, res, next) => { + try { + // first need to check that redis is running! + if (!redisClient) { + throw new Error('Redis client does not exist'); + process.exit(1); + } + // options for the rate limiter are set below. Opts can be found + // at https://github.com/animir/node-rate-limiter-flexible/wiki/Options + const opts = { + storeClient: redisClient, + points: 5, // Number of points + duration: 5, // Number of seconds before consumed points are reset. + + // Custom + execEvenly: true, // delay actions after first action - this may need adjusting (leaky bucket) + blockDuration: 0, // Do not block if consumed more than points + keyPrefix: 'rlflx', // must be unique for limiters with different purpose + }; + + const rateLimiterRedis = new RateLimiterRedis(opts); + + rateLimiterRedis + .consume(req.ip) + .then((rateLimiterRes) => { + // everything is ok - can put addition logic in there later for users etc + next(); + }) + .catch((rejRes) => { + if (rejRes instanceof Error) { + throw new Error('Redis insurance limiter not set up'); + process.exit(1); + } else { + // If there is no error, rateLimiterRedis promise rejected with number of ms before next request allowed + const secs = Math.round(rejRes.msBeforeNext / 1000) || 1; + res.set('Retry-After', String(secs)); + res.status(429).send('Too Many Requests'); + throw new Error(`To many requests, retry after ${secs}.`); + process.exit(1); + } + }); + } catch (error) { + next(error); + } +}; + +export default rateLimiter;