misc: added handling of automatic config sync

backend/package-lock.json

@@ -36,6 +36,7 @@
         "bcrypt": "^5.1.1",
         "bullmq": "^5.4.2",
         "cassandra-driver": "^4.7.2",
+        "cron": "^3.1.7",
         "dotenv": "^16.4.1",
         "fastify": "^4.26.0",
         "fastify-plugin": "^4.5.1",
@@ -4806,6 +4807,11 @@
         "long": "*"
       }
     },
+    "node_modules/@types/luxon": {
+      "version": "3.4.2",
+      "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.4.2.tgz",
+      "integrity": "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA=="
+    },
     "node_modules/@types/mime": {
       "version": "1.3.5",
       "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz",
@@ -6689,6 +6695,15 @@
       "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
       "dev": true
     },
+    "node_modules/cron": {
+      "version": "3.1.7",
+      "resolved": "https://registry.npmjs.org/cron/-/cron-3.1.7.tgz",
+      "integrity": "sha512-tlBg7ARsAMQLzgwqVxy8AZl/qlTc5nibqYwtNGoCrd+cV+ugI+tvZC1oT/8dFH8W455YrywGykx/KMmAqOr7Jw==",
+      "dependencies": {
+        "@types/luxon": "~3.4.0",
+        "luxon": "~3.4.0"
+      }
+    },
     "node_modules/cron-parser": {
       "version": "4.9.0",
       "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz",

backend/package.json

@@ -97,6 +97,7 @@
     "bcrypt": "^5.1.1",
     "bullmq": "^5.4.2",
     "cassandra-driver": "^4.7.2",
+    "cron": "^3.1.7",
     "dotenv": "^16.4.1",
     "fastify": "^4.26.0",
     "fastify-plugin": "^4.5.1",

backend/src/main.ts

@@ -22,12 +22,13 @@ const run = async () => {
   const queue = queueServiceFactory(appCfg.REDIS_URL);
   const keyStore = keyStoreFactory(appCfg.REDIS_URL);
 
-  const server = await main({ db, smtp, logger, queue, keyStore });
+  const { server, jobs } = await main({ db, smtp, logger, queue, keyStore });
   const bootstrap = await bootstrapCheck({ db });
   // eslint-disable-next-line
   process.on("SIGINT", async () => {
     await server.close();
     await db.destroy();
+    jobs.forEach((job) => job.stop());
     process.exit(0);
   });
 
@@ -35,6 +36,7 @@ const run = async () => {
   process.on("SIGTERM", async () => {
     await server.close();
     await db.destroy();
+    jobs.forEach((job) => job.stop());
     process.exit(0);
   });
 

backend/src/server/app.ts

@@ -10,6 +10,7 @@ import fastifyFormBody from "@fastify/formbody";
 import helmet from "@fastify/helmet";
 import type { FastifyRateLimitOptions } from "@fastify/rate-limit";
 import ratelimiter from "@fastify/rate-limit";
+import { CronJob } from "cron";
 import fasitfy from "fastify";
 import { Knex } from "knex";
 import { Logger } from "pino";
@@ -41,6 +42,7 @@ type TMain = {
 // Run the server!
 export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => {
   const appCfg = getConfig();
+  const cronJobs: CronJob[] = [];
   const server = fasitfy({
     logger: appCfg.NODE_ENV === "test" ? false : logger,
     trustProxy: true,
@@ -72,7 +74,13 @@ export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => {
     // Rate limiters and security headers
     if (appCfg.isProductionMode) {
       const rateLimitDAL = rateLimitDALFactory(db);
-      const rateLimits = await rateLimitServiceFactory({ rateLimitDAL }).getRateLimits();
+      const rateLimitService = rateLimitServiceFactory({ rateLimitDAL });
+      const rateLimits = await rateLimitService.getRateLimits();
+
+      if (rateLimits) {
+        cronJobs.push(rateLimitService.initializeBackgroundSync());
+      }
+
       await server.register<FastifyRateLimitOptions>(ratelimiter, globalRateLimiterCfg(rateLimits));
     }
 
@@ -92,7 +100,7 @@ export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => {
 
     await server.ready();
     server.swagger();
-    return server;
+    return { server, jobs: cronJobs };
   } catch (err) {
     server.log.error(err);
     await queue.shutdown();

backend/src/server/config/rateLimiter.ts

@@ -4,17 +4,28 @@ import { Redis } from "ioredis";
 import { TRateLimit } from "@app/db/schemas";
 import { getConfig } from "@app/lib/config/env";
 
+export const rateLimitMaxConfiguration = {
+  readLimit: 60,
+  publicEndpointLimit: 30,
+  writeLimit: 200,
+  secretsLimit: 60,
+  authRateLimit: 60,
+  inviteUserRateLimit: 30,
+  mfaRateLimit: 20,
+  creationLimit: 30
+};
+
 // GET endpoints
 export const readLimit: RateLimitOptions = {
   timeWindow: 60 * 1000,
-  max: 600,
+  max: () => rateLimitMaxConfiguration.readLimit,
   keyGenerator: (req) => req.realIp
 };
 
 // POST, PATCH, PUT, DELETE endpoints
 export const writeLimit: RateLimitOptions = {
   timeWindow: 60 * 1000,
-  max: 200, // (too low, FA having issues so increasing it - maidul)
+  max: () => rateLimitMaxConfiguration.writeLimit, // (too low, FA having issues so increasing it - maidul)
   keyGenerator: (req) => req.realIp
 };
 
@@ -22,25 +33,25 @@ export const writeLimit: RateLimitOptions = {
 export const secretsLimit: RateLimitOptions = {
   // secrets, folders, secret imports
   timeWindow: 60 * 1000,
-  max: 60,
+  max: () => rateLimitMaxConfiguration.secretsLimit,
   keyGenerator: (req) => req.realIp
 };
 
 export const authRateLimit: RateLimitOptions = {
   timeWindow: 60 * 1000,
-  max: 60,
+  max: () => rateLimitMaxConfiguration.authRateLimit,
   keyGenerator: (req) => req.realIp
 };
 
 export const inviteUserRateLimit: RateLimitOptions = {
   timeWindow: 60 * 1000,
-  max: 30,
+  max: () => rateLimitMaxConfiguration.inviteUserRateLimit,
   keyGenerator: (req) => req.realIp
 };
 
 export const mfaRateLimit: RateLimitOptions = {
   timeWindow: 60 * 1000,
-  max: 20,
+  max: () => rateLimitMaxConfiguration.mfaRateLimit,
   keyGenerator: (req) => {
     return req.headers.authorization?.split(" ")[1] || req.realIp;
   }
@@ -49,7 +60,7 @@ export const mfaRateLimit: RateLimitOptions = {
 export const creationLimit: RateLimitOptions = {
   // identity, project, org
   timeWindow: 60 * 1000,
-  max: 30,
+  max: () => rateLimitMaxConfiguration.creationLimit,
   keyGenerator: (req) => req.realIp
 };
 
@@ -57,25 +68,25 @@ export const creationLimit: RateLimitOptions = {
 export const publicEndpointLimit: RateLimitOptions = {
   // Shared Secrets
   timeWindow: 60 * 1000,
-  max: 30,
+  max: () => rateLimitMaxConfiguration.publicEndpointLimit,
   keyGenerator: (req) => req.realIp
 };
 
-export const globalRateLimiterCfg = async (rateLimits?: TRateLimit): Promise<RateLimitPluginOptions> => {
+export const globalRateLimiterCfg = async (customRateLimits?: TRateLimit): Promise<RateLimitPluginOptions> => {
   const appCfg = getConfig();
   const redis = appCfg.isRedisConfigured
     ? new Redis(appCfg.REDIS_URL, { connectTimeout: 500, maxRetriesPerRequest: 1 })
     : null;
 
-  if (rateLimits) {
-    readLimit.max = rateLimits.readRateLimit;
-    publicEndpointLimit.max = rateLimits.publicEndpointLimit;
-    writeLimit.max = rateLimits.writeRateLimit;
-    secretsLimit.max = rateLimits.secretsRateLimit;
-    authRateLimit.max = rateLimits.authRateLimit;
-    inviteUserRateLimit.max = rateLimits.inviteUserRateLimit;
-    mfaRateLimit.max = rateLimits.mfaRateLimit;
-    creationLimit.max = rateLimits.creationLimit;
+  if (customRateLimits) {
+    rateLimitMaxConfiguration.readLimit = customRateLimits.readRateLimit;
+    rateLimitMaxConfiguration.publicEndpointLimit = customRateLimits.publicEndpointLimit;
+    rateLimitMaxConfiguration.writeLimit = customRateLimits.writeRateLimit;
+    rateLimitMaxConfiguration.secretsLimit = customRateLimits.secretsRateLimit;
+    rateLimitMaxConfiguration.authRateLimit = customRateLimits.authRateLimit;
+    rateLimitMaxConfiguration.inviteUserRateLimit = customRateLimits.inviteUserRateLimit;
+    rateLimitMaxConfiguration.mfaRateLimit = customRateLimits.mfaRateLimit;
+    rateLimitMaxConfiguration.creationLimit = customRateLimits.creationLimit;
   }
 
   return {

backend/src/services/rate-limit/rate-limit-service.ts

@@ -1,3 +1,8 @@
+import { CronJob } from "cron";
+
+import { logger } from "@app/lib/logger";
+import { rateLimitMaxConfiguration } from "@app/server/config/rateLimiter";
+
 import { TRateLimitDALFactory } from "./rate-limit-dal";
 import { TRateLimit, TRateLimitUpdateDTO } from "./rate-limit-types";
 
@@ -20,8 +25,35 @@ export const rateLimitServiceFactory = ({ rateLimitDAL }: TRateLimitServiceFacto
     return rateLimitDAL.updateById("00000000-0000-0000-0000-000000000000", updates);
   };
 
+  const initializeBackgroundSync = () => {
+    const rateLimitSync = async () => {
+      try {
+        const rateLimit = await getRateLimits();
+        if (rateLimit) {
+          rateLimitMaxConfiguration.readLimit = rateLimit.readRateLimit;
+          rateLimitMaxConfiguration.publicEndpointLimit = rateLimit.publicEndpointLimit;
+          rateLimitMaxConfiguration.writeLimit = rateLimit.writeRateLimit;
+          rateLimitMaxConfiguration.secretsLimit = rateLimit.secretsRateLimit;
+          rateLimitMaxConfiguration.authRateLimit = rateLimit.authRateLimit;
+          rateLimitMaxConfiguration.inviteUserRateLimit = rateLimit.inviteUserRateLimit;
+          rateLimitMaxConfiguration.mfaRateLimit = rateLimit.mfaRateLimit;
+          rateLimitMaxConfiguration.creationLimit = rateLimit.creationLimit;
+        }
+      } catch (error) {
+        logger.error(`Error syncing rate limit configurations: %o`, error);
+      }
+    };
+
+    // sync rate limits configuration every 10 minutes
+    const job = new CronJob("*/10 * * * *", rateLimitSync);
+    job.start();
+
+    return job;
+  };
+
   return {
     getRateLimits,
-    updateRateLimit
+    updateRateLimit,
+    initializeBackgroundSync
   };
 };