feaet(e2e-tests): secret rotation tests

.github/workflows/run-backend-tests.yml

@@ -34,6 +34,8 @@ jobs:
         working-directory: backend
       - name: Start postgres and redis
         run: touch .env && docker compose -f docker-compose.dev.yml up -d db redis
+      - name: Start Secret Rotation testing databases
+        run: docker compose -f docker-compose.e2e-dbs.yml up -d
       - name: Run unit test
         run: npm run test:unit
         working-directory: backend

backend/e2e-test/mocks/queue.ts

@@ -1,34 +0,0 @@
-import { TQueueServiceFactory } from "@app/queue";
-
-export const mockQueue = (): TQueueServiceFactory => {
-  const queues: Record<string, unknown> = {};
-  const workers: Record<string, unknown> = {};
-  const job: Record<string, unknown> = {};
-  const events: Record<string, unknown> = {};
-
-  return {
-    queue: async (name, jobData) => {
-      job[name] = jobData;
-    },
-    queuePg: async () => {},
-    schedulePg: async () => {},
-    initialize: async () => {},
-    shutdown: async () => undefined,
-    stopRepeatableJob: async () => true,
-    start: (name, jobFn) => {
-      queues[name] = jobFn;
-      workers[name] = jobFn;
-    },
-    startPg: async () => {},
-    listen: (name, event) => {
-      events[name] = event;
-    },
-    getRepeatableJobs: async () => [],
-    getDelayedJobs: async () => [],
-    clearQueue: async () => {},
-    stopJobById: async () => {},
-    stopJobByIdPg: async () => {},
-    stopRepeatableJobByJobId: async () => true,
-    stopRepeatableJobByKey: async () => true
-  };
-};

backend/e2e-test/routes/v3/secret-rotations.spec.ts

@@ -0,0 +1,442 @@
+/* eslint-disable no-promise-executor-return */
+/* eslint-disable no-await-in-loop */
+import knex from "knex";
+import { v4 as uuidv4 } from "uuid";
+
+import { seedData1 } from "@app/db/seed-data";
+
+enum SecretRotationType {
+  OracleDb = "oracledb",
+  MySQL = "mysql"
+}
+
+type TGenericSqlCredentials = {
+  host: string;
+  port: number;
+  username: string;
+  password: string;
+  database: string;
+};
+
+type TSecretMapping = {
+  username: string;
+  password: string;
+};
+
+type TDatabaseUserCredentials = {
+  username: string;
+};
+
+const formatSqlUsername = (username: string) => `${username}_${uuidv4().slice(0, 8).replace(/-/g, "").toUpperCase()}`;
+
+const getSecretValue = async (secretKey: string) => {
+  const passwordSecret = await testServer.inject({
+    url: `/api/v3/secrets/raw/${secretKey}`,
+    method: "GET",
+    query: {
+      workspaceId: seedData1.projectV3.id,
+      environment: seedData1.environment.slug
+    },
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+
+  expect(passwordSecret.statusCode).toBe(200);
+  expect(passwordSecret.json().secret).toBeDefined();
+
+  const passwordSecretJson = JSON.parse(passwordSecret.payload);
+
+  return passwordSecretJson.secret.secretValue as string;
+};
+
+const deleteSecretRotation = async (id: string, type: SecretRotationType) => {
+  const res = await testServer.inject({
+    method: "DELETE",
+    query: {
+      deleteSecrets: "true",
+      revokeGeneratedCredentials: "true"
+    },
+    url: `/api/v2/secret-rotations/${type}-credentials/${id}`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+
+  expect(res.statusCode).toBe(200);
+};
+
+const deleteAppConnection = async (id: string, type: SecretRotationType) => {
+  const res = await testServer.inject({
+    method: "DELETE",
+    url: `/api/v1/app-connections/${type}/${id}`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+
+  expect(res.statusCode).toBe(200);
+};
+
+const createOracleDBAppConnection = async (credentials: TGenericSqlCredentials) => {
+  const createOracleDBAppConnectionReqBody = {
+    credentials: {
+      database: credentials.database,
+      host: credentials.host,
+      username: credentials.username,
+      password: credentials.password,
+      port: credentials.port,
+      sslEnabled: true,
+      sslRejectUnauthorized: true
+    },
+    name: `oracle-db-${uuidv4()}`,
+    description: "Test OracleDB App Connection",
+    gatewayId: null,
+    isPlatformManagedCredentials: false,
+    method: "username-and-password"
+  };
+
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v1/app-connections/oracledb`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: createOracleDBAppConnectionReqBody
+  });
+
+  const json = JSON.parse(res.payload);
+
+  expect(res.statusCode).toBe(200);
+  expect(json.appConnection).toBeDefined();
+
+  return json.appConnection.id as string;
+};
+
+const createMySQLAppConnection = async (credentials: TGenericSqlCredentials) => {
+  const createMySQLAppConnectionReqBody = {
+    name: `mysql-test-${uuidv4()}`,
+    description: "test-mysql",
+    gatewayId: null,
+    method: "username-and-password",
+    credentials: {
+      host: credentials.host,
+      port: credentials.port,
+      database: credentials.database,
+      username: credentials.username,
+      password: credentials.password,
+      sslEnabled: false,
+      sslRejectUnauthorized: true
+    }
+  };
+
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v1/app-connections/mysql`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: createMySQLAppConnectionReqBody
+  });
+
+  const json = JSON.parse(res.payload);
+
+  expect(res.statusCode).toBe(200);
+  expect(json.appConnection).toBeDefined();
+
+  return json.appConnection.id as string;
+};
+
+const createOracleInfisicalUsers = async (
+  credentials: TGenericSqlCredentials,
+  userCredentials: TDatabaseUserCredentials[]
+) => {
+  const client = knex({
+    client: "oracledb",
+    connection: {
+      database: credentials.database,
+      port: credentials.port,
+      host: credentials.host,
+      user: credentials.username,
+      password: credentials.password,
+      connectionTimeoutMillis: 10000,
+      ssl: {
+        // @ts-expect-error - this is a valid property for the ssl object
+        sslServerDNMatch: true
+      }
+    }
+  });
+
+  for await (const { username } of userCredentials) {
+    // check if user exists, and if it does, don't create it
+    const existingUser = await client.raw(`SELECT * FROM all_users WHERE username = '${username.toUpperCase()}'`);
+
+    if (!existingUser.length) {
+      await client.raw(`CREATE USER ${username} IDENTIFIED BY "temporary_password"`);
+    }
+    await client.raw(`GRANT ALL PRIVILEGES TO ${username} WITH ADMIN OPTION`);
+  }
+
+  await client.destroy();
+};
+
+const createMySQLInfisicalUsers = async (
+  credentials: TGenericSqlCredentials,
+  userCredentials: TDatabaseUserCredentials[]
+) => {
+  const client = knex({
+    client: "mysql2",
+    connection: {
+      database: credentials.database,
+      port: credentials.port,
+      host: credentials.host,
+      user: credentials.username,
+      password: credentials.password,
+      connectionTimeoutMillis: 10000
+    }
+  });
+
+  // Fix: Ensure root has GRANT OPTION privileges
+  try {
+    await client.raw("GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION;");
+    await client.raw("FLUSH PRIVILEGES;");
+  } catch (error) {
+    // Ignore if already has privileges
+  }
+
+  for await (const { username } of userCredentials) {
+    // check if user exists, and if it does, dont create it
+
+    const existingUser = await client.raw(`SELECT * FROM mysql.user WHERE user = '${username}'`);
+
+    if (!existingUser[0].length) {
+      await client.raw(`CREATE USER '${username}'@'%' IDENTIFIED BY 'temporary_password';`);
+    }
+
+    await client.raw(`GRANT ALL PRIVILEGES ON \`${credentials.database}\`.* TO '${username}'@'%';`);
+    await client.raw("FLUSH PRIVILEGES;");
+  }
+
+  await client.destroy();
+};
+
+const createOracleDBSecretRotation = async (
+  appConnectionId: string,
+  credentials: TGenericSqlCredentials,
+  userCredentials: TDatabaseUserCredentials[],
+  secretMapping: TSecretMapping
+) => {
+  const now = new Date();
+  const rotationTime = new Date(now.getTime() - 2 * 60 * 1000); // 2 minutes ago
+
+  await createOracleInfisicalUsers(credentials, userCredentials);
+
+  const createOracleDBSecretRotationReqBody = {
+    parameters: userCredentials.reduce(
+      (acc, user, index) => {
+        acc[`username${index + 1}`] = user.username;
+        return acc;
+      },
+      {} as Record<string, string>
+    ),
+    secretsMapping: {
+      username: secretMapping.username,
+      password: secretMapping.password
+    },
+    name: `test-oracle-${uuidv4()}`,
+    description: "Test OracleDB Secret Rotation",
+    secretPath: "/",
+    isAutoRotationEnabled: true,
+    rotationInterval: 5, // 5 seconds for testing
+    rotateAtUtc: {
+      hours: rotationTime.getUTCHours(),
+      minutes: rotationTime.getUTCMinutes()
+    },
+    connectionId: appConnectionId,
+    environment: seedData1.environment.slug,
+    projectId: seedData1.projectV3.id
+  };
+
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v2/secret-rotations/oracledb-credentials`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: createOracleDBSecretRotationReqBody
+  });
+
+  expect(res.statusCode).toBe(200);
+  expect(res.json().secretRotation).toBeDefined();
+
+  return res;
+};
+
+const createMySQLSecretRotation = async (
+  appConnectionId: string,
+  credentials: TGenericSqlCredentials,
+  userCredentials: TDatabaseUserCredentials[],
+  secretMapping: TSecretMapping
+) => {
+  const now = new Date();
+  const rotationTime = new Date(now.getTime() - 2 * 60 * 1000); // 2 minutes ago
+
+  await createMySQLInfisicalUsers(credentials, userCredentials);
+
+  const createMySQLSecretRotationReqBody = {
+    parameters: userCredentials.reduce(
+      (acc, user, index) => {
+        acc[`username${index + 1}`] = user.username;
+        return acc;
+      },
+      {} as Record<string, string>
+    ),
+    secretsMapping: {
+      username: secretMapping.username,
+      password: secretMapping.password
+    },
+    name: `test-mysql-rotation-${uuidv4()}`,
+    description: "Test MySQL Secret Rotation",
+    secretPath: "/",
+    isAutoRotationEnabled: true,
+    rotationInterval: 5,
+    rotateAtUtc: {
+      hours: rotationTime.getUTCHours(),
+      minutes: rotationTime.getUTCMinutes()
+    },
+    connectionId: appConnectionId,
+    environment: seedData1.environment.slug,
+    projectId: seedData1.projectV3.id
+  };
+
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v2/secret-rotations/mysql-credentials`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: createMySQLSecretRotationReqBody
+  });
+
+  expect(res.statusCode).toBe(200);
+  expect(res.json().secretRotation).toBeDefined();
+
+  return res;
+};
+
+describe("Secret Rotations", async () => {
+  const testCases = [
+    {
+      type: SecretRotationType.MySQL,
+      name: "MySQL (8.4.6) Secret Rotation",
+      dbCredentials: {
+        database: "mysql-test",
+        host: "127.0.0.1",
+        username: "root",
+        password: "mysql-test",
+        port: 3306
+      },
+      secretMapping: {
+        username: formatSqlUsername("MYSQL_USERNAME"),
+        password: formatSqlUsername("MYSQL_PASSWORD")
+      },
+      userCredentials: [
+        {
+          username: formatSqlUsername("MYSQL_USER_1")
+        },
+        {
+          username: formatSqlUsername("MYSQL_USER_2")
+        }
+      ]
+    },
+    {
+      type: SecretRotationType.OracleDb,
+      name: "OracleDB (23.8) Secret Rotation",
+      dbCredentials: {
+        database: "FREEPDB1",
+        host: "127.0.0.1",
+        username: "system",
+        password: "pdb-password",
+        port: 1521
+      },
+      secretMapping: {
+        username: formatSqlUsername("ORACLEDB_USERNAME"),
+        password: formatSqlUsername("ORACLEDB_PASSWORD")
+      },
+      userCredentials: [
+        {
+          username: formatSqlUsername("INFISICAL_USER_1")
+        },
+        {
+          username: formatSqlUsername("INFISICAL_USER_2")
+        }
+      ]
+    }
+  ] as {
+    type: SecretRotationType;
+    name: string;
+    dbCredentials: TGenericSqlCredentials;
+    secretMapping: TSecretMapping;
+    userCredentials: TDatabaseUserCredentials[];
+  }[];
+
+  const createAppConnectionMap = {
+    [SecretRotationType.OracleDb]: createOracleDBAppConnection,
+    [SecretRotationType.MySQL]: createMySQLAppConnection
+  };
+
+  const createRotationMap = {
+    [SecretRotationType.OracleDb]: createOracleDBSecretRotation,
+    [SecretRotationType.MySQL]: createMySQLSecretRotation
+  };
+
+  const appConnectionIds: { id: string; type: SecretRotationType }[] = [];
+  const secretRotationIds: { id: string; type: SecretRotationType }[] = [];
+
+  afterAll(async () => {
+    for (const { id, type } of secretRotationIds) {
+      await deleteSecretRotation(id, type);
+    }
+
+    for (const { id, type } of appConnectionIds) {
+      await deleteAppConnection(id, type);
+    }
+  });
+
+  test.concurrent.each(testCases)(
+    "Create secret rotation for $name",
+    async ({ dbCredentials, secretMapping, userCredentials, type }) => {
+      const appConnectionId = await createAppConnectionMap[type](dbCredentials);
+
+      if (appConnectionId) {
+        appConnectionIds.push({ id: appConnectionId, type });
+      }
+
+      const res = await createRotationMap[type](appConnectionId, dbCredentials, userCredentials, secretMapping);
+
+      const resJson = JSON.parse(res.payload);
+
+      if (resJson.secretRotation) {
+        secretRotationIds.push({ id: resJson.secretRotation.id, type });
+      }
+
+      const startSecretValue = await getSecretValue(secretMapping.password);
+      expect(startSecretValue).toBeDefined();
+
+      let attempts = 0;
+      while (attempts < 60) {
+        const currentSecretValue = await getSecretValue(secretMapping.password);
+
+        if (currentSecretValue !== startSecretValue) {
+          break;
+        }
+
+        attempts += 1;
+        await new Promise((resolve) => setTimeout(resolve, 2_500));
+      }
+    },
+    {
+      timeout: 300_000
+    }
+  );
+});

backend/e2e-test/vitest-environment-knex.ts

@@ -18,6 +18,7 @@ import { keyStoreFactory } from "@app/keystore/keystore";
 import { initializeHsmModule } from "@app/ee/services/hsm/hsm-fns";
 import { buildRedisFromConfig } from "@app/lib/config/redis";
 import { superAdminDALFactory } from "@app/services/super-admin/super-admin-dal";
+import { bootstrapCheck } from "@app/server/boot-strap-check";
 
 dotenv.config({ path: path.join(__dirname, "../../.env.test"), debug: true });
 export default {
@@ -63,6 +64,8 @@ export default {
       const queue = queueServiceFactory(envCfg, { dbConnectionUrl: envCfg.DB_CONNECTION_URI });
       const keyStore = keyStoreFactory(envCfg);
 
+      await queue.initialize();
+
       const hsmModule = initializeHsmModule(envCfg);
       hsmModule.initialize();
 
@@ -78,9 +81,13 @@ export default {
         envConfig: envCfg
       });
 
+      await bootstrapCheck({ db });
+
       // @ts-expect-error type
       globalThis.testServer = server;
       // @ts-expect-error type
+      globalThis.testQueue = queue;
+      // @ts-expect-error type
       globalThis.testSuperAdminDAL = superAdminDAL;
       // @ts-expect-error type
       globalThis.jwtAuthToken = crypto.jwt().sign(
@@ -105,6 +112,8 @@ export default {
     // custom setup
     return {
       async teardown() {
+        // @ts-expect-error type
+        await globalThis.testQueue.shutdown();
         // @ts-expect-error type
         await globalThis.testServer.close();
         // @ts-expect-error type
@@ -113,6 +122,8 @@ export default {
         delete globalThis.testSuperAdminDAL;
         // @ts-expect-error type
         delete globalThis.jwtToken;
+        // @ts-expect-error type
+        delete globalThis.testQueue;
         // called after all tests with this env have been run
         await db.migrate.rollback(
           {

backend/src/ee/services/dynamic-secret/dynamic-secret-fns.ts

@@ -9,7 +9,7 @@ import { getDbConnectionHost } from "@app/lib/knex";
 export const verifyHostInputValidity = async (host: string, isGateway = false) => {
   const appCfg = getConfig();
 
-  if (appCfg.isDevelopmentMode) return [host];
+  if (appCfg.isDevelopmentMode || appCfg.isTestMode) return [host];
 
   if (isGateway) return [host];
 

backend/src/ee/services/license/__mocks__/license-fns.ts

@@ -31,7 +31,7 @@ export const getDefaultOnPremFeatures = () => {
     caCrl: false,
     sshHostGroups: false,
     enterpriseSecretSyncs: false,
-    enterpriseAppConnections: false
+    enterpriseAppConnections: true
   };
 };
 

backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-fns.ts

@@ -14,14 +14,17 @@ import { OKTA_CLIENT_SECRET_ROTATION_LIST_OPTION } from "./okta-client-secret";
 import { ORACLEDB_CREDENTIALS_ROTATION_LIST_OPTION } from "./oracledb-credentials";
 import { POSTGRES_CREDENTIALS_ROTATION_LIST_OPTION } from "./postgres-credentials";
 import { SecretRotation, SecretRotationStatus } from "./secret-rotation-v2-enums";
-import { TSecretRotationV2ServiceFactoryDep } from "./secret-rotation-v2-service";
+import { TSecretRotationV2ServiceFactory, TSecretRotationV2ServiceFactoryDep } from "./secret-rotation-v2-service";
 import {
+  TSecretRotationRotateSecretsJobPayload,
   TSecretRotationV2,
   TSecretRotationV2GeneratedCredentials,
   TSecretRotationV2ListItem,
   TSecretRotationV2Raw,
   TUpdateSecretRotationV2DTO
 } from "./secret-rotation-v2-types";
+import { logger } from "@app/lib/logger";
+import { TSecretRotationV2DALFactory } from "./secret-rotation-v2-dal";
 
 const SECRET_ROTATION_LIST_OPTIONS: Record<SecretRotation, TSecretRotationV2ListItem> = {
   [SecretRotation.PostgresCredentials]: POSTGRES_CREDENTIALS_ROTATION_LIST_OPTION,
@@ -74,6 +77,10 @@ export const getNextUtcRotationInterval = (rotateAtUtc?: TSecretRotationV2["rota
   const appCfg = getConfig();
 
   if (appCfg.isRotationDevelopmentMode) {
+    if (appCfg.isTestMode) {
+      // if its test mode, it should always rotate
+      return new Date(Date.now() + 365 * 24 * 60 * 60 * 1000); // Current time + 1 year
+    }
     return getNextUTCMinuteInterval(rotateAtUtc);
   }
 
@@ -263,3 +270,51 @@ export const throwOnImmutableParameterUpdate = (
     // do nothing
   }
 };
+
+export const rotateSecretsFns = async ({
+  job,
+  secretRotationV2DAL,
+  secretRotationV2Service
+}: {
+  job: {
+    data: TSecretRotationRotateSecretsJobPayload;
+    id: string;
+    retryCount: number;
+    retryLimit: number;
+  };
+  secretRotationV2DAL: Pick<TSecretRotationV2DALFactory, "findById">;
+  secretRotationV2Service: Pick<TSecretRotationV2ServiceFactory, "rotateGeneratedCredentials">;
+}) => {
+  const { rotationId, queuedAt, isManualRotation } = job.data;
+  const { retryCount, retryLimit } = job;
+
+  const logDetails = `[rotationId=${rotationId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
+
+  try {
+    const secretRotation = await secretRotationV2DAL.findById(rotationId);
+
+    if (!secretRotation) throw new Error(`Secret rotation ${rotationId} not found`);
+
+    if (!secretRotation.isAutoRotationEnabled) {
+      logger.info(`secretRotationV2Queue: Skipping Rotation - Auto-Rotation Disabled Since Queue ${logDetails}`);
+    }
+
+    if (new Date(secretRotation.lastRotatedAt).getTime() >= new Date(queuedAt).getTime()) {
+      // rotated since being queued, skip rotation
+      logger.info(`secretRotationV2Queue: Skipping Rotation - Rotated Since Queue ${logDetails}`);
+      return;
+    }
+
+    await secretRotationV2Service.rotateGeneratedCredentials(secretRotation, {
+      jobId: job.id,
+      shouldSendNotification: true,
+      isFinalAttempt: retryCount === retryLimit,
+      isManualRotation
+    });
+
+    logger.info(`secretRotationV2Queue: Secrets Rotated ${logDetails}`);
+  } catch (error) {
+    logger.error(error, `secretRotationV2Queue: Failed to Rotate Secrets ${logDetails}`);
+    throw error;
+  }
+};

backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts

@@ -1,9 +1,12 @@
+import { v4 as uuidv4 } from "uuid";
+
 import { ProjectMembershipRole } from "@app/db/schemas";
 import { TSecretRotationV2DALFactory } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-dal";
 import { SecretRotation } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-enums";
 import {
   getNextUtcRotationInterval,
-  getSecretRotationRotateSecretJobOptions
+  getSecretRotationRotateSecretJobOptions,
+  rotateSecretsFns
 } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-fns";
 import { SECRET_ROTATION_NAME_MAP } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-maps";
 import { TSecretRotationV2ServiceFactory } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-service";
@@ -63,6 +66,26 @@ export const secretRotationV2QueueServiceFactory = async ({
               rotation.lastRotatedAt
             ).toISOString()}] [rotateAt=${new Date(rotation.nextRotationAt!).toISOString()}]`
           );
+
+          const data = {
+            rotationId: rotation.id,
+            queuedAt: currentTime
+          } as TSecretRotationRotateSecretsJobPayload;
+
+          if (appCfg.isTestMode) {
+            logger.warn("secretRotationV2Queue: Manually rotating secrets for test mode");
+            await rotateSecretsFns({
+              job: {
+                id: uuidv4(),
+                data,
+                retryCount: 0,
+                retryLimit: 0
+              },
+              secretRotationV2DAL,
+              secretRotationV2Service
+            });
+          }
+
           await queueService.queuePg(
             QueueJobs.SecretRotationV2RotateSecrets,
             {
@@ -87,38 +110,14 @@ export const secretRotationV2QueueServiceFactory = async ({
   await queueService.startPg<QueueName.SecretRotationV2>(
     QueueJobs.SecretRotationV2RotateSecrets,
     async ([job]) => {
-      const { rotationId, queuedAt, isManualRotation } = job.data as TSecretRotationRotateSecretsJobPayload;
-      const { retryCount, retryLimit } = job;
-
-      const logDetails = `[rotationId=${rotationId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
-
-      try {
-        const secretRotation = await secretRotationV2DAL.findById(rotationId);
-
-        if (!secretRotation) throw new Error(`Secret rotation ${rotationId} not found`);
-
-        if (!secretRotation.isAutoRotationEnabled) {
-          logger.info(`secretRotationV2Queue: Skipping Rotation - Auto-Rotation Disabled Since Queue ${logDetails}`);
-        }
-
-        if (new Date(secretRotation.lastRotatedAt).getTime() >= new Date(queuedAt).getTime()) {
-          // rotated since being queued, skip rotation
-          logger.info(`secretRotationV2Queue: Skipping Rotation - Rotated Since Queue ${logDetails}`);
-          return;
-        }
-
-        await secretRotationV2Service.rotateGeneratedCredentials(secretRotation, {
-          jobId: job.id,
-          shouldSendNotification: true,
-          isFinalAttempt: retryCount === retryLimit,
-          isManualRotation
-        });
-
-        logger.info(`secretRotationV2Queue: Secrets Rotated ${logDetails}`);
-      } catch (error) {
-        logger.error(error, `secretRotationV2Queue: Failed to Rotate Secrets ${logDetails}`);
-        throw error;
-      }
+      await rotateSecretsFns({
+        job: {
+          ...job,
+          data: job.data as TSecretRotationRotateSecretsJobPayload
+        },
+        secretRotationV2DAL,
+        secretRotationV2Service
+      });
     },
     {
       batchSize: 1,

backend/src/lib/config/env.ts

@@ -345,7 +345,9 @@ const envSchema = z
     isSmtpConfigured: Boolean(data.SMTP_HOST),
     isRedisConfigured: Boolean(data.REDIS_URL || data.REDIS_SENTINEL_HOSTS),
     isDevelopmentMode: data.NODE_ENV === "development",
-    isRotationDevelopmentMode: data.NODE_ENV === "development" && data.ROTATION_DEVELOPMENT_MODE,
+    isTestMode: data.NODE_ENV === "test",
+    isRotationDevelopmentMode:
+      (data.NODE_ENV === "development" && data.ROTATION_DEVELOPMENT_MODE) || data.NODE_ENV === "test",
     isProductionMode: data.NODE_ENV === "production" || IS_PACKAGED,
     isRedisSentinelMode: Boolean(data.REDIS_SENTINEL_HOSTS),
     REDIS_SENTINEL_HOSTS: data.REDIS_SENTINEL_HOSTS?.trim()

backend/vitest.e2e.config.ts

@@ -8,14 +8,18 @@ export default defineConfig({
       NODE_ENV: "test"
     },
     environment: "./e2e-test/vitest-environment-knex.ts",
-    include: ["./e2e-test/**/*.spec.ts"],
+    include: ["./e2e-test/**/secret-rotations.spec.ts"],
+
+    pool: "forks",
     poolOptions: {
-      threads: {
-        singleThread: true,
-        useAtomics: true,
+      forks: {
+        singleFork: true,
         isolate: false
       }
     },
+    isolate: false,
+    fileParallelism: false,
+
     alias: {
       "./license-fns": path.resolve(__dirname, "./src/ee/services/license/__mocks__/license-fns")
     }

docker-compose.e2e-dbs.yml

@@ -0,0 +1,43 @@
+version: '3.8'
+
+services:
+  oracle-db-23.8:
+    image: container-registry.oracle.com/database/free:23.8.0.0
+    container_name: oracle-db-23.8
+    ports:
+      - "1521:1521"
+    environment:
+      - ORACLE_PDB=pdb
+      - ORACLE_PWD=pdb-password
+    volumes:
+      - oracle-data-23.8:/opt/oracle/oradata
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "sqlplus", "-L", "system/pdb-password@//localhost:1521/FREEPDB1", "<<<", "SELECT 1 FROM DUAL;"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+
+  mysql-8.4.6:
+    image: mysql:8.4.6
+    container_name: mysql-8.4.6
+    ports:
+      - "3306:3306"
+    environment:
+      - MYSQL_ROOT_PASSWORD=mysql-test
+      - MYSQL_DATABASE=mysql-test
+      - MYSQL_ROOT_HOST=% 
+      - MYSQL_USER=mysql-test
+      - MYSQL_PASSWORD=mysql-test
+    volumes:
+      - mysql-data-8.4.6:/var/lib/mysql
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "mysql-test", "-pmysql-test"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+
+volumes:
+  oracle-data-23.8:
+  mysql-data-8.4.6: