feat: updated migration to be auto matic

backend/e2e-test/vitest-environment-knex.ts

@@ -23,7 +23,7 @@ export default {
   name: "knex-env",
   transformMode: "ssr",
   async setup() {
-    const logger = await initLogger();
+    const logger = initLogger();
     const envConfig = initEnvConfig(logger);
     const db = initDbConnection({
       dbConnectionUri: envConfig.DB_CONNECTION_URI,
@@ -119,4 +119,5 @@ export default {
       }
     };
   }
-};
+};
+

backend/src/auto-start-migrations.ts

@@ -0,0 +1,79 @@
+import path from "node:path";
+
+import dotenv from "dotenv";
+import { Knex } from "knex";
+import { Logger } from "pino";
+
+import { PgSqlLock } from "./keystore/keystore";
+
+dotenv.config();
+
+type TArgs = {
+  auditLogDb?: Knex;
+  applicationDb: Knex;
+  logger: Logger;
+};
+
+const migrationConfig = {
+  directory: path.join(__dirname, "./db/migrations"),
+  extension: "ts",
+  tableName: "infisical_migrations"
+};
+
+const migrationStatusCheckErrorHandler = (err: Error) => {
+  // happens for first time  in which the migration table itself is not created yet
+  //    error: select * from "infisical_migrations" - relation "infisical_migrations" does not exist
+  if (err?.message?.includes("does not exist")) {
+    return true;
+  }
+  throw err;
+};
+
+export const runMigrations = async ({ applicationDb, auditLogDb, logger }: TArgs) => {
+  try {
+    const shouldRunMigration = Boolean(
+      await applicationDb.migrate.status(migrationConfig).catch(migrationStatusCheckErrorHandler)
+    ); // db.length - code.length
+    if (!shouldRunMigration) {
+      logger.info("No migrations pending: Skipping migration process.");
+      return;
+    }
+
+    if (auditLogDb) {
+      await auditLogDb.transaction(async (tx) => {
+        await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]);
+        logger.info("Running audit log migrations.");
+
+        const didPreviousInstanceRunMigration = !(await auditLogDb.migrate
+          .status(migrationConfig)
+          .catch(migrationStatusCheckErrorHandler));
+        if (didPreviousInstanceRunMigration) {
+          logger.info("No audit log migrations pending: Applied by previous instance. Skipping migration process.");
+          return;
+        }
+
+        await auditLogDb.migrate.latest(migrationConfig);
+        logger.info("Finished audit log migrations.");
+      });
+    }
+
+    await applicationDb.transaction(async (tx) => {
+      await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.BootUpMigration]);
+      logger.info("Running application migrations.");
+
+      const didPreviousInstanceRunMigration = !(await applicationDb.migrate
+        .status(migrationConfig)
+        .catch(migrationStatusCheckErrorHandler));
+      if (didPreviousInstanceRunMigration) {
+        logger.info("No application migrations pending: Applied by previous instance. Skipping migration process.");
+        return;
+      }
+
+      await applicationDb.migrate.latest(migrationConfig);
+      logger.info("Finished application migrations.");
+    });
+  } catch (err) {
+    logger.error(err, "Boot up migration failed");
+    process.exit(1);
+  }
+};

backend/src/db/instance.ts

@@ -49,6 +49,9 @@ export const initDbConnection = ({
             ca: Buffer.from(dbRootCert, "base64").toString("ascii")
           }
         : false
+    },
+    migrations: {
+      tableName: "infisical_migrations"
     }
   });
 
@@ -64,6 +67,9 @@ export const initDbConnection = ({
               ca: Buffer.from(replicaDbCertificate, "base64").toString("ascii")
             }
           : false
+      },
+      migrations: {
+        tableName: "infisical_migrations"
       }
     });
   });
@@ -98,6 +104,9 @@ export const initAuditLogDbConnection = ({
             ca: Buffer.from(dbRootCert, "base64").toString("ascii")
           }
         : false
+    },
+    migrations: {
+      tableName: "infisical_migrations"
     }
   });
 

backend/src/db/migrations/20250109104500_webhook-to-kms.ts

@@ -25,7 +25,7 @@ export async function up(knex: Knex): Promise<void> {
     });
   }
 
-  await initLogger();
+  initLogger();
   const envConfig = getMigrationEnvConfig();
   const keyStore = inMemoryKeyStore();
   const { kmsService } = await getMigrationEncryptionServices({ envConfig, keyStore, db: knex });

backend/src/db/migrations/20250109104501_secret-rotation-to-kms.ts

@@ -22,7 +22,7 @@ export async function up(knex: Knex): Promise<void> {
     });
   }
 
-  await initLogger();
+  initLogger();
   const envConfig = getMigrationEnvConfig();
   const keyStore = inMemoryKeyStore();
   const { kmsService } = await getMigrationEncryptionServices({ envConfig, keyStore, db: knex });

backend/src/db/migrations/20250109104502_identity-k8-auth-to-kms.ts

@@ -53,7 +53,7 @@ const reencryptIdentityK8sAuth = async (knex: Knex) => {
     });
   }
 
-  await initLogger();
+  initLogger();
   const envConfig = getMigrationEnvConfig();
   const keyStore = inMemoryKeyStore();
   const { kmsService } = await getMigrationEncryptionServices({ envConfig, keyStore, db: knex });

backend/src/db/migrations/20250109104502_identity-oidc-auth-to-kms.ts

@@ -33,7 +33,7 @@ const reencryptIdentityOidcAuth = async (knex: Knex) => {
     });
   }
 
-  await initLogger();
+  initLogger();
   const envConfig = getMigrationEnvConfig();
   const keyStore = inMemoryKeyStore();
   const { kmsService } = await getMigrationEncryptionServices({ envConfig, keyStore, db: knex });

backend/src/db/migrations/20250109104503_dynamic-secret-root-to-kms.ts

@@ -28,7 +28,7 @@ export async function up(knex: Knex): Promise<void> {
     });
   }
 
-  await initLogger();
+  initLogger();
   const envConfig = getMigrationEnvConfig();
   const keyStore = inMemoryKeyStore();
   const { kmsService } = await getMigrationEncryptionServices({ envConfig, keyStore, db: knex });

backend/src/db/migrations/20250109104508_directory-config-to-kms.ts

@@ -26,7 +26,7 @@ const reencryptSamlConfig = async (knex: Knex) => {
     });
   }
 
-  await initLogger();
+  initLogger();
   const envConfig = getMigrationEnvConfig();
   const keyStore = inMemoryKeyStore();
   const { kmsService } = await getMigrationEncryptionServices({ envConfig, keyStore, db: knex });
@@ -181,7 +181,7 @@ const reencryptLdapConfig = async (knex: Knex) => {
     });
   }
 
-  await initLogger();
+  initLogger();
   const envConfig = getMigrationEnvConfig();
   const keyStore = inMemoryKeyStore();
   const { kmsService } = await getMigrationEncryptionServices({ envConfig, keyStore, db: knex });
@@ -330,7 +330,7 @@ const reencryptOidcConfig = async (knex: Knex) => {
     });
   }
 
-  await initLogger();
+  initLogger();
   const envConfig = getMigrationEnvConfig();
   const keyStore = inMemoryKeyStore();
   const { kmsService } = await getMigrationEncryptionServices({ envConfig, keyStore, db: knex });

backend/src/db/migrations/utils/services.ts

@@ -20,7 +20,6 @@ type TDependencies = {
 
 export const getMigrationEncryptionServices = async ({ envConfig, db, keyStore }: TDependencies) => {
   // eslint-disable-next-line no-param-reassign
-  db.replicaNode = () => db;
   const hsmModule = initializeHsmModule(envConfig);
   hsmModule.initialize();
 

backend/src/keystore/keystore.ts

@@ -2,6 +2,11 @@ import { Redis } from "ioredis";
 
 import { Redlock, Settings } from "@app/lib/red-lock";
 
+export enum PgSqlLock {
+  BootUpMigration = 2023,
+  SuperAdminInit = 2024
+}
+
 export type TKeyStoreFactory = ReturnType<typeof keyStoreFactory>;
 
 // all the key prefixes used must be set here to avoid conflict

backend/src/lib/logger/logger.ts

@@ -98,7 +98,7 @@ const extractReqId = () => {
   }
 };
 
-export const initLogger = async () => {
+export const initLogger = () => {
   const cfg = loggerConfig.parse(process.env);
   const targets: pino.TransportMultiOptions["targets"][number][] = [
     {

backend/src/main.ts

@@ -2,14 +2,13 @@ import "./lib/telemetry/instrumentation";
 
 import dotenv from "dotenv";
 import { Redis } from "ioredis";
-import path from "path";
 
 import { initializeHsmModule } from "@app/ee/services/hsm/hsm-fns";
 
+import { runMigrations } from "./auto-start-migrations";
 import { initAuditLogDbConnection, initDbConnection } from "./db";
 import { keyStoreFactory } from "./keystore/keystore";
-import { formatSmtpConfig, initEnvConfig, IS_PACKAGED } from "./lib/config/env";
-import { isMigrationMode } from "./lib/fn";
+import { formatSmtpConfig, initEnvConfig } from "./lib/config/env";
 import { initLogger } from "./lib/logger";
 import { queueServiceFactory } from "./queue";
 import { main } from "./server/app";
@@ -19,7 +18,7 @@ import { smtpServiceFactory } from "./services/smtp/smtp-service";
 dotenv.config();
 
 const run = async () => {
-  const logger = await initLogger();
+  const logger = initLogger();
   const envConfig = initEnvConfig(logger);
 
   const db = initDbConnection({
@@ -38,22 +37,7 @@ const run = async () => {
       })
     : undefined;
 
-  // Case: App is running in packaged mode (binary), and migration mode is enabled.
-  // Run the migrations and exit the process after completion.
-  if (IS_PACKAGED && isMigrationMode()) {
-    try {
-      logger.info("Running Postgres migrations..");
-      await db.migrate.latest({
-        directory: path.join(__dirname, "./db/migrations")
-      });
-      logger.info("Postgres migrations completed");
-    } catch (err) {
-      logger.error(err, "Failed to run migrations");
-      process.exit(1);
-    }
-
-    process.exit(0);
-  }
+  await runMigrations({ applicationDb: db, auditLogDb, logger });
 
   const smtp = smtpServiceFactory(formatSmtpConfig());
 

backend/src/services/super-admin/super-admin-service.ts

@@ -2,7 +2,7 @@ import bcrypt from "bcrypt";
 
 import { TSuperAdmin, TSuperAdminUpdate } from "@app/db/schemas";
 import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
-import { TKeyStoreFactory } from "@app/keystore/keystore";
+import { PgSqlLock, TKeyStoreFactory } from "@app/keystore/keystore";
 import { getConfig } from "@app/lib/config/env";
 import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
 import { getUserPrivateKey } from "@app/lib/crypto/srp";
@@ -87,17 +87,24 @@ export const superAdminServiceFactory = ({
 
     // reset on initialized
     await keyStore.deleteItem(ADMIN_CONFIG_KEY);
-    const serverCfg = await serverCfgDAL.findById(ADMIN_CONFIG_DB_UUID);
-    if (serverCfg) return;
+    const serverCfg = await serverCfgDAL.transaction(async (tx) => {
+      await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.SuperAdminInit]);
+      const serverCfgInDB = await serverCfgDAL.findById(ADMIN_CONFIG_DB_UUID, tx);
+      if (serverCfgInDB) return serverCfgInDB;
 
-    const newCfg = await serverCfgDAL.create({
-      // @ts-expect-error id is kept as fixed for idempotence and to avoid race condition
-      id: ADMIN_CONFIG_DB_UUID,
-      initialized: false,
-      allowSignUp: true,
-      defaultAuthOrgId: null
+      const newCfg = await serverCfgDAL.create(
+        {
+          // @ts-expect-error id is kept as fixed for idempotence and to avoid race condition
+          id: ADMIN_CONFIG_DB_UUID,
+          initialized: false,
+          allowSignUp: true,
+          defaultAuthOrgId: null
+        },
+        tx
+      );
+      return newCfg;
     });
-    return newCfg;
+    return serverCfg;
   };
 
   const updateServerCfg = async (

docker-compose.dev.yml

@@ -56,22 +56,8 @@ services:
       POSTGRES_USER: infisical
       POSTGRES_DB: infisical-test
 
-  db-migration:
-    container_name: infisical-db-migration
-    depends_on:
-      - db
-    build:
-      context: ./backend
-      dockerfile: Dockerfile.dev
-    env_file: .env
-    environment:
-      - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable
-    command: npm run migration:latest
-    volumes:
-      - ./backend/src:/app/src
-
   backend:
-    container_name: infisical-dev-api
+    # container_name: infisical-dev-api
     build:
       context: ./backend
       dockerfile: Dockerfile.dev
@@ -80,13 +66,11 @@ services:
         condition: service_started
       redis:
         condition: service_started
-      db-migration:
-        condition: service_completed_successfully
     env_file:
       - .env
     ports:
-      - 4000:4000
-      - 9464:9464 # for OTEL collection of Prometheus metrics
+      - 4000-4010:4000
+      # - 9464:9464 # for OTEL collection of Prometheus metrics
     environment:
       - NODE_ENV=development
       - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable
@@ -192,7 +176,7 @@ services:
     depends_on:
       - openldap
     profiles: [ldap]
-    
+
   keycloak:
     image: quay.io/keycloak/keycloak:26.1.0
     restart: always
@@ -202,7 +186,7 @@ services:
     command: start-dev
     ports:
       - 8088:8080
-    profiles: [ sso ]
+    profiles: [sso]
 
 volumes:
   postgres-data:

docker-compose.prod.yml

@@ -1,18 +1,6 @@
 version: "3"
 
 services:
-  db-migration:
-    container_name: infisical-db-migration
-    depends_on:
-      db:
-        condition: service_healthy
-    image: infisical/infisical:latest-postgres
-    env_file: .env
-    command: npm run migration:latest
-    pull_policy: always
-    networks:
-      - infisical
-
   backend:
     container_name: infisical-backend
     restart: unless-stopped
@@ -21,8 +9,6 @@ services:
         condition: service_healthy
       redis:
         condition: service_started
-      db-migration:
-        condition: service_completed_successfully
     image: infisical/infisical:latest-postgres
     pull_policy: always
     env_file: .env
@@ -69,4 +55,5 @@ volumes:
     driver: local
 
 networks:
-  infisical:
+  infisical:
+