From 259c01c11077a67ceef78f4d2215b63d9fe23f24 Mon Sep 17 00:00:00 2001 From: = Date: Mon, 24 Jun 2024 23:59:10 +0530 Subject: [PATCH] feat: added read replica option in config and extended knex to choose --- backend/e2e-test/vitest-environment-knex.ts | 37 +- backend/src/@types/knex-tables.d.ts | 593 ++++++++++++++++++++ backend/src/@types/knex.d.ts | 11 +- backend/src/db/instance.ts | 48 +- backend/src/lib/config/env.ts | 12 + backend/src/main.ts | 6 +- docker-compose.dev-read-replica.yml | 191 +++++++ 7 files changed, 877 insertions(+), 21 deletions(-) create mode 100644 backend/src/@types/knex-tables.d.ts create mode 100644 docker-compose.dev-read-replica.yml diff --git a/backend/e2e-test/vitest-environment-knex.ts b/backend/e2e-test/vitest-environment-knex.ts index 09ab054436..cc1f9afc28 100644 --- a/backend/e2e-test/vitest-environment-knex.ts +++ b/backend/e2e-test/vitest-environment-knex.ts @@ -3,7 +3,6 @@ import "ts-node/register"; import dotenv from "dotenv"; import jwt from "jsonwebtoken"; -import knex from "knex"; import path from "path"; import { seedData1 } from "@app/db/seed-data"; @@ -15,6 +14,7 @@ import { AuthMethod, AuthTokenType } from "@app/services/auth/auth-type"; import { mockQueue } from "./mocks/queue"; import { mockSmtpServer } from "./mocks/smtp"; import { mockKeyStore } from "./mocks/keystore"; +import { initDbConnection } from "@app/db"; dotenv.config({ path: path.join(__dirname, "../../.env.test"), debug: true }); export default { @@ -23,23 +23,21 @@ export default { async setup() { const logger = await initLogger(); const cfg = initEnvConfig(logger); - const db = knex({ - client: "pg", - connection: cfg.DB_CONNECTION_URI, - migrations: { - directory: path.join(__dirname, "../src/db/migrations"), - extension: "ts", - tableName: "infisical_migrations" - }, - seeds: { - directory: path.join(__dirname, "../src/db/seeds"), - extension: "ts" - } + const db = initDbConnection({ + dbConnectionUri: cfg.DB_CONNECTION_URI, + dbRootCert: cfg.DB_ROOT_CERT }); try { - await db.migrate.latest(); - await db.seed.run(); + await db.migrate.latest({ + directory: path.join(__dirname, "../src/db/migrations"), + extension: "ts", + tableName: "infisical_migrations" + }); + await db.seed.run({ + directory: path.join(__dirname, "../src/db/seeds"), + extension: "ts" + }); const smtp = mockSmtpServer(); const queue = mockQueue(); const keyStore = mockKeyStore(); @@ -74,7 +72,14 @@ export default { // @ts-expect-error type delete globalThis.jwtToken; // called after all tests with this env have been run - await db.migrate.rollback({}, true); + await db.migrate.rollback( + { + directory: path.join(__dirname, "../src/db/migrations"), + extension: "ts", + tableName: "infisical_migrations" + }, + true + ); await db.destroy(); } }; diff --git a/backend/src/@types/knex-tables.d.ts b/backend/src/@types/knex-tables.d.ts new file mode 100644 index 0000000000..79342ea276 --- /dev/null +++ b/backend/src/@types/knex-tables.d.ts @@ -0,0 +1,593 @@ +import { Knex } from "knex"; + +import { + TableName, + TAccessApprovalPolicies, + TAccessApprovalPoliciesApprovers, + TAccessApprovalPoliciesApproversInsert, + TAccessApprovalPoliciesApproversUpdate, + TAccessApprovalPoliciesInsert, + TAccessApprovalPoliciesUpdate, + TAccessApprovalRequests, + TAccessApprovalRequestsInsert, + TAccessApprovalRequestsReviewers, + TAccessApprovalRequestsReviewersInsert, + TAccessApprovalRequestsReviewersUpdate, + TAccessApprovalRequestsUpdate, + TApiKeys, + TApiKeysInsert, + TApiKeysUpdate, + TAuditLogs, + TAuditLogsInsert, + TAuditLogStreams, + TAuditLogStreamsInsert, + TAuditLogStreamsUpdate, + TAuditLogsUpdate, + TAuthTokens, + TAuthTokenSessions, + TAuthTokenSessionsInsert, + TAuthTokenSessionsUpdate, + TAuthTokensInsert, + TAuthTokensUpdate, + TBackupPrivateKey, + TBackupPrivateKeyInsert, + TBackupPrivateKeyUpdate, + TCertificateAuthorities, + TCertificateAuthoritiesInsert, + TCertificateAuthoritiesUpdate, + TCertificateAuthorityCerts, + TCertificateAuthorityCertsInsert, + TCertificateAuthorityCertsUpdate, + TCertificateAuthorityCrl, + TCertificateAuthorityCrlInsert, + TCertificateAuthorityCrlUpdate, + TCertificateAuthoritySecret, + TCertificateAuthoritySecretInsert, + TCertificateAuthoritySecretUpdate, + TCertificateBodies, + TCertificateBodiesInsert, + TCertificateBodiesUpdate, + TCertificates, + TCertificateSecrets, + TCertificateSecretsInsert, + TCertificateSecretsUpdate, + TCertificatesInsert, + TCertificatesUpdate, + TDynamicSecretLeases, + TDynamicSecretLeasesInsert, + TDynamicSecretLeasesUpdate, + TDynamicSecrets, + TDynamicSecretsInsert, + TDynamicSecretsUpdate, + TGitAppInstallSessions, + TGitAppInstallSessionsInsert, + TGitAppInstallSessionsUpdate, + TGitAppOrg, + TGitAppOrgInsert, + TGitAppOrgUpdate, + TGroupProjectMembershipRoles, + TGroupProjectMembershipRolesInsert, + TGroupProjectMembershipRolesUpdate, + TGroupProjectMemberships, + TGroupProjectMembershipsInsert, + TGroupProjectMembershipsUpdate, + TGroups, + TGroupsInsert, + TGroupsUpdate, + TIdentities, + TIdentitiesInsert, + TIdentitiesUpdate, + TIdentityAccessTokens, + TIdentityAccessTokensInsert, + TIdentityAccessTokensUpdate, + TIdentityAwsAuths, + TIdentityAwsAuthsInsert, + TIdentityAwsAuthsUpdate, + TIdentityAzureAuths, + TIdentityAzureAuthsInsert, + TIdentityAzureAuthsUpdate, + TIdentityGcpAuths, + TIdentityGcpAuthsInsert, + TIdentityGcpAuthsUpdate, + TIdentityKubernetesAuths, + TIdentityKubernetesAuthsInsert, + TIdentityKubernetesAuthsUpdate, + TIdentityOrgMemberships, + TIdentityOrgMembershipsInsert, + TIdentityOrgMembershipsUpdate, + TIdentityProjectAdditionalPrivilege, + TIdentityProjectAdditionalPrivilegeInsert, + TIdentityProjectAdditionalPrivilegeUpdate, + TIdentityProjectMembershipRole, + TIdentityProjectMembershipRoleInsert, + TIdentityProjectMembershipRoleUpdate, + TIdentityProjectMemberships, + TIdentityProjectMembershipsInsert, + TIdentityProjectMembershipsUpdate, + TIdentityUaClientSecrets, + TIdentityUaClientSecretsInsert, + TIdentityUaClientSecretsUpdate, + TIdentityUniversalAuths, + TIdentityUniversalAuthsInsert, + TIdentityUniversalAuthsUpdate, + TIncidentContacts, + TIncidentContactsInsert, + TIncidentContactsUpdate, + TIntegrationAuths, + TIntegrationAuthsInsert, + TIntegrationAuthsUpdate, + TIntegrations, + TIntegrationsInsert, + TIntegrationsUpdate, + TKmsKeys, + TKmsKeysInsert, + TKmsKeysUpdate, + TKmsKeyVersions, + TKmsKeyVersionsInsert, + TKmsKeyVersionsUpdate, + TKmsRootConfig, + TKmsRootConfigInsert, + TKmsRootConfigUpdate, + TLdapConfigs, + TLdapConfigsInsert, + TLdapConfigsUpdate, + TLdapGroupMaps, + TLdapGroupMapsInsert, + TLdapGroupMapsUpdate, + TOrganizations, + TOrganizationsInsert, + TOrganizationsUpdate, + TOrgBots, + TOrgBotsInsert, + TOrgBotsUpdate, + TOrgMemberships, + TOrgMembershipsInsert, + TOrgMembershipsUpdate, + TOrgRoles, + TOrgRolesInsert, + TOrgRolesUpdate, + TProjectBots, + TProjectBotsInsert, + TProjectBotsUpdate, + TProjectEnvironments, + TProjectEnvironmentsInsert, + TProjectEnvironmentsUpdate, + TProjectKeys, + TProjectKeysInsert, + TProjectKeysUpdate, + TProjectMemberships, + TProjectMembershipsInsert, + TProjectMembershipsUpdate, + TProjectRoles, + TProjectRolesInsert, + TProjectRolesUpdate, + TProjects, + TProjectsInsert, + TProjectsUpdate, + TProjectUserAdditionalPrivilege, + TProjectUserAdditionalPrivilegeInsert, + TProjectUserAdditionalPrivilegeUpdate, + TProjectUserMembershipRoles, + TProjectUserMembershipRolesInsert, + TProjectUserMembershipRolesUpdate, + TRateLimit, + TRateLimitInsert, + TRateLimitUpdate, + TSamlConfigs, + TSamlConfigsInsert, + TSamlConfigsUpdate, + TScimTokens, + TScimTokensInsert, + TScimTokensUpdate, + TSecretApprovalPolicies, + TSecretApprovalPoliciesApprovers, + TSecretApprovalPoliciesApproversInsert, + TSecretApprovalPoliciesApproversUpdate, + TSecretApprovalPoliciesInsert, + TSecretApprovalPoliciesUpdate, + TSecretApprovalRequests, + TSecretApprovalRequestSecretTags, + TSecretApprovalRequestSecretTagsInsert, + TSecretApprovalRequestSecretTagsUpdate, + TSecretApprovalRequestsInsert, + TSecretApprovalRequestsReviewers, + TSecretApprovalRequestsReviewersInsert, + TSecretApprovalRequestsReviewersUpdate, + TSecretApprovalRequestsSecrets, + TSecretApprovalRequestsSecretsInsert, + TSecretApprovalRequestsSecretsUpdate, + TSecretApprovalRequestsUpdate, + TSecretBlindIndexes, + TSecretBlindIndexesInsert, + TSecretBlindIndexesUpdate, + TSecretFolders, + TSecretFoldersInsert, + TSecretFoldersUpdate, + TSecretFolderVersions, + TSecretFolderVersionsInsert, + TSecretFolderVersionsUpdate, + TSecretImports, + TSecretImportsInsert, + TSecretImportsUpdate, + TSecretReferences, + TSecretReferencesInsert, + TSecretReferencesUpdate, + TSecretRotationOutputs, + TSecretRotationOutputsInsert, + TSecretRotationOutputsUpdate, + TSecretRotations, + TSecretRotationsInsert, + TSecretRotationsUpdate, + TSecrets, + TSecretScanningGitRisks, + TSecretScanningGitRisksInsert, + TSecretScanningGitRisksUpdate, + TSecretSharing, + TSecretSharingInsert, + TSecretSharingUpdate, + TSecretsInsert, + TSecretSnapshotFolders, + TSecretSnapshotFoldersInsert, + TSecretSnapshotFoldersUpdate, + TSecretSnapshots, + TSecretSnapshotSecrets, + TSecretSnapshotSecretsInsert, + TSecretSnapshotSecretsUpdate, + TSecretSnapshotsInsert, + TSecretSnapshotsUpdate, + TSecretsUpdate, + TSecretTagJunction, + TSecretTagJunctionInsert, + TSecretTagJunctionUpdate, + TSecretTags, + TSecretTagsInsert, + TSecretTagsUpdate, + TSecretVersions, + TSecretVersionsInsert, + TSecretVersionsUpdate, + TSecretVersionTagJunction, + TSecretVersionTagJunctionInsert, + TSecretVersionTagJunctionUpdate, + TServiceTokens, + TServiceTokensInsert, + TServiceTokensUpdate, + TSuperAdmin, + TSuperAdminInsert, + TSuperAdminUpdate, + TTrustedIps, + TTrustedIpsInsert, + TTrustedIpsUpdate, + TUserActions, + TUserActionsInsert, + TUserActionsUpdate, + TUserAliases, + TUserAliasesInsert, + TUserAliasesUpdate, + TUserEncryptionKeys, + TUserEncryptionKeysInsert, + TUserEncryptionKeysUpdate, + TUserGroupMembership, + TUserGroupMembershipInsert, + TUserGroupMembershipUpdate, + TUsers, + TUsersInsert, + TUsersUpdate, + TWebhooks, + TWebhooksInsert, + TWebhooksUpdate +} from "@app/db/schemas"; + +declare module "knex/types/tables" { + interface Tables { + [TableName.Users]: Knex.CompositeTableType; + [TableName.Groups]: Knex.CompositeTableType; + [TableName.CertificateAuthority]: Knex.CompositeTableType< + TCertificateAuthorities, + TCertificateAuthoritiesInsert, + TCertificateAuthoritiesUpdate + >; + [TableName.CertificateAuthorityCert]: Knex.CompositeTableType< + TCertificateAuthorityCerts, + TCertificateAuthorityCertsInsert, + TCertificateAuthorityCertsUpdate + >; + [TableName.CertificateAuthoritySecret]: Knex.CompositeTableType< + TCertificateAuthoritySecret, + TCertificateAuthoritySecretInsert, + TCertificateAuthoritySecretUpdate + >; + [TableName.CertificateAuthorityCrl]: Knex.CompositeTableType< + TCertificateAuthorityCrl, + TCertificateAuthorityCrlInsert, + TCertificateAuthorityCrlUpdate + >; + [TableName.Certificate]: Knex.CompositeTableType; + [TableName.CertificateBody]: Knex.CompositeTableType< + TCertificateBodies, + TCertificateBodiesInsert, + TCertificateBodiesUpdate + >; + [TableName.CertificateSecret]: Knex.CompositeTableType< + TCertificateSecrets, + TCertificateSecretsInsert, + TCertificateSecretsUpdate + >; + [TableName.UserGroupMembership]: Knex.CompositeTableType< + TUserGroupMembership, + TUserGroupMembershipInsert, + TUserGroupMembershipUpdate + >; + [TableName.GroupProjectMembership]: Knex.CompositeTableType< + TGroupProjectMemberships, + TGroupProjectMembershipsInsert, + TGroupProjectMembershipsUpdate + >; + [TableName.GroupProjectMembershipRole]: Knex.CompositeTableType< + TGroupProjectMembershipRoles, + TGroupProjectMembershipRolesInsert, + TGroupProjectMembershipRolesUpdate + >; + [TableName.UserAliases]: Knex.CompositeTableType; + [TableName.UserEncryptionKey]: Knex.CompositeTableType< + TUserEncryptionKeys, + TUserEncryptionKeysInsert, + TUserEncryptionKeysUpdate + >; + [TableName.AuthTokens]: Knex.CompositeTableType; + [TableName.AuthTokenSession]: Knex.CompositeTableType< + TAuthTokenSessions, + TAuthTokenSessionsInsert, + TAuthTokenSessionsUpdate + >; + [TableName.BackupPrivateKey]: Knex.CompositeTableType< + TBackupPrivateKey, + TBackupPrivateKeyInsert, + TBackupPrivateKeyUpdate + >; + [TableName.Organization]: Knex.CompositeTableType; + [TableName.OrgMembership]: Knex.CompositeTableType; + [TableName.OrgRoles]: Knex.CompositeTableType; + [TableName.IncidentContact]: Knex.CompositeTableType< + TIncidentContacts, + TIncidentContactsInsert, + TIncidentContactsUpdate + >; + [TableName.UserAction]: Knex.CompositeTableType; + [TableName.SuperAdmin]: Knex.CompositeTableType; + [TableName.ApiKey]: Knex.CompositeTableType; + [TableName.Project]: Knex.CompositeTableType; + [TableName.ProjectMembership]: Knex.CompositeTableType< + TProjectMemberships, + TProjectMembershipsInsert, + TProjectMembershipsUpdate + >; + [TableName.Environment]: Knex.CompositeTableType< + TProjectEnvironments, + TProjectEnvironmentsInsert, + TProjectEnvironmentsUpdate + >; + [TableName.ProjectBot]: Knex.CompositeTableType; + [TableName.ProjectUserMembershipRole]: Knex.CompositeTableType< + TProjectUserMembershipRoles, + TProjectUserMembershipRolesInsert, + TProjectUserMembershipRolesUpdate + >; + [TableName.ProjectRoles]: Knex.CompositeTableType; + [TableName.ProjectUserAdditionalPrivilege]: Knex.CompositeTableType< + TProjectUserAdditionalPrivilege, + TProjectUserAdditionalPrivilegeInsert, + TProjectUserAdditionalPrivilegeUpdate + >; + [TableName.ProjectKeys]: Knex.CompositeTableType; + [TableName.Secret]: Knex.CompositeTableType; + [TableName.SecretReference]: Knex.CompositeTableType< + TSecretReferences, + TSecretReferencesInsert, + TSecretReferencesUpdate + >; + [TableName.SecretBlindIndex]: Knex.CompositeTableType< + TSecretBlindIndexes, + TSecretBlindIndexesInsert, + TSecretBlindIndexesUpdate + >; + [TableName.SecretVersion]: Knex.CompositeTableType; + [TableName.SecretFolder]: Knex.CompositeTableType; + [TableName.SecretFolderVersion]: Knex.CompositeTableType< + TSecretFolderVersions, + TSecretFolderVersionsInsert, + TSecretFolderVersionsUpdate + >; + [TableName.SecretSharing]: Knex.CompositeTableType; + [TableName.RateLimit]: Knex.CompositeTableType; + [TableName.SecretTag]: Knex.CompositeTableType; + [TableName.SecretImport]: Knex.CompositeTableType; + [TableName.Integration]: Knex.CompositeTableType; + [TableName.Webhook]: Knex.CompositeTableType; + [TableName.ServiceToken]: Knex.CompositeTableType; + [TableName.IntegrationAuth]: Knex.CompositeTableType< + TIntegrationAuths, + TIntegrationAuthsInsert, + TIntegrationAuthsUpdate + >; + [TableName.Identity]: Knex.CompositeTableType; + [TableName.IdentityUniversalAuth]: Knex.CompositeTableType< + TIdentityUniversalAuths, + TIdentityUniversalAuthsInsert, + TIdentityUniversalAuthsUpdate + >; + [TableName.IdentityKubernetesAuth]: Knex.CompositeTableType< + TIdentityKubernetesAuths, + TIdentityKubernetesAuthsInsert, + TIdentityKubernetesAuthsUpdate + >; + [TableName.IdentityGcpAuth]: Knex.CompositeTableType< + TIdentityGcpAuths, + TIdentityGcpAuthsInsert, + TIdentityGcpAuthsUpdate + >; + [TableName.IdentityAwsAuth]: Knex.CompositeTableType< + TIdentityAwsAuths, + TIdentityAwsAuthsInsert, + TIdentityAwsAuthsUpdate + >; + [TableName.IdentityAzureAuth]: Knex.CompositeTableType< + TIdentityAzureAuths, + TIdentityAzureAuthsInsert, + TIdentityAzureAuthsUpdate + >; + [TableName.IdentityUaClientSecret]: Knex.CompositeTableType< + TIdentityUaClientSecrets, + TIdentityUaClientSecretsInsert, + TIdentityUaClientSecretsUpdate + >; + [TableName.IdentityAccessToken]: Knex.CompositeTableType< + TIdentityAccessTokens, + TIdentityAccessTokensInsert, + TIdentityAccessTokensUpdate + >; + [TableName.IdentityOrgMembership]: Knex.CompositeTableType< + TIdentityOrgMemberships, + TIdentityOrgMembershipsInsert, + TIdentityOrgMembershipsUpdate + >; + [TableName.IdentityProjectMembership]: Knex.CompositeTableType< + TIdentityProjectMemberships, + TIdentityProjectMembershipsInsert, + TIdentityProjectMembershipsUpdate + >; + [TableName.IdentityProjectMembershipRole]: Knex.CompositeTableType< + TIdentityProjectMembershipRole, + TIdentityProjectMembershipRoleInsert, + TIdentityProjectMembershipRoleUpdate + >; + [TableName.IdentityProjectAdditionalPrivilege]: Knex.CompositeTableType< + TIdentityProjectAdditionalPrivilege, + TIdentityProjectAdditionalPrivilegeInsert, + TIdentityProjectAdditionalPrivilegeUpdate + >; + + [TableName.AccessApprovalPolicy]: Knex.CompositeTableType< + TAccessApprovalPolicies, + TAccessApprovalPoliciesInsert, + TAccessApprovalPoliciesUpdate + >; + + [TableName.AccessApprovalPolicyApprover]: Knex.CompositeTableType< + TAccessApprovalPoliciesApprovers, + TAccessApprovalPoliciesApproversInsert, + TAccessApprovalPoliciesApproversUpdate + >; + + [TableName.AccessApprovalRequest]: Knex.CompositeTableType< + TAccessApprovalRequests, + TAccessApprovalRequestsInsert, + TAccessApprovalRequestsUpdate + >; + + [TableName.AccessApprovalRequestReviewer]: Knex.CompositeTableType< + TAccessApprovalRequestsReviewers, + TAccessApprovalRequestsReviewersInsert, + TAccessApprovalRequestsReviewersUpdate + >; + + [TableName.ScimToken]: Knex.CompositeTableType; + [TableName.SecretApprovalPolicy]: Knex.CompositeTableType< + TSecretApprovalPolicies, + TSecretApprovalPoliciesInsert, + TSecretApprovalPoliciesUpdate + >; + [TableName.SecretApprovalPolicyApprover]: Knex.CompositeTableType< + TSecretApprovalPoliciesApprovers, + TSecretApprovalPoliciesApproversInsert, + TSecretApprovalPoliciesApproversUpdate + >; + [TableName.SecretApprovalRequest]: Knex.CompositeTableType< + TSecretApprovalRequests, + TSecretApprovalRequestsInsert, + TSecretApprovalRequestsUpdate + >; + [TableName.SecretApprovalRequestReviewer]: Knex.CompositeTableType< + TSecretApprovalRequestsReviewers, + TSecretApprovalRequestsReviewersInsert, + TSecretApprovalRequestsReviewersUpdate + >; + [TableName.SecretApprovalRequestSecret]: Knex.CompositeTableType< + TSecretApprovalRequestsSecrets, + TSecretApprovalRequestsSecretsInsert, + TSecretApprovalRequestsSecretsUpdate + >; + [TableName.SecretApprovalRequestSecretTag]: Knex.CompositeTableType< + TSecretApprovalRequestSecretTags, + TSecretApprovalRequestSecretTagsInsert, + TSecretApprovalRequestSecretTagsUpdate + >; + [TableName.SecretRotation]: Knex.CompositeTableType< + TSecretRotations, + TSecretRotationsInsert, + TSecretRotationsUpdate + >; + [TableName.SecretRotationOutput]: Knex.CompositeTableType< + TSecretRotationOutputs, + TSecretRotationOutputsInsert, + TSecretRotationOutputsUpdate + >; + [TableName.Snapshot]: Knex.CompositeTableType; + [TableName.SnapshotSecret]: Knex.CompositeTableType< + TSecretSnapshotSecrets, + TSecretSnapshotSecretsInsert, + TSecretSnapshotSecretsUpdate + >; + [TableName.SnapshotFolder]: Knex.CompositeTableType< + TSecretSnapshotFolders, + TSecretSnapshotFoldersInsert, + TSecretSnapshotFoldersUpdate + >; + [TableName.DynamicSecret]: Knex.CompositeTableType; + [TableName.DynamicSecretLease]: Knex.CompositeTableType< + TDynamicSecretLeases, + TDynamicSecretLeasesInsert, + TDynamicSecretLeasesUpdate + >; + [TableName.SamlConfig]: Knex.CompositeTableType; + [TableName.LdapConfig]: Knex.CompositeTableType; + [TableName.LdapGroupMap]: Knex.CompositeTableType; + [TableName.OrgBot]: Knex.CompositeTableType; + [TableName.AuditLog]: Knex.CompositeTableType; + [TableName.AuditLogStream]: Knex.CompositeTableType< + TAuditLogStreams, + TAuditLogStreamsInsert, + TAuditLogStreamsUpdate + >; + [TableName.GitAppInstallSession]: Knex.CompositeTableType< + TGitAppInstallSessions, + TGitAppInstallSessionsInsert, + TGitAppInstallSessionsUpdate + >; + [TableName.GitAppOrg]: Knex.CompositeTableType; + [TableName.SecretScanningGitRisk]: Knex.CompositeTableType< + TSecretScanningGitRisks, + TSecretScanningGitRisksInsert, + TSecretScanningGitRisksUpdate + >; + [TableName.TrustedIps]: Knex.CompositeTableType; + // Junction tables + [TableName.JnSecretTag]: Knex.CompositeTableType< + TSecretTagJunction, + TSecretTagJunctionInsert, + TSecretTagJunctionUpdate + >; + [TableName.SecretVersionTag]: Knex.CompositeTableType< + TSecretVersionTagJunction, + TSecretVersionTagJunctionInsert, + TSecretVersionTagJunctionUpdate + >; + // KMS service + [TableName.KmsServerRootConfig]: Knex.CompositeTableType< + TKmsRootConfig, + TKmsRootConfigInsert, + TKmsRootConfigUpdate + >; + [TableName.KmsKey]: Knex.CompositeTableType; + [TableName.KmsKeyVersion]: Knex.CompositeTableType; + } +} diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 4fdfda70c0..edd1cb7739 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -1,4 +1,4 @@ -import { Knex } from "knex"; +import { Knex as KnexOriginal } from "knex"; import { TableName, @@ -595,3 +595,12 @@ declare module "knex/types/tables" { [TableName.KmsKeyVersion]: Knex.CompositeTableType; } } + +declare module "knex" { + namespace Knex { + interface QueryInterface { + primaryNode(): KnexOriginal; + replicaNode(): KnexOriginal; + } + } +} diff --git a/backend/src/db/instance.ts b/backend/src/db/instance.ts index bd4ce99c16..8c63b53169 100644 --- a/backend/src/db/instance.ts +++ b/backend/src/db/instance.ts @@ -1,8 +1,34 @@ -import knex from "knex"; +import knex, { Knex } from "knex"; export type TDbClient = ReturnType; -export const initDbConnection = ({ dbConnectionUri, dbRootCert }: { dbConnectionUri: string; dbRootCert?: string }) => { - const db = knex({ +export const initDbConnection = ({ + dbConnectionUri, + dbRootCert, + readReplicas = [] +}: { + dbConnectionUri: string; + dbRootCert?: string; + readReplicas?: { + dbConnectionUri: string; + dbRootCert?: string; + }[]; +}) => { + let db: Knex; + let readReplicaDbs: Knex[]; + // @ts-expect-error the querybuilder type is expected but our intension is to return a knex instance + knex.QueryBuilder.extend("primaryNode", () => { + return db; + }); + + // @ts-expect-error the querybuilder type is expected but our intension is to return a knex instance + knex.QueryBuilder.extend("replicaNode", () => { + if (!readReplicaDbs.length) return db; + + const selectedReplica = readReplicaDbs[Math.floor(Math.random() * readReplicaDbs.length)]; + return selectedReplica; + }); + + db = knex({ client: "pg", connection: { connectionString: dbConnectionUri, @@ -22,5 +48,21 @@ export const initDbConnection = ({ dbConnectionUri, dbRootCert }: { dbConnection } }); + readReplicaDbs = readReplicas.map((el) => { + const replicaDbCertificate = el.dbRootCert || dbRootCert; + return knex({ + client: "pg", + connection: { + connectionString: el.dbConnectionUri, + ssl: replicaDbCertificate + ? { + rejectUnauthorized: true, + ca: Buffer.from(replicaDbCertificate, "base64").toString("ascii") + } + : false + } + }); + }); + return db; }; diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index f4da712936..e6e8a42e29 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -10,6 +10,14 @@ const zodStrBool = z .optional() .transform((val) => val === "true"); +const databaseReadReplicaSchema = z + .object({ + DB_CONNECTION_URI: z.string().describe("Postgres read replica database connection string"), + DB_ROOT_CERT: zpStr(z.string().optional().describe("Postgres read replica database certificate string")) + }) + .array() + .optional(); + const envSchema = z .object({ PORT: z.coerce.number().default(4000), @@ -29,6 +37,7 @@ const envSchema = z DB_USER: zpStr(z.string().describe("Postgres database username").optional()), DB_PASSWORD: zpStr(z.string().describe("Postgres database password").optional()), DB_NAME: zpStr(z.string().describe("Postgres database name").optional()), + DB_READ_REPLICAS: zpStr(z.string().describe("Postgres read replicas").optional()), BCRYPT_SALT_ROUND: z.number().default(12), NODE_ENV: z.enum(["development", "test", "production"]).default("production"), SALT_ROUNDS: z.coerce.number().default(10), @@ -127,6 +136,9 @@ const envSchema = z }) .transform((data) => ({ ...data, + DB_READ_REPLICAS: data.DB_READ_REPLICAS + ? databaseReadReplicaSchema.parse(JSON.parse(data.DB_READ_REPLICAS)) + : undefined, isCloud: Boolean(data.LICENSE_SERVER_KEY), isSmtpConfigured: Boolean(data.SMTP_HOST), isRedisConfigured: Boolean(data.REDIS_URL), diff --git a/backend/src/main.ts b/backend/src/main.ts index 86681ef337..dac189b87d 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -15,7 +15,11 @@ const run = async () => { const appCfg = initEnvConfig(logger); const db = initDbConnection({ dbConnectionUri: appCfg.DB_CONNECTION_URI, - dbRootCert: appCfg.DB_ROOT_CERT + dbRootCert: appCfg.DB_ROOT_CERT, + readReplicas: appCfg.DB_READ_REPLICAS?.map((el) => ({ + dbRootCert: el.DB_ROOT_CERT, + dbConnectionUri: el.DB_CONNECTION_URI + })) }); const smtp = smtpServiceFactory(formatSmtpConfig()); diff --git a/docker-compose.dev-read-replica.yml b/docker-compose.dev-read-replica.yml new file mode 100644 index 0000000000..7d1e6e7feb --- /dev/null +++ b/docker-compose.dev-read-replica.yml @@ -0,0 +1,191 @@ +version: "3.9" + +services: + nginx: + container_name: infisical-dev-nginx + image: nginx + restart: always + ports: + - 8080:80 + volumes: + - ./nginx/default.dev.conf:/etc/nginx/conf.d/default.conf:ro + depends_on: + - backend + - frontend + + db: + image: bitnami/postgresql:14 + ports: + - "5432:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + environment: + POSTGRESQL_PASSWORD: infisical + POSTGRESQL_USERNAME: infisical + POSTGRESQL_DATABASE: infisical + POSTGRESQL_REPLICATION_MODE: master + POSTGRESQL_REPLICATION_USER: repl_user + POSTGRESQL_REPLICATION_PASSWORD: repl_password + POSTGRESQL_SYNCHRONOUS_COMMIT_MODE: on + POSTGRESQL_NUM_SYNCHRONOUS_REPLICAS: 1 + + db-slave: + image: bitnami/postgresql:14 + ports: + - "5433:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + environment: + POSTGRESQL_PASSWORD: infisical + POSTGRESQL_USERNAME: infisical + POSTGRESQL_DATABASE: infisical + POSTGRESQL_REPLICATION_MODE: slave + POSTGRESQL_REPLICATION_USER: repl_user + POSTGRESQL_REPLICATION_PASSWORD: repl_password + POSTGRESQL_MASTER_HOST: db + POSTGRESQL_MASTER_PORT_NUMBER: 5432 + + + redis: + image: redis + container_name: infisical-dev-redis + environment: + - ALLOW_EMPTY_PASSWORD=yes + ports: + - 6379:6379 + volumes: + - redis_data:/data + + redis-commander: + container_name: infisical-dev-redis-commander + image: rediscommander/redis-commander + restart: always + depends_on: + - redis + environment: + - REDIS_HOSTS=local:redis:6379 + ports: + - "8085:8081" + + db-test: + profiles: ["test"] + image: postgres:14-alpine + ports: + - "5430:5432" + environment: + POSTGRES_PASSWORD: infisical + POSTGRES_USER: infisical + POSTGRES_DB: infisical-test + + db-migration: + container_name: infisical-db-migration + depends_on: + - db + build: + context: ./backend + dockerfile: Dockerfile.dev + env_file: .env + environment: + - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable + command: npm run migration:latest + volumes: + - ./backend/src:/app/src + + backend: + container_name: infisical-dev-api + build: + context: ./backend + dockerfile: Dockerfile.dev + depends_on: + db: + condition: service_started + redis: + condition: service_started + db-migration: + condition: service_completed_successfully + env_file: + - .env + ports: + - 4000:4000 + environment: + - NODE_ENV=development + - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable + - TELEMETRY_ENABLED=false + volumes: + - ./backend/src:/app/src + extra_hosts: + - "host.docker.internal:host-gateway" + + frontend: + container_name: infisical-dev-frontend + restart: unless-stopped + depends_on: + - backend + build: + context: ./frontend + dockerfile: Dockerfile.dev + volumes: + - ./frontend/src:/app/src/ # mounted whole src to avoid missing reload on new files + - ./frontend/public:/app/public + env_file: .env + environment: + - NEXT_PUBLIC_ENV=development + - INFISICAL_TELEMETRY_ENABLED=false + + pgadmin: + image: dpage/pgadmin4 + restart: always + environment: + PGADMIN_DEFAULT_EMAIL: admin@example.com + PGADMIN_DEFAULT_PASSWORD: pass + ports: + - 5050:80 + depends_on: + - db + + smtp-server: + container_name: infisical-dev-smtp-server + image: lytrax/mailhog:latest # https://github.com/mailhog/MailHog/issues/353#issuecomment-821137362 + restart: always + logging: + driver: "none" # disable saving logs + ports: + - 1025:1025 # SMTP server + - 8025:8025 # Web UI + + openldap: # note: more advanced configuration is available + image: osixia/openldap:1.5.0 + restart: always + environment: + LDAP_ORGANISATION: Acme + LDAP_DOMAIN: acme.com + LDAP_ADMIN_PASSWORD: admin + ports: + - 389:389 + - 636:636 + volumes: + - ldap_data:/var/lib/ldap + - ldap_config:/etc/ldap/slapd.d + profiles: [ldap] + + phpldapadmin: # username: cn=admin,dc=acme,dc=com, pass is admin + image: osixia/phpldapadmin:latest + restart: always + environment: + - PHPLDAPADMIN_LDAP_HOSTS=openldap + - PHPLDAPADMIN_HTTPS=false + ports: + - 6433:80 + depends_on: + - openldap + profiles: [ldap] + +volumes: + postgres-data: + driver: local + postgres-slave-data: + driver: local + redis_data: + driver: local + ldap_data: + ldap_config: