diff --git a/backend/scripts/generate-schema-types.ts b/backend/scripts/generate-schema-types.ts index fc398c2acf..c0e18a7632 100644 --- a/backend/scripts/generate-schema-types.ts +++ b/backend/scripts/generate-schema-types.ts @@ -84,6 +84,11 @@ const getZodDefaultValue = (type: unknown, value: string | number | boolean | Ob } }; +const bigIntegerColumns: Record = { + "folder_commits": ["commitId"] +}; + + const main = async () => { const tables = ( await db("information_schema.tables") @@ -108,6 +113,9 @@ const main = async () => { const columnName = columnNames[colNum]; const colInfo = columns[columnName]; let ztype = getZodPrimitiveType(colInfo.type); + if (bigIntegerColumns[tableName]?.includes(columnName)) { + ztype = "z.coerce.bigint()"; + } if (["zodBuffer"].includes(ztype)) { zodImportSet.add(ztype); } diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index 344d1e02eb..9f825b6b62 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -26,6 +26,7 @@ import { TLdapConfigServiceFactory } from "@app/ee/services/ldap-config/ldap-con import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { TOidcConfigServiceFactory } from "@app/ee/services/oidc/oidc-config-service"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { TPitServiceFactory } from "@app/ee/services/pit/pit-service"; import { TProjectTemplateServiceFactory } from "@app/ee/services/project-template/project-template-service"; import { TProjectUserAdditionalPrivilegeServiceFactory } from "@app/ee/services/project-user-additional-privilege/project-user-additional-privilege-service"; import { TRateLimitServiceFactory } from "@app/ee/services/rate-limit/rate-limit-service"; @@ -59,6 +60,7 @@ import { TCertificateTemplateServiceFactory } from "@app/services/certificate-te import { TCmekServiceFactory } from "@app/services/cmek/cmek-service"; import { TExternalGroupOrgRoleMappingServiceFactory } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-service"; import { TExternalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TGroupProjectServiceFactory } from "@app/services/group-project/group-project-service"; import { THsmServiceFactory } from "@app/services/hsm/hsm-service"; import { TIdentityServiceFactory } from "@app/services/identity/identity-service"; @@ -276,6 +278,8 @@ declare module "fastify" { microsoftTeams: TMicrosoftTeamsServiceFactory; assumePrivileges: TAssumePrivilegeServiceFactory; githubOrgSync: TGithubOrgSyncServiceFactory; + folderCommit: TFolderCommitServiceFactory; + pit: TPitServiceFactory; secretScanningV2: TSecretScanningV2ServiceFactory; internalCertificateAuthority: TInternalCertificateAuthorityServiceFactory; pkiTemplate: TPkiTemplatesServiceFactory; diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 44cd6bc79f..8fd073a49a 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -80,6 +80,24 @@ import { TExternalKms, TExternalKmsInsert, TExternalKmsUpdate, + TFolderCheckpointResources, + TFolderCheckpointResourcesInsert, + TFolderCheckpointResourcesUpdate, + TFolderCheckpoints, + TFolderCheckpointsInsert, + TFolderCheckpointsUpdate, + TFolderCommitChanges, + TFolderCommitChangesInsert, + TFolderCommitChangesUpdate, + TFolderCommits, + TFolderCommitsInsert, + TFolderCommitsUpdate, + TFolderTreeCheckpointResources, + TFolderTreeCheckpointResourcesInsert, + TFolderTreeCheckpointResourcesUpdate, + TFolderTreeCheckpoints, + TFolderTreeCheckpointsInsert, + TFolderTreeCheckpointsUpdate, TGateways, TGatewaysInsert, TGatewaysUpdate, @@ -1122,6 +1140,36 @@ declare module "knex/types/tables" { TGithubOrgSyncConfigsInsert, TGithubOrgSyncConfigsUpdate >; + [TableName.FolderCommit]: KnexOriginal.CompositeTableType< + TFolderCommits, + TFolderCommitsInsert, + TFolderCommitsUpdate + >; + [TableName.FolderCommitChanges]: KnexOriginal.CompositeTableType< + TFolderCommitChanges, + TFolderCommitChangesInsert, + TFolderCommitChangesUpdate + >; + [TableName.FolderCheckpoint]: KnexOriginal.CompositeTableType< + TFolderCheckpoints, + TFolderCheckpointsInsert, + TFolderCheckpointsUpdate + >; + [TableName.FolderCheckpointResources]: KnexOriginal.CompositeTableType< + TFolderCheckpointResources, + TFolderCheckpointResourcesInsert, + TFolderCheckpointResourcesUpdate + >; + [TableName.FolderTreeCheckpoint]: KnexOriginal.CompositeTableType< + TFolderTreeCheckpoints, + TFolderTreeCheckpointsInsert, + TFolderTreeCheckpointsUpdate + >; + [TableName.FolderTreeCheckpointResources]: KnexOriginal.CompositeTableType< + TFolderTreeCheckpointResources, + TFolderTreeCheckpointResourcesInsert, + TFolderTreeCheckpointResourcesUpdate + >; [TableName.SecretScanningDataSource]: KnexOriginal.CompositeTableType< TSecretScanningDataSources, TSecretScanningDataSourcesInsert, diff --git a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts new file mode 100644 index 0000000000..30d1ec49b4 --- /dev/null +++ b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts @@ -0,0 +1,166 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; +import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils"; + +export async function up(knex: Knex): Promise { + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + if (!hasFolderCommitTable) { + await knex.schema.createTable(TableName.FolderCommit, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.bigIncrements("commitId"); + t.jsonb("actorMetadata").notNullable(); + t.string("actorType").notNullable(); + t.string("message"); + t.uuid("folderId").notNullable(); + t.uuid("envId").notNullable(); + t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderId"); + t.index("envId"); + }); + } + + const hasFolderCommitChangesTable = await knex.schema.hasTable(TableName.FolderCommitChanges); + if (!hasFolderCommitChangesTable) { + await knex.schema.createTable(TableName.FolderCommitChanges, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.string("changeType").notNullable(); + t.boolean("isUpdate").notNullable().defaultTo(false); + t.uuid("secretVersionId"); + t.foreign("secretVersionId").references("id").inTable(TableName.SecretVersionV2).onDelete("CASCADE"); + t.uuid("folderVersionId"); + t.foreign("folderVersionId").references("id").inTable(TableName.SecretFolderVersion).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCommitId"); + t.index("secretVersionId"); + t.index("folderVersionId"); + }); + } + + const hasFolderCheckpointTable = await knex.schema.hasTable(TableName.FolderCheckpoint); + if (!hasFolderCheckpointTable) { + await knex.schema.createTable(TableName.FolderCheckpoint, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCommitId"); + }); + } + + const hasFolderCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderCheckpointResources); + if (!hasFolderCheckpointResourcesTable) { + await knex.schema.createTable(TableName.FolderCheckpointResources, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderCheckpointId").notNullable(); + t.foreign("folderCheckpointId").references("id").inTable(TableName.FolderCheckpoint).onDelete("CASCADE"); + t.uuid("secretVersionId"); + t.foreign("secretVersionId").references("id").inTable(TableName.SecretVersionV2).onDelete("CASCADE"); + t.uuid("folderVersionId"); + t.foreign("folderVersionId").references("id").inTable(TableName.SecretFolderVersion).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCheckpointId"); + t.index("secretVersionId"); + t.index("folderVersionId"); + }); + } + + const hasFolderTreeCheckpointTable = await knex.schema.hasTable(TableName.FolderTreeCheckpoint); + if (!hasFolderTreeCheckpointTable) { + await knex.schema.createTable(TableName.FolderTreeCheckpoint, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCommitId"); + }); + } + + const hasFolderTreeCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderTreeCheckpointResources); + if (!hasFolderTreeCheckpointResourcesTable) { + await knex.schema.createTable(TableName.FolderTreeCheckpointResources, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderTreeCheckpointId").notNullable(); + t.foreign("folderTreeCheckpointId").references("id").inTable(TableName.FolderTreeCheckpoint).onDelete("CASCADE"); + t.uuid("folderId").notNullable(); + t.uuid("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderTreeCheckpointId"); + t.index("folderId"); + t.index("folderCommitId"); + }); + } + + if (!hasFolderCommitTable) { + await createOnUpdateTrigger(knex, TableName.FolderCommit); + } + + if (!hasFolderCommitChangesTable) { + await createOnUpdateTrigger(knex, TableName.FolderCommitChanges); + } + + if (!hasFolderCheckpointTable) { + await createOnUpdateTrigger(knex, TableName.FolderCheckpoint); + } + + if (!hasFolderCheckpointResourcesTable) { + await createOnUpdateTrigger(knex, TableName.FolderCheckpointResources); + } + + if (!hasFolderTreeCheckpointTable) { + await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); + } + + if (!hasFolderTreeCheckpointResourcesTable) { + await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); + } +} + +export async function down(knex: Knex): Promise { + const hasFolderCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderCheckpointResources); + const hasFolderTreeCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderTreeCheckpointResources); + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + const hasFolderCommitChangesTable = await knex.schema.hasTable(TableName.FolderCommitChanges); + const hasFolderTreeCheckpointTable = await knex.schema.hasTable(TableName.FolderTreeCheckpoint); + const hasFolderCheckpointTable = await knex.schema.hasTable(TableName.FolderCheckpoint); + + if (hasFolderTreeCheckpointResourcesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpointResources); + } + + if (hasFolderCheckpointResourcesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources); + } + + if (hasFolderTreeCheckpointTable) { + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpoint); + } + + if (hasFolderCheckpointTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderCheckpoint); + } + + if (hasFolderCommitChangesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCommitChanges); + await knex.schema.dropTableIfExists(TableName.FolderCommitChanges); + } + + if (hasFolderCommitTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCommit); + await knex.schema.dropTableIfExists(TableName.FolderCommit); + } +} diff --git a/backend/src/db/migrations/20250528110936_add-folder-description-to-versioning.ts b/backend/src/db/migrations/20250528110936_add-folder-description-to-versioning.ts new file mode 100644 index 0000000000..8c1da7ebc7 --- /dev/null +++ b/backend/src/db/migrations/20250528110936_add-folder-description-to-versioning.ts @@ -0,0 +1,19 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasColumn(TableName.SecretFolderVersion, "description"))) { + await knex.schema.alterTable(TableName.SecretFolderVersion, (t) => { + t.string("description").nullable(); + }); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasColumn(TableName.SecretFolderVersion, "description")) { + await knex.schema.alterTable(TableName.SecretFolderVersion, (t) => { + t.dropColumn("description"); + }); + } +} diff --git a/backend/src/db/migrations/20250602155451_fix-secret-versions.ts b/backend/src/db/migrations/20250602155451_fix-secret-versions.ts new file mode 100644 index 0000000000..f525e85f36 --- /dev/null +++ b/backend/src/db/migrations/20250602155451_fix-secret-versions.ts @@ -0,0 +1,139 @@ +/* eslint-disable no-await-in-loop */ +import { Knex } from "knex"; + +import { chunkArray } from "@app/lib/fn"; +import { selectAllTableCols } from "@app/lib/knex"; +import { logger } from "@app/lib/logger"; + +import { SecretType, TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + logger.info("Starting secret version fix migration"); + + // Get all shared secret IDs first to optimize versions query + const secretIds = await knex(TableName.SecretV2) + .where("type", SecretType.Shared) + .select("id") + .then((rows) => rows.map((row) => row.id)); + + logger.info(`Found ${secretIds.length} shared secrets to process`); + + if (secretIds.length === 0) { + logger.info("No shared secrets found"); + return; + } + + const secretIdChunks = chunkArray(secretIds, 5000); + + for (let chunkIndex = 0; chunkIndex < secretIdChunks.length; chunkIndex += 1) { + const currentSecretIds = secretIdChunks[chunkIndex]; + logger.info(`Processing chunk ${chunkIndex + 1} of ${secretIdChunks.length}`); + + // Get secrets and versions for current chunk + const [sharedSecrets, allVersions] = await Promise.all([ + knex(TableName.SecretV2).whereIn("id", currentSecretIds).select(selectAllTableCols(TableName.SecretV2)), + knex(TableName.SecretVersionV2).whereIn("secretId", currentSecretIds).select("secretId", "version") + ]); + + const versionsBySecretId = new Map(); + + allVersions.forEach((v) => { + const versions = versionsBySecretId.get(v.secretId); + if (versions) { + versions.push(v.version); + } else { + versionsBySecretId.set(v.secretId, [v.version]); + } + }); + + const versionsToAdd = []; + const secretsToUpdate = []; + + // Process each shared secret + for (const secret of sharedSecrets) { + const existingVersions = versionsBySecretId.get(secret.id) || []; + + if (existingVersions.length === 0) { + // No versions exist - add current version + versionsToAdd.push({ + secretId: secret.id, + version: secret.version, + key: secret.key, + encryptedValue: secret.encryptedValue, + encryptedComment: secret.encryptedComment, + reminderNote: secret.reminderNote, + reminderRepeatDays: secret.reminderRepeatDays, + skipMultilineEncoding: secret.skipMultilineEncoding, + metadata: secret.metadata, + folderId: secret.folderId, + actorType: "platform" + }); + } else { + const latestVersion = Math.max(...existingVersions); + + if (latestVersion !== secret.version) { + // Latest version doesn't match - create new version and update secret + const nextVersion = latestVersion + 1; + + versionsToAdd.push({ + secretId: secret.id, + version: nextVersion, + key: secret.key, + encryptedValue: secret.encryptedValue, + encryptedComment: secret.encryptedComment, + reminderNote: secret.reminderNote, + reminderRepeatDays: secret.reminderRepeatDays, + skipMultilineEncoding: secret.skipMultilineEncoding, + metadata: secret.metadata, + folderId: secret.folderId, + actorType: "platform" + }); + + secretsToUpdate.push({ + id: secret.id, + newVersion: nextVersion + }); + } + } + } + + logger.info( + `Chunk ${chunkIndex + 1}: Adding ${versionsToAdd.length} versions, updating ${secretsToUpdate.length} secrets` + ); + + // Batch insert new versions + if (versionsToAdd.length > 0) { + const insertBatches = chunkArray(versionsToAdd, 9000); + for (let i = 0; i < insertBatches.length; i += 1) { + await knex.batchInsert(TableName.SecretVersionV2, insertBatches[i]); + } + } + + if (secretsToUpdate.length > 0) { + const updateBatches = chunkArray(secretsToUpdate, 1000); + + for (const updateBatch of updateBatches) { + const ids = updateBatch.map((u) => u.id); + const versionCases = updateBatch.map((u) => `WHEN '${u.id}' THEN ${u.newVersion}`).join(" "); + + await knex.raw( + ` + UPDATE ${TableName.SecretV2} + SET version = CASE id ${versionCases} END, + "updatedAt" = NOW() + WHERE id IN (${ids.map(() => "?").join(",")}) + `, + ids + ); + } + } + } + + logger.info("Secret version fix migration completed"); +} + +export async function down(): Promise { + logger.info("Rollback not implemented for secret version fix migration"); + // Note: Rolling back this migration would be complex and potentially destructive + // as it would require tracking which version entries were added +} diff --git a/backend/src/db/migrations/20250602155452_pit-projects-commits-initialization.ts b/backend/src/db/migrations/20250602155452_pit-projects-commits-initialization.ts new file mode 100644 index 0000000000..dd4034c572 --- /dev/null +++ b/backend/src/db/migrations/20250602155452_pit-projects-commits-initialization.ts @@ -0,0 +1,345 @@ +import { Knex } from "knex"; + +import { chunkArray } from "@app/lib/fn"; +import { selectAllTableCols } from "@app/lib/knex"; +import { logger } from "@app/lib/logger"; +import { ActorType } from "@app/services/auth/auth-type"; +import { ChangeType } from "@app/services/folder-commit/folder-commit-service"; + +import { + ProjectType, + SecretType, + TableName, + TFolderCheckpoints, + TFolderCommits, + TFolderTreeCheckpoints, + TSecretFolders +} from "../schemas"; + +const sortFoldersByHierarchy = (folders: TSecretFolders[]) => { + // Create a map for quick lookup of children by parent ID + const childrenMap = new Map(); + + // Set of all folder IDs + const allFolderIds = new Set(); + + // Build the set of all folder IDs + folders.forEach((folder) => { + if (folder.id) { + allFolderIds.add(folder.id); + } + }); + + // Group folders by their parentId + folders.forEach((folder) => { + if (folder.parentId) { + const children = childrenMap.get(folder.parentId) || []; + children.push(folder); + childrenMap.set(folder.parentId, children); + } + }); + + // Find root folders - those with no parentId or with a parentId that doesn't exist + const rootFolders = folders.filter((folder) => !folder.parentId || !allFolderIds.has(folder.parentId)); + + // Process each level of the hierarchy + const result = []; + let currentLevel = rootFolders; + + while (currentLevel.length > 0) { + result.push(...currentLevel); + + const nextLevel = []; + for (const folder of currentLevel) { + if (folder.id) { + const children = childrenMap.get(folder.id) || []; + nextLevel.push(...children); + } + } + + currentLevel = nextLevel; + } + + return result.reverse(); +}; + +const getSecretsByFolderIds = async (knex: Knex, folderIds: string[]): Promise> => { + const secrets = await knex(TableName.SecretV2) + .whereIn(`${TableName.SecretV2}.folderId`, folderIds) + .where(`${TableName.SecretV2}.type`, SecretType.Shared) + .join(TableName.SecretVersionV2, (queryBuilder) => { + void queryBuilder + .on(`${TableName.SecretVersionV2}.secretId`, `${TableName.SecretV2}.id`) + .andOn(`${TableName.SecretVersionV2}.version`, `${TableName.SecretV2}.version`); + }) + .select(selectAllTableCols(TableName.SecretV2)) + .select(knex.ref("id").withSchema(TableName.SecretVersionV2).as("secretVersionId")); + + const secretsMap: Record = {}; + + secrets.forEach((secret) => { + if (!secretsMap[secret.folderId]) { + secretsMap[secret.folderId] = []; + } + secretsMap[secret.folderId].push(secret.secretVersionId); + }); + + return secretsMap; +}; + +const getFoldersByParentIds = async (knex: Knex, parentIds: string[]): Promise> => { + const folders = await knex(TableName.SecretFolder) + .whereIn(`${TableName.SecretFolder}.parentId`, parentIds) + .where(`${TableName.SecretFolder}.isReserved`, false) + .join(TableName.SecretFolderVersion, (queryBuilder) => { + void queryBuilder + .on(`${TableName.SecretFolderVersion}.folderId`, `${TableName.SecretFolder}.id`) + .andOn(`${TableName.SecretFolderVersion}.version`, `${TableName.SecretFolder}.version`); + }) + .select(selectAllTableCols(TableName.SecretFolder)) + .select(knex.ref("id").withSchema(TableName.SecretFolderVersion).as("folderVersionId")); + + const foldersMap: Record = {}; + + folders.forEach((folder) => { + if (!folder.parentId) { + return; + } + if (!foldersMap[folder.parentId]) { + foldersMap[folder.parentId] = []; + } + foldersMap[folder.parentId].push(folder.folderVersionId); + }); + + return foldersMap; +}; + +export async function up(knex: Knex): Promise { + logger.info("Initializing folder commits"); + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + if (hasFolderCommitTable) { + // Get Projects to Initialize + const projects = await knex(TableName.Project) + .where(`${TableName.Project}.version`, 3) + .where(`${TableName.Project}.type`, ProjectType.SecretManager) + .select(selectAllTableCols(TableName.Project)); + logger.info(`Found ${projects.length} projects to initialize`); + + // Process Projects in batches of 100 + const batches = chunkArray(projects, 100); + let i = 0; + for (const batch of batches) { + i += 1; + logger.info(`Processing project batch ${i} of ${batches.length}`); + let foldersCommitsList = []; + + const rootFoldersMap: Record = {}; + const envRootFoldersMap: Record = {}; + + // Get All Folders for the Project + // eslint-disable-next-line no-await-in-loop + const folders = await knex(TableName.SecretFolder) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .whereIn( + `${TableName.Environment}.projectId`, + batch.map((project) => project.id) + ) + .where(`${TableName.SecretFolder}.isReserved`, false) + .select(selectAllTableCols(TableName.SecretFolder)); + logger.info(`Found ${folders.length} folders to initialize in project batch ${i} of ${batches.length}`); + + // Sort Folders by Hierarchy (parents before nested folders) + const sortedFolders = sortFoldersByHierarchy(folders); + + // eslint-disable-next-line no-await-in-loop + const folderSecretsMap = await getSecretsByFolderIds( + knex, + sortedFolders.map((folder) => folder.id) + ); + // eslint-disable-next-line no-await-in-loop + const folderFoldersMap = await getFoldersByParentIds( + knex, + sortedFolders.map((folder) => folder.id) + ); + + // Get folder commit changes + for (const folder of sortedFolders) { + const subFolderVersionIds = folderFoldersMap[folder.id]; + const secretVersionIds = folderSecretsMap[folder.id]; + const changes = []; + if (subFolderVersionIds) { + changes.push( + ...subFolderVersionIds.map((folderVersionId) => ({ + folderId: folder.id, + changeType: ChangeType.ADD, + secretVersionId: undefined, + folderVersionId, + isUpdate: false + })) + ); + } + if (secretVersionIds) { + changes.push( + ...secretVersionIds.map((secretVersionId) => ({ + folderId: folder.id, + changeType: ChangeType.ADD, + secretVersionId, + folderVersionId: undefined, + isUpdate: false + })) + ); + } + if (changes.length > 0) { + const folderCommit = { + commit: { + actorMetadata: {}, + actorType: ActorType.PLATFORM, + message: "Initialized folder", + folderId: folder.id, + envId: folder.envId + }, + changes + }; + foldersCommitsList.push(folderCommit); + if (!folder.parentId) { + rootFoldersMap[folder.id] = folder.envId; + envRootFoldersMap[folder.envId] = folder.id; + } + } + } + logger.info(`Retrieved folder changes for project batch ${i} of ${batches.length}`); + + const filteredBrokenProjectFolders: string[] = []; + + foldersCommitsList = foldersCommitsList.filter((folderCommit) => { + if (!envRootFoldersMap[folderCommit.commit.envId]) { + filteredBrokenProjectFolders.push(folderCommit.commit.folderId); + return false; + } + return true; + }); + + logger.info( + `Filtered ${filteredBrokenProjectFolders.length} broken project folders: ${JSON.stringify(filteredBrokenProjectFolders)}` + ); + + // Insert New Commits in batches of 9000 + const newCommits = foldersCommitsList.map((folderCommit) => folderCommit.commit); + const commitBatches = chunkArray(newCommits, 9000); + + let j = 0; + for (const commitBatch of commitBatches) { + j += 1; + logger.info(`Inserting folder commits - batch ${j} of ${commitBatches.length}`); + // Create folder commit + // eslint-disable-next-line no-await-in-loop + const newCommitsInserted = (await knex + .batchInsert(TableName.FolderCommit, commitBatch) + .returning("*")) as TFolderCommits[]; + + logger.info(`Finished inserting folder commits - batch ${j} of ${commitBatches.length}`); + + const newCommitsMap: Record = {}; + const newCommitsMapInverted: Record = {}; + const newCheckpointsMap: Record = {}; + newCommitsInserted.forEach((commit) => { + newCommitsMap[commit.folderId] = commit.id; + newCommitsMapInverted[commit.id] = commit.folderId; + }); + + // Create folder checkpoints + // eslint-disable-next-line no-await-in-loop + const newCheckpoints = (await knex + .batchInsert( + TableName.FolderCheckpoint, + Object.values(newCommitsMap).map((commitId) => ({ + folderCommitId: commitId + })) + ) + .returning("*")) as TFolderCheckpoints[]; + + logger.info(`Finished inserting folder checkpoints - batch ${j} of ${commitBatches.length}`); + + newCheckpoints.forEach((checkpoint) => { + newCheckpointsMap[newCommitsMapInverted[checkpoint.folderCommitId]] = checkpoint.id; + }); + + // Create folder commit changes + // eslint-disable-next-line no-await-in-loop + await knex.batchInsert( + TableName.FolderCommitChanges, + foldersCommitsList + .map((folderCommit) => folderCommit.changes) + .flat() + .map((change) => ({ + folderCommitId: newCommitsMap[change.folderId], + changeType: change.changeType, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId, + isUpdate: false + })) + ); + + logger.info(`Finished inserting folder commit changes - batch ${j} of ${commitBatches.length}`); + + // Create folder checkpoint resources + // eslint-disable-next-line no-await-in-loop + await knex.batchInsert( + TableName.FolderCheckpointResources, + foldersCommitsList + .map((folderCommit) => folderCommit.changes) + .flat() + .map((change) => ({ + folderCheckpointId: newCheckpointsMap[change.folderId], + folderVersionId: change.folderVersionId, + secretVersionId: change.secretVersionId + })) + ); + + logger.info(`Finished inserting folder checkpoint resources - batch ${j} of ${commitBatches.length}`); + + // Create Folder Tree Checkpoint + // eslint-disable-next-line no-await-in-loop + const newTreeCheckpoints = (await knex + .batchInsert( + TableName.FolderTreeCheckpoint, + Object.keys(rootFoldersMap).map((folderId) => ({ + folderCommitId: newCommitsMap[folderId] + })) + ) + .returning("*")) as TFolderTreeCheckpoints[]; + + logger.info(`Finished inserting folder tree checkpoints - batch ${j} of ${commitBatches.length}`); + + const newTreeCheckpointsMap: Record = {}; + newTreeCheckpoints.forEach((checkpoint) => { + newTreeCheckpointsMap[rootFoldersMap[newCommitsMapInverted[checkpoint.folderCommitId]]] = checkpoint.id; + }); + + // Create Folder Tree Checkpoint Resources + // eslint-disable-next-line no-await-in-loop + await knex + .batchInsert( + TableName.FolderTreeCheckpointResources, + newCommitsInserted.map((folderCommit) => ({ + folderTreeCheckpointId: newTreeCheckpointsMap[folderCommit.envId], + folderId: folderCommit.folderId, + folderCommitId: folderCommit.id + })) + ) + .returning("*"); + + logger.info(`Finished inserting folder tree checkpoint resources - batch ${j} of ${commitBatches.length}`); + } + } + } + logger.info("Folder commits initialized"); +} + +export async function down(knex: Knex): Promise { + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + if (hasFolderCommitTable) { + // delete all existing entries + await knex(TableName.FolderCommit).del(); + } +} diff --git a/backend/src/db/migrations/20250606134139_add-project-snapshots-legacy-option.ts b/backend/src/db/migrations/20250606134139_add-project-snapshots-legacy-option.ts new file mode 100644 index 0000000000..f5f73d7fea --- /dev/null +++ b/backend/src/db/migrations/20250606134139_add-project-snapshots-legacy-option.ts @@ -0,0 +1,21 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + const hasShowSnapshotsLegacyColumn = await knex.schema.hasColumn(TableName.Project, "showSnapshotsLegacy"); + if (!hasShowSnapshotsLegacyColumn) { + await knex.schema.table(TableName.Project, (table) => { + table.boolean("showSnapshotsLegacy").notNullable().defaultTo(false); + }); + } +} + +export async function down(knex: Knex): Promise { + const hasShowSnapshotsLegacyColumn = await knex.schema.hasColumn(TableName.Project, "showSnapshotsLegacy"); + if (hasShowSnapshotsLegacyColumn) { + await knex.schema.table(TableName.Project, (table) => { + table.dropColumn("showSnapshotsLegacy"); + }); + } +} diff --git a/backend/src/db/migrations/utils/services.ts b/backend/src/db/migrations/utils/services.ts index 731f703e2c..0e071e6fe5 100644 --- a/backend/src/db/migrations/utils/services.ts +++ b/backend/src/db/migrations/utils/services.ts @@ -3,12 +3,27 @@ import { Knex } from "knex"; import { initializeHsmModule } from "@app/ee/services/hsm/hsm-fns"; import { hsmServiceFactory } from "@app/ee/services/hsm/hsm-service"; import { TKeyStoreFactory } from "@app/keystore/keystore"; +import { folderCheckpointDALFactory } from "@app/services/folder-checkpoint/folder-checkpoint-dal"; +import { folderCheckpointResourcesDALFactory } from "@app/services/folder-checkpoint-resources/folder-checkpoint-resources-dal"; +import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal"; +import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; +import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal"; +import { folderTreeCheckpointDALFactory } from "@app/services/folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { folderTreeCheckpointResourcesDALFactory } from "@app/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; +import { identityDALFactory } from "@app/services/identity/identity-dal"; import { internalKmsDALFactory } from "@app/services/kms/internal-kms-dal"; import { kmskeyDALFactory } from "@app/services/kms/kms-key-dal"; import { kmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal"; import { kmsServiceFactory } from "@app/services/kms/kms-service"; import { orgDALFactory } from "@app/services/org/org-dal"; import { projectDALFactory } from "@app/services/project/project-dal"; +import { resourceMetadataDALFactory } from "@app/services/resource-metadata/resource-metadata-dal"; +import { secretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal"; +import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; +import { secretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; +import { secretVersionV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; +import { userDALFactory } from "@app/services/user/user-dal"; import { TMigrationEnvConfig } from "./env-config"; @@ -50,3 +65,77 @@ export const getMigrationEncryptionServices = async ({ envConfig, db, keyStore } return { kmsService }; }; + +export const getMigrationPITServices = async ({ + db, + keyStore, + envConfig +}: { + db: Knex; + keyStore: TKeyStoreFactory; + envConfig: TMigrationEnvConfig; +}) => { + const projectDAL = projectDALFactory(db); + const folderCommitDAL = folderCommitDALFactory(db); + const folderCommitChangesDAL = folderCommitChangesDALFactory(db); + const folderCheckpointDAL = folderCheckpointDALFactory(db); + const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db); + const userDAL = userDALFactory(db); + const identityDAL = identityDALFactory(db); + const folderDAL = secretFolderDALFactory(db); + const folderVersionDAL = secretFolderVersionDALFactory(db); + const secretVersionV2BridgeDAL = secretVersionV2BridgeDALFactory(db); + const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db); + const secretV2BridgeDAL = secretV2BridgeDALFactory({ db, keyStore }); + const folderTreeCheckpointResourcesDAL = folderTreeCheckpointResourcesDALFactory(db); + const secretTagDAL = secretTagDALFactory(db); + + const orgDAL = orgDALFactory(db); + const kmsRootConfigDAL = kmsRootConfigDALFactory(db); + const kmsDAL = kmskeyDALFactory(db); + const internalKmsDAL = internalKmsDALFactory(db); + const resourceMetadataDAL = resourceMetadataDALFactory(db); + + const hsmModule = initializeHsmModule(envConfig); + hsmModule.initialize(); + + const hsmService = hsmServiceFactory({ + hsmModule: hsmModule.getModule(), + envConfig + }); + + const kmsService = kmsServiceFactory({ + kmsRootConfigDAL, + keyStore, + kmsDAL, + internalKmsDAL, + orgDAL, + projectDAL, + hsmService, + envConfig + }); + + await hsmService.startService(); + await kmsService.startService(); + + const folderCommitService = folderCommitServiceFactory({ + folderCommitDAL, + folderCommitChangesDAL, + folderCheckpointDAL, + folderTreeCheckpointDAL, + userDAL, + identityDAL, + folderDAL, + folderVersionDAL, + secretVersionV2BridgeDAL, + projectDAL, + folderCheckpointResourcesDAL, + secretV2BridgeDAL, + folderTreeCheckpointResourcesDAL, + kmsService, + secretTagDAL, + resourceMetadataDAL + }); + + return { folderCommitService }; +}; diff --git a/backend/src/db/schemas/folder-checkpoint-resources.ts b/backend/src/db/schemas/folder-checkpoint-resources.ts new file mode 100644 index 0000000000..5fa8215cf3 --- /dev/null +++ b/backend/src/db/schemas/folder-checkpoint-resources.ts @@ -0,0 +1,23 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCheckpointResourcesSchema = z.object({ + id: z.string().uuid(), + folderCheckpointId: z.string().uuid(), + secretVersionId: z.string().uuid().nullable().optional(), + folderVersionId: z.string().uuid().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCheckpointResources = z.infer; +export type TFolderCheckpointResourcesInsert = Omit, TImmutableDBKeys>; +export type TFolderCheckpointResourcesUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/folder-checkpoints.ts b/backend/src/db/schemas/folder-checkpoints.ts new file mode 100644 index 0000000000..ba0ce6f71f --- /dev/null +++ b/backend/src/db/schemas/folder-checkpoints.ts @@ -0,0 +1,19 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCheckpointsSchema = z.object({ + id: z.string().uuid(), + folderCommitId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCheckpoints = z.infer; +export type TFolderCheckpointsInsert = Omit, TImmutableDBKeys>; +export type TFolderCheckpointsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/folder-commit-changes.ts b/backend/src/db/schemas/folder-commit-changes.ts new file mode 100644 index 0000000000..2bee0c5b3f --- /dev/null +++ b/backend/src/db/schemas/folder-commit-changes.ts @@ -0,0 +1,23 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCommitChangesSchema = z.object({ + id: z.string().uuid(), + folderCommitId: z.string().uuid(), + changeType: z.string(), + isUpdate: z.boolean().default(false), + secretVersionId: z.string().uuid().nullable().optional(), + folderVersionId: z.string().uuid().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCommitChanges = z.infer; +export type TFolderCommitChangesInsert = Omit, TImmutableDBKeys>; +export type TFolderCommitChangesUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/folder-commits.ts b/backend/src/db/schemas/folder-commits.ts new file mode 100644 index 0000000000..ade480eda8 --- /dev/null +++ b/backend/src/db/schemas/folder-commits.ts @@ -0,0 +1,24 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCommitsSchema = z.object({ + id: z.string().uuid(), + commitId: z.coerce.bigint(), + actorMetadata: z.unknown(), + actorType: z.string(), + message: z.string().nullable().optional(), + folderId: z.string().uuid(), + envId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCommits = z.infer; +export type TFolderCommitsInsert = Omit, TImmutableDBKeys>; +export type TFolderCommitsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/folder-tree-checkpoint-resources.ts b/backend/src/db/schemas/folder-tree-checkpoint-resources.ts new file mode 100644 index 0000000000..06d5770cc4 --- /dev/null +++ b/backend/src/db/schemas/folder-tree-checkpoint-resources.ts @@ -0,0 +1,26 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderTreeCheckpointResourcesSchema = z.object({ + id: z.string().uuid(), + folderTreeCheckpointId: z.string().uuid(), + folderId: z.string().uuid(), + folderCommitId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderTreeCheckpointResources = z.infer; +export type TFolderTreeCheckpointResourcesInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TFolderTreeCheckpointResourcesUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/folder-tree-checkpoints.ts b/backend/src/db/schemas/folder-tree-checkpoints.ts new file mode 100644 index 0000000000..ea500af6ba --- /dev/null +++ b/backend/src/db/schemas/folder-tree-checkpoints.ts @@ -0,0 +1,19 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderTreeCheckpointsSchema = z.object({ + id: z.string().uuid(), + folderCommitId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderTreeCheckpoints = z.infer; +export type TFolderTreeCheckpointsInsert = Omit, TImmutableDBKeys>; +export type TFolderTreeCheckpointsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/index.ts b/backend/src/db/schemas/index.ts index 6743a23cc4..7a27caf9e1 100644 --- a/backend/src/db/schemas/index.ts +++ b/backend/src/db/schemas/index.ts @@ -24,6 +24,12 @@ export * from "./dynamic-secrets"; export * from "./external-certificate-authorities"; export * from "./external-group-org-role-mappings"; export * from "./external-kms"; +export * from "./folder-checkpoint-resources"; +export * from "./folder-checkpoints"; +export * from "./folder-commit-changes"; +export * from "./folder-commits"; +export * from "./folder-tree-checkpoint-resources"; +export * from "./folder-tree-checkpoints"; export * from "./gateways"; export * from "./git-app-install-sessions"; export * from "./git-app-org"; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 6722ce235c..df0a858b99 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -160,6 +160,12 @@ export enum TableName { ProjectMicrosoftTeamsConfigs = "project_microsoft_teams_configs", SecretReminderRecipients = "secret_reminder_recipients", GithubOrgSyncConfig = "github_org_sync_configs", + FolderCommit = "folder_commits", + FolderCommitChanges = "folder_commit_changes", + FolderCheckpoint = "folder_checkpoints", + FolderCheckpointResources = "folder_checkpoint_resources", + FolderTreeCheckpoint = "folder_tree_checkpoints", + FolderTreeCheckpointResources = "folder_tree_checkpoint_resources", SecretScanningDataSource = "secret_scanning_data_sources", SecretScanningResource = "secret_scanning_resources", SecretScanningScan = "secret_scanning_scans", @@ -167,7 +173,7 @@ export enum TableName { SecretScanningConfig = "secret_scanning_configs" } -export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt"; +export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt" | "commitId"; export const UserDeviceSchema = z .object({ diff --git a/backend/src/db/schemas/projects.ts b/backend/src/db/schemas/projects.ts index c1e96e8ced..b4c98d8a28 100644 --- a/backend/src/db/schemas/projects.ts +++ b/backend/src/db/schemas/projects.ts @@ -28,7 +28,8 @@ export const ProjectsSchema = z.object({ type: z.string(), enforceCapitalization: z.boolean().default(false), hasDeleteProtection: z.boolean().default(false).nullable().optional(), - secretSharing: z.boolean().default(true) + secretSharing: z.boolean().default(true), + showSnapshotsLegacy: z.boolean().default(false) }); export type TProjects = z.infer; diff --git a/backend/src/db/schemas/secret-folder-versions.ts b/backend/src/db/schemas/secret-folder-versions.ts index 8bef6e83f5..3d444c566c 100644 --- a/backend/src/db/schemas/secret-folder-versions.ts +++ b/backend/src/db/schemas/secret-folder-versions.ts @@ -14,7 +14,8 @@ export const SecretFolderVersionsSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), envId: z.string().uuid(), - folderId: z.string().uuid() + folderId: z.string().uuid(), + description: z.string().nullable().optional() }); export type TSecretFolderVersions = z.infer; diff --git a/backend/src/ee/routes/v1/index.ts b/backend/src/ee/routes/v1/index.ts index 0b8c78586a..9ba7f734e0 100644 --- a/backend/src/ee/routes/v1/index.ts +++ b/backend/src/ee/routes/v1/index.ts @@ -18,6 +18,7 @@ import { registerLdapRouter } from "./ldap-router"; import { registerLicenseRouter } from "./license-router"; import { registerOidcRouter } from "./oidc-router"; import { registerOrgRoleRouter } from "./org-role-router"; +import { registerPITRouter } from "./pit-router"; import { registerProjectRoleRouter } from "./project-role-router"; import { registerProjectRouter } from "./project-router"; import { registerRateLimitRouter } from "./rate-limit-router"; @@ -53,6 +54,7 @@ export const registerV1EERoutes = async (server: FastifyZodProvider) => { { prefix: "/workspace" } ); await server.register(registerSnapshotRouter, { prefix: "/secret-snapshot" }); + await server.register(registerPITRouter, { prefix: "/pit" }); await server.register(registerSecretApprovalPolicyRouter, { prefix: "/secret-approvals" }); await server.register(registerSecretApprovalRequestRouter, { prefix: "/secret-approval-requests" diff --git a/backend/src/ee/routes/v1/pit-router.ts b/backend/src/ee/routes/v1/pit-router.ts new file mode 100644 index 0000000000..f993e31d73 --- /dev/null +++ b/backend/src/ee/routes/v1/pit-router.ts @@ -0,0 +1,416 @@ +/* eslint-disable @typescript-eslint/no-base-to-string */ +import { z } from "zod"; + +import { EventType } from "@app/ee/services/audit-log/audit-log-types"; +import { removeTrailingSlash } from "@app/lib/fn"; +import { readLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { booleanSchema } from "@app/server/routes/sanitizedSchemas"; +import { AuthMode } from "@app/services/auth/auth-type"; +import { commitChangesResponseSchema, resourceChangeSchema } from "@app/services/folder-commit/folder-commit-schemas"; + +const commitHistoryItemSchema = z.object({ + id: z.string(), + folderId: z.string(), + actorType: z.string(), + actorMetadata: z.unknown().optional(), + message: z.string().optional().nullable(), + commitId: z.string(), + createdAt: z.string().or(z.date()), + envId: z.string() +}); + +const folderStateSchema = z.array( + z.object({ + type: z.string(), + id: z.string(), + versionId: z.string(), + secretKey: z.string().optional(), + secretVersion: z.number().optional(), + folderName: z.string().optional(), + folderVersion: z.number().optional() + }) +); + +export const registerPITRouter = async (server: FastifyZodProvider) => { + // Get commits count for a folder + server.route({ + method: "GET", + url: "/commits/count", + config: { + rateLimit: readLimit + }, + schema: { + querystring: z.object({ + environment: z.string().trim(), + path: z.string().trim().default("/").transform(removeTrailingSlash), + projectId: z.string().trim() + }), + response: { + 200: z.object({ + count: z.number(), + folderId: z.string() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.getCommitsCount({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.query.projectId, + environment: req.query.environment, + path: req.query.path + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.query.projectId, + event: { + type: EventType.GET_PROJECT_PIT_COMMIT_COUNT, + metadata: { + environment: req.query.environment, + path: req.query.path, + commitCount: result.count.toString() + } + } + }); + + return result; + } + }); + + // Get all commits for a folder + server.route({ + method: "GET", + url: "/commits", + config: { + rateLimit: readLimit + }, + schema: { + querystring: z.object({ + environment: z.string().trim(), + path: z.string().trim().default("/").transform(removeTrailingSlash), + projectId: z.string().trim(), + offset: z.coerce.number().min(0).default(0), + limit: z.coerce.number().min(1).max(100).default(20), + search: z.string().trim().optional(), + sort: z.enum(["asc", "desc"]).default("desc") + }), + response: { + 200: z.object({ + commits: commitHistoryItemSchema.array(), + total: z.number(), + hasMore: z.boolean() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.getCommitsForFolder({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.query.projectId, + environment: req.query.environment, + path: req.query.path, + offset: req.query.offset, + limit: req.query.limit, + search: req.query.search, + sort: req.query.sort + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.query.projectId, + event: { + type: EventType.GET_PROJECT_PIT_COMMITS, + metadata: { + environment: req.query.environment, + path: req.query.path, + commitCount: result.commits.length.toString(), + offset: req.query.offset.toString(), + limit: req.query.limit.toString(), + search: req.query.search, + sort: req.query.sort + } + } + }); + + return result; + } + }); + + // Get commit changes for a specific commit + server.route({ + method: "GET", + url: "/commits/:commitId/changes", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + querystring: z.object({ + projectId: z.string().trim() + }), + response: { + 200: commitChangesResponseSchema + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.getCommitChanges({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.query.projectId, + commitId: req.params.commitId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.query.projectId, + event: { + type: EventType.GET_PROJECT_PIT_COMMIT_CHANGES, + metadata: { + commitId: req.params.commitId, + changesCount: (result.changes.changes?.length || 0).toString() + } + } + }); + + return result; + } + }); + + // Retrieve rollback changes for a commit + server.route({ + method: "GET", + url: "/commits/:commitId/compare", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + querystring: z.object({ + folderId: z.string().trim(), + environment: z.string().trim(), + deepRollback: booleanSchema.default(false), + secretPath: z.string().trim().default("/").transform(removeTrailingSlash), + projectId: z.string().trim() + }), + response: { + 200: z.array( + z.object({ + folderId: z.string(), + folderName: z.string(), + folderPath: z.string().optional(), + changes: z.array(resourceChangeSchema) + }) + ) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.compareCommitChanges({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.query.projectId, + commitId: req.params.commitId, + folderId: req.query.folderId, + environment: req.query.environment, + deepRollback: req.query.deepRollback, + secretPath: req.query.secretPath + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.query.projectId, + event: { + type: EventType.PIT_COMPARE_FOLDER_STATES, + metadata: { + targetCommitId: req.params.commitId, + folderId: req.query.folderId, + deepRollback: req.query.deepRollback, + diffsCount: result.length.toString(), + environment: req.query.environment, + folderPath: req.query.secretPath + } + } + }); + + return result; + } + }); + + // Rollback to a previous commit + server.route({ + method: "POST", + url: "/commits/:commitId/rollback", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + body: z.object({ + folderId: z.string().trim(), + deepRollback: z.boolean().default(false), + message: z.string().max(256).trim().optional(), + environment: z.string().trim(), + projectId: z.string().trim() + }), + response: { + 200: z.object({ + success: z.boolean(), + secretChangesCount: z.number().optional(), + folderChangesCount: z.number().optional(), + totalChanges: z.number().optional() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.rollbackToCommit({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.body.projectId, + commitId: req.params.commitId, + folderId: req.body.folderId, + deepRollback: req.body.deepRollback, + message: req.body.message, + environment: req.body.environment + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.body.projectId, + event: { + type: EventType.PIT_ROLLBACK_COMMIT, + metadata: { + targetCommitId: req.params.commitId, + environment: req.body.environment, + folderId: req.body.folderId, + deepRollback: req.body.deepRollback, + message: req.body.message || "Rollback to previous commit", + totalChanges: result.totalChanges?.toString() || "0" + } + } + }); + + return result; + } + }); + + // Revert commit + server.route({ + method: "POST", + url: "/commits/:commitId/revert", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + body: z.object({ + projectId: z.string().trim() + }), + response: { + 200: z.object({ + success: z.boolean(), + message: z.string(), + originalCommitId: z.string(), + revertCommitId: z.string().optional(), + changesReverted: z.number().optional() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.revertCommit({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.body.projectId, + commitId: req.params.commitId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.body.projectId, + event: { + type: EventType.PIT_REVERT_COMMIT, + metadata: { + commitId: req.params.commitId, + revertCommitId: result.revertCommitId, + changesReverted: result.changesReverted?.toString() + } + } + }); + + return result; + } + }); + + // Folder state at commit + server.route({ + method: "GET", + url: "/commits/:commitId", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + querystring: z.object({ + folderId: z.string().trim(), + projectId: z.string().trim() + }), + response: { + 200: folderStateSchema + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.getFolderStateAtCommit({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.query.projectId, + commitId: req.params.commitId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.query.projectId, + event: { + type: EventType.PIT_GET_FOLDER_STATE, + metadata: { + commitId: req.params.commitId, + folderId: req.query.folderId, + resourceCount: result.length.toString() + } + } + }); + + return result; + } + }); +}; diff --git a/backend/src/ee/routes/v1/snapshot-router.ts b/backend/src/ee/routes/v1/snapshot-router.ts index 3ee80adce8..c14d97192d 100644 --- a/backend/src/ee/routes/v1/snapshot-router.ts +++ b/backend/src/ee/routes/v1/snapshot-router.ts @@ -65,9 +65,10 @@ export const registerSnapshotRouter = async (server: FastifyZodProvider) => { rateLimit: writeLimit }, schema: { - hide: false, + hide: true, + deprecated: true, tags: [ApiDocsTags.Projects], - description: "Roll back project secrets to those captured in a secret snapshot version.", + description: "(Deprecated) Roll back project secrets to those captured in a secret snapshot version.", security: [ { bearerAuth: [] @@ -84,6 +85,10 @@ export const registerSnapshotRouter = async (server: FastifyZodProvider) => { }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { + throw new Error( + "This endpoint is deprecated. Please use the new PIT recovery system. More information is available at: https://infisical.com/docs/documentation/platform/pit-recovery." + ); + const secretSnapshot = await server.services.snapshot.rollbackSnapshot({ actor: req.permission.type, actorId: req.permission.id, diff --git a/backend/src/ee/services/audit-log/audit-log-types.ts b/backend/src/ee/services/audit-log/audit-log-types.ts index cfc01741f5..5fdcb7be41 100644 --- a/backend/src/ee/services/audit-log/audit-log-types.ts +++ b/backend/src/ee/services/audit-log/audit-log-types.ts @@ -393,6 +393,13 @@ export enum EventType { PROJECT_ASSUME_PRIVILEGE_SESSION_START = "project-assume-privileges-session-start", PROJECT_ASSUME_PRIVILEGE_SESSION_END = "project-assume-privileges-session-end", + GET_PROJECT_PIT_COMMITS = "get-project-pit-commits", + GET_PROJECT_PIT_COMMIT_CHANGES = "get-project-pit-commit-changes", + GET_PROJECT_PIT_COMMIT_COUNT = "get-project-pit-commit-count", + PIT_ROLLBACK_COMMIT = "pit-rollback-commit", + PIT_REVERT_COMMIT = "pit-revert-commit", + PIT_GET_FOLDER_STATE = "pit-get-folder-state", + PIT_COMPARE_FOLDER_STATES = "pit-compare-folder-states", SECRET_SCANNING_DATA_SOURCE_LIST = "secret-scanning-data-source-list", SECRET_SCANNING_DATA_SOURCE_CREATE = "secret-scanning-data-source-create", SECRET_SCANNING_DATA_SOURCE_UPDATE = "secret-scanning-data-source-update", @@ -2979,6 +2986,78 @@ interface MicrosoftTeamsWorkflowIntegrationUpdateEvent { }; } +interface GetProjectPitCommitsEvent { + type: EventType.GET_PROJECT_PIT_COMMITS; + metadata: { + commitCount: string; + environment: string; + path: string; + offset: string; + limit: string; + search?: string; + sort: string; + }; +} + +interface GetProjectPitCommitChangesEvent { + type: EventType.GET_PROJECT_PIT_COMMIT_CHANGES; + metadata: { + changesCount: string; + commitId: string; + }; +} + +interface GetProjectPitCommitCountEvent { + type: EventType.GET_PROJECT_PIT_COMMIT_COUNT; + metadata: { + environment: string; + path: string; + commitCount: string; + }; +} + +interface PitRollbackCommitEvent { + type: EventType.PIT_ROLLBACK_COMMIT; + metadata: { + targetCommitId: string; + folderId: string; + deepRollback: boolean; + message: string; + totalChanges: string; + environment: string; + }; +} + +interface PitRevertCommitEvent { + type: EventType.PIT_REVERT_COMMIT; + metadata: { + commitId: string; + revertCommitId?: string; + changesReverted?: string; + }; +} + +interface PitGetFolderStateEvent { + type: EventType.PIT_GET_FOLDER_STATE; + metadata: { + commitId: string; + folderId: string; + resourceCount: string; + }; +} + +interface PitCompareFolderStatesEvent { + type: EventType.PIT_COMPARE_FOLDER_STATES; + metadata: { + targetCommitId: string; + folderId: string; + deepRollback: boolean; + diffsCount: string; + environment: string; + folderPath: string; + }; +} + interface SecretScanningDataSourceListEvent { type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST; metadata: { @@ -3397,6 +3476,13 @@ export type Event = | MicrosoftTeamsWorkflowIntegrationGetEvent | MicrosoftTeamsWorkflowIntegrationListEvent | MicrosoftTeamsWorkflowIntegrationUpdateEvent + | GetProjectPitCommitsEvent + | GetProjectPitCommitChangesEvent + | PitRollbackCommitEvent + | GetProjectPitCommitCountEvent + | PitRevertCommitEvent + | PitCompareFolderStatesEvent + | PitGetFolderStateEvent | SecretScanningDataSourceListEvent | SecretScanningDataSourceGetEvent | SecretScanningDataSourceCreateEvent diff --git a/backend/src/ee/services/permission/default-roles.ts b/backend/src/ee/services/permission/default-roles.ts index 40c5310ae5..cca4efaf24 100644 --- a/backend/src/ee/services/permission/default-roles.ts +++ b/backend/src/ee/services/permission/default-roles.ts @@ -4,6 +4,7 @@ import { ProjectPermissionActions, ProjectPermissionCertificateActions, ProjectPermissionCmekActions, + ProjectPermissionCommitsActions, ProjectPermissionDynamicSecretActions, ProjectPermissionGroupActions, ProjectPermissionIdentityActions, @@ -90,6 +91,11 @@ const buildAdminPermissionRules = () => { ProjectPermissionSub.Certificates ); + can( + [ProjectPermissionCommitsActions.Read, ProjectPermissionCommitsActions.PerformRollback], + ProjectPermissionSub.Commits + ); + can( [ ProjectPermissionSshHostActions.Edit, @@ -292,6 +298,11 @@ const buildMemberPermissionRules = () => { ProjectPermissionSub.SecretImports ); + can( + [ProjectPermissionCommitsActions.Read, ProjectPermissionCommitsActions.PerformRollback], + ProjectPermissionSub.Commits + ); + can([ProjectPermissionActions.Read], ProjectPermissionSub.SecretApproval); can([ProjectPermissionSecretRotationActions.Read], ProjectPermissionSub.SecretRotation); @@ -479,6 +490,7 @@ const buildViewerPermissionRules = () => { can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificates); can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificateTemplates); can(ProjectPermissionSecretSyncActions.Read, ProjectPermissionSub.SecretSyncs); + can(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); can( [ diff --git a/backend/src/ee/services/permission/project-permission.ts b/backend/src/ee/services/permission/project-permission.ts index d1ae695741..f61c4b1a4a 100644 --- a/backend/src/ee/services/permission/project-permission.ts +++ b/backend/src/ee/services/permission/project-permission.ts @@ -17,6 +17,11 @@ export enum ProjectPermissionActions { Delete = "delete" } +export enum ProjectPermissionCommitsActions { + Read = "read", + PerformRollback = "perform-rollback" +} + export enum ProjectPermissionCertificateActions { Read = "read", Create = "create", @@ -172,6 +177,7 @@ export enum ProjectPermissionSub { SecretRollback = "secret-rollback", SecretApproval = "secret-approval", SecretRotation = "secret-rotation", + Commits = "commits", Identity = "identity", CertificateAuthorities = "certificate-authorities", Certificates = "certificates", @@ -325,6 +331,7 @@ export type ProjectPermissionSet = | [ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback] | [ProjectPermissionActions.Create, ProjectPermissionSub.SecretRollback] | [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms] + | [ProjectPermissionCommitsActions, ProjectPermissionSub.Commits] | [ProjectPermissionSecretScanningDataSourceActions, ProjectPermissionSub.SecretScanningDataSources] | [ProjectPermissionSecretScanningFindingActions, ProjectPermissionSub.SecretScanningFindings] | [ProjectPermissionSecretScanningConfigActions, ProjectPermissionSub.SecretScanningConfigs]; @@ -676,6 +683,12 @@ const GeneralPermissionSchema = [ "Describe what action an entity can take." ) }), + z.object({ + subject: z.literal(ProjectPermissionSub.Commits).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionCommitsActions).describe( + "Describe what action an entity can take." + ) + }), z.object({ subject: z .literal(ProjectPermissionSub.SecretScanningDataSources) diff --git a/backend/src/ee/services/pit/pit-service.ts b/backend/src/ee/services/pit/pit-service.ts new file mode 100644 index 0000000000..1607291238 --- /dev/null +++ b/backend/src/ee/services/pit/pit-service.ts @@ -0,0 +1,485 @@ +/* eslint-disable no-await-in-loop */ +import { ForbiddenError } from "@casl/ability"; + +import { ActionProjectType } from "@app/db/schemas"; +import { ProjectPermissionCommitsActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { NotFoundError } from "@app/lib/errors"; +import { logger } from "@app/lib/logger"; +import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type"; +import { ResourceType, TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; +import { + isFolderCommitChange, + isSecretCommitChange +} from "@app/services/folder-commit-changes/folder-commit-changes-dal"; +import { TProjectEnvDALFactory } from "@app/services/project-env/project-env-dal"; +import { TSecretServiceFactory } from "@app/services/secret/secret-service"; +import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service"; + +import { TPermissionServiceFactory } from "../permission/permission-service"; + +type TPitServiceFactoryDep = { + folderCommitService: TFolderCommitServiceFactory; + secretService: Pick; + folderService: Pick; + permissionService: Pick; + folderDAL: Pick; + projectEnvDAL: Pick; +}; + +export type TPitServiceFactory = ReturnType; + +export const pitServiceFactory = ({ + folderCommitService, + secretService, + folderService, + permissionService, + folderDAL, + projectEnvDAL +}: TPitServiceFactoryDep) => { + const getCommitsCount = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + environment, + path + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + environment: string; + path: string; + }) => { + const result = await folderCommitService.getCommitsCount({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + environment, + path + }); + + return result; + }; + + const getCommitsForFolder = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + environment, + path, + offset, + limit, + search, + sort + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + environment: string; + path: string; + offset: number; + limit: number; + search?: string; + sort: "asc" | "desc"; + }) => { + const result = await folderCommitService.getCommitsForFolder({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + environment, + path, + offset, + limit, + search, + sort + }); + + return { + commits: result.commits.map((commit) => ({ + ...commit, + commitId: commit.commitId.toString() + })), + total: result.total, + hasMore: result.hasMore + }; + }; + + const getCommitChanges = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + commitId: string; + }) => { + const changes = await folderCommitService.getCommitChanges({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId + }); + + const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(projectId, [changes.folderId]); + + for (const change of changes.changes) { + if (isSecretCommitChange(change)) { + change.versions = await secretService.getChangeVersions( + { + secretVersion: change.secretVersion, + secretId: change.secretId, + id: change.id, + isUpdate: change.isUpdate, + changeType: change.changeType + }, + (Number.parseInt(change.secretVersion, 10) - 1).toString(), + actorId, + actor, + actorOrgId, + actorAuthMethod, + changes.envId, + projectId, + folderWithPath?.path || "" + ); + } else if (isFolderCommitChange(change)) { + change.versions = await folderService.getFolderVersions( + change, + (Number.parseInt(change.folderVersion, 10) - 1).toString(), + change.folderChangeId + ); + } + } + + return { + changes: { + ...changes, + commitId: changes.commitId.toString() + } + }; + }; + + const compareCommitChanges = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId, + folderId, + environment, + deepRollback, + secretPath + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + commitId: string; + folderId: string; + environment: string; + deepRollback: boolean; + secretPath: string; + }) => { + const latestCommit = await folderCommitService.getLatestCommit({ + folderId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId + }); + + const targetCommit = await folderCommitService.getCommitById({ + commitId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId + }); + + const env = await projectEnvDAL.findOne({ + projectId, + slug: environment + }); + + if (!latestCommit) { + throw new NotFoundError({ message: "Latest commit not found" }); + } + + let diffs; + if (deepRollback) { + diffs = await folderCommitService.deepCompareFolder({ + targetCommitId: targetCommit.id, + envId: env.id, + projectId + }); + } else { + const folderData = await folderService.getFolderById({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + id: folderId + }); + + diffs = [ + { + folderId: folderData.id, + folderName: folderData.name, + folderPath: secretPath, + changes: await folderCommitService.compareFolderStates({ + targetCommitId: commitId, + currentCommitId: latestCommit.id + }) + } + ]; + } + + for (const diff of diffs) { + for (const change of diff.changes) { + // Use discriminated union type checking + if (change.type === ResourceType.SECRET) { + // TypeScript now knows this is a SecretChange + if (change.secretKey && change.secretVersion && change.secretId) { + change.versions = await secretService.getChangeVersions( + { + secretVersion: change.secretVersion, + secretId: change.secretId, + id: change.id, + isUpdate: change.isUpdate, + changeType: change.changeType + }, + change.fromVersion || "1", + actorId, + actor, + actorOrgId, + actorAuthMethod, + env.id, + projectId, + diff.folderPath || "" + ); + } + } else if (change.type === ResourceType.FOLDER) { + // TypeScript now knows this is a FolderChange + if (change.folderVersion) { + change.versions = await folderService.getFolderVersions(change, change.fromVersion || "1", change.id); + } + } + } + } + + return diffs; + }; + + const rollbackToCommit = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId, + folderId, + deepRollback, + message, + environment + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + commitId: string; + folderId: string; + deepRollback: boolean; + message?: string; + environment: string; + }) => { + const { permission: userPermission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.SecretManager + }); + + ForbiddenError.from(userPermission).throwUnlessCan( + ProjectPermissionCommitsActions.PerformRollback, + ProjectPermissionSub.Commits + ); + + const latestCommit = await folderCommitService.getLatestCommit({ + folderId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId + }); + + if (!latestCommit) { + throw new NotFoundError({ message: "Latest commit not found" }); + } + + logger.info(`PIT - Attempting to rollback folder ${folderId} from commit ${latestCommit.id} to commit ${commitId}`); + + const targetCommit = await folderCommitService.getCommitById({ + commitId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + + const env = await projectEnvDAL.findOne({ + projectId, + slug: environment + }); + + if (!targetCommit || targetCommit.folderId !== folderId || targetCommit.envId !== env.id) { + throw new NotFoundError({ message: "Target commit not found" }); + } + + if (!latestCommit || latestCommit.envId !== env.id) { + throw new NotFoundError({ message: "Latest commit not found" }); + } + + if (deepRollback) { + await folderCommitService.deepRollbackFolder(commitId, env.id, actorId, actor, projectId, message); + return { success: true }; + } + + const diff = await folderCommitService.compareFolderStates({ + currentCommitId: latestCommit.id, + targetCommitId: commitId + }); + + const response = await folderCommitService.applyFolderStateDifferences({ + differences: diff, + actorInfo: { + actorType: actor, + actorId, + message: message || "Rollback to previous commit" + }, + folderId, + projectId, + reconstructNewFolders: deepRollback + }); + + return { + success: true, + secretChangesCount: response.secretChangesCount, + folderChangesCount: response.folderChangesCount, + totalChanges: response.totalChanges + }; + }; + + const revertCommit = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + commitId: string; + }) => { + const response = await folderCommitService.revertCommitChanges({ + commitId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + + return response; + }; + + const getFolderStateAtCommit = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + commitId: string; + }) => { + const commit = await folderCommitService.getCommitById({ + commitId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId + }); + + if (!commit) { + throw new NotFoundError({ message: `Commit with ID ${commitId} not found` }); + } + + const response = await folderCommitService.reconstructFolderState(commitId); + + return response.map((item) => { + if (item.type === ResourceType.SECRET) { + return { + ...item, + secretVersion: Number(item.secretVersion) + }; + } + + if (item.type === ResourceType.FOLDER) { + return { + ...item, + folderVersion: Number(item.folderVersion) + }; + } + + return item; + }); + }; + + return { + getCommitsCount, + getCommitsForFolder, + getCommitChanges, + compareCommitChanges, + rollbackToCommit, + revertCommit, + getFolderStateAtCommit + }; +}; diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 2171812812..2a88136c28 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -20,6 +20,7 @@ import { EnforcementLevel } from "@app/lib/types"; import { triggerWorkflowIntegrationNotification } from "@app/lib/workflow-integrations/trigger-notification"; import { TriggerFeature } from "@app/lib/workflow-integrations/types"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TMicrosoftTeamsServiceFactory } from "@app/services/microsoft-teams/microsoft-teams-service"; @@ -130,6 +131,7 @@ type TSecretApprovalRequestServiceFactoryDep = { licenseService: Pick; projectMicrosoftTeamsConfigDAL: Pick; microsoftTeamsService: Pick; + folderCommitService: Pick; }; export type TSecretApprovalRequestServiceFactory = ReturnType; @@ -161,7 +163,8 @@ export const secretApprovalRequestServiceFactory = ({ projectSlackConfigDAL, resourceMetadataDAL, projectMicrosoftTeamsConfigDAL, - microsoftTeamsService + microsoftTeamsService, + folderCommitService }: TSecretApprovalRequestServiceFactoryDep) => { const requestCount = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod }: TApprovalRequestCountDTO) => { if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" }); @@ -597,6 +600,10 @@ export const secretApprovalRequestServiceFactory = ({ ? await fnSecretV2BridgeBulkInsert({ tx, folderId, + actor: { + actorId, + type: actor + }, orgId: actorOrgId, inputSecrets: secretCreationCommits.map((el) => ({ tagIds: el?.tags.map(({ id }) => id), @@ -619,13 +626,18 @@ export const secretApprovalRequestServiceFactory = ({ secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, - secretVersionTagDAL: secretVersionTagV2BridgeDAL + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService }) : []; const updatedSecrets = secretUpdationCommits.length ? await fnSecretV2BridgeBulkUpdate({ folderId, orgId: actorOrgId, + actor: { + actorId, + type: actor + }, tx, inputSecrets: secretUpdationCommits.map((el) => { const encryptedValue = @@ -659,7 +671,8 @@ export const secretApprovalRequestServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }) : []; const deletedSecret = secretDeletionCommits.length @@ -667,10 +680,13 @@ export const secretApprovalRequestServiceFactory = ({ projectId, folderId, tx, - actorId: "", + actorId, + actorType: actor, secretDAL: secretV2BridgeDAL, secretQueueService, - inputSecrets: secretDeletionCommits.map(({ key }) => ({ secretKey: key, type: SecretType.Shared })) + inputSecrets: secretDeletionCommits.map(({ key }) => ({ secretKey: key, type: SecretType.Shared })), + folderCommitService, + secretVersionDAL: secretVersionV2BridgeDAL }) : []; const updatedSecretApproval = await secretApprovalRequestDAL.updateById( diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index 90fdf561e1..628f8e310c 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -10,6 +10,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueName, TQueueServiceFactory } from "@app/queue"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -87,6 +88,7 @@ type TSecretReplicationServiceFactoryDep = { projectBotService: Pick; kmsService: Pick; + folderCommitService: Pick; }; export type TSecretReplicationServiceFactory = ReturnType; @@ -132,6 +134,7 @@ export const secretReplicationServiceFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, kmsService, + folderCommitService, resourceMetadataDAL }: TSecretReplicationServiceFactoryDep) => { const $getReplicatedSecrets = ( @@ -419,7 +422,7 @@ export const secretReplicationServiceFactory = ({ return { op: operation, requestId: approvalRequestDoc.id, - metadata: doc.metadata, + metadata: doc.metadata ? JSON.stringify(doc.metadata) : [], secretMetadata: JSON.stringify(doc.secretMetadata), key: doc.key, encryptedValue: doc.encryptedValue, @@ -446,11 +449,12 @@ export const secretReplicationServiceFactory = ({ tx, secretTagDAL, resourceMetadataDAL, + folderCommitService, secretVersionTagDAL: secretVersionV2TagBridgeDAL, inputSecrets: locallyCreatedSecrets.map((doc) => { return { type: doc.type, - metadata: doc.metadata, + metadata: doc.metadata ? JSON.stringify(doc.metadata) : [], key: doc.key, encryptedValue: doc.encryptedValue, encryptedComment: doc.encryptedComment, @@ -466,6 +470,7 @@ export const secretReplicationServiceFactory = ({ orgId, folderId: destinationReplicationFolderId, secretVersionDAL: secretVersionV2BridgeDAL, + folderCommitService, secretDAL: secretV2BridgeDAL, tx, resourceMetadataDAL, @@ -479,7 +484,7 @@ export const secretReplicationServiceFactory = ({ }, data: { type: doc.type, - metadata: doc.metadata, + metadata: doc.metadata ? JSON.stringify(doc.metadata) : [], key: doc.key, encryptedValue: doc.encryptedValue as Buffer, encryptedComment: doc.encryptedComment, diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts index 6bf9c9b774..84a3435b6b 100644 --- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts +++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts @@ -63,6 +63,7 @@ import { TAppConnectionDALFactory } from "@app/services/app-connection/app-conne import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns"; import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -98,7 +99,7 @@ export type TSecretRotationV2ServiceFactoryDep = { TSecretV2BridgeDALFactory, "bulkUpdate" | "insertMany" | "deleteMany" | "upsertSecretReferences" | "find" | "invalidateSecretCacheByProjectId" >; - secretVersionV2BridgeDAL: Pick; + secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; secretTagDAL: Pick; @@ -106,6 +107,7 @@ export type TSecretRotationV2ServiceFactoryDep = { snapshotService: Pick; queueService: Pick; appConnectionDAL: Pick; + folderCommitService: Pick; }; export type TSecretRotationV2ServiceFactory = ReturnType; @@ -145,6 +147,7 @@ export const secretRotationV2ServiceFactory = ({ snapshotService, keyStore, queueService, + folderCommitService, appConnectionDAL }: TSecretRotationV2ServiceFactoryDep) => { const $queueSendSecretRotationStatusNotification = async (secretRotation: TSecretRotationV2Raw) => { @@ -538,7 +541,12 @@ export const secretRotationV2ServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, secretTagDAL, - resourceMetadataDAL + folderCommitService, + resourceMetadataDAL, + actor: { + type: actor.type, + actorId: actor.id + } }); await secretRotationV2DAL.insertSecretMappings( @@ -674,7 +682,12 @@ export const secretRotationV2ServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, secretTagDAL, - resourceMetadataDAL + folderCommitService, + resourceMetadataDAL, + actor: { + type: actor.type, + actorId: actor.id + } }); secretsMappingUpdated = true; @@ -792,6 +805,9 @@ export const secretRotationV2ServiceFactory = ({ projectId, folderId, actorId: actor.id, // not actually used since rotated secrets are shared + actorType: actor.type, + folderCommitService, + secretVersionDAL: secretVersionV2BridgeDAL, tx }); } @@ -935,6 +951,10 @@ export const secretRotationV2ServiceFactory = ({ secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService, + actor: { + type: ActorType.PLATFORM + }, secretTagDAL, resourceMetadataDAL }); diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts index 2c6124348a..d792ac6e65 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts @@ -14,6 +14,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { ActorType } from "@app/services/auth/auth-type"; +import { CommitType, TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -53,6 +54,7 @@ type TSecretRotationQueueFactoryDep = { secretVersionV2BridgeDAL: Pick; telemetryService: Pick; kmsService: Pick; + folderCommitService: Pick; }; // These error should stop the repeatable job and ask user to reconfigure rotation @@ -77,6 +79,7 @@ export const secretRotationQueueFactory = ({ telemetryService, secretV2BridgeDAL, secretVersionV2BridgeDAL, + folderCommitService, kmsService }: TSecretRotationQueueFactoryDep) => { const addToQueue = async (rotationId: string, interval: number) => { @@ -330,7 +333,7 @@ export const secretRotationQueueFactory = ({ })), tx ); - await secretVersionV2BridgeDAL.insertMany( + const secretVersions = await secretVersionV2BridgeDAL.insertMany( updatedSecrets.map(({ id, updatedAt, createdAt, ...el }) => ({ ...el, actorType: ActorType.PLATFORM, @@ -338,6 +341,22 @@ export const secretRotationQueueFactory = ({ })), tx ); + + await folderCommitService.createCommit( + { + actor: { + type: ActorType.PLATFORM + }, + message: "Changed by Secret rotation", + folderId: secretVersions[0].folderId, + changes: secretVersions.map((sv) => ({ + type: CommitType.ADD, + isUpdate: true, + secretVersionId: sv.id + })) + }, + tx + ); }); await secretV2BridgeDAL.invalidateSecretCacheByProjectId(secretRotation.projectId); diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index 015a8d4202..8cae3dfb58 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -8,6 +8,7 @@ import { InternalServerError, NotFoundError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { ActorType } from "@app/services/auth/auth-type"; +import { CommitType, TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -51,8 +52,8 @@ type TSecretSnapshotServiceFactoryDep = { snapshotSecretV2BridgeDAL: TSnapshotSecretV2DALFactory; snapshotFolderDAL: TSnapshotFolderDALFactory; secretVersionDAL: Pick; - secretVersionV2BridgeDAL: Pick; - folderVersionDAL: Pick; + secretVersionV2BridgeDAL: Pick; + folderVersionDAL: Pick; secretDAL: Pick; secretV2BridgeDAL: Pick; secretTagDAL: Pick; @@ -63,6 +64,7 @@ type TSecretSnapshotServiceFactoryDep = { licenseService: Pick; kmsService: Pick; projectBotService: Pick; + folderCommitService: Pick; }; export type TSecretSnapshotServiceFactory = ReturnType; @@ -84,7 +86,8 @@ export const secretSnapshotServiceFactory = ({ snapshotSecretV2BridgeDAL, secretVersionV2TagBridgeDAL, kmsService, - projectBotService + projectBotService, + folderCommitService }: TSecretSnapshotServiceFactoryDep) => { const projectSecretSnapshotCount = async ({ environment, @@ -403,6 +406,18 @@ export const secretSnapshotServiceFactory = ({ .filter((el) => el.isRotatedSecret) .map((el) => el.secretId); + const deletedSecretsChanges = new Map(); // secretId -> version info + const deletedFoldersChanges = new Map(); // folderId -> version info + const addedSecretsChanges = new Map(); // secretId -> version info + const addedFoldersChanges = new Map(); // folderId -> version info + const commitChanges: { + type: string; + secretVersionId?: string; + folderVersionId?: string; + isUpdate?: boolean; + folderId?: string; + }[] = []; + // this will remove all secrets in current folder except rotated secrets which we ignore const deletedTopLevelSecs = await secretV2BridgeDAL.delete( { @@ -424,7 +439,35 @@ export const secretSnapshotServiceFactory = ({ }, tx ); + + await Promise.all( + deletedTopLevelSecs.map(async (sec) => { + const version = await secretVersionV2BridgeDAL.findOne({ secretId: sec.id, version: sec.version }, tx); + deletedSecretsChanges.set(sec.id, { + id: sec.id, + version: sec.version, + // Store the version ID if available from the snapshot + versionId: version?.id + }); + }) + ); + const deletedTopLevelSecsGroupById = groupBy(deletedTopLevelSecs, (item) => item.id); + + const deletedFoldersData = await folderDAL.delete({ parentId: snapshot.folderId, isReserved: false }, tx); + + await Promise.all( + deletedFoldersData.map(async (folder) => { + const version = await folderVersionDAL.findOne({ folderId: folder.id, version: folder.version }, tx); + deletedFoldersChanges.set(folder.id, { + id: folder.id, + version: folder.version, + // Store the version ID if available + versionId: version?.id + }); + }) + ); + // this will remove all secrets and folders on child // due to sql foreign key and link list connection removing the folders removes everything below too const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId, isReserved: false }, tx); @@ -489,14 +532,21 @@ export const secretSnapshotServiceFactory = ({ }); await secretTagDAL.saveTagsToSecretV2(secretTagsToBeInsert, tx); const folderVersions = await folderVersionDAL.insertMany( - folders.map(({ version, name, id, envId }) => ({ + folders.map(({ version, name, id, envId, description }) => ({ name, version, folderId: id, - envId + envId, + description })), tx ); + + // Track added folders + folderVersions.forEach((fv) => { + addedFoldersChanges.set(fv.folderId, fv); + }); + const userActorId = actor === ActorType.USER ? actorId : undefined; const identityActorId = actor !== ActorType.USER ? actorId : undefined; const actorType = actor || ActorType.PLATFORM; @@ -511,6 +561,11 @@ export const secretSnapshotServiceFactory = ({ })), tx ); + + secretVersions.forEach((sv) => { + addedSecretsChanges.set(sv.secretId, sv); + }); + await secretVersionV2TagBridgeDAL.insertMany( secretVersions.flatMap(({ secretId, id }) => secretVerTagToBeInsert?.[secretId]?.length @@ -522,6 +577,70 @@ export const secretSnapshotServiceFactory = ({ ), tx ); + + // Compute commit changes + // Handle secrets + deletedSecretsChanges.forEach((deletedInfo, secretId) => { + const addedSecret = addedSecretsChanges.get(secretId); + if (addedSecret) { + // Secret was deleted and re-added - this is an update only if versions are different + if (deletedInfo.versionId !== addedSecret.id) { + commitChanges.push({ + type: CommitType.ADD, // In the commit system, updates are tracked as "add" with isUpdate=true + secretVersionId: addedSecret.id, + isUpdate: true + }); + } + // Remove from addedSecrets since we've handled it + addedSecretsChanges.delete(secretId); + } else if (deletedInfo.versionId) { + // Secret was only deleted + commitChanges.push({ + type: CommitType.DELETE, + secretVersionId: deletedInfo.versionId + }); + } + }); + // Add remaining new secrets (not updates) + addedSecretsChanges.forEach((addedSecret) => { + commitChanges.push({ + type: CommitType.ADD, + secretVersionId: addedSecret.id + }); + }); + + // Handle folders + deletedFoldersChanges.forEach((deletedInfo, folderId) => { + const addedFolder = addedFoldersChanges.get(folderId); + if (addedFolder) { + // Folder was deleted and re-added - this is an update only if versions are different + if (deletedInfo.versionId !== addedFolder.id) { + commitChanges.push({ + type: CommitType.ADD, + folderVersionId: addedFolder.id, + isUpdate: true + }); + } + // Remove from addedFolders since we've handled it + addedFoldersChanges.delete(folderId); + } else if (deletedInfo.versionId) { + // Folder was only deleted + commitChanges.push({ + type: CommitType.DELETE, + folderVersionId: deletedInfo.versionId, + folderId: deletedInfo.id + }); + } + }); + + // Add remaining new folders (not updates) + addedFoldersChanges.forEach((addedFolder) => { + commitChanges.push({ + type: CommitType.ADD, + folderVersionId: addedFolder.id + }); + }); + const newSnapshot = await snapshotDAL.create( { folderId: snapshot.folderId, @@ -550,6 +669,22 @@ export const secretSnapshotServiceFactory = ({ })), tx ); + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType, + metadata: { + id: userActorId || identityActorId + } + }, + message: "Rollback to snapshot", + folderId: snapshot.folderId, + changes: commitChanges + }, + tx + ); + } return { ...newSnapshot, snapshotSecrets, snapshotFolders }; }); @@ -609,11 +744,12 @@ export const secretSnapshotServiceFactory = ({ }); await secretTagDAL.saveTagsToSecret(secretTagsToBeInsert, tx); const folderVersions = await folderVersionDAL.insertMany( - folders.map(({ version, name, id, envId }) => ({ + folders.map(({ version, name, id, envId, description }) => ({ name, version, folderId: id, - envId + envId, + description })), tx ); diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index c6ec5dccbe..d3f19168a2 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -27,6 +27,7 @@ export const KeyStorePrefixes = { KmsOrgDataKeyCreation: "kms-org-data-key-creation-lock", WaitUntilReadyKmsOrgKeyCreation: "wait-until-ready-kms-org-key-creation-", WaitUntilReadyKmsOrgDataKeyCreation: "wait-until-ready-kms-org-data-key-creation-", + FolderTreeCheckpoint: (envId: string) => `folder-tree-checkpoint-${envId}`, WaitUntilReadyProjectEnvironmentOperation: (projectId: string) => `wait-until-ready-project-environments-operation-${projectId}`, diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index a10fcc67ec..642ba453ae 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -626,7 +626,8 @@ export const PROJECTS = { autoCapitalization: "Disable or enable auto-capitalization for the project.", slug: "An optional slug for the project. (must be unique within the organization)", hasDeleteProtection: "Enable or disable delete protection for the project.", - secretSharing: "Enable or disable secret sharing for the project." + secretSharing: "Enable or disable secret sharing for the project.", + showSnapshotsLegacy: "Enable or disable legacy snapshots for the project." }, GET_KEY: { workspaceId: "The ID of the project to get the key from." diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index e2fc73d8a6..6b9d33c2ac 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -261,6 +261,10 @@ const envSchema = z DATADOG_SERVICE: zpStr(z.string().optional().default("infisical-core")), DATADOG_HOSTNAME: zpStr(z.string().optional()), + // PIT + PIT_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("2")), + PIT_TREE_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("30")), + /* CORS ----------------------------------------------------------------------------- */ CORS_ALLOWED_ORIGINS: zpStr( z diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index e4d6549980..25677841d4 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -60,6 +60,7 @@ export enum QueueName { ImportSecretsFromExternalSource = "import-secrets-from-external-source", AppConnectionSecretSync = "app-connection-secret-sync", SecretRotationV2 = "secret-rotation-v2", + FolderTreeCheckpoint = "folder-tree-checkpoint", InvalidateCache = "invalidate-cache", SecretScanningV2 = "secret-scanning-v2" } @@ -94,6 +95,7 @@ export enum QueueJobs { SecretRotationV2QueueRotations = "secret-rotation-v2-queue-rotations", SecretRotationV2RotateSecrets = "secret-rotation-v2-rotate-secrets", SecretRotationV2SendNotification = "secret-rotation-v2-send-notification", + CreateFolderTreeCheckpoint = "create-folder-tree-checkpoint", InvalidateCache = "invalidate-cache", SecretScanningV2FullScan = "secret-scanning-v2-full-scan", SecretScanningV2DiffScan = "secret-scanning-v2-diff-scan", @@ -209,6 +211,12 @@ export type TQueueJobTypes = { name: QueueJobs.ProjectV3Migration; payload: { projectId: string }; }; + [QueueName.FolderTreeCheckpoint]: { + name: QueueJobs.CreateFolderTreeCheckpoint; + payload: { + envId: string; + }; + }; [QueueName.ImportSecretsFromExternalSource]: { name: QueueJobs.ImportSecretsFromExternalSource; payload: { diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index aa38bb0e8c..5513fb49b0 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -60,6 +60,7 @@ import { oidcConfigDALFactory } from "@app/ee/services/oidc/oidc-config-dal"; import { oidcConfigServiceFactory } from "@app/ee/services/oidc/oidc-config-service"; import { permissionDALFactory } from "@app/ee/services/permission/permission-dal"; import { permissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { pitServiceFactory } from "@app/ee/services/pit/pit-service"; import { projectTemplateDALFactory } from "@app/ee/services/project-template/project-template-dal"; import { projectTemplateServiceFactory } from "@app/ee/services/project-template/project-template-service"; import { projectUserAdditionalPrivilegeDALFactory } from "@app/ee/services/project-user-additional-privilege/project-user-additional-privilege-dal"; @@ -154,6 +155,14 @@ import { externalGroupOrgRoleMappingDALFactory } from "@app/services/external-gr import { externalGroupOrgRoleMappingServiceFactory } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-service"; import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue"; import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; +import { folderCheckpointDALFactory } from "@app/services/folder-checkpoint/folder-checkpoint-dal"; +import { folderCheckpointResourcesDALFactory } from "@app/services/folder-checkpoint-resources/folder-checkpoint-resources-dal"; +import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal"; +import { folderCommitQueueServiceFactory } from "@app/services/folder-commit/folder-commit-queue"; +import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; +import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal"; +import { folderTreeCheckpointDALFactory } from "@app/services/folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { folderTreeCheckpointResourcesDALFactory } from "@app/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal"; import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal"; import { groupProjectServiceFactory } from "@app/services/group-project/group-project-service"; @@ -583,6 +592,41 @@ export const registerRoutes = async ( projectRoleDAL, permissionService }); + + const folderCommitChangesDAL = folderCommitChangesDALFactory(db); + const folderCheckpointDAL = folderCheckpointDALFactory(db); + const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db); + const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db); + const folderCommitDAL = folderCommitDALFactory(db); + const folderTreeCheckpointResourcesDAL = folderTreeCheckpointResourcesDALFactory(db); + const folderCommitQueueService = folderCommitQueueServiceFactory({ + queueService, + folderTreeCheckpointDAL, + keyStore, + folderTreeCheckpointResourcesDAL, + folderCommitDAL, + folderDAL + }); + const folderCommitService = folderCommitServiceFactory({ + folderCommitDAL, + folderCommitChangesDAL, + folderCheckpointDAL, + folderTreeCheckpointDAL, + userDAL, + identityDAL, + folderDAL, + folderVersionDAL, + secretVersionV2BridgeDAL, + projectDAL, + folderCheckpointResourcesDAL, + secretV2BridgeDAL, + folderTreeCheckpointResourcesDAL, + folderCommitQueueService, + permissionService, + kmsService, + secretTagDAL, + resourceMetadataDAL + }); const scimService = scimServiceFactory({ licenseService, scimDAL, @@ -987,6 +1031,7 @@ export const registerRoutes = async ( projectMembershipDAL, projectBotDAL, secretDAL, + folderCommitService, secretBlindIndexDAL, secretVersionDAL, secretTagDAL, @@ -1034,6 +1079,7 @@ export const registerRoutes = async ( secretReminderRecipientsDAL, orgService, resourceMetadataDAL, + folderCommitService, secretSyncQueue }); @@ -1110,6 +1156,7 @@ export const registerRoutes = async ( snapshotDAL, snapshotFolderDAL, snapshotSecretDAL, + folderCommitService, secretVersionDAL, folderVersionDAL, secretTagDAL, @@ -1136,7 +1183,8 @@ export const registerRoutes = async ( folderVersionDAL, projectEnvDAL, snapshotService, - projectDAL + projectDAL, + folderCommitService }); const secretImportService = secretImportServiceFactory({ @@ -1161,6 +1209,7 @@ export const registerRoutes = async ( const secretV2BridgeService = secretV2BridgeServiceFactory({ folderDAL, secretVersionDAL: secretVersionV2BridgeDAL, + folderCommitService, secretQueueService, secretDAL: secretV2BridgeDAL, permissionService, @@ -1204,7 +1253,8 @@ export const registerRoutes = async ( projectSlackConfigDAL, resourceMetadataDAL, projectMicrosoftTeamsConfigDAL, - microsoftTeamsService + microsoftTeamsService, + folderCommitService }); const secretService = secretServiceFactory({ @@ -1291,7 +1341,8 @@ export const registerRoutes = async ( secretV2BridgeDAL, secretVersionV2TagBridgeDAL: secretVersionTagV2BridgeDAL, secretVersionV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const secretRotationQueue = secretRotationQueueFactory({ @@ -1303,6 +1354,7 @@ export const registerRoutes = async ( projectBotService, secretVersionV2BridgeDAL, secretV2BridgeDAL, + folderCommitService, kmsService }); @@ -1454,6 +1506,15 @@ export const registerRoutes = async ( permissionService }); + const pitService = pitServiceFactory({ + folderCommitService, + secretService, + folderService, + permissionService, + folderDAL, + projectEnvDAL + }); + const identityOidcAuthService = identityOidcAuthServiceFactory({ identityOidcAuthDAL, identityOrgMembershipDAL, @@ -1597,7 +1658,9 @@ export const registerRoutes = async ( secretDAL: secretV2BridgeDAL, queueService, secretV2BridgeService, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService, + folderVersionDAL }); const migrationService = externalMigrationServiceFactory({ @@ -1707,6 +1770,7 @@ export const registerRoutes = async ( auditLogService, secretV2BridgeDAL, secretTagDAL, + folderCommitService, secretVersionTagV2BridgeDAL, secretVersionV2BridgeDAL, keyStore, @@ -1895,6 +1959,7 @@ export const registerRoutes = async ( certificateTemplate: certificateTemplateService, certificateAuthorityCrl: certificateAuthorityCrlService, certificateEst: certificateEstService, + pit: pitService, pkiAlert: pkiAlertService, pkiCollection: pkiCollectionService, pkiSubscriber: pkiSubscriberService, @@ -1929,6 +1994,7 @@ export const registerRoutes = async ( microsoftTeams: microsoftTeamsService, assumePrivileges: assumePrivilegeService, githubOrgSync: githubOrgSyncConfigService, + folderCommit: folderCommitService, secretScanningV2: secretScanningV2Service }); diff --git a/backend/src/server/routes/sanitizedSchemas.ts b/backend/src/server/routes/sanitizedSchemas.ts index a26293ac83..ce51b1079e 100644 --- a/backend/src/server/routes/sanitizedSchemas.ts +++ b/backend/src/server/routes/sanitizedSchemas.ts @@ -262,7 +262,8 @@ export const SanitizedProjectSchema = ProjectsSchema.pick({ kmsCertificateKeyId: true, auditLogsRetentionDays: true, hasDeleteProtection: true, - secretSharing: true + secretSharing: true, + showSnapshotsLegacy: true }); export const SanitizedTagSchema = SecretTagsSchema.pick({ diff --git a/backend/src/server/routes/v1/project-router.ts b/backend/src/server/routes/v1/project-router.ts index 2a868864e4..cc94adede8 100644 --- a/backend/src/server/routes/v1/project-router.ts +++ b/backend/src/server/routes/v1/project-router.ts @@ -376,7 +376,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { }) .optional() .describe(PROJECTS.UPDATE.slug), - secretSharing: z.boolean().optional().describe(PROJECTS.UPDATE.secretSharing) + secretSharing: z.boolean().optional().describe(PROJECTS.UPDATE.secretSharing), + showSnapshotsLegacy: z.boolean().optional().describe(PROJECTS.UPDATE.showSnapshotsLegacy) }), response: { 200: z.object({ @@ -397,7 +398,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { autoCapitalization: req.body.autoCapitalization, hasDeleteProtection: req.body.hasDeleteProtection, slug: req.body.slug, - secretSharing: req.body.secretSharing + secretSharing: req.body.secretSharing, + showSnapshotsLegacy: req.body.showSnapshotsLegacy }, actorAuthMethod: req.permission.authMethod, actorId: req.permission.id, diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 856b39012f..018d7bd43e 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -10,6 +10,7 @@ import { chunkArray } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; +import { CommitType, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { KmsDataKey } from "../kms/kms-types"; import { TProjectDALFactory } from "../project/project-dal"; @@ -18,6 +19,7 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { fnSecretBulkInsert, getAllSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns"; @@ -42,6 +44,8 @@ export type TImportDataIntoInfisicalDTO = { projectService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; + folderCommitService: Pick; + folderVersionDAL: Pick; input: TImportInfisicalDataCreate; }; @@ -507,6 +511,8 @@ export const importDataIntoInfisicalFn = async ({ secretVersionTagDAL, folderDAL, resourceMetadataDAL, + folderVersionDAL, + folderCommitService, input: { data, actor, actorId, actorOrgId, actorAuthMethod } }: TImportDataIntoInfisicalDTO) => { // Import data to infisical @@ -599,6 +605,36 @@ export const importDataIntoInfisicalFn = async ({ tx ); + const newFolderVersion = await folderVersionDAL.create( + { + name: newFolder.name, + envId: newFolder.envId, + version: newFolder.version, + folderId: newFolder.id + }, + tx + ); + + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Changed by external migration", + folderId: parentEnv.rootFolderId, + changes: [ + { + type: CommitType.ADD, + folderVersionId: newFolderVersion.id + } + ] + }, + tx + ); + originalToNewFolderId.set(folder.id, { folderId: newFolder.id, projectId: parentEnv.projectId @@ -772,6 +808,7 @@ export const importDataIntoInfisicalFn = async ({ secretVersionDAL, secretTagDAL, secretVersionTagDAL, + folderCommitService, actor: { type: actor, actorId diff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.ts index 8aa46b94c1..66f2c73e77 100644 --- a/backend/src/services/external-migration/external-migration-queue.ts +++ b/backend/src/services/external-migration/external-migration-queue.ts @@ -3,6 +3,7 @@ import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectServiceFactory } from "../project/project-service"; @@ -10,6 +11,7 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; @@ -36,6 +38,8 @@ export type TExternalMigrationQueueFactoryDep = { projectService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; + folderCommitService: Pick; + folderVersionDAL: Pick; resourceMetadataDAL: Pick; }; @@ -56,6 +60,8 @@ export const externalMigrationQueueFactory = ({ secretTagDAL, secretVersionTagDAL, folderDAL, + folderCommitService, + folderVersionDAL, resourceMetadataDAL }: TExternalMigrationQueueFactoryDep) => { const startImport = async (dto: { @@ -114,6 +120,8 @@ export const externalMigrationQueueFactory = ({ projectService, projectEnvService, secretV2BridgeService, + folderCommitService, + folderVersionDAL, resourceMetadataDAL }); diff --git a/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts new file mode 100644 index 0000000000..3e2d09d0bf --- /dev/null +++ b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts @@ -0,0 +1,118 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { + TableName, + TFolderCheckpointResources, + TFolderCheckpoints, + TSecretFolderVersions, + TSecretVersionsV2 +} from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCheckpointResourcesDALFactory = ReturnType; + +export type ResourceWithCheckpointInfo = TFolderCheckpointResources & { + folderCommitId: string; +}; + +export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { + const folderCheckpointResourcesOrm = ormify(db, TableName.FolderCheckpointResources); + + const findByCheckpointId = async ( + folderCheckpointId: string, + tx?: Knex + ): Promise< + (TFolderCheckpointResources & { + referencedSecretId?: string; + referencedFolderId?: string; + folderName?: string; + folderVersion?: string; + secretKey?: string; + secretVersion?: string; + })[] + > => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCheckpointResources) + .where({ folderCheckpointId }) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCheckpointResources}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCheckpointResources}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.FolderCheckpointResources)) + .select( + db.ref("secretId").withSchema(TableName.SecretVersionV2).as("referencedSecretId"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId"), + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), + db.ref("version").withSchema(TableName.SecretFolderVersion).as("folderVersion"), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secretKey"), + db.ref("version").withSchema(TableName.SecretVersionV2).as("secretVersion") + ); + return docs.map((doc) => ({ + ...doc, + folderVersion: doc.folderVersion?.toString(), + secretVersion: doc.secretVersion?.toString() + })); + } catch (error) { + throw new DatabaseError({ error, name: "FindByCheckpointId" }); + } + }; + + const findBySecretVersionId = async (secretVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCheckpointResources & Pick + >(TableName.FolderCheckpointResources) + .where({ secretVersionId }) + .select(selectAllTableCols(TableName.FolderCheckpointResources)) + .join( + TableName.FolderCheckpoint, + `${TableName.FolderCheckpointResources}.folderCheckpointId`, + `${TableName.FolderCheckpoint}.id` + ) + .select( + db.ref("folderCommitId").withSchema(TableName.FolderCheckpoint), + db.ref("createdAt").withSchema(TableName.FolderCheckpoint) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindBySecretVersionId" }); + } + }; + + const findByFolderVersionId = async (folderVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCheckpointResources & Pick + >(TableName.FolderCheckpointResources) + .where({ folderVersionId }) + .select(selectAllTableCols(TableName.FolderCheckpointResources)) + .join( + TableName.FolderCheckpoint, + `${TableName.FolderCheckpointResources}.folderCheckpointId`, + `${TableName.FolderCheckpoint}.id` + ) + .select( + db.ref("folderCommitId").withSchema(TableName.FolderCheckpoint), + db.ref("createdAt").withSchema(TableName.FolderCheckpoint) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderVersionId" }); + } + }; + + return { + ...folderCheckpointResourcesOrm, + findByCheckpointId, + findBySecretVersionId, + findByFolderVersionId + }; +}; diff --git a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts new file mode 100644 index 0000000000..51bba9cc0f --- /dev/null +++ b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts @@ -0,0 +1,129 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCheckpoints, TFolderCommits } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCheckpointDALFactory = ReturnType; + +type CheckpointWithCommitInfo = TFolderCheckpoints & { + actorMetadata: unknown; + actorType: string; + message?: string | null; + commitDate: Date; + folderId: string; +}; + +export const folderCheckpointDALFactory = (db: TDbClient) => { + const folderCheckpointOrm = ormify(db, TableName.FolderCheckpoint); + + const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderCommitId }, TableName.FolderCheckpoint)) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindByCommitId" }); + } + }; + + const findByFolderId = async (folderId: string, limit?: number, tx?: Knex): Promise => { + try { + let query = (tx || db.replicaNode())(TableName.FolderCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderId }, TableName.FolderCommit)) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderCheckpoint}.createdAt`, "desc"); + + if (limit !== undefined) { + query = query.limit(limit); + } + + return await query; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderId" }); + } + }; + + const findLatestByFolderId = async (folderId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderId }, TableName.FolderCommit)) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderCheckpoint}.createdAt`, "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestByFolderId" }); + } + }; + + const findNearestCheckpoint = async ( + folderCommitId: bigint, + folderId: string, + tx?: Knex + ): Promise<(CheckpointWithCommitInfo & { commitId: bigint }) | undefined> => { + try { + // Get the checkpoint with the highest commitId that's still less than or equal to our commit + const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .where(`${TableName.FolderCommit}.folderId`, "=", folderId) + .where(`${TableName.FolderCommit}.commitId`, "<=", folderCommitId.toString()) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("commitId").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderCommit}.commitId`, "desc") + .first(); + return nearestCheckpoint; + } catch (error) { + throw new DatabaseError({ error, name: "FindNearestCheckpoint" }); + } + }; + + return { + ...folderCheckpointOrm, + findByCommitId, + findByFolderId, + findLatestByFolderId, + findNearestCheckpoint + }; +}; diff --git a/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts new file mode 100644 index 0000000000..b2d74c5afd --- /dev/null +++ b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts @@ -0,0 +1,233 @@ +/* eslint-disable @typescript-eslint/no-misused-promises */ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { + TableName, + TFolderCommitChanges, + TFolderCommits, + TProjectEnvironments, + TSecretFolderVersions, + TSecretVersionsV2 +} from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCommitChangesDALFactory = ReturnType; + +// Base type with common fields +type BaseCommitChangeInfo = TFolderCommitChanges & { + actorMetadata: unknown; + actorType: string; + message?: string | null; + folderId: string; + createdAt: Date; +}; + +// Secret-specific change +export type SecretCommitChange = BaseCommitChangeInfo & { + resourceType: "secret"; + secretKey: string; + changeType: string; + secretVersionId?: string | null; + secretVersion: string; + secretId: string; + versions?: { + secretKey: string; + secretComment: string; + skipMultilineEncoding?: boolean | null; + secretReminderRepeatDays?: number | null; + secretReminderNote?: string | null; + metadata?: unknown; + tags?: string[] | null; + secretReminderRecipients?: string[] | null; + secretValue: string; + }[]; +}; + +// Folder-specific change +export type FolderCommitChange = BaseCommitChangeInfo & { + resourceType: "folder"; + folderName: string; + folderVersion: string; + folderChangeId: string; + versions?: { + version: string; + name?: string; + }[]; +}; + +// Discriminated union +export type CommitChangeWithCommitInfo = SecretCommitChange | FolderCommitChange; + +// Type guards +export const isSecretCommitChange = (change: CommitChangeWithCommitInfo): change is SecretCommitChange => + change.resourceType === "secret"; + +export const isFolderCommitChange = (change: CommitChangeWithCommitInfo): change is FolderCommitChange => + change.resourceType === "folder"; + +export const folderCommitChangesDALFactory = (db: TDbClient) => { + const folderCommitChangesOrm = ormify(db, TableName.FolderCommitChanges); + + const findByCommitId = async ( + folderCommitId: string, + projectId: string, + tx?: Knex + ): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCommitChanges) + .where(buildFindFilter({ folderCommitId }, TableName.FolderCommitChanges)) + .leftJoin( + TableName.FolderCommit, + `${TableName.FolderCommitChanges}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCommitChanges}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCommitChanges}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .leftJoin( + TableName.Environment, + `${TableName.FolderCommit}.envId`, + `${TableName.Environment}.id` + ) + .where((qb) => { + if (projectId) { + void qb.where(`${TableName.Environment}.projectId`, "=", projectId); + } + }) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .select( + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderChangeId"), + db.ref("version").withSchema(TableName.SecretFolderVersion).as("folderVersion"), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secretKey"), + db.ref("version").withSchema(TableName.SecretVersionV2).as("secretVersion"), + db.ref("secretId").withSchema(TableName.SecretVersionV2), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit), + db.ref("folderId").withSchema(TableName.FolderCommit) + ); + + return docs.map((doc) => { + // Determine if this is a secret or folder change based on populated fields + if (doc.secretKey && doc.secretVersion && doc.secretId) { + return { + ...doc, + resourceType: "secret", + secretKey: doc.secretKey, + secretVersion: doc.secretVersion.toString(), + secretId: doc.secretId + } as SecretCommitChange; + } + return { + ...doc, + resourceType: "folder", + folderName: doc.folderName, + folderVersion: doc.folderVersion.toString(), + folderChangeId: doc.folderChangeId + } as FolderCommitChange; + }); + } catch (error) { + throw new DatabaseError({ error, name: "FindByCommitId" }); + } + }; + + const findBySecretVersionId = async (secretVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCommitChanges & + Pick + >(TableName.FolderCommitChanges) + .where(buildFindFilter({ secretVersionId }, TableName.FolderCommitChanges)) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCommitChanges}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit), + db.ref("folderId").withSchema(TableName.FolderCommit), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secretKey"), + db.ref("version").withSchema(TableName.SecretVersionV2).as("secretVersion"), + db.ref("secretId").withSchema(TableName.SecretVersionV2) + ); + + return docs + .filter((doc) => doc.secretKey && doc.secretVersion && doc.secretId) + .map( + (doc): SecretCommitChange => ({ + ...doc, + resourceType: "secret", + secretKey: doc.secretKey, + secretVersion: doc.secretVersion.toString(), + secretId: doc.secretId + }) + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindBySecretVersionId" }); + } + }; + + const findByFolderVersionId = async (folderVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCommitChanges & + Pick + >(TableName.FolderCommitChanges) + .where(buildFindFilter({ folderVersionId }, TableName.FolderCommitChanges)) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCommitChanges}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit), + db.ref("folderId").withSchema(TableName.FolderCommit), + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderChangeId"), + db.ref("version").withSchema(TableName.SecretFolderVersion).as("folderVersion") + ); + + return docs + .filter((doc) => doc.folderName && doc.folderVersion && doc.folderChangeId) + .map( + (doc): FolderCommitChange => ({ + ...doc, + resourceType: "folder", + folderName: doc.folderName, + folderVersion: doc.folderVersion!.toString(), + folderChangeId: doc.folderChangeId + }) + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderVersionId" }); + } + }; + + return { + ...folderCommitChangesOrm, + findByCommitId, + findBySecretVersionId, + findByFolderVersionId + }; +}; diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts new file mode 100644 index 0000000000..e95dbb8393 --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -0,0 +1,513 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { + TableName, + TFolderCommitChanges, + TFolderCommits, + TProjectEnvironments, + TSecretFolderVersions, + TSecretVersionsV2 +} from "@app/db/schemas"; +import { DatabaseError, NotFoundError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCommitDALFactory = ReturnType; + +export const folderCommitDALFactory = (db: TDbClient) => { + const folderCommitOrm = ormify(db, TableName.FolderCommit); + const { delete: deleteOp, deleteById, ...restOfOrm } = folderCommitOrm; + + const findByFolderId = async (folderId: string, tx?: Knex): Promise => { + try { + const trx = tx || db.replicaNode(); + + // First, get all folder commits + const folderCommits = await trx(TableName.FolderCommit) + .where({ folderId }) + .select("*") + .orderBy("createdAt", "desc"); + + if (folderCommits.length === 0) return []; + + // Get all commit IDs + const commitIds = folderCommits.map((commit) => commit.id); + + // Then get all related changes + const changes = await trx(TableName.FolderCommitChanges).whereIn("folderCommitId", commitIds).select("*"); + + const changesMap = changes.reduce( + (acc, change) => { + const { folderCommitId } = change; + if (!acc[folderCommitId]) acc[folderCommitId] = []; + acc[folderCommitId].push(change); + return acc; + }, + {} as Record + ); + + return folderCommits.map((commit) => ({ + ...commit, + changes: changesMap[commit.id] || [] + })); + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderId" }); + } + }; + + const findLatestCommit = async ( + folderId: string, + projectId?: string, + tx?: Knex + ): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .leftJoin(TableName.Environment, `${TableName.FolderCommit}.envId`, `${TableName.Environment}.id`) + .where((qb) => { + if (projectId) { + void qb.where(`${TableName.Environment}.projectId`, "=", projectId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommit" }); + } + }; + + const findLatestCommitByFolderIds = async (folderIds: string[], tx?: Knex): Promise => { + try { + // First get max commitId for each folderId + const maxCommitIdSubquery = (tx || db.replicaNode())(TableName.FolderCommit) + .select("folderId") + .max("commitId as maxCommitId") + .whereIn("folderId", folderIds) + .groupBy("folderId"); + + // Join with main table to get complete records for each max commitId + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + .select(selectAllTableCols(TableName.FolderCommit)) + // eslint-disable-next-line func-names + .join(maxCommitIdSubquery.as("latest"), function () { + this.on(`${TableName.FolderCommit}.folderId`, "=", "latest.folderId").andOn( + `${TableName.FolderCommit}.commitId`, + "=", + "latest.maxCommitId" + ); + }); + + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitByFolderIds" }); + } + }; + + const findLatestEnvCommit = async (envId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where(`${TableName.FolderCommit}.envId`, "=", envId) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommit" }); + } + }; + + const findMultipleLatestCommits = async (folderIds: string[], tx?: Knex): Promise => { + try { + const knexInstance = tx || db.replicaNode(); + + // Get the latest commitId for each folderId + const subquery = knexInstance(TableName.FolderCommit) + .whereIn("folderId", folderIds) + .groupBy("folderId") + .select("folderId") + .max("commitId as maxCommitId"); + + // Then fetch the complete rows matching those latest commits + const docs = await knexInstance(TableName.FolderCommit) + // eslint-disable-next-line func-names + .innerJoin(subquery.as("latest"), function () { + this.on(`${TableName.FolderCommit}.folderId`, "=", "latest.folderId").andOn( + `${TableName.FolderCommit}.commitId`, + "=", + "latest.maxCommitId" + ); + }) + .select(selectAllTableCols(TableName.FolderCommit)); + + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindMultipleLatestCommits" }); + } + }; + + const getNumberOfCommitsSince = async (folderId: string, folderCommitId: string, tx?: Knex): Promise => { + try { + const referencedCommit = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ id: folderCommitId }) + .select("commitId") + .first(); + + if (referencedCommit?.commitId) { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .where("commitId", ">", referencedCommit.commitId) + .count(); + return Number(doc?.[0].count); + } + return 0; + } catch (error) { + throw new DatabaseError({ error, name: "getNumberOfCommitsSince" }); + } + }; + + const getEnvNumberOfCommitsSince = async (envId: string, folderCommitId: string, tx?: Knex): Promise => { + try { + const referencedCommit = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ id: folderCommitId }) + .select("commitId") + .first(); + + if (referencedCommit?.commitId) { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where(`${TableName.FolderCommit}.envId`, "=", envId) + .where("commitId", ">", referencedCommit.commitId) + .count(); + return Number(doc?.[0].count); + } + return 0; + } catch (error) { + throw new DatabaseError({ error, name: "getNumberOfCommitsSince" }); + } + }; + + const findCommitsToRecreate = async ( + folderId: string, + targetCommitNumber: bigint, + checkpointCommitNumber: bigint, + tx?: Knex + ): Promise< + (TFolderCommits & { + changes: (TFolderCommitChanges & { + referencedSecretId?: string; + referencedFolderId?: string; + folderName?: string; + folderVersion?: string; + secretKey?: string; + secretVersion?: string; + })[]; + })[] + > => { + try { + // First get all the commits in the range + const commits = await (tx || db.replicaNode())(TableName.FolderCommit) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderId }, TableName.FolderCommit)) + .andWhere(`${TableName.FolderCommit}.commitId`, ">", checkpointCommitNumber.toString()) + .andWhere(`${TableName.FolderCommit}.commitId`, "<=", targetCommitNumber.toString()) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy(`${TableName.FolderCommit}.commitId`, "asc"); + + // If no commits found, return empty array + if (!commits.length) { + return []; + } + + // Get all the commit IDs + const commitIds = commits.map((commit) => commit.id); + + // Get all changes for these commits in a single query + const allChanges = await (tx || db.replicaNode())(TableName.FolderCommitChanges) + .whereIn(`${TableName.FolderCommitChanges}.folderCommitId`, commitIds) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCommitChanges}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCommitChanges}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .select( + db.ref("secretId").withSchema(TableName.SecretVersionV2).as("referencedSecretId"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId"), + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), + db.ref("version").withSchema(TableName.SecretFolderVersion).as("folderVersion"), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secretKey"), + db.ref("version").withSchema(TableName.SecretVersionV2).as("secretVersion") + ); + + // Organize changes by commit ID + const changesByCommitId = allChanges.reduce( + (acc, change) => { + if (!acc[change.folderCommitId]) { + acc[change.folderCommitId] = []; + } + acc[change.folderCommitId].push(change); + return acc; + }, + {} as Record + ); + + // Attach changes to each commit + return commits.map((commit) => ({ + ...commit, + changes: changesByCommitId[commit.id] || [] + })); + } catch (error) { + throw new DatabaseError({ error, name: "FindCommitsToRecreate" }); + } + }; + + const findLatestCommitBetween = async ({ + folderId, + startCommitId, + endCommitId, + tx + }: { + folderId: string; + startCommitId?: string; + endCommitId: string; + tx?: Knex; + }): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where("commitId", "<=", endCommitId) + .where({ folderId }) + .where((qb) => { + if (startCommitId) { + void qb.where("commitId", ">=", startCommitId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitBetween" }); + } + }; + + const findAllCommitsBetween = async ({ + envId, + startCommitId, + endCommitId, + tx + }: { + envId?: string; + startCommitId?: string; + endCommitId?: string; + tx?: Knex; + }): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + .where((qb) => { + if (envId) { + void qb.where(`${TableName.FolderCommit}.envId`, "=", envId); + } + if (startCommitId) { + void qb.where("commitId", ">=", startCommitId); + } + if (endCommitId) { + void qb.where("commitId", "<=", endCommitId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc"); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitBetween" }); + } + }; + + const findAllFolderCommitsAfter = async ({ + envId, + startCommitId, + tx + }: { + envId?: string; + startCommitId?: string; + tx?: Knex; + }): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + .where((qb) => { + if (envId) { + void qb.where(`${TableName.FolderCommit}.envId`, "=", envId); + } + if (startCommitId) { + void qb.where("commitId", ">=", startCommitId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc"); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitBetween" }); + } + }; + + const findPreviousCommitTo = async ( + folderId: string, + commitId: string, + tx?: Knex + ): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .where("commitId", "<=", commitId) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindPreviousCommitTo" }); + } + }; + + const findById = async (id: string, tx?: Knex, projectId?: string): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ id }, TableName.FolderCommit)) + .leftJoin( + TableName.Environment, + `${TableName.FolderCommit}.envId`, + `${TableName.Environment}.id` + ) + .where((qb) => { + if (projectId) { + void qb.where(`${TableName.Environment}.projectId`, "=", projectId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + if (!doc) { + throw new NotFoundError({ + message: `Folder commit not found for ID ${id}` + }); + } + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindById" }); + } + }; + + const findByFolderIdPaginated = async ( + folderId: string, + options: { + offset?: number; + limit?: number; + search?: string; + sort?: "asc" | "desc"; + } = {}, + tx?: Knex + ): Promise<{ + commits: TFolderCommits[]; + total: number; + hasMore: boolean; + }> => { + try { + const { offset = 0, limit = 20, search, sort = "desc" } = options; + const trx = tx || db.replicaNode(); + + // Build base query + let baseQuery = trx(TableName.FolderCommit).where({ folderId }); + + // Add search functionality + if (search) { + baseQuery = baseQuery.where((qb) => { + void qb.whereILike("message", `%${search}%`); + }); + } + + // Get total count + const totalResult = await baseQuery.clone().count("*", { as: "count" }).first(); + const total = Number(totalResult?.count || 0); + + // Get paginated commits + const folderCommits = await baseQuery.select("*").orderBy("createdAt", sort).limit(limit).offset(offset); + + if (folderCommits.length === 0) { + return { commits: [], total, hasMore: false }; + } + + // Get all commit IDs for changes + const commitIds = folderCommits.map((commit) => commit.id); + + // Get all related changes + const changes = await trx(TableName.FolderCommitChanges).whereIn("folderCommitId", commitIds).select("*"); + + const changesMap = changes.reduce( + (acc, change) => { + const { folderCommitId } = change; + if (!acc[folderCommitId]) acc[folderCommitId] = []; + acc[folderCommitId].push(change); + return acc; + }, + {} as Record + ); + + const commitsWithChanges = folderCommits.map((commit) => ({ + ...commit, + changes: changesMap[commit.id] || [] + })); + + const hasMore = offset + limit < total; + + return { + commits: commitsWithChanges, + total, + hasMore + }; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderIdPaginated" }); + } + }; + + const findCommitBefore = async ( + folderId: string, + commitId: bigint, + tx?: Knex + ): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .where("commitId", "<", commitId.toString()) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindCommitBefore" }); + } + }; + + return { + ...restOfOrm, + findByFolderId, + findLatestCommit, + getNumberOfCommitsSince, + findCommitsToRecreate, + findMultipleLatestCommits, + findAllCommitsBetween, + findLatestCommitBetween, + findLatestEnvCommit, + getEnvNumberOfCommitsSince, + findLatestCommitByFolderIds, + findAllFolderCommitsAfter, + findPreviousCommitTo, + findById, + findByFolderIdPaginated, + findCommitBefore + }; +}; diff --git a/backend/src/services/folder-commit/folder-commit-queue.ts b/backend/src/services/folder-commit/folder-commit-queue.ts new file mode 100644 index 0000000000..fcb3487844 --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-queue.ts @@ -0,0 +1,282 @@ +import { Knex } from "knex"; + +import { TSecretFolders } from "@app/db/schemas"; +import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore"; +import { getConfig } from "@app/lib/config/env"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; + +import { TFolderTreeCheckpointDALFactory } from "../folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { TFolderTreeCheckpointResourcesDALFactory } from "../folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TFolderCommitDALFactory } from "./folder-commit-dal"; + +// Define types for job data +type TCreateFolderTreeCheckpointDTO = { + envId: string; + failedToAcquireLockCount?: number; + folderCommitId?: string; +}; + +type TFolderCommitQueueServiceFactoryDep = { + queueService: TQueueServiceFactory; + keyStore: Pick; + folderTreeCheckpointDAL: Pick< + TFolderTreeCheckpointDALFactory, + "create" | "findLatestByEnvId" | "findNearestCheckpoint" + >; + folderTreeCheckpointResourcesDAL: Pick< + TFolderTreeCheckpointResourcesDALFactory, + "insertMany" | "findByTreeCheckpointId" + >; + folderCommitDAL: Pick< + TFolderCommitDALFactory, + "findLatestEnvCommit" | "getEnvNumberOfCommitsSince" | "findMultipleLatestCommits" | "findById" + >; + folderDAL: Pick; +}; + +export type TFolderCommitQueueServiceFactory = ReturnType; + +export const folderCommitQueueServiceFactory = ({ + queueService, + keyStore, + folderTreeCheckpointDAL, + folderTreeCheckpointResourcesDAL, + folderCommitDAL, + folderDAL +}: TFolderCommitQueueServiceFactoryDep) => { + const appCfg = getConfig(); + + // Helper function to calculate delay for requeuing + const getRequeueDelay = (failureCount?: number) => { + if (!failureCount) return 0; + + const baseDelay = 5000; + const maxDelay = 30000; + + const delay = Math.min(baseDelay * 2 ** failureCount, maxDelay); + const jitter = delay * (0.5 + Math.random() * 0.5); + + return jitter; + }; + + const scheduleTreeCheckpoint = async (payload: TCreateFolderTreeCheckpointDTO) => { + const { envId, failedToAcquireLockCount = 0 } = payload; + + // Create a unique jobId for each retry to prevent conflicts + const jobId = + failedToAcquireLockCount > 0 ? `${envId}-retry-${failedToAcquireLockCount}-${Date.now()}` : `${envId}`; + + await queueService.queue(QueueName.FolderTreeCheckpoint, QueueJobs.CreateFolderTreeCheckpoint, payload, { + jobId, + delay: getRequeueDelay(failedToAcquireLockCount), + backoff: { + type: "exponential", + delay: 3000 + }, + removeOnFail: { + count: 3 + }, + removeOnComplete: true + }); + }; + + // Sort folders by hierarchy (copied from the source code) + const sortFoldersByHierarchy = (folders: TSecretFolders[]) => { + const childrenMap = new Map(); + const allFolderIds = new Set(); + + folders.forEach((folder) => { + if (folder.id) allFolderIds.add(folder.id); + }); + + folders.forEach((folder) => { + if (folder.parentId) { + const children = childrenMap.get(folder.parentId) || []; + children.push(folder); + childrenMap.set(folder.parentId, children); + } + }); + + const rootFolders = folders.filter((folder) => !folder.parentId || !allFolderIds.has(folder.parentId)); + + const result = []; + let currentLevel = rootFolders; + + while (currentLevel.length > 0) { + result.push(...currentLevel); + + const nextLevel = []; + for (const folder of currentLevel) { + if (folder.id) { + const children = childrenMap.get(folder.id) || []; + nextLevel.push(...children); + } + } + + currentLevel = nextLevel; + } + + return result; + }; + + const createFolderTreeCheckpoint = async (jobData: TCreateFolderTreeCheckpointDTO, tx?: Knex) => { + const { envId, folderCommitId, failedToAcquireLockCount = 0 } = jobData; + + logger.info(`Folder tree checkpoint creation started [envId=${envId}] [attempt=${failedToAcquireLockCount + 1}]`); + + // First, try to clear any stale locks before attempting to acquire + if (failedToAcquireLockCount > 1) { + try { + await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId)); + logger.info(`Cleared potential stale lock for envId ${envId} before attempt ${failedToAcquireLockCount + 1}`); + } catch (error) { + // This is fine if it fails, we'll still try to acquire the lock + logger.info(`No stale lock found for envId ${envId}`); + } + } + + let lock: Awaited> | undefined; + + try { + // Attempt to acquire the lock with a shorter timeout for first attempts + const timeout = failedToAcquireLockCount > 3 ? 60 * 1000 : 15 * 1000; + + logger.info(`Attempting to acquire lock for envId=${envId} with timeout ${timeout}ms`); + + lock = await keyStore.acquireLock([KeyStorePrefixes.FolderTreeCheckpoint(envId)], timeout); + + logger.info(`Successfully acquired lock for envId=${envId}`); + } catch (e) { + logger.info( + `Failed to acquire lock for folder tree checkpoint [envId=${envId}] [attempt=${failedToAcquireLockCount + 1}]` + ); + + // Requeue with incremented failure count if under max attempts + if (failedToAcquireLockCount < 10) { + // Force a delay between retries + const nextRetryCount = failedToAcquireLockCount + 1; + + logger.info(`Scheduling retry #${nextRetryCount} for folder tree checkpoint [envId=${envId}]`); + + // Create a new job with incremented counter + await scheduleTreeCheckpoint({ + envId, + folderCommitId, + failedToAcquireLockCount: nextRetryCount + }); + } else { + // Max retries reached + logger.error(`Maximum lock acquisition attempts (10) reached for envId ${envId}. Giving up.`); + // Try to force-clear the lock for next time + try { + await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId)); + } catch (clearError) { + logger.error(clearError, `Failed to clear lock after maximum retries for envId=${envId}`); + } + } + return; + } + + if (!lock) { + logger.error(`Lock is undefined after acquisition for envId=${envId}. This should never happen.`); + return; + } + + try { + logger.info(`Processing tree checkpoint data for envId=${envId}`); + + const latestTreeCheckpoint = await folderTreeCheckpointDAL.findLatestByEnvId(envId, tx); + + let latestCommit; + if (folderCommitId) { + latestCommit = await folderCommitDAL.findById(folderCommitId, tx); + } else { + latestCommit = await folderCommitDAL.findLatestEnvCommit(envId, tx); + } + if (!latestCommit) { + logger.info(`Latest commit ID not found for envId ${envId}`); + return; + } + const latestCommitId = latestCommit.id; + + if (latestTreeCheckpoint) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getEnvNumberOfCommitsSince( + envId, + latestTreeCheckpoint.folderCommitId, + tx + ); + if (commitsSinceLastCheckpoint < Number(appCfg.PIT_TREE_CHECKPOINT_WINDOW)) { + logger.info( + `Commits since last checkpoint ${commitsSinceLastCheckpoint} is less than ${appCfg.PIT_TREE_CHECKPOINT_WINDOW}` + ); + return; + } + } + + const folders = await folderDAL.findByEnvId(envId, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + const filteredFoldersIds = sortedFolders.filter((folder) => !folder.isReserved).map((folder) => folder.id); + + const folderCommits = await folderCommitDAL.findMultipleLatestCommits(filteredFoldersIds, tx); + const folderTreeCheckpoint = await folderTreeCheckpointDAL.create( + { + folderCommitId: latestCommitId + }, + tx + ); + + await folderTreeCheckpointResourcesDAL.insertMany( + folderCommits.map((folderCommit) => ({ + folderTreeCheckpointId: folderTreeCheckpoint.id, + folderId: folderCommit.folderId, + folderCommitId: folderCommit.id + })), + tx + ); + + logger.info(`Folder tree checkpoint created successfully: ${folderTreeCheckpoint.id}`); + } catch (error) { + logger.error(error, `Error processing folder tree checkpoint [envId=${envId}]`); + throw error; + } finally { + // Always release the lock + try { + if (lock) { + await lock.release(); + logger.info(`Released lock for folder tree checkpoint [envId=${envId}]`); + } else { + logger.error(`No lock to release for envId=${envId}. This should never happen.`); + } + } catch (releaseError) { + logger.error(releaseError, `Error releasing lock for folder tree checkpoint [envId=${envId}]`); + // Try to force delete the lock if release fails + try { + await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId)); + logger.info(`Force deleted lock after release failure for envId=${envId}`); + } catch (deleteError) { + logger.error(deleteError, `Failed to force delete lock after release failure for envId=${envId}`); + } + } + } + }; + + queueService.start(QueueName.FolderTreeCheckpoint, async (job) => { + try { + if (job.name === QueueJobs.CreateFolderTreeCheckpoint) { + const jobData = job.data as TCreateFolderTreeCheckpointDTO; + await createFolderTreeCheckpoint(jobData); + } + } catch (error) { + logger.error(error, "Error creating folder tree checkpoint:"); + throw error; + } + }); + + return { + scheduleTreeCheckpoint: (envId: string) => scheduleTreeCheckpoint({ envId }), + createFolderTreeCheckpoint: (envId: string, folderCommitId?: string, tx?: Knex) => + createFolderTreeCheckpoint({ envId, folderCommitId }, tx) + }; +}; diff --git a/backend/src/services/folder-commit/folder-commit-schemas.ts b/backend/src/services/folder-commit/folder-commit-schemas.ts new file mode 100644 index 0000000000..9f99bd2ccd --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-schemas.ts @@ -0,0 +1,143 @@ +import { z } from "zod"; + +// Base schema shared by both secret and folder changes +const baseChangeSchema = z.object({ + id: z.string(), + folderCommitId: z.string(), + changeType: z.string(), + isUpdate: z.boolean().optional(), + createdAt: z.union([z.string(), z.date()]), + updatedAt: z.union([z.string(), z.date()]), + actorMetadata: z + .union([ + z.object({ + id: z.string().optional(), + name: z.string().optional() + }), + z.unknown() + ]) + .optional(), + actorType: z.string(), + message: z.string().nullable().optional(), + folderId: z.string() +}); + +// Secret-specific versions schema +const secretVersionSchema = z.object({ + secretKey: z.string(), + secretComment: z.string(), + skipMultilineEncoding: z.boolean().nullable().optional(), + tags: z.array(z.string()).nullable().optional(), + metadata: z.unknown().nullable().optional(), + secretValue: z.string() +}); + +// Folder-specific versions schema +const folderVersionSchema = z.object({ + version: z.string().optional(), + name: z.string().optional(), + description: z.string().optional().nullable() +}); + +// Secret commit change schema +const secretCommitChangeSchema = baseChangeSchema.extend({ + resourceType: z.literal("secret"), + secretVersionId: z.string().optional().nullable(), + secretKey: z.string(), + secretVersion: z.union([z.string(), z.number()]), + secretId: z.string(), + versions: z.array(secretVersionSchema).optional() +}); + +// Folder commit change schema +const folderCommitChangeSchema = baseChangeSchema.extend({ + resourceType: z.literal("folder"), + folderVersionId: z.string().optional().nullable(), + folderName: z.string(), + folderChangeId: z.string(), + folderVersion: z.union([z.string(), z.number()]), + versions: z.array(folderVersionSchema).optional() +}); + +// Discriminated union for commit changes +export const commitChangeSchema = z.discriminatedUnion("resourceType", [ + secretCommitChangeSchema, + folderCommitChangeSchema +]); + +// Commit schema +const commitSchema = z.object({ + id: z.string(), + commitId: z.string(), + actorMetadata: z + .union([ + z.object({ + id: z.string().optional(), + name: z.string().optional() + }), + z.unknown() + ]) + .optional(), + actorType: z.string(), + message: z.string().nullable().optional(), + folderId: z.string(), + envId: z.string(), + createdAt: z.union([z.string(), z.date()]), + updatedAt: z.union([z.string(), z.date()]), + isLatest: z.boolean().default(false), + changes: z.array(commitChangeSchema).optional() +}); + +// Response schema +export const commitChangesResponseSchema = z.object({ + changes: commitSchema +}); + +// Base resource change schema for comparison results +const baseResourceChangeSchema = z.object({ + id: z.string(), + versionId: z.string(), + oldVersionId: z.string().optional(), + changeType: z.enum(["add", "delete", "update", "create"]), + commitId: z.union([z.string(), z.bigint()]), + createdAt: z.union([z.string(), z.date()]).optional(), + parentId: z.string().optional(), + isUpdate: z.boolean().optional(), + fromVersion: z.union([z.string(), z.number()]).optional() +}); + +// Secret resource change schema +const secretResourceChangeSchema = baseResourceChangeSchema.extend({ + type: z.literal("secret"), + secretKey: z.string(), + secretVersion: z.union([z.string(), z.number()]), + secretId: z.string(), + versions: z + .array( + z.object({ + secretKey: z.string().optional(), + secretComment: z.string().optional(), + skipMultilineEncoding: z.boolean().nullable().optional(), + secretReminderRepeatDays: z.number().nullable().optional(), + tags: z.array(z.string()).nullable().optional(), + metadata: z.unknown().nullable().optional(), + secretReminderNote: z.string().nullable().optional(), + secretValue: z.string().optional() + }) + ) + .optional() +}); + +// Folder resource change schema +const folderResourceChangeSchema = baseResourceChangeSchema.extend({ + type: z.literal("folder"), + folderName: z.string(), + folderVersion: z.union([z.string(), z.number()]), + versions: z.array(folderVersionSchema).optional() +}); + +// Discriminated union for resource changes +export const resourceChangeSchema = z.discriminatedUnion("type", [ + secretResourceChangeSchema, + folderResourceChangeSchema +]); diff --git a/backend/src/services/folder-commit/folder-commit-service.test.ts b/backend/src/services/folder-commit/folder-commit-service.test.ts new file mode 100644 index 0000000000..1879cf493c --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-service.test.ts @@ -0,0 +1,671 @@ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/return-await */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +import { Knex } from "knex"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ProjectType, TSecretFolderVersions, TSecretVersionsV2 } from "@app/db/schemas"; +import { BadRequestError, NotFoundError } from "@app/lib/errors"; + +import { ActorType } from "../auth/auth-type"; +import { + ChangeType, + CommitType, + folderCommitServiceFactory, + ResourceChange, + TFolderCommitServiceFactory +} from "./folder-commit-service"; + +// Mock config +vi.mock("@app/lib/config/env", () => ({ + getConfig: () => ({ + PIT_CHECKPOINT_WINDOW: 5, + PIT_TREE_CHECKPOINT_WINDOW: 10 + }) +})); + +// Mock logger +vi.mock("@app/lib/logger", () => ({ + logger: { + info: vi.fn(), + error: vi.fn() + } +})); + +describe("folderCommitServiceFactory", () => { + // Properly type the mock functions + type TransactionCallback = (trx: Knex) => Promise; + + // Mock dependencies + const mockFolderCommitDAL = { + create: vi.fn().mockResolvedValue({}), + findById: vi.fn().mockResolvedValue({}), + findByFolderId: vi.fn().mockResolvedValue([]), + findLatestCommit: vi.fn().mockResolvedValue({}), + transaction: vi.fn().mockImplementation((callback: TransactionCallback) => callback({} as Knex)), + getNumberOfCommitsSince: vi.fn().mockResolvedValue(0), + getEnvNumberOfCommitsSince: vi.fn().mockResolvedValue(0), + findCommitsToRecreate: vi.fn().mockResolvedValue([]), + findMultipleLatestCommits: vi.fn().mockResolvedValue([]), + findLatestCommitBetween: vi.fn().mockResolvedValue({}), + findAllCommitsBetween: vi.fn().mockResolvedValue([]), + findLatestEnvCommit: vi.fn().mockResolvedValue({}), + findLatestCommitByFolderIds: vi.fn().mockResolvedValue({}) + }; + + const mockKmsService = { + createCipherPairWithDataKey: vi.fn().mockResolvedValue({}) + }; + + const mockFolderCommitChangesDAL = { + create: vi.fn().mockResolvedValue({}), + findByCommitId: vi.fn().mockResolvedValue([]), + insertMany: vi.fn().mockResolvedValue([]) + }; + + const mockFolderCheckpointDAL = { + create: vi.fn().mockResolvedValue({}), + findByFolderId: vi.fn().mockResolvedValue([]), + findLatestByFolderId: vi.fn().mockResolvedValue(null), + findNearestCheckpoint: vi.fn().mockResolvedValue({}) + }; + + const mockFolderCheckpointResourcesDAL = { + insertMany: vi.fn().mockResolvedValue([]), + findByCheckpointId: vi.fn().mockResolvedValue([]) + }; + + const mockFolderTreeCheckpointDAL = { + create: vi.fn().mockResolvedValue({}), + findByProjectId: vi.fn().mockResolvedValue([]), + findLatestByProjectId: vi.fn().mockResolvedValue({}), + findNearestCheckpoint: vi.fn().mockResolvedValue({}), + findLatestByEnvId: vi.fn().mockResolvedValue({}) + }; + + const mockFolderTreeCheckpointResourcesDAL = { + insertMany: vi.fn().mockResolvedValue([]), + findByTreeCheckpointId: vi.fn().mockResolvedValue([]) + }; + + const mockUserDAL = { + findById: vi.fn().mockResolvedValue({}) + }; + + const mockIdentityDAL = { + findById: vi.fn().mockResolvedValue({}) + }; + + const mockFolderDAL = { + findByParentId: vi.fn().mockResolvedValue([]), + findByProjectId: vi.fn().mockResolvedValue([]), + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + update: vi.fn().mockResolvedValue({}), + find: vi.fn().mockResolvedValue([]), + findById: vi.fn().mockResolvedValue({}), + findByEnvId: vi.fn().mockResolvedValue([]), + findFoldersByRootAndIds: vi.fn().mockResolvedValue([]) + }; + + const mockFolderVersionDAL = { + findLatestFolderVersions: vi.fn().mockResolvedValue({}), + findById: vi.fn().mockResolvedValue({}), + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + find: vi.fn().mockResolvedValue({}), // Changed from [] to {} to match Object.values() expectation + findByIdsWithLatestVersion: vi.fn().mockResolvedValue({}) + }; + + const mockSecretVersionV2BridgeDAL = { + findLatestVersionByFolderId: vi.fn().mockResolvedValue([]), + findById: vi.fn().mockResolvedValue({}), + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + find: vi.fn().mockResolvedValue([]), + findByIdsWithLatestVersion: vi.fn().mockResolvedValue({}), + findLatestVersionMany: vi.fn().mockResolvedValue({}) + }; + + const mockSecretV2BridgeDAL = { + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + update: vi.fn().mockResolvedValue({}), + insertMany: vi.fn().mockResolvedValue([]), + invalidateSecretCacheByProjectId: vi.fn().mockResolvedValue({}) + }; + + const mockProjectDAL = { + findById: vi.fn().mockResolvedValue({}), + findProjectByEnvId: vi.fn().mockResolvedValue({}) + }; + + const mockFolderCommitQueueService = { + scheduleTreeCheckpoint: vi.fn().mockResolvedValue({}), + createFolderTreeCheckpoint: vi.fn().mockResolvedValue({}) + }; + + const mockPermissionService = { + getProjectPermission: vi.fn().mockResolvedValue({}) + }; + + const mockSecretTagDAL = { + findSecretTagsByVersionId: vi.fn().mockResolvedValue([]), + saveTagsToSecretV2: vi.fn().mockResolvedValue([]), + findSecretTagsBySecretId: vi.fn().mockResolvedValue([]), + deleteTagsToSecretV2: vi.fn().mockResolvedValue([]), + saveTagsToSecretVersionV2: vi.fn().mockResolvedValue([]) + }; + + const mockResourceMetadataDAL = { + find: vi.fn().mockResolvedValue([]), + insertMany: vi.fn().mockResolvedValue([]), + delete: vi.fn().mockResolvedValue([]) + }; + + let folderCommitService: TFolderCommitServiceFactory; + + beforeEach(() => { + vi.clearAllMocks(); + + folderCommitService = folderCommitServiceFactory({ + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderCommitDAL: mockFolderCommitDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderCommitChangesDAL: mockFolderCommitChangesDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderCheckpointDAL: mockFolderCheckpointDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderCheckpointResourcesDAL: mockFolderCheckpointResourcesDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderTreeCheckpointDAL: mockFolderTreeCheckpointDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderTreeCheckpointResourcesDAL: mockFolderTreeCheckpointResourcesDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + userDAL: mockUserDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + identityDAL: mockIdentityDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderDAL: mockFolderDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderVersionDAL: mockFolderVersionDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + secretVersionV2BridgeDAL: mockSecretVersionV2BridgeDAL, + projectDAL: mockProjectDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + secretV2BridgeDAL: mockSecretV2BridgeDAL, + folderCommitQueueService: mockFolderCommitQueueService, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + permissionService: mockPermissionService, + kmsService: mockKmsService, + secretTagDAL: mockSecretTagDAL, + resourceMetadataDAL: mockResourceMetadataDAL + }); + }); + + afterEach(() => { + vi.resetAllMocks(); + }); + + describe("createCommit", () => { + it("should successfully create a commit with user actor", async () => { + // Arrange + const userData = { id: "user-id", username: "testuser" }; + const folderData = { id: "folder-id", envId: "env-id" }; + const commitData = { id: "commit-id", folderId: "folder-id" }; + + mockUserDAL.findById.mockResolvedValue(userData); + mockFolderDAL.findById.mockResolvedValue(folderData); + mockFolderCommitDAL.create.mockResolvedValue(commitData); + mockFolderCheckpointDAL.findLatestByFolderId.mockResolvedValue(null); + mockFolderCommitDAL.findLatestCommit.mockResolvedValue({ id: "latest-commit-id" }); + mockFolderDAL.findByParentId.mockResolvedValue([]); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([]); + + const data = { + actor: { + type: ActorType.USER, + metadata: { id: userData.id } + }, + message: "Test commit", + folderId: folderData.id, + changes: [ + { + type: CommitType.ADD, + secretVersionId: "secret-version-1" + } + ] + }; + + // Act + const result = await folderCommitService.createCommit(data); + + // Assert + expect(mockUserDAL.findById).toHaveBeenCalledWith(userData.id, undefined); + expect(mockFolderDAL.findById).toHaveBeenCalledWith(folderData.id, undefined); + expect(mockFolderCommitDAL.create).toHaveBeenCalledWith( + expect.objectContaining({ + actorType: ActorType.USER, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + actorMetadata: expect.objectContaining({ name: userData.username }), + message: data.message, + folderId: data.folderId, + envId: folderData.envId + }), + undefined + ); + expect(mockFolderCommitChangesDAL.insertMany).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ + folderCommitId: commitData.id, + changeType: data.changes[0].type, + secretVersionId: data.changes[0].secretVersionId + }) + ]), + undefined + ); + expect(mockFolderCommitQueueService.scheduleTreeCheckpoint).toHaveBeenCalledWith(folderData.envId); + expect(result).toEqual(commitData); + }); + + it("should successfully create a commit with identity actor", async () => { + // Arrange + const identityData = { id: "identity-id", name: "testidentity" }; + const folderData = { id: "folder-id", envId: "env-id" }; + const commitData = { id: "commit-id", folderId: "folder-id" }; + + mockIdentityDAL.findById.mockResolvedValue(identityData); + mockFolderDAL.findById.mockResolvedValue(folderData); + mockFolderCommitDAL.create.mockResolvedValue(commitData); + mockFolderCheckpointDAL.findLatestByFolderId.mockResolvedValue(null); + mockFolderCommitDAL.findLatestCommit.mockResolvedValue({ id: "latest-commit-id" }); + mockFolderDAL.findByParentId.mockResolvedValue([]); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([]); + + // Mock folderVersionDAL.find to return an object with folder version data + mockFolderVersionDAL.find.mockResolvedValue({ + "folder-version-1": { + id: "folder-version-1", + folderId: "sub-folder-id", + envId: "env-id", + name: "Test Folder", + version: 1 + } + }); + + const data = { + actor: { + type: ActorType.IDENTITY, + metadata: { id: identityData.id } + }, + message: "Test commit", + folderId: folderData.id, + changes: [ + { + type: CommitType.ADD, + folderVersionId: "folder-version-1" + } + ], + omitIgnoreFilter: true + }; + + // Act + const result = await folderCommitService.createCommit(data); + + // Assert + expect(mockIdentityDAL.findById).toHaveBeenCalledWith(identityData.id, undefined); + expect(mockFolderDAL.findById).toHaveBeenCalledWith(folderData.id, undefined); + expect(mockFolderCommitDAL.create).toHaveBeenCalledWith( + expect.objectContaining({ + actorType: ActorType.IDENTITY, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + actorMetadata: expect.objectContaining({ name: identityData.name }), + message: data.message, + folderId: data.folderId, + envId: folderData.envId + }), + undefined + ); + expect(mockFolderCommitChangesDAL.insertMany).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ + folderCommitId: commitData.id, + changeType: data.changes[0].type, + folderVersionId: data.changes[0].folderVersionId + }) + ]), + undefined + ); + expect(mockFolderCommitQueueService.scheduleTreeCheckpoint).toHaveBeenCalledWith(folderData.envId); + expect(result).toEqual(commitData); + }); + + it("should throw NotFoundError when folder does not exist", async () => { + // Arrange + mockFolderDAL.findById.mockResolvedValue(null); + + const data = { + actor: { + type: ActorType.PLATFORM + }, + message: "Test commit", + folderId: "non-existent-folder", + changes: [] + }; + + // Act & Assert + await expect(folderCommitService.createCommit(data)).rejects.toThrow(NotFoundError); + expect(mockFolderDAL.findById).toHaveBeenCalledWith("non-existent-folder", undefined); + }); + }); + + describe("addCommitChange", () => { + it("should successfully add a change to an existing commit", async () => { + // Arrange + const commitData = { id: "commit-id", folderId: "folder-id" }; + const changeData = { id: "change-id", folderCommitId: "commit-id" }; + + mockFolderCommitDAL.findById.mockResolvedValue(commitData); + mockFolderCommitChangesDAL.create.mockResolvedValue(changeData); + + const data = { + folderCommitId: commitData.id, + changeType: CommitType.ADD, + secretVersionId: "secret-version-1" + }; + + // Act + const result = await folderCommitService.addCommitChange(data); + + // Assert + expect(mockFolderCommitDAL.findById).toHaveBeenCalledWith(commitData.id, undefined); + expect(mockFolderCommitChangesDAL.create).toHaveBeenCalledWith(data, undefined); + expect(result).toEqual(changeData); + }); + + it("should throw BadRequestError when neither secretVersionId nor folderVersionId is provided", async () => { + // Arrange + const data = { + folderCommitId: "commit-id", + changeType: CommitType.ADD + }; + + // Act & Assert + await expect(folderCommitService.addCommitChange(data)).rejects.toThrow(BadRequestError); + }); + + it("should throw NotFoundError when commit does not exist", async () => { + // Arrange + mockFolderCommitDAL.findById.mockResolvedValue(null); + + const data = { + folderCommitId: "non-existent-commit", + changeType: CommitType.ADD, + secretVersionId: "secret-version-1" + }; + + // Act & Assert + await expect(folderCommitService.addCommitChange(data)).rejects.toThrow(NotFoundError); + expect(mockFolderCommitDAL.findById).toHaveBeenCalledWith("non-existent-commit", undefined); + }); + }); + + // Note: reconstructFolderState is an internal function not exposed in the public API + // We'll test it indirectly through compareFolderStates + + describe("compareFolderStates", () => { + it("should mark all resources as creates when currentCommitId is not provided", async () => { + // Arrange + const targetCommitId = "target-commit-id"; + const targetCommit = { id: targetCommitId, commitId: 1, folderId: "folder-id" }; + + mockFolderCommitDAL.findById.mockResolvedValue(targetCommit); + // Mock how compareFolderStates would process the results internally + mockFolderCheckpointDAL.findNearestCheckpoint.mockResolvedValue({ id: "checkpoint-id", commitId: "hash-0" }); + mockFolderCheckpointResourcesDAL.findByCheckpointId.mockResolvedValue([ + { secretVersionId: "secret-version-1", referencedSecretId: "secret-1" }, + { folderVersionId: "folder-version-1", referencedFolderId: "folder-1" } + ]); + mockFolderCommitDAL.findCommitsToRecreate.mockResolvedValue([]); + mockProjectDAL.findProjectByEnvId.mockResolvedValue({ + id: "project-id", + name: "test-project", + type: ProjectType.SecretManager + }); + + // Act + const result = await folderCommitService.compareFolderStates({ + targetCommitId + }); + + // Assert + expect(mockFolderCommitDAL.findById).toHaveBeenCalledWith(targetCommitId, undefined); + + // Verify we get resources marked as create + expect(result).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + changeType: "create", + commitId: targetCommit.commitId + }) + ]) + ); + }); + }); + + describe("createFolderCheckpoint", () => { + it("should successfully create a checkpoint when force is true", async () => { + // Arrange + const folderCommitId = "commit-id"; + const folderId = "folder-id"; + const checkpointData = { id: "checkpoint-id", folderCommitId }; + + mockFolderDAL.findByParentId.mockResolvedValue([{ id: "subfolder-id" }]); + mockFolderVersionDAL.findLatestFolderVersions.mockResolvedValue({ "subfolder-id": { id: "folder-version-1" } }); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([{ id: "secret-version-1" }]); + mockFolderCheckpointDAL.create.mockResolvedValue(checkpointData); + + // Act + const result = await folderCommitService.createFolderCheckpoint({ + folderId, + folderCommitId, + force: true + }); + + // Assert + expect(mockFolderCheckpointDAL.create).toHaveBeenCalledWith({ folderCommitId }, undefined); + expect(mockFolderCheckpointResourcesDAL.insertMany).toHaveBeenCalled(); + expect(result).toBe(folderCommitId); + }); + }); + + describe("deepRollbackFolder", () => { + it("should throw NotFoundError when commit doesn't exist", async () => { + // Arrange + const targetCommitId = "non-existent-commit"; + const envId = "env-id"; + const actorId = "user-id"; + const actorType = ActorType.USER; + const projectId = "project-id"; + + // Mock the transaction to properly handle the error + mockFolderCommitDAL.transaction.mockImplementation(async (callback) => { + return await callback({} as Knex); + }); + + // Mock findById to return null inside the transaction + mockFolderCommitDAL.findById.mockResolvedValue(null); + + // Act & Assert + await expect( + folderCommitService.deepRollbackFolder(targetCommitId, envId, actorId, actorType, projectId) + ).rejects.toThrow(NotFoundError); + }); + }); + + describe("createFolderTreeCheckpoint", () => { + it("should create a tree checkpoint when checkpoint window is exceeded", async () => { + // Arrange + const envId = "env-id"; + const folderCommitId = "commit-id"; + const latestCommit = { id: folderCommitId }; + const latestTreeCheckpoint = { id: "tree-checkpoint-id", folderCommitId: "old-commit-id" }; + const folders = [ + { id: "folder-1", isReserved: false }, + { id: "folder-2", isReserved: false }, + { id: "folder-3", isReserved: true } // Reserved folders should be filtered out + ]; + const folderCommits = [ + { folderId: "folder-1", id: "commit-1" }, + { folderId: "folder-2", id: "commit-2" } + ]; + const treeCheckpoint = { id: "new-tree-checkpoint-id" }; + + mockFolderCommitDAL.findLatestEnvCommit.mockResolvedValue(latestCommit); + mockFolderTreeCheckpointDAL.findLatestByEnvId.mockResolvedValue(latestTreeCheckpoint); + mockFolderCommitDAL.getEnvNumberOfCommitsSince.mockResolvedValue(15); // More than PIT_TREE_CHECKPOINT_WINDOW (10) + mockFolderDAL.findByEnvId.mockResolvedValue(folders); + mockFolderCommitDAL.findMultipleLatestCommits.mockResolvedValue(folderCommits); + mockFolderTreeCheckpointDAL.create.mockResolvedValue(treeCheckpoint); + + // Act + await folderCommitService.createFolderTreeCheckpoint(envId); + + // Assert + expect(mockFolderCommitDAL.findLatestEnvCommit).toHaveBeenCalledWith(envId, undefined); + expect(mockFolderTreeCheckpointDAL.create).toHaveBeenCalledWith({ folderCommitId }, undefined); + }); + }); + + describe("applyFolderStateDifferences", () => { + it("should process changes correctly", async () => { + // Arrange + const folderId = "folder-id"; + const projectId = "project-id"; + const actorId = "user-id"; + const actorType = ActorType.USER; + + const differences = [ + { + id: "secret-1", + versionId: "v1", + changeType: ChangeType.CREATE, + commitId: BigInt(1) + } as ResourceChange, + { + id: "folder-1", + versionId: "v2", + changeType: ChangeType.UPDATE, + commitId: BigInt(1), + folderName: "Test Folder", + folderVersion: "v2" + } as ResourceChange + ]; + + const secretVersions = { + "secret-1": { + id: "secret-version-1", + createdAt: new Date(), + updatedAt: new Date(), + type: "shared", + folderId: "folder-1", + secretId: "secret-1", + version: 1, + key: "SECRET_KEY", + encryptedValue: Buffer.from("encrypted"), + encryptedComment: Buffer.from("comment"), + skipMultilineEncoding: false, + userId: "user-1", + envId: "env-1", + metadata: {} + } as TSecretVersionsV2 + }; + + const folderVersions = { + "folder-1": { + folderId: "folder-1", + version: 1, + name: "Test Folder", + envId: "env-1" + } as TSecretFolderVersions + }; + + // Mock folder lookup for the folder being processed + mockFolderDAL.findById.mockImplementation((id) => { + if (id === folderId) { + return Promise.resolve({ id: folderId, envId: "env-1" }); + } + return Promise.resolve(null); + }); + + // Mock latest commit lookup + mockFolderCommitDAL.findLatestCommit.mockImplementation((id) => { + if (id === folderId) { + return Promise.resolve({ id: "latest-commit-id", folderId }); + } + return Promise.resolve(null); + }); + + // Make sure findByParentId returns an array, not undefined + mockFolderDAL.findByParentId.mockResolvedValue([]); + + // Make sure other required functions return appropriate values + mockFolderCheckpointDAL.findLatestByFolderId.mockResolvedValue(null); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([]); + + // These mocks need to return objects with an id field + mockSecretVersionV2BridgeDAL.findByIdsWithLatestVersion.mockResolvedValue(Object.values(secretVersions)); + mockFolderVersionDAL.findByIdsWithLatestVersion.mockResolvedValue(Object.values(folderVersions)); + mockSecretV2BridgeDAL.insertMany.mockResolvedValue([{ id: "new-secret-1" }]); + mockSecretVersionV2BridgeDAL.create.mockResolvedValue({ id: "new-secret-version-1" }); + mockFolderDAL.updateById.mockResolvedValue({ id: "updated-folder-1" }); + mockFolderVersionDAL.create.mockResolvedValue({ id: "new-folder-version-1" }); + mockFolderCommitDAL.create.mockResolvedValue({ id: "new-commit-id" }); + mockSecretVersionV2BridgeDAL.findLatestVersionMany.mockResolvedValue([ + { + id: "secret-version-1", + createdAt: new Date(), + updatedAt: new Date(), + type: "shared", + folderId: "folder-1", + secretId: "secret-1", + version: 1, + key: "SECRET_KEY", + encryptedValue: Buffer.from("encrypted"), + encryptedComment: Buffer.from("comment"), + skipMultilineEncoding: false, + userId: "user-1", + envId: "env-1", + metadata: {} + } + ]); + + // Mock transaction + mockFolderCommitDAL.transaction.mockImplementation((callback: TransactionCallback) => callback({} as Knex)); + + // Act + const result = await folderCommitService.applyFolderStateDifferences({ + differences, + actorInfo: { + actorType, + actorId, + message: "Applying changes" + }, + folderId, + projectId, + reconstructNewFolders: false + }); + + // Assert + expect(mockFolderCommitDAL.create).toHaveBeenCalled(); + expect(mockSecretV2BridgeDAL.invalidateSecretCacheByProjectId).toHaveBeenCalledWith(projectId); + + // Check that we got the right counts + expect(result.totalChanges).toEqual(2); + }); + }); +}); diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts new file mode 100644 index 0000000000..aac3804aa2 --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -0,0 +1,2173 @@ +/* eslint-disable no-await-in-loop */ +import { ForbiddenError } from "@casl/ability"; +import { Knex } from "knex"; + +import { + ActionProjectType, + TSecretFolders, + TSecretFolderVersions, + TSecretV2TagJunctionInsert, + TSecretVersionsV2 +} from "@app/db/schemas"; +import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { ProjectPermissionCommitsActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { getConfig } from "@app/lib/config/env"; +import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; +import { chunkArray } from "@app/lib/fn"; +import { logger } from "@app/lib/logger"; + +import { ActorAuthMethod, ActorType } from "../auth/auth-type"; +import { TFolderCheckpointDALFactory } from "../folder-checkpoint/folder-checkpoint-dal"; +import { TFolderCheckpointResourcesDALFactory } from "../folder-checkpoint-resources/folder-checkpoint-resources-dal"; +import { TFolderCommitChangesDALFactory } from "../folder-commit-changes/folder-commit-changes-dal"; +import { TFolderTreeCheckpointDALFactory } from "../folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { TFolderTreeCheckpointResourcesDALFactory } from "../folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; +import { TIdentityDALFactory } from "../identity/identity-dal"; +import { TKmsServiceFactory } from "../kms/kms-service"; +import { KmsDataKey } from "../kms/kms-types"; +import { TProjectDALFactory } from "../project/project-dal"; +import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; +import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; +import { TUserDALFactory } from "../user/user-dal"; +import { TFolderCommitDALFactory } from "./folder-commit-dal"; +import { TFolderCommitQueueServiceFactory } from "./folder-commit-queue"; + +export enum ChangeType { + ADD = "add", + DELETE = "delete", + UPDATE = "update", + CREATE = "create" +} + +export enum CommitType { + ADD = "add", + DELETE = "delete" +} + +export enum ResourceType { + SECRET = "secret", + FOLDER = "folder" +} + +type TCreateCommitDTO = { + actor: { + type: string; + metadata?: { + name?: string; + id?: string; + }; + }; + message?: string; + folderId: string; + changes: { + type: string; + secretVersionId?: string; + folderVersionId?: string; + isUpdate?: boolean; + folderId?: string; + }[]; + omitIgnoreFilter?: boolean; +}; + +type TCommitChangeDTO = { + folderCommitId: string; + changeType: string; + secretVersionId?: string; + folderVersionId?: string; +}; + +type BaseChange = { + id: string; + versionId: string; + oldVersionId?: string; + changeType: ChangeType; + commitId: bigint; + createdAt?: Date; + parentId?: string; + isUpdate?: boolean; + fromVersion?: string; +}; + +type SecretChange = BaseChange & { + type: ResourceType.SECRET; + secretKey: string; + secretVersion: string; + secretId: string; + versions?: { + secretKey?: string; + secretComment?: string; + skipMultilineEncoding?: boolean | null; + metadata?: unknown; + tags?: string[] | null; + secretValue?: string; + }[]; +}; + +type FolderChange = BaseChange & { + type: ResourceType.FOLDER; + folderName: string; + folderVersion: string; + versions?: { + name: string; + description?: string | null; + }[]; +}; + +type SecretTargetChange = { + type: ResourceType.SECRET; + id: string; + versionId: string; + secretKey: string; + secretVersion: string; + fromVersion?: string; +}; + +type FolderTargetChange = { + type: ResourceType.FOLDER; + id: string; + versionId: string; + folderName: string; + folderVersion: string; + fromVersion?: string; +}; + +export type ResourceChange = SecretChange | FolderChange; + +type ActorInfo = { + actorType: string; + actorId?: string; + message?: string; +}; + +type StateChangeResult = { + secretChangesCount: number; + folderChangesCount: number; + totalChanges: number; +}; + +type TFolderCommitServiceFactoryDep = { + folderCommitDAL: TFolderCommitDALFactory; + folderCommitChangesDAL: TFolderCommitChangesDALFactory; + folderCheckpointDAL: TFolderCheckpointDALFactory; + folderCheckpointResourcesDAL: TFolderCheckpointResourcesDALFactory; + folderTreeCheckpointDAL: TFolderTreeCheckpointDALFactory; + folderTreeCheckpointResourcesDAL: TFolderTreeCheckpointResourcesDALFactory; + userDAL: TUserDALFactory; + identityDAL: TIdentityDALFactory; + folderDAL: TSecretFolderDALFactory; + folderVersionDAL: TSecretFolderVersionDALFactory; + secretVersionV2BridgeDAL: TSecretVersionV2DALFactory; + secretV2BridgeDAL: TSecretV2BridgeDALFactory; + projectDAL: Pick; + folderCommitQueueService?: Pick< + TFolderCommitQueueServiceFactory, + "scheduleTreeCheckpoint" | "createFolderTreeCheckpoint" + >; + permissionService?: TPermissionServiceFactory; + kmsService: Pick; + secretTagDAL: Pick< + TSecretTagDALFactory, + | "findSecretTagsByVersionId" + | "saveTagsToSecretV2" + | "findSecretTagsBySecretId" + | "deleteTagsToSecretV2" + | "saveTagsToSecretVersionV2" + >; + resourceMetadataDAL: Pick; +}; + +export const folderCommitServiceFactory = ({ + folderCommitDAL, + folderCommitChangesDAL, + folderCheckpointDAL, + folderTreeCheckpointDAL, + folderCheckpointResourcesDAL, + userDAL, + identityDAL, + folderDAL, + folderVersionDAL, + secretVersionV2BridgeDAL, + projectDAL, + secretV2BridgeDAL, + folderTreeCheckpointResourcesDAL, + folderCommitQueueService, + permissionService, + kmsService, + secretTagDAL, + resourceMetadataDAL +}: TFolderCommitServiceFactoryDep) => { + const appCfg = getConfig(); + + const checkProjectCommitReadPermission = async ({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + }: { + actor: ActorType; + actorId: string; + projectId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + }) => { + if (!permissionService) { + throw new Error("Permission service not initialized"); + } + const { permission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.SecretManager + }); + + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); + }; + + /** + * Fetches all resources within a folder + */ + const getFolderResources = async (folderId: string, tx?: Knex) => { + const resources = []; + const subFolders = await folderDAL.findByParentId(folderId, tx); + + if (subFolders.length > 0) { + const subFolderIds = subFolders.map((folder) => folder.id); + const folderVersions = await folderVersionDAL.findLatestFolderVersions(subFolderIds, tx); + resources.push( + ...Object.values(folderVersions).map((folderVersion) => ({ + folderVersionId: folderVersion.id, + secretVersionId: undefined + })) + ); + } + + const secretVersions = await secretVersionV2BridgeDAL.findLatestVersionByFolderId(folderId, tx); + if (secretVersions.length > 0) { + resources.push( + ...secretVersions.map((secretVersion) => ({ secretVersionId: secretVersion.id, folderVersionId: undefined })) + ); + } + + return resources; + }; + + /** + * Creates a checkpoint for a folder if necessary + */ + const createFolderCheckpoint = async ({ + folderId, + folderCommitId, + force = false, + tx + }: { + folderId: string; + folderCommitId?: string; + force?: boolean; + tx?: Knex; + }) => { + let latestCommitId = folderCommitId; + const latestCheckpoint = await folderCheckpointDAL.findLatestByFolderId(folderId, tx); + + if (!latestCommitId) { + const latestCommit = await folderCommitDAL.findLatestCommit(folderId, undefined, tx); + if (!latestCommit) { + throw new BadRequestError({ message: "Latest commit ID not found" }); + } + latestCommitId = latestCommit.id; + } + + if (!force && latestCheckpoint) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getNumberOfCommitsSince( + folderId, + latestCheckpoint.folderCommitId, + tx + ); + if (commitsSinceLastCheckpoint < Number(appCfg.PIT_CHECKPOINT_WINDOW)) { + return; + } + } + + const checkpointResources = await getFolderResources(folderId, tx); + + const newCheckpoint = await folderCheckpointDAL.create( + { + folderCommitId: latestCommitId + }, + tx + ); + const batchSize = 500; + const chunks = chunkArray(checkpointResources, batchSize); + + await Promise.all( + chunks.map(async (chunk) => { + await folderCheckpointResourcesDAL.insertMany( + chunk.map((resource) => ({ folderCheckpointId: newCheckpoint.id, ...resource })), + tx + ); + }) + ); + + return latestCommitId; + }; + + /** + * Reconstructs the state of a folder at a specific commit + */ + const reconstructFolderState = async ( + folderCommitId: string, + tx?: Knex + ): Promise<(SecretTargetChange | FolderTargetChange)[]> => { + const targetCommit = await folderCommitDAL.findById(folderCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `Commit with ID ${folderCommitId} not found` }); + } + + const nearestCheckpoint = await folderCheckpointDAL.findNearestCheckpoint( + targetCommit.commitId, + targetCommit.folderId, + tx + ); + if (!nearestCheckpoint) { + throw new NotFoundError({ message: `Nearest checkpoint not found for commit ${folderCommitId}` }); + } + + const checkpointResources = await folderCheckpointResourcesDAL.findByCheckpointId(nearestCheckpoint.id, tx); + + const folderState: Record = {}; + + // Add all checkpoint resources to initial state + checkpointResources.forEach((resource) => { + if (resource.secretVersionId && resource.referencedSecretId) { + folderState[`secret-${resource.referencedSecretId}`] = { + type: ResourceType.SECRET, + id: resource.referencedSecretId, + versionId: resource.secretVersionId, + secretKey: resource.secretKey, + secretVersion: resource.secretVersion + } as SecretTargetChange; + } else if (resource.folderVersionId && resource.referencedFolderId) { + folderState[`folder-${resource.referencedFolderId}`] = { + type: ResourceType.FOLDER, + id: resource.referencedFolderId, + versionId: resource.folderVersionId, + folderName: resource.folderName, + folderVersion: resource.folderVersion + } as FolderTargetChange; + } + }); + + const commitsToRecreate = await folderCommitDAL.findCommitsToRecreate( + targetCommit.folderId, + targetCommit.commitId, + nearestCheckpoint.commitId, + tx + ); + + // Process commits to recreate final state + for (const commit of commitsToRecreate) { + // eslint-disable-next-line no-continue + if (!commit.changes) continue; + + for (const change of commit.changes) { + if (change.secretVersionId && change.referencedSecretId) { + const key = `secret-${change.referencedSecretId}`; + + if (change.changeType.toLowerCase() === "add") { + folderState[key] = { + type: ResourceType.SECRET, + id: change.referencedSecretId, + versionId: change.secretVersionId, + secretKey: change.secretKey, + secretVersion: change.secretVersion + } as SecretTargetChange; + } else if (change.changeType.toLowerCase() === "delete") { + delete folderState[key]; + } + } else if (change.folderVersionId && change.referencedFolderId) { + const key = `folder-${change.referencedFolderId}`; + + if (change.changeType.toLowerCase() === "add") { + folderState[key] = { + type: ResourceType.FOLDER, + id: change.referencedFolderId, + versionId: change.folderVersionId, + folderName: change.folderName, + folderVersion: change.folderVersion + } as FolderTargetChange; + } else if (change.changeType.toLowerCase() === "delete") { + delete folderState[key]; + } + } + } + } + return Object.values(folderState); + }; + + /** + * Compares folder states between two commits and returns the differences + */ + const compareFolderStates = async ({ + currentCommitId, + targetCommitId, + defaultOperation = "create", + tx + }: { + currentCommitId?: string; + targetCommitId: string; + defaultOperation?: "create" | "update" | "delete"; + tx?: Knex; + }): Promise => { + const targetCommit = await folderCommitDAL.findById(targetCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `Commit with ID ${targetCommitId} not found` }); + } + + const project = await projectDAL.findProjectByEnvId(targetCommit.envId, tx); + + if (!project) { + throw new NotFoundError({ message: `No project found for envId ${targetCommit.envId}` }); + } + + // If currentCommitId is not provided, mark all resources in target as creates + if (!currentCommitId) { + const targetState = await reconstructFolderState(targetCommitId, tx); + + return targetState + .map((resource): ResourceChange | null => { + if (resource.type === ResourceType.SECRET) { + return { + type: ResourceType.SECRET, + id: resource.id, + versionId: resource.versionId, + changeType: defaultOperation as ChangeType, + commitId: targetCommit.commitId, + secretKey: resource.secretKey, + secretVersion: resource.secretVersion, + secretId: resource.id + }; + } + if (resource.type === ResourceType.FOLDER) { + return { + type: ResourceType.FOLDER, + id: resource.id, + versionId: resource.versionId, + changeType: defaultOperation as ChangeType, + commitId: targetCommit.commitId, + folderName: resource.folderName, + folderVersion: resource.folderVersion + }; + } + return null; + }) + .filter((change): change is ResourceChange => !!change); + } + + // Original logic for when currentCommitId is provided + const currentState = await reconstructFolderState(currentCommitId, tx); + const targetState = await reconstructFolderState(targetCommitId, tx); + + // Create lookup maps for easier comparison + const currentMap: Record = {}; + const targetMap: Record< + string, + { + type: string; + id: string; + versionId: string; + secretKey?: string; + secretVersion?: string; + folderName?: string; + folderVersion?: string; + fromVersion?: string; + } + > = {}; + + // Build lookup maps + currentState.forEach((resource) => { + const key = `${resource.type}-${resource.id}`; + currentMap[key] = resource; + }); + + targetState.forEach((resource) => { + const key = `${resource.type}-${resource.id}`; + targetMap[key] = resource; + }); + + // Track differences + const differences: ResourceChange[] = []; + + // Find deletes and updates + Object.keys(currentMap).forEach((key) => { + const currentResource = currentMap[key]; + const targetResource = targetMap[key]; + + if (!targetResource) { + // Resource was deleted + if (currentResource.type === ResourceType.SECRET) { + differences.push({ + type: ResourceType.SECRET, + id: currentResource.id, + versionId: currentResource.versionId, + changeType: ChangeType.DELETE, + commitId: targetCommit.commitId, + secretKey: currentResource.secretKey, + secretVersion: currentResource.secretVersion, + secretId: currentResource.id, + fromVersion: currentResource.versionId + }); + } else if (currentResource.type === ResourceType.FOLDER) { + differences.push({ + type: ResourceType.FOLDER, + id: currentResource.id, + versionId: currentResource.versionId, + changeType: ChangeType.DELETE, + commitId: targetCommit.commitId, + folderName: currentResource.folderName, + folderVersion: currentResource.folderVersion, + fromVersion: currentResource.versionId + }); + } + } else if (currentResource.versionId !== targetResource.versionId) { + // Resource was updated + if (targetResource.type === ResourceType.SECRET) { + const secretCurrentResource = currentResource as SecretTargetChange; + const secretTargetResource = targetResource as SecretTargetChange; + differences.push({ + type: ResourceType.SECRET, + id: secretTargetResource.id, + versionId: secretTargetResource.versionId, + changeType: ChangeType.UPDATE, + commitId: targetCommit.commitId, + secretKey: secretTargetResource.secretKey, + secretVersion: secretTargetResource.secretVersion, + secretId: secretTargetResource.id, + fromVersion: secretCurrentResource.secretVersion + }); + } else if (targetResource.type === ResourceType.FOLDER) { + const folderCurrentResource = currentResource as FolderTargetChange; + const folderTargetResource = targetResource as FolderTargetChange; + + differences.push({ + type: ResourceType.FOLDER, + id: folderTargetResource.id, + versionId: folderTargetResource.versionId, + changeType: ChangeType.UPDATE, + commitId: targetCommit.commitId, + folderName: folderTargetResource.folderName, + folderVersion: folderTargetResource.folderVersion, + fromVersion: folderCurrentResource.folderVersion + }); + } + } + }); + + // Find new resources + Object.keys(targetMap).forEach((key) => { + if (!currentMap[key]) { + const targetResource = targetMap[key]; + if (targetResource.type === ResourceType.SECRET) { + const secretTargetResource = targetResource as SecretTargetChange; + differences.push({ + type: ResourceType.SECRET, + id: secretTargetResource.id, + versionId: secretTargetResource.versionId, + changeType: ChangeType.CREATE, + commitId: targetCommit.commitId, + createdAt: targetCommit.createdAt, + secretKey: secretTargetResource.secretKey, + secretVersion: secretTargetResource.secretVersion, + secretId: secretTargetResource.id + }); + } else if (targetResource.type === ResourceType.FOLDER) { + const folderTargetResource = targetResource as FolderTargetChange; + differences.push({ + type: ResourceType.FOLDER, + id: folderTargetResource.id, + versionId: folderTargetResource.versionId, + changeType: ChangeType.CREATE, + commitId: targetCommit.commitId, + createdAt: targetCommit.createdAt, + folderName: folderTargetResource.folderName, + folderVersion: folderTargetResource.folderVersion + }); + } + } + }); + + const removeNoChangeUpdate: string[] = []; + + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId: project.id + }); + + await Promise.all( + differences.map(async (change) => { + if (change.changeType === ChangeType.UPDATE) { + if (change.type === ResourceType.FOLDER && change.folderVersion && change.fromVersion) { + const versions = await folderVersionDAL.find({ + folderId: change.id, + $in: { + version: [Number(change.folderVersion), Number(change.fromVersion)] + } + }); + const versionsShaped = versions.map((version) => ({ + name: version.name, + description: version.description + })); + const uniqueVersions = versionsShaped.filter( + (item, index, arr) => + arr.findIndex((other) => + Object.entries(item).every( + ([key, value]) => JSON.stringify(value) === JSON.stringify(other[key as keyof typeof other]) + ) + ) === index + ); + if (uniqueVersions.length === 1) { + removeNoChangeUpdate.push(change.id); + } + } else if (change.type === ResourceType.SECRET && change.secretVersion && change.fromVersion) { + const versions = await secretVersionV2BridgeDAL.findVersionsBySecretIdWithActors({ + secretId: change.id, + projectId: project.id, + secretVersions: [change.secretVersion, change.fromVersion] + }); + const versionsShaped = versions.map((el) => ({ + secretKey: el.key, + secretComment: el.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() + : "", + skipMultilineEncoding: el.skipMultilineEncoding, + secretReminderRepeatDays: el.reminderRepeatDays, + tags: el.tags, + metadata: el.metadata, + secretReminderNote: el.reminderNote, + secretValue: el.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() + : "" + })); + const uniqueVersions = versionsShaped.filter( + (item, index, arr) => + arr.findIndex((other) => + Object.entries(item).every( + ([key, value]) => JSON.stringify(value) === JSON.stringify(other[key as keyof typeof other]) + ) + ) === index + ); + if (uniqueVersions.length === 1) { + removeNoChangeUpdate.push(change.id); + } + } + } + }) + ); + return differences.filter((change) => !removeNoChangeUpdate.includes(change.id)); + }; + + /** + * Adds a change to an existing commit + */ + const addCommitChange = async (data: TCommitChangeDTO, tx?: Knex) => { + try { + if (!data.secretVersionId && !data.folderVersionId) { + throw new BadRequestError({ message: "Either secretVersionId or folderVersionId must be provided" }); + } + + const commit = await folderCommitDAL.findById(data.folderCommitId, tx); + if (!commit) { + throw new NotFoundError({ message: `Commit with ID ${data.folderCommitId} not found` }); + } + + return await folderCommitChangesDAL.create(data, tx); + } catch (error) { + if (error instanceof NotFoundError || error instanceof BadRequestError) { + throw error; + } + throw new DatabaseError({ error, name: "AddCommitChange" }); + } + }; + + const createDeleteCommitForNestedFolders = async ({ + folderId, + actorMetadata, + actorType, + envId, + parentFolderName, + step = 1, + tx + }: { + folderId: string; + actorMetadata: Record; + actorType: string; + envId: string; + parentFolderName: string; + step?: number; + tx?: Knex; + }) => { + if (step > 20) { + logger.info(`createDeleteCommitForNestedFolders - Max step reached for folder ${folderId}`); + return; + } + logger.info(`Creating delete commit for nested folders ${folderId}`); + const folderVersion = await folderVersionDAL.findLatestVersion(folderId, tx); + if (!folderVersion) { + logger.info(`No folder version found for ${folderId}`); + return; + } + const lastFolderCommit = await folderCommitDAL.findLatestCommit(folderId, undefined, tx); + if (!lastFolderCommit) { + logger.info(`No commit found for folder ${folderId}`); + return; + } + const folderState = await reconstructFolderState(lastFolderCommit.id, tx); + const changes = folderState.map((resource) => ({ + type: ChangeType.DELETE, + folderId: resource.id, + folderName: resource.type === ResourceType.FOLDER ? resource.folderName : undefined, + secretVersionId: resource.type === ResourceType.SECRET ? resource.versionId : undefined, + folderVersionId: resource.type === ResourceType.FOLDER ? resource.versionId : undefined, + secretKey: resource.type === ResourceType.SECRET ? resource.secretKey : undefined + })); + logger.info(`Found ${changes.length} changes for ${folderId}`); + + const newCommit = await folderCommitDAL.create( + { + actorMetadata, + actorType, + message: `Parent folder ${parentFolderName} deleted`, + folderId, + envId + }, + tx + ); + + const batchSize = 500; + const chunks = chunkArray(changes, batchSize); + + await Promise.all( + chunks.map(async (chunk) => { + await folderCommitChangesDAL.insertMany( + chunk.map((change) => ({ + folderCommitId: newCommit.id, + changeType: CommitType.DELETE, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId, + isUpdate: false + })), + tx + ); + }) + ); + + await Promise.all( + changes + .filter((change) => change.type === ChangeType.DELETE && change.folderVersionId) + .map(async (change) => { + await createDeleteCommitForNestedFolders({ + folderId: change.folderId, + actorMetadata, + actorType, + envId, + parentFolderName: folderVersion.name, + step: step + 1, + tx + }); + }) + ); + }; + + const compareSecretVersions = async ( + version1: TSecretVersionsV2 & { tags: { id: string }[] }, + version2: TSecretVersionsV2 & { tags: { id: string }[] }, + projectId: string + ) => { + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const objectsEqual = (o1: unknown, o2: unknown): boolean => { + if (typeof o1 !== "object" || o1 === null || typeof o2 !== "object" || o2 === null) { + return o1 === o2; + } + + const obj1 = o1 as Record; + const obj2 = o2 as Record; + return ( + Object.keys(obj1).length === Object.keys(obj2).length && Object.keys(obj1).every((p) => obj1[p] === obj2[p]) + ); + }; + + const arraysEqual = (a1: unknown[], a2: unknown[]) => + a1.length === a2.length && a1.every((obj1) => a2.some((obj2) => objectsEqual(obj1, obj2))); + + const version1Reshaped = { + ...version1, + encryptedValue: version1.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: version1.encryptedValue }).toString() + : "", + encryptedComment: version1.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: version1.encryptedComment }).toString() + : "", + metadata: version1.metadata as { key: string; value: string }[], + tags: version1.tags.map((tag) => tag.id) + }; + const version2Reshaped = { + ...version2, + encryptedValue: version2.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: version2.encryptedValue }).toString() + : "", + encryptedComment: version2.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: version2.encryptedComment }).toString() + : "", + metadata: version2.metadata as { key: string; value: string }[], + tags: version2.tags.map((tag) => tag.id) + }; + return ( + version1Reshaped.key === version2Reshaped.key && + version1Reshaped.encryptedValue === version2Reshaped.encryptedValue && + version1Reshaped.encryptedComment === version2Reshaped.encryptedComment && + version1Reshaped.skipMultilineEncoding === version2Reshaped.skipMultilineEncoding && + arraysEqual(version1Reshaped.metadata, version2Reshaped.metadata) && + version1Reshaped.tags.length === version2Reshaped.tags.length && + version1Reshaped.tags.every((tag) => version2Reshaped.tags.includes(tag)) + ); + }; + + const filterIgnoredChanges = async ( + changes: { + type: string; + secretVersionId?: string; + folderVersionId?: string; + isUpdate?: boolean; + folderId?: string; + }[], + projectId: string, + tx?: Knex + ) => { + let filteredChanges = [...changes]; + for (const change of changes) { + if (change.type === ChangeType.ADD && change.isUpdate && change.secretVersionId) { + const secretVersions = await secretVersionV2BridgeDAL.findByIdAndPreviousVersion(change.secretVersionId, tx); + const comparison = await compareSecretVersions(secretVersions[0], secretVersions[1], projectId); + if (comparison) { + filteredChanges = filteredChanges.filter( + (filteredChange) => filteredChange.secretVersionId !== change.secretVersionId + ); + } + } + } + return filteredChanges; + }; + + /** + * Creates a new commit with the provided changes + */ + const createCommit = async (data: TCreateCommitDTO, tx?: Knex) => { + try { + const metadata = { ...data.actor.metadata } || {}; + + if (data.actor.type === ActorType.USER && data.actor.metadata?.id) { + const user = await userDAL.findById(data.actor.metadata?.id, tx); + metadata.name = user?.username; + } + + if (data.actor.type === ActorType.IDENTITY && data.actor.metadata?.id) { + const identity = await identityDAL.findById(data.actor.metadata?.id, tx); + metadata.name = identity?.name; + } + + const folder = await folderDAL.findById(data.folderId, tx); + if (!folder) { + throw new NotFoundError({ message: `Folder with ID ${data.folderId} not found` }); + } + + let { changes } = data; + if (!data.omitIgnoreFilter) { + const project = await projectDAL.findProjectByEnvId(folder.envId, tx); + + if (!project) { + return; + } + + changes = await filterIgnoredChanges(data.changes, project.id, tx); + if (changes.length === 0) { + return; + } + } + + const newCommit = await folderCommitDAL.create( + { + actorMetadata: metadata, + actorType: data.actor.type, + message: data.message, + folderId: data.folderId, + envId: folder.envId + }, + tx + ); + + const batchSize = 500; + const chunks = chunkArray(changes, batchSize); + + await Promise.all( + chunks.map(async (chunk) => { + await folderCommitChangesDAL.insertMany( + chunk.map((change) => ({ + folderCommitId: newCommit.id, + changeType: change.type, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId, + isUpdate: change.isUpdate || false + })), + tx + ); + }) + ); + + await Promise.all( + changes.map(async (change) => { + if (change.type === ChangeType.DELETE && change.folderId) { + await createDeleteCommitForNestedFolders({ + folderId: change.folderId, + actorMetadata: metadata, + actorType: data.actor.type, + envId: folder.envId, + parentFolderName: folder.name, + tx + }); + } + }) + ); + + await createFolderCheckpoint({ folderId: data.folderId, folderCommitId: newCommit.id, tx }); + if (folderCommitQueueService) { + if (!folder.parentId) { + const previousTreeCommit = await folderTreeCheckpointDAL.findLatestByEnvId(folder.envId); + if (!previousTreeCommit) { + await folderCommitQueueService.createFolderTreeCheckpoint(folder.envId, newCommit.id, tx); + } + } + await folderCommitQueueService.scheduleTreeCheckpoint(folder.envId); + } + return newCommit; + } catch (error) { + if (error instanceof NotFoundError || error instanceof BadRequestError) { + throw error; + } + throw new DatabaseError({ error, name: "CreateCommit" }); + } + }; + + /** + * Process secret changes when applying folder state differences + */ + const processSecretChanges = async ( + changes: ResourceChange[], + secretVersions: Record, + actorInfo: ActorInfo, + folderId: string, + tx?: Knex + ) => { + const commitChanges = []; + const folder = await folderDAL.findById(folderId, tx); + if (!folder) { + return []; + } + const project = await projectDAL.findById(folder.projectId, tx); + + // Filter only secret changes using discriminated union + const secretChanges = changes.filter( + (change): change is ResourceChange & SecretChange => change.type === ResourceType.SECRET + ); + + // Collect all secretIds for batch lookup + const secretIds = secretChanges.map((change) => secretVersions[change.id]?.secretId).filter(Boolean); + + // Fetch all latest versions in one call + const latestVersionsMap = await secretVersionV2BridgeDAL.findLatestVersionMany(folderId, secretIds, tx); + + for (const change of secretChanges) { + const secretVersion = secretVersions[change.id]; + // eslint-disable-next-line no-continue + if (!secretVersion) continue; + + // Get the latest version from our batch result + const latestVersion = latestVersionsMap[secretVersion.secretId]; + const nextVersion = latestVersion ? latestVersion.version + 1 : 1; + + switch (change.changeType) { + case "create": + { + const newSecret = [ + { + id: change.id, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + version: nextVersion, + type: secretVersion.type, + key: secretVersion.key, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + encryptedValue: secretVersion.encryptedValue, + encryptedComment: secretVersion.encryptedComment, + userId: secretVersion.userId, + folderId + } + ]; + await secretV2BridgeDAL.insertMany(newSecret, tx); + + const metadata: { key: string; value: string }[] = + (secretVersion.metadata as { key: string; value: string }[]) || []; + if (metadata.length > 0) { + await resourceMetadataDAL.insertMany( + metadata.map(({ key, value }) => ({ + key, + value, + secretId: change.id, + orgId: project.orgId + })), + tx + ); + } + + const newVersion = await secretVersionV2BridgeDAL.create( + { + folderId, + secretId: secretVersion.secretId, + version: nextVersion, + encryptedValue: secretVersion.encryptedValue, + key: secretVersion.key, + encryptedComment: secretVersion.encryptedComment, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + userId: secretVersion.userId, + actorType: actorInfo.actorType, + envId: secretVersion.envId, + metadata: JSON.stringify(metadata), + ...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }), + ...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId }) + }, + tx + ); + + const secretTagsToBeInsert: TSecretV2TagJunctionInsert[] = []; + const secretTags = await secretTagDAL.findSecretTagsByVersionId(secretVersion.id, tx); + secretTags.forEach((tag) => { + secretTagsToBeInsert.push({ secrets_v2Id: change.id, secret_tagsId: tag.secret_tagsId }); + }); + await secretTagDAL.saveTagsToSecretV2(secretTagsToBeInsert, tx); + await secretTagDAL.saveTagsToSecretVersionV2( + secretTagsToBeInsert.map((tag) => ({ + secret_tagsId: tag.secret_tagsId, + secret_versions_v2Id: newVersion.id + })), + tx + ); + + commitChanges.push({ + type: ChangeType.ADD, + secretVersionId: newVersion.id + }); + } + break; + + case "update": + { + await secretV2BridgeDAL.updateById( + change.id, + { + skipMultilineEncoding: secretVersion?.skipMultilineEncoding, + version: nextVersion, + type: secretVersion?.type, + key: secretVersion?.key, + reminderNote: secretVersion?.reminderNote, + reminderRepeatDays: secretVersion?.reminderRepeatDays, + encryptedValue: secretVersion?.encryptedValue, + encryptedComment: secretVersion?.encryptedComment, + userId: secretVersion?.userId + }, + tx + ); + + const metadata: { key: string; value: string }[] = + (secretVersion.metadata as { key: string; value: string }[]) || []; + await resourceMetadataDAL.delete({ secretId: change.id }, tx); + if (metadata.length > 0) { + await resourceMetadataDAL.insertMany( + metadata.map(({ key, value }) => ({ + key, + value, + secretId: change.id, + orgId: project.orgId + })), + tx + ); + } + + const newVersion = await secretVersionV2BridgeDAL.create( + { + version: nextVersion, + encryptedValue: secretVersion.encryptedValue, + key: secretVersion.key, + encryptedComment: secretVersion.encryptedComment, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + userId: secretVersion.userId, + metadata: JSON.stringify(metadata), + actorType: actorInfo.actorType, + envId: secretVersion.envId, + folderId, + secretId: secretVersion.secretId, + ...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }), + ...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId }) + }, + tx + ); + + let secretTagsToBeInsert: TSecretV2TagJunctionInsert[] = []; + const secretTagsToBeDelete: string[] = []; + const secretTags = await secretTagDAL.findSecretTagsByVersionId(secretVersion.id, tx); + secretTags.forEach((tag) => { + secretTagsToBeInsert.push({ secrets_v2Id: change.id, secret_tagsId: tag.secret_tagsId }); + }); + const currentTags = await secretTagDAL.findSecretTagsBySecretId(change.id, tx); + currentTags.forEach((tag) => { + if (!secretTagsToBeInsert.find((t) => t.secret_tagsId === tag.secret_tagsId)) { + secretTagsToBeDelete.push(tag.secret_tagsId); + secretTagsToBeInsert = secretTagsToBeInsert.filter((t) => t.secret_tagsId !== tag.secret_tagsId); + } + }); + await secretTagDAL.saveTagsToSecretV2(secretTagsToBeInsert, tx); + await secretTagDAL.saveTagsToSecretVersionV2( + secretTagsToBeInsert.map((tag) => ({ + secret_tagsId: tag.secret_tagsId, + secret_versions_v2Id: newVersion.id + })), + tx + ); + await secretTagDAL.deleteTagsToSecretV2( + { $in: { secret_tagsId: secretTagsToBeDelete }, secrets_v2Id: change.id }, + tx + ); + + commitChanges.push({ + type: ChangeType.ADD, + isUpdate: true, + secretVersionId: newVersion.id + }); + } + break; + + // Delete case remains unchanged + case "delete": + await secretV2BridgeDAL.deleteById(change.id, tx); + commitChanges.push({ + type: ChangeType.DELETE, + secretVersionId: change.versionId + }); + break; + + default: + throw new BadRequestError({ message: `Unknown change type: ${change.changeType}` }); + } + } + + return commitChanges; + }; + + /** + * Core function to apply folder state differences + */ + const applyFolderStateDifferencesFn = async ({ + differences, + actorInfo, + folderId, + projectId, + reconstructNewFolders, + reconstructUpToCommit, + step = 0, + tx + }: { + differences: ResourceChange[]; + actorInfo: ActorInfo; + folderId: string; + projectId: string; + reconstructNewFolders: boolean; + reconstructUpToCommit?: string; + step: number; + tx?: Knex; + }): Promise => { + /** + * Process folder changes when applying folder state differences + */ + const processFolderChanges = async ( + changes: ResourceChange[], + folderVersions: Record + ) => { + const commitChanges = []; + + // Filter only folder changes using discriminated union + const folderChanges = changes.filter( + (change): change is ResourceChange & FolderChange => change.type === ResourceType.FOLDER + ); + + for (const change of folderChanges) { + const folderVersion = folderVersions[change.id]; + + switch (change.changeType) { + case "create": + if (folderVersion) { + const newFolder = { + id: change.id, + parentId: folderId, + envId: folderVersion.envId, + version: (folderVersion.version || 1) + 1, + name: folderVersion.name, + description: folderVersion.description + }; + await folderDAL.create(newFolder, tx); + + const newFolderVersion = await folderVersionDAL.create( + { + folderId: change.id, + version: (folderVersion.version || 1) + 1, + name: folderVersion.name, + description: folderVersion.description, + envId: folderVersion.envId + }, + tx + ); + + if (reconstructNewFolders && reconstructUpToCommit && step < 20) { + const subFolderLatestCommit = await folderCommitDAL.findLatestCommitBetween({ + folderId: change.id, + endCommitId: reconstructUpToCommit, + tx + }); + if (subFolderLatestCommit) { + const subFolderDiff = await compareFolderStates({ + targetCommitId: subFolderLatestCommit.id, + tx + }); + if (subFolderDiff?.length > 0) { + await applyFolderStateDifferencesFn({ + differences: subFolderDiff, + actorInfo, + folderId: change.id, + projectId, + reconstructNewFolders, + reconstructUpToCommit, + step: step + 1, + tx + }); + } + } + } + + commitChanges.push({ + type: ChangeType.ADD, + folderVersionId: newFolderVersion.id + }); + } + break; + + case "update": + if (change.versionId) { + const latestVersionDetails = await folderVersionDAL.findByIdsWithLatestVersion( + [change.id], + [change.versionId], + tx + ); + if (latestVersionDetails && Object.keys(latestVersionDetails).length > 0) { + const versionDetails = Object.values(latestVersionDetails)[0]; + await folderDAL.updateById( + change.id, + { + parentId: folderId, + envId: versionDetails.envId, + version: (versionDetails.version || 1) + 1, + name: versionDetails.name, + description: versionDetails.description + }, + tx + ); + + const newFolderVersion = await folderVersionDAL.create( + { + folderId: change.id, + version: (versionDetails.version || 1) + 1, + name: versionDetails.name, + description: versionDetails.description, + envId: versionDetails.envId + }, + tx + ); + + commitChanges.push({ + type: ChangeType.ADD, + isUpdate: true, + folderVersionId: newFolderVersion.id + }); + } + } + break; + + case "delete": + await folderDAL.deleteById(change.id, tx); + + commitChanges.push({ + type: ChangeType.DELETE, + folderVersionId: change.versionId, + folderId: change.id + }); + break; + + default: + throw new BadRequestError({ message: `Unknown change type: ${change.changeType}` }); + } + } + return commitChanges; + }; + + // Group differences by type for more efficient processing using discriminated unions + const secretChanges = differences.filter( + (diff): diff is ResourceChange & SecretChange => diff.type === ResourceType.SECRET + ); + const folderChanges = differences.filter( + (diff): diff is ResourceChange & FolderChange => diff.type === ResourceType.FOLDER + ); + + // Batch fetch necessary data + const secretVersions = await secretVersionV2BridgeDAL.findByIdsWithLatestVersion( + folderId, + secretChanges.map((diff) => diff.id), + secretChanges.map((diff) => diff.versionId), + tx + ); + + const folderVersions = await folderVersionDAL.findByIdsWithLatestVersion( + folderChanges.map((diff) => diff.id), + folderChanges.map((diff) => diff.versionId), + tx + ); + + // Process changes in parallel + const [secretCommitChanges, folderCommitChanges] = await Promise.all([ + processSecretChanges(differences, secretVersions, actorInfo, folderId, tx), + processFolderChanges(differences, folderVersions) + ]); + + // Combine all changes + const allCommitChanges = [...secretCommitChanges, ...folderCommitChanges]; + + // Create a commit with all the changes + await createCommit( + { + actor: { + type: actorInfo.actorType, + metadata: { id: actorInfo.actorId } + }, + message: actorInfo.message || "Rolled back folder state", + folderId, + changes: allCommitChanges, + omitIgnoreFilter: true + }, + tx + ); + + // Invalidate cache to reflect the changes + await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId); + + return { + secretChangesCount: secretChanges.length, + folderChangesCount: folderChanges.length, + totalChanges: differences.length + }; + }; + + /** + * Apply folder state differences with transaction handling + */ + const applyFolderStateDifferences = async (params: { + differences: ResourceChange[]; + actorInfo: ActorInfo; + folderId: string; + projectId: string; + reconstructNewFolders: boolean; + reconstructUpToCommit?: string; + tx?: Knex; + }): Promise => { + // If a transaction was provided, use it directly + if (params.tx) { + return applyFolderStateDifferencesFn({ ...params, step: 0 }); + } + + // Otherwise, start a new transaction + return folderCommitDAL.transaction((newTx) => applyFolderStateDifferencesFn({ ...params, tx: newTx, step: 0 })); + }; + + /** + * Retrieve a commit by ID + */ + const getCommitById = async ({ + commitId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + tx + }: { + commitId: string; + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + tx?: Knex; + }) => { + await checkProjectCommitReadPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + return folderCommitDAL.findById(commitId, tx, projectId); + }; + + /** + * Get all commits for a folder + */ + const getCommitsByFolderId = async (folderId: string, tx?: Knex) => { + return folderCommitDAL.findByFolderId(folderId, tx); + }; + + const getCommitsForFolder = async ({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + environment, + path, + offset = 0, + limit = 20, + search, + sort = "desc" + }: { + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + environment: string; + path: string; + offset: number; + limit: number; + search?: string; + sort: "asc" | "desc"; + }) => { + await checkProjectCommitReadPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + const folder = await folderDAL.findBySecretPath(projectId, environment, path); + if (!folder) { + throw new NotFoundError({ + message: `Folder not found for project ID ${projectId}, environment ${environment}, path ${path}` + }); + } + const folderCommits = await folderCommitDAL.findByFolderIdPaginated(folder.id, { + offset, + limit, + search, + sort + }); + return folderCommits; + }; + + const getCommitsCount = async ({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + environment, + path + }: { + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + environment: string; + path: string; + }) => { + await checkProjectCommitReadPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + + const folder = await folderDAL.findBySecretPath(projectId, environment, path); + if (!folder) { + throw new NotFoundError({ + message: `Folder not found for project ID ${projectId}, environment ${environment}, path ${path}` + }); + } + const folderCommits = await folderCommitDAL.findByFolderId(folder.id); + return { count: folderCommits.length, folderId: folder.id }; + }; + + /** + * Get changes for a commit + */ + const getCommitChanges = async ({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + commitId + }: { + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + commitId: string; + }) => { + await checkProjectCommitReadPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + const changes = await folderCommitChangesDAL.findByCommitId(commitId, projectId); + const commit = await folderCommitDAL.findById(commitId, undefined, projectId); + const latestCommit = await folderCommitDAL.findLatestCommit(commit.folderId, projectId); + return { ...commit, changes, isLatest: commit.id === latestCommit?.id }; + }; + + /** + * Get checkpoints for a folder + */ + const getCheckpointsByFolderId = async (folderId: string, limit?: number, tx?: Knex) => { + return folderCheckpointDAL.findByFolderId(folderId, limit, tx); + }; + + /** + * Get the latest checkpoint for a folder + */ + const getLatestCheckpoint = async (folderId: string, tx?: Knex) => { + return folderCheckpointDAL.findLatestByFolderId(folderId, tx); + }; + + /** + * Initialize a folder with its current state + */ + const getFolderInitialChanges = async (folderId: string, envId: string, tx?: Knex) => { + const folderResources = await getFolderResources(folderId, tx); + const changes = folderResources.map((resource) => ({ type: ChangeType.ADD, ...resource })); + + if (changes.length > 0) { + return { + commit: { + actorMetadata: {}, + actorType: ActorType.PLATFORM, + message: "Initialized folder", + folderId, + envId + }, + changes: changes.map((change) => ({ + folderId, + changeType: change.type, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId, + isUpdate: false + })) + }; + } + return {}; + }; + + /** + * Sort folders by hierarchy (parents before children) + */ + const sortFoldersByHierarchy = (folders: TSecretFolders[]) => { + // Create a map for quick lookup of children by parent ID + const childrenMap = new Map(); + + // Set of all folder IDs + const allFolderIds = new Set(); + + // Build the set of all folder IDs + folders.forEach((folder) => { + if (folder.id) { + allFolderIds.add(folder.id); + } + }); + + // Group folders by their parentId + folders.forEach((folder) => { + if (folder.parentId) { + const children = childrenMap.get(folder.parentId) || []; + children.push(folder); + childrenMap.set(folder.parentId, children); + } + }); + + // Find root folders - those with no parentId or with a parentId that doesn't exist + const rootFolders = folders.filter((folder) => !folder.parentId || !allFolderIds.has(folder.parentId)); + + // Process each level of the hierarchy + const result = []; + let currentLevel = rootFolders; + + while (currentLevel.length > 0) { + result.push(...currentLevel); + + const nextLevel = []; + for (const folder of currentLevel) { + if (folder.id) { + const children = childrenMap.get(folder.id) || []; + nextLevel.push(...children); + } + } + + currentLevel = nextLevel; + } + + return result; + }; + + /** + * Create a checkpoint for a folder tree + */ + const createFolderTreeCheckpoint = async (envId: string, folderCommitId?: string, tx?: Knex) => { + let latestCommitId = folderCommitId; + const latestTreeCheckpoint = await folderTreeCheckpointDAL.findLatestByEnvId(envId, tx); + + if (!latestCommitId) { + const latestCommit = await folderCommitDAL.findLatestEnvCommit(envId, tx); + if (!latestCommit) { + logger.info(`createFolderTreeCheckpoint - Latest commit ID not found for envId ${envId}`); + return; + } + latestCommitId = latestCommit.id; + } + + if (latestTreeCheckpoint) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getEnvNumberOfCommitsSince( + envId, + latestTreeCheckpoint.folderCommitId, + tx + ); + if (commitsSinceLastCheckpoint < Number(appCfg.PIT_TREE_CHECKPOINT_WINDOW)) { + logger.info( + `createFolderTreeCheckpoint - Commits since last checkpoint ${commitsSinceLastCheckpoint} is less than ${appCfg.PIT_TREE_CHECKPOINT_WINDOW}` + ); + return; + } + } + + const folders = await folderDAL.findByEnvId(envId, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + const filteredFoldersIds = sortedFolders.filter((folder) => !folder.isReserved).map((folder) => folder.id); + const folderCommits = await folderCommitDAL.findMultipleLatestCommits(filteredFoldersIds, tx); + const folderTreeCheckpoint = await folderTreeCheckpointDAL.create( + { + folderCommitId: latestCommitId + }, + tx + ); + await folderTreeCheckpointResourcesDAL.insertMany( + folderCommits.map((folderCommit) => ({ + folderTreeCheckpointId: folderTreeCheckpoint.id, + folderId: folderCommit.folderId, + folderCommitId: folderCommit.id + })), + tx + ); + }; + + const addNestedFolderChanges = async ({ + changes, + beforeCommit, + folderId, + folderName, + folderPath, + step = 1, + tx + }: { + changes: { + folderId: string; + folderName: string; + changes: ResourceChange[]; + folderPath?: string; + }[]; + beforeCommit: bigint; + folderId: string; + folderName?: string; + folderPath?: string; + step?: number; + tx?: Knex; + }) => { + if (step > 20) { + return; + } + const latestFolderCommit = await folderCommitDAL.findCommitBefore(folderId, beforeCommit, tx); + if (!latestFolderCommit) { + return; + } + const diff = await compareFolderStates({ + targetCommitId: latestFolderCommit.id, + tx + }); + changes.push({ + folderId, + folderName: folderName || "", + changes: diff, + folderPath: folderPath || "" + }); + await Promise.all( + diff.map(async (change) => { + if (change.type === ResourceType.FOLDER && change.changeType === ChangeType.CREATE) { + await addNestedFolderChanges({ + changes, + beforeCommit, + folderId: change.id, + folderName: change.folderName, + folderPath: `${folderPath}/${change.folderName}`, + step: step + 1, + tx + }); + } + }) + ); + }; + + const deepCompareFolder = async ({ + targetCommitId, + envId, + projectId, + tx + }: { + targetCommitId: string; + envId: string; + projectId: string; + tx?: Knex; + }) => { + const targetCommit = await folderCommitDAL.findById(targetCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `No commit found for commit ID ${targetCommitId}` }); + } + + const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommit.commitId, envId, tx); + if (!checkpoint) { + throw new NotFoundError({ message: `No checkpoint found for commit ID ${targetCommitId}` }); + } + + const folderCheckpointCommits = await folderTreeCheckpointResourcesDAL.findByTreeCheckpointId(checkpoint.id, tx); + const folderCommits = await folderCommitDAL.findAllCommitsBetween({ + envId, + startCommitId: checkpoint.commitId.toString(), + tx + }); + + // Group commits by folderId and keep only the latest + const folderGroups = new Map(); + + if (folderCheckpointCommits && folderCheckpointCommits.length > 0) { + for (const commit of folderCheckpointCommits) { + if (commit.commitId > targetCommit.commitId) { + folderGroups.set(commit.folderId, { + commitId: commit.commitId, + id: commit.folderCommitId + }); + } + } + } + + if (folderCommits && folderCommits.length > 0) { + for (const commit of folderCommits) { + const { folderId, commitId, id } = commit; + const existingCommit = folderGroups.get(folderId); + + if ((!existingCommit || commitId > existingCommit.commitId) && commitId > targetCommit.commitId) { + folderGroups.set(folderId, { commitId, id }); + } + } + } + + const folderDiffs = new Map(); + + // Process each folder to determine differences + await Promise.all( + Array.from(folderGroups.entries()).map(async ([folderId, commit]) => { + const previousCommit = await folderCommitDAL.findPreviousCommitTo( + folderId, + targetCommit.commitId.toString(), + tx + ); + let diff = []; + if (previousCommit && previousCommit.id !== commit.id) { + diff = await compareFolderStates({ + currentCommitId: commit.id, + targetCommitId: previousCommit.id, + tx + }); + } else { + diff = await compareFolderStates({ + targetCommitId: commit.id, + defaultOperation: "delete", + tx + }); + } + if (diff?.length > 0) { + folderDiffs.set(folderId, diff); + } + }) + ); + + // Apply changes in hierarchical order + const folderIds = Array.from(folderDiffs.keys()); + const folders = await folderDAL.findFoldersByRootAndIds({ rootId: targetCommit.folderId, folderIds }, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + + const response: { + folderId: string; + folderName: string; + changes: ResourceChange[]; + folderPath?: string; + }[] = []; + for (const folder of sortedFolders) { + const diff = folderDiffs.get(folder.id); + if (diff) { + const folderPath = await folderDAL.findSecretPathByFolderIds(projectId, [folder.id]); + response.push({ + folderId: folder.id, + folderName: folder.name, + changes: diff, + folderPath: folderPath?.[0]?.path + }); + const recreatedFolders = diff + .filter( + (change): change is FolderChange => + change.type === ResourceType.FOLDER && change.changeType === ChangeType.CREATE + ) + .map((change) => ({ + id: change.id, + folderName: change.folderName, + folderPath: folderPath?.[0]?.path + })); + await Promise.all( + recreatedFolders.map(async (change) => { + const nestedFolderPath = folderPath?.[0]?.path; + await addNestedFolderChanges({ + changes: response, + beforeCommit: targetCommit.commitId, + folderId: change.id, + folderName: change.folderName, + folderPath: `${nestedFolderPath !== "/" ? nestedFolderPath : ""}/${change.folderName}`, + tx + }); + }) + ); + } + } + return response; + }; + + /** + * Roll back a folder tree to a specific commit + */ + const deepRollbackFolder = async ( + targetCommitId: string, + envId: string, + actorId: string, + actorType: ActorType, + projectId: string, + message?: string + ) => { + await folderCommitDAL.transaction(async (tx) => { + const targetCommit = await folderCommitDAL.findById(targetCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `No commit found for commit ID ${targetCommitId}` }); + } + + const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommit.commitId, envId, tx); + if (!checkpoint) { + throw new NotFoundError({ message: `No checkpoint found for commit ID ${targetCommitId}` }); + } + + const folderCheckpointCommits = await folderTreeCheckpointResourcesDAL.findByTreeCheckpointId(checkpoint.id, tx); + const folderCommits = await folderCommitDAL.findAllCommitsBetween({ + envId, + startCommitId: checkpoint.commitId.toString(), + tx + }); + + // Group commits by folderId and keep only the latest + const folderGroups = new Map(); + + if (folderCheckpointCommits && folderCheckpointCommits.length > 0) { + for (const commit of folderCheckpointCommits) { + if (commit.commitId > targetCommit.commitId) { + folderGroups.set(commit.folderId, { + commitId: commit.commitId, + id: commit.folderCommitId + }); + } + } + } + + if (folderCommits && folderCommits.length > 0) { + for (const commit of folderCommits) { + const { folderId, commitId, id } = commit; + const existingCommit = folderGroups.get(folderId); + + if ((!existingCommit || commitId > existingCommit.commitId) && commitId > targetCommit.commitId) { + folderGroups.set(folderId, { commitId, id }); + } + } + } + + const folderDiffs = new Map(); + + // Process each folder to determine differences + await Promise.all( + Array.from(folderGroups.entries()).map(async ([folderId, { id }]) => { + const previousCommit = await folderCommitDAL.findPreviousCommitTo( + folderId, + targetCommit.commitId.toString(), + tx + ); + if (previousCommit && previousCommit.id !== id) { + const diff = await compareFolderStates({ + currentCommitId: id, + targetCommitId: previousCommit.id, + tx + }); + if (diff?.length > 0) { + folderDiffs.set(folderId, diff); + } + } + }) + ); + + const foldersToDelete = new Set(); + + // Process all DELETE operations to build a complete set of folders to be deleted + for (const changes of folderDiffs.values()) { + for (const change of changes) { + if (change.changeType === ChangeType.DELETE && change.type === ResourceType.FOLDER) { + foldersToDelete.add(change.id); + } + } + } + + // Now, remove any folder that is being deleted from the folderDiffs map + // before applying any changes + for (const folderId of foldersToDelete) { + folderDiffs.delete(folderId); + } + + // Apply changes in hierarchical order + const folderIds = Array.from(folderDiffs.keys()); + const folders = await folderDAL.findFoldersByRootAndIds({ rootId: targetCommit.folderId, folderIds }, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + + for (const folder of sortedFolders) { + const diff = folderDiffs.get(folder.id); + if (diff) { + await applyFolderStateDifferences({ + differences: diff, + actorInfo: { + actorType, + actorId, + message: message || "Deep rollback" + }, + folderId: folder.id, + projectId, + reconstructNewFolders: true, + reconstructUpToCommit: targetCommit.commitId.toString(), + tx + }); + } + } + }); + }; + + const getLatestCommit = async ({ + folderId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }: { + folderId: string; + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + }) => { + await checkProjectCommitReadPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + return folderCommitDAL.findLatestCommit(folderId, projectId); + }; + + /** + * Revert changes made in a specific commit + */ + const revertCommitChanges = async ({ + commitId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + message = "Revert commit changes" + }: { + commitId: string; + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + message?: string; + }) => { + if (!permissionService) { + throw new Error("Permission service not initialized"); + } + const { permission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.SecretManager + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionCommitsActions.PerformRollback, + ProjectPermissionSub.Commits + ); + // Check permissions first + await checkProjectCommitReadPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + }); + + // Get the commit to revert + const commitToRevert = await folderCommitDAL.findById(commitId, undefined, projectId); + if (!commitToRevert) { + throw new NotFoundError({ message: `Commit with ID ${commitId} not found` }); + } + + const previousCommit = await folderCommitDAL.findCommitBefore(commitToRevert.folderId, commitToRevert.commitId); + + if (!previousCommit) { + throw new BadRequestError({ message: "Cannot revert the first commit" }); + } + + // Calculate the changes needed to go from current commit back to the previous one + const inverseChanges = await compareFolderStates({ + currentCommitId: commitToRevert.id, + targetCommitId: previousCommit.id + }); + + const latestCommit = await folderCommitDAL.findLatestCommit(commitToRevert.folderId); + if (!latestCommit) { + throw new NotFoundError({ message: `Latest commit not found for folder ${commitToRevert.folderId}` }); + } + const currentState = await reconstructFolderState(latestCommit.id); + + const filteredChanges = inverseChanges.filter( + (change) => + ((change.changeType === ChangeType.DELETE || change.changeType === ChangeType.UPDATE) && + (currentState.some((c) => c.id === change.id) || currentState.some((c) => c.id === change.id))) || + (change.changeType === ChangeType.CREATE && + (currentState.every((c) => c.id !== change.id) || currentState.every((c) => c.id !== change.id))) + ); + + if (!filteredChanges || filteredChanges.length === 0) { + return { + success: true, + message: "No changes to revert", + originalCommitId: commitId + }; + } + + // Apply the changes to revert the commit + const revertResult = await applyFolderStateDifferences({ + differences: filteredChanges, + actorInfo: { + actorType: actor, + actorId, + message: message || `Reverted changes from commit ${commitId}` + }, + folderId: commitToRevert.folderId, + projectId, + reconstructNewFolders: true, + reconstructUpToCommit: commitToRevert.commitId.toString() + }); + + return { + success: true, + message: "Changes reverted successfully", + originalCommitId: commitId, + revertCommitId: latestCommit?.id, + changesReverted: revertResult.totalChanges + }; + }; + + return { + createCommit, + addCommitChange, + getCommitById, + getCommitsByFolderId, + getCommitChanges, + getCheckpointsByFolderId, + getLatestCheckpoint, + getFolderInitialChanges, + createFolderCheckpoint, + compareFolderStates, + applyFolderStateDifferences, + createFolderTreeCheckpoint, + deepRollbackFolder, + getCommitsCount, + getLatestCommit, + deepCompareFolder, + reconstructFolderState, + getCommitsForFolder, + revertCommitChanges + }; +}; + +export type TFolderCommitServiceFactory = ReturnType; diff --git a/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts new file mode 100644 index 0000000000..58651a48a6 --- /dev/null +++ b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts @@ -0,0 +1,44 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderTreeCheckpointResources } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderTreeCheckpointResourcesDALFactory = ReturnType; + +type TFolderTreeCheckpointResourcesWithCommitId = TFolderTreeCheckpointResources & { + commitId: bigint; +}; + +export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { + const folderTreeCheckpointResourcesOrm = ormify(db, TableName.FolderTreeCheckpointResources); + + const findByTreeCheckpointId = async ( + folderTreeCheckpointId: string, + tx?: Knex + ): Promise => { + try { + const docs = await (tx || db.replicaNode())( + TableName.FolderTreeCheckpointResources + ) + .join( + TableName.FolderCommit, + `${TableName.FolderTreeCheckpointResources}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderTreeCheckpointId }, TableName.FolderTreeCheckpointResources)) + .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) + .select(db.ref("commitId").withSchema(TableName.FolderCommit).as("commitId")); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByTreeCheckpointId" }); + } + }; + + return { + ...folderTreeCheckpointResourcesOrm, + findByTreeCheckpointId + }; +}; diff --git a/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts new file mode 100644 index 0000000000..cc0634e746 --- /dev/null +++ b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts @@ -0,0 +1,79 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCommits, TFolderTreeCheckpoints } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderTreeCheckpointDALFactory = ReturnType; + +type TreeCheckpointWithCommitInfo = TFolderTreeCheckpoints & { + commitId: bigint; +}; + +export const folderTreeCheckpointDALFactory = (db: TDbClient) => { + const folderTreeCheckpointOrm = ormify(db, TableName.FolderTreeCheckpoint); + + const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderCommitId }, TableName.FolderTreeCheckpoint)) + .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindByCommitId" }); + } + }; + + const findNearestCheckpoint = async ( + folderCommitId: bigint, + envId: string, + tx?: Knex + ): Promise => { + try { + const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderTreeCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(`${TableName.FolderCommit}.envId`, "=", envId) + .andWhere(`${TableName.FolderCommit}.commitId`, "<=", folderCommitId.toString()) + .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) + .select(db.ref("commitId").withSchema(TableName.FolderCommit)) + .orderBy(`${TableName.FolderCommit}.commitId`, "desc") + .first(); + + return nearestCheckpoint; + } catch (error) { + throw new DatabaseError({ error, name: "FindNearestCheckpoint" }); + } + }; + + const findLatestByEnvId = async (envId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderTreeCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .where(`${TableName.FolderCommit}.envId`, "=", envId) + .orderBy(`${TableName.FolderTreeCheckpoint}.createdAt`, "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestByEnvId" }); + } + }; + + return { + ...folderTreeCheckpointOrm, + findByCommitId, + findNearestCheckpoint, + findLatestByEnvId + }; +}; diff --git a/backend/src/services/project/project-dal.ts b/backend/src/services/project/project-dal.ts index 54bef02d13..7f733503c8 100644 --- a/backend/src/services/project/project-dal.ts +++ b/backend/src/services/project/project-dal.ts @@ -12,7 +12,7 @@ import { TProjectsUpdate } from "@app/db/schemas"; import { BadRequestError, DatabaseError, NotFoundError, UnauthorizedError } from "@app/lib/errors"; -import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; +import { buildFindFilter, ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; import { ActorType } from "../auth/auth-type"; import { Filter, ProjectFilterType, SearchProjectSortBy } from "./project-types"; @@ -475,6 +475,16 @@ export const projectDALFactory = (db: TDbClient) => { return { docs, totalCount: Number(docs?.[0]?.count ?? 0) }; }; + const findProjectByEnvId = async (envId: string, tx?: Knex) => { + const project = await (tx || db.replicaNode())(TableName.Project) + .leftJoin(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ id: envId }, TableName.Environment)) + .select(selectAllTableCols(TableName.Project)) + .first(); + return project; + }; + const countOfOrgProjects = async (orgId: string | null, tx?: Knex) => { try { const doc = await (tx || db.replicaNode())(TableName.Project) @@ -504,6 +514,7 @@ export const projectDALFactory = (db: TDbClient) => { checkProjectUpgradeStatus, getProjectFromSplitId, searchProjects, + findProjectByEnvId, countOfOrgProjects }; }; diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index d8eee188a4..680b0187c5 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -671,7 +671,8 @@ export const projectServiceFactory = ({ enforceCapitalization: update.autoCapitalization, hasDeleteProtection: update.hasDeleteProtection, slug: update.slug, - secretSharing: update.secretSharing + secretSharing: update.secretSharing, + showSnapshotsLegacy: update.showSnapshotsLegacy }); return updatedProject; diff --git a/backend/src/services/project/project-types.ts b/backend/src/services/project/project-types.ts index be052f1cbe..8ef72492a7 100644 --- a/backend/src/services/project/project-types.ts +++ b/backend/src/services/project/project-types.ts @@ -94,6 +94,7 @@ export type TUpdateProjectDTO = { hasDeleteProtection?: boolean; slug?: string; secretSharing?: boolean; + showSnapshotsLegacy?: boolean; }; } & Omit; diff --git a/backend/src/services/secret-folder/secret-folder-dal.ts b/backend/src/services/secret-folder/secret-folder-dal.ts index e136c5a50f..7dfeaddcf7 100644 --- a/backend/src/services/secret-folder/secret-folder-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-dal.ts @@ -488,6 +488,75 @@ export const secretFolderDALFactory = (db: TDbClient) => { } }; + const findFoldersByRootAndIds = async ({ rootId, folderIds }: { rootId: string; folderIds: string[] }, tx?: Knex) => { + try { + // First, get all descendant folders of rootId + const descendants = await (tx || db.replicaNode()) + .withRecursive("descendants", (qb) => + qb + .select( + selectAllTableCols(TableName.SecretFolder), + db.raw("0 as depth"), + db.raw(`'/' as path`), + db.ref(`${TableName.Environment}.slug`).as("environment") + ) + .from(TableName.SecretFolder) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where(`${TableName.SecretFolder}.id`, rootId) + .union((un) => { + void un + .select( + selectAllTableCols(TableName.SecretFolder), + db.raw("descendants.depth + 1 as depth"), + db.raw( + `CONCAT( + CASE WHEN descendants.path = '/' THEN '' ELSE descendants.path END, + CASE WHEN ${TableName.SecretFolder}."parentId" is NULL THEN '' ELSE CONCAT('/', secret_folders.name) END + )` + ), + db.ref("descendants.environment") + ) + .from(TableName.SecretFolder) + .where(`${TableName.SecretFolder}.isReserved`, false) + .join("descendants", `${TableName.SecretFolder}.parentId`, "descendants.id"); + }) + ) + .select<(TSecretFolders & { path: string; depth: number; environment: string })[]>("*") + .from("descendants") + .whereIn(`id`, folderIds) + .orderBy("depth") + .orderBy(`name`); + + return descendants; + } catch (error) { + throw new DatabaseError({ error, name: "FindFoldersByRootAndIds" }); + } + }; + + const findByParentId = async (parentId: string, tx?: Knex) => { + try { + const folders = await (tx || db.replicaNode())(TableName.SecretFolder) + .where({ parentId }) + .andWhere({ isReserved: false }) + .select(selectAllTableCols(TableName.SecretFolder)); + return folders; + } catch (error) { + throw new DatabaseError({ error, name: "findByParentId" }); + } + }; + + const findByEnvId = async (envId: string, tx?: Knex) => { + try { + const folders = await (tx || db.replicaNode())(TableName.SecretFolder) + .where({ envId }) + .andWhere({ isReserved: false }) + .select(selectAllTableCols(TableName.SecretFolder)); + return folders; + } catch (error) { + throw new DatabaseError({ error, name: "findByEnvId" }); + } + }; + return { ...secretFolderOrm, update, @@ -499,6 +568,9 @@ export const secretFolderDALFactory = (db: TDbClient) => { findClosestFolder, findByProjectId, findByMultiEnv, - findByEnvsDeep + findByEnvsDeep, + findByParentId, + findByEnvId, + findFoldersByRootAndIds }; }; diff --git a/backend/src/services/secret-folder/secret-folder-service.ts b/backend/src/services/secret-folder/secret-folder-service.ts index 842eb2bb7c..5722734f8f 100644 --- a/backend/src/services/secret-folder/secret-folder-service.ts +++ b/backend/src/services/secret-folder/secret-folder-service.ts @@ -10,6 +10,7 @@ import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { OrderByDirection, OrgServiceActor } from "@app/lib/types"; import { buildFolderPath } from "@app/services/secret-folder/secret-folder-fns"; +import { ChangeType, CommitType, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretFolderDALFactory } from "./secret-folder-dal"; @@ -29,7 +30,8 @@ type TSecretFolderServiceFactoryDep = { snapshotService: Pick; folderDAL: TSecretFolderDALFactory; projectEnvDAL: Pick; - folderVersionDAL: TSecretFolderVersionDALFactory; + folderVersionDAL: Pick; + folderCommitService: Pick; projectDAL: Pick; }; @@ -41,6 +43,7 @@ export const secretFolderServiceFactory = ({ permissionService, projectEnvDAL, folderVersionDAL, + folderCommitService, projectDAL }: TSecretFolderServiceFactoryDep) => { const createFolder = async ({ @@ -111,15 +114,33 @@ export const secretFolderServiceFactory = ({ }); parentFolderId = newFolders.at(-1)?.id as string; const docs = await folderDAL.insertMany(newFolders, tx); - await folderVersionDAL.insertMany( + const folderVersions = await folderVersionDAL.insertMany( docs.map((doc) => ({ name: doc.name, envId: doc.envId, version: doc.version, - folderId: doc.id + folderId: doc.id, + description: doc.description })), tx ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder created", + folderId: parentFolderId, + changes: folderVersions.map((fv) => ({ + type: CommitType.ADD, + folderVersionId: fv.id + })) + }, + tx + ); } } @@ -127,12 +148,32 @@ export const secretFolderServiceFactory = ({ { name, envId: env.id, version: 1, parentId: parentFolderId, description }, tx ); - await folderVersionDAL.create( + const folderVersion = await folderVersionDAL.create( { name: doc.name, envId: doc.envId, version: doc.version, - folderId: doc.id + folderId: doc.id, + description: doc.description + }, + tx + ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder created", + folderId: parentFolderId, + changes: [ + { + type: CommitType.ADD, + folderVersionId: folderVersion.id + } + ] }, tx ); @@ -225,12 +266,33 @@ export const secretFolderServiceFactory = ({ { name, description }, tx ); - await folderVersionDAL.create( + const folderVersion = await folderVersionDAL.create( { name: doc.name, envId: doc.envId, version: doc.version, - folderId: doc.id + folderId: doc.id, + description: doc.description + }, + tx + ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder updated", + folderId: parentFolder.id, + changes: [ + { + type: CommitType.ADD, + isUpdate: true, + folderVersionId: folderVersion.id + } + ] }, tx ); @@ -321,12 +383,33 @@ export const secretFolderServiceFactory = ({ { name, description }, tx ); - await folderVersionDAL.create( + const folderVersion = await folderVersionDAL.create( { name: doc.name, envId: doc.envId, version: doc.version, - folderId: doc.id + folderId: doc.id, + description: doc.description + }, + tx + ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder updated", + folderId: parentFolder.id, + changes: [ + { + type: CommitType.ADD, + isUpdate: true, + folderVersionId: folderVersion.id + } + ] }, tx ); @@ -381,7 +464,31 @@ export const secretFolderServiceFactory = ({ }, tx ); + if (!doc) throw new NotFoundError({ message: `Failed to delete folder with ID '${idOrName}', not found` }); + + const folderVersions = await folderVersionDAL.findLatestFolderVersions([doc.id], tx); + + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder deleted", + folderId: parentFolder.id, + changes: [ + { + type: CommitType.DELETE, + folderVersionId: folderVersions[doc.id].id, + folderId: doc.id + } + ] + }, + tx + ); return doc; }); @@ -665,6 +772,45 @@ export const secretFolderServiceFactory = ({ return environmentFolders; }; + const getFolderVersionsByIds = async ({ + folderId, + folderVersions + }: { + folderId: string; + folderVersions: string[]; + }) => { + const versions = await folderVersionDAL.find({ + folderId, + $in: { + version: folderVersions.map((v) => Number.parseInt(v, 10)) + } + }); + return versions; + }; + + const getFolderVersions = async ( + change: { + folderVersion?: string; + isUpdate?: boolean; + changeType?: string; + }, + fromVersion: string, + folderId: string + ) => { + const currentVersion = change.folderVersion || "1"; + // eslint-disable-next-line no-await-in-loop + const versions = await getFolderVersionsByIds({ + folderId, + folderVersions: + change.isUpdate || change.changeType === ChangeType.UPDATE ? [currentVersion, fromVersion] : [currentVersion] + }); + return versions.map((v) => ({ + version: v.version?.toString() || "1", + name: v.name, + description: v.description + })); + }; + return { createFolder, updateFolder, @@ -675,6 +821,8 @@ export const secretFolderServiceFactory = ({ getProjectFolderCount, getFoldersMultiEnv, getFoldersDeepByEnvs, - getProjectEnvironmentsFolders + getProjectEnvironmentsFolders, + getFolderVersionsByIds, + getFolderVersions }; }; diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index 78186333d6..46ff49692f 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -43,7 +43,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { const docs: Array = await (tx || db.replicaNode())( TableName.SecretFolderVersion ) - .whereIn("folderId", folderIds) + .whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds) .join( (tx || db)(TableName.SecretFolderVersion) .groupBy("folderId") @@ -85,6 +85,8 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) .join("folder_cte", "folder_cte.id", `${TableName.SecretFolderVersion}.id`) .whereRaw(`folder_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + // Projects with version >= 3 will require to have all folder versions for PIT + .andWhere(`${TableName.Project}.version`, "<", 3) .delete(); } catch (error) { throw new DatabaseError({ @@ -95,5 +97,107 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret folder versions completed`); }; - return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId, pruneExcessVersions }; + // Get latest versions by folderIds + const getLatestFolderVersions = async (folderIds: string[], tx?: Knex): Promise> => { + if (!folderIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + return knexInstance(TableName.SecretFolderVersion) + .whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds) + .join( + knexInstance(TableName.SecretFolderVersion) + .groupBy("folderId") + .max("version") + .select("folderId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretFolderVersion}.folderId`, "latestVersion.folderId").andOn( + `${TableName.SecretFolderVersion}.version`, + "latestVersion.max" + ); + } + ); + }; + + // Get specific versions and update with max version + const getSpecificFolderVersionsWithLatest = async ( + versionIds: string[], + tx?: Knex + ): Promise> => { + if (!versionIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + + // Get specific versions + const specificVersions = await knexInstance(TableName.SecretFolderVersion).whereIn("id", versionIds); + + // Get folderIds from these versions + const specificFolderIds = [...new Set(specificVersions.map((v) => v.folderId).filter(Boolean))]; + + if (!specificFolderIds.length) return specificVersions; + + // Get max versions for these folderIds + const maxVersionsQuery = await knexInstance(TableName.SecretFolderVersion) + .whereIn("folderId", specificFolderIds) + .groupBy("folderId") + .select("folderId") + .max("version", { as: "maxVersion" }); + + // Create lookup map for max versions + const maxVersionMap = maxVersionsQuery.reduce>((acc, item) => { + if (item.maxVersion) { + acc[item.folderId] = item.maxVersion; + } + return acc; + }, {}); + + // Replace version with max version + return specificVersions.map((version) => ({ + ...version, + version: maxVersionMap[version.folderId] || version.version + })); + }; + + const findByIdsWithLatestVersion = async (folderIds: string[], versionIds?: string[], tx?: Knex) => { + try { + if (!folderIds.length && (!versionIds || !versionIds.length)) return {}; + + // Run both queries in parallel + const [latestVersions, specificVersionsWithLatest] = await Promise.all([ + folderIds.length ? getLatestFolderVersions(folderIds, tx) : [], + versionIds?.length ? getSpecificFolderVersionsWithLatest(versionIds, tx) : [] + ]); + + const allDocs = [...latestVersions, ...specificVersionsWithLatest]; + + // Convert array to record with folderId as key + return allDocs.reduce>( + (prev, curr) => ({ ...prev, [curr.folderId || ""]: curr }), + {} + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindByIdsWithLatestVersion" }); + } + }; + + const findLatestVersion = async (folderId: string, tx?: Knex) => { + try { + const doc = await (tx || db.replicaNode())(TableName.SecretFolderVersion) + .where(`${TableName.SecretFolderVersion}.folderId`, folderId) + .select(selectAllTableCols(TableName.SecretFolderVersion)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "findLatestVersion" }); + } + }; + + return { + ...secretFolderVerOrm, + findLatestFolderVersions, + findLatestVersionByFolderId, + pruneExcessVersions, + findByIdsWithLatestVersion, + findLatestVersion + }; }; diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts index 6f627c24e3..66e83661f4 100644 --- a/backend/src/services/secret-sync/secret-sync-queue.ts +++ b/backend/src/services/secret-sync/secret-sync-queue.ts @@ -59,6 +59,7 @@ import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/se import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; import { TAppConnectionDALFactory } from "../app-connection/app-connection-dal"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; export type TSecretSyncQueueFactory = ReturnType; @@ -94,6 +95,7 @@ type TSecretSyncQueueFactoryDep = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; licenseService: Pick; }; @@ -136,6 +138,7 @@ export const secretSyncQueueFactory = ({ secretVersionV2BridgeDAL, secretVersionTagV2BridgeDAL, resourceMetadataDAL, + folderCommitService, licenseService }: TSecretSyncQueueFactoryDep) => { const appCfg = getConfig(); @@ -167,7 +170,8 @@ export const secretSyncQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const $updateManySecretsRawFn = updateManySecretsRawFnFactory({ @@ -183,7 +187,8 @@ export const secretSyncQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const $getInfisicalSecrets = async ( @@ -373,7 +378,7 @@ export const secretSyncQueueFactory = ({ if (Object.hasOwn(secretMap, key)) { // Only update secrets if the source value is not empty - if (value) { + if (value && value !== secretMap[key].value) { secretsToUpdate.push(secret); if (importBehavior === SecretSyncImportBehavior.PrioritizeDestination) importedSecretMap[key] = secretData; } diff --git a/backend/src/services/secret-tag/secret-tag-dal.ts b/backend/src/services/secret-tag/secret-tag-dal.ts index 3b9151557e..8768990fef 100644 --- a/backend/src/services/secret-tag/secret-tag-dal.ts +++ b/backend/src/services/secret-tag/secret-tag-dal.ts @@ -11,6 +11,7 @@ export const secretTagDALFactory = (db: TDbClient) => { const secretTagOrm = ormify(db, TableName.SecretTag); const secretJnTagOrm = ormify(db, TableName.JnSecretTag); const secretV2JnTagOrm = ormify(db, TableName.SecretV2JnTag); + const secretVersionV2TagOrm = ormify(db, TableName.SecretVersionV2Tag); const findManyTagsById = async (projectId: string, ids: string[], tx?: Knex) => { try { @@ -48,14 +49,39 @@ export const secretTagDALFactory = (db: TDbClient) => { } }; + const findSecretTagsByVersionId = async (versionId: string, tx?: Knex) => { + try { + const tags = await (tx || db.replicaNode())(TableName.SecretVersionV2Tag) + .where(`${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id`, versionId) + .select(selectAllTableCols(TableName.SecretVersionV2Tag)); + return tags; + } catch (error) { + throw new DatabaseError({ error, name: "Find all by version id" }); + } + }; + + const findSecretTagsBySecretId = async (secretId: string, tx?: Knex) => { + try { + const tags = await (tx || db.replicaNode())(TableName.SecretV2JnTag) + .where(`${TableName.SecretV2JnTag}.${TableName.SecretV2}Id`, secretId) + .select(selectAllTableCols(TableName.SecretV2JnTag)); + return tags; + } catch (error) { + throw new DatabaseError({ error, name: "Find all by secret id" }); + } + }; + return { ...secretTagOrm, saveTagsToSecret: secretJnTagOrm.insertMany, deleteTagsToSecret: secretJnTagOrm.delete, saveTagsToSecretV2: secretV2JnTagOrm.batchInsert, deleteTagsToSecretV2: secretV2JnTagOrm.delete, + saveTagsToSecretVersionV2: secretVersionV2TagOrm.insertMany, findSecretTagsByProjectId, deleteTagsManySecret, - findManyTagsById + findManyTagsById, + findSecretTagsByVersionId, + findSecretTagsBySecretId }; }; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts index 6fdcadefff..1adfac22c5 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts @@ -8,6 +8,7 @@ import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { ActorType } from "../auth/auth-type"; +import { CommitType } from "../folder-commit/folder-commit-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; import { INFISICAL_SECRET_VALUE_HIDDEN_MASK } from "../secret/secret-fns"; @@ -73,6 +74,7 @@ export const fnSecretBulkInsert = async ({ resourceMetadataDAL, secretTagDAL, secretVersionTagDAL, + folderCommitService, actor, tx }: TFnSecretBulkInsert) => { @@ -126,11 +128,36 @@ export const fnSecretBulkInsert = async ({ userActorId, identityActorId, actorType, + metadata: el.metadata ? JSON.stringify(el.metadata) : [], secretId: newSecretGroupedByKeyName[el.key][0].id })), tx ); + const commitChanges = secretVersions + .filter(({ type }) => type === SecretType.Shared) + .map((sv) => ({ + type: CommitType.ADD, + secretVersionId: sv.id + })); + + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType || ActorType.PLATFORM, + metadata: { + id: actor?.actorId + } + }, + message: "Secret Created", + folderId, + changes: commitChanges + }, + tx + ); + } + await secretDAL.upsertSecretReferences( inputSecrets.map(({ references = [], key }) => ({ secretId: newSecretGroupedByKeyName[key][0].id, @@ -185,6 +212,7 @@ export const fnSecretBulkUpdate = async ({ orgId, secretDAL, secretVersionDAL, + folderCommitService, secretTagDAL, secretVersionTagDAL, resourceMetadataDAL, @@ -246,7 +274,7 @@ export const fnSecretBulkUpdate = async ({ userId, encryptedComment, version, - metadata, + metadata: metadata ? JSON.stringify(metadata) : [], reminderNote, encryptedValue, reminderRepeatDays, @@ -259,6 +287,7 @@ export const fnSecretBulkUpdate = async ({ ), tx ); + await secretDAL.upsertSecretReferences( inputSecrets .filter(({ data: { references } }) => Boolean(references)) @@ -329,6 +358,31 @@ export const fnSecretBulkUpdate = async ({ }, { tx } ); + + const commitChanges = secretVersions + .filter(({ type }) => type === SecretType.Shared) + .map((sv) => ({ + type: CommitType.ADD, + isUpdate: true, + secretVersionId: sv.id + })); + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType || ActorType.PLATFORM, + metadata: { + id: actor?.actorId + } + }, + message: "Secret Updated", + folderId, + changes: commitChanges + }, + tx + ); + } + return secretsWithTags.map((secret) => ({ ...secret, _id: secret.id })); }; @@ -337,8 +391,11 @@ export const fnSecretBulkDelete = async ({ inputSecrets, tx, actorId, + actorType, secretDAL, - secretQueueService + secretQueueService, + folderCommitService, + secretVersionDAL }: TFnSecretBulkDelete) => { const deletedSecrets = await secretDAL.deleteMany( inputSecrets.map(({ type, secretKey }) => ({ @@ -358,6 +415,35 @@ export const fnSecretBulkDelete = async ({ ) ); + const secretVersions = await secretVersionDAL.findLatestVersionMany( + folderId, + deletedSecrets.map(({ id }) => id), + tx + ); + + const commitChanges = deletedSecrets + .filter(({ type }) => type === SecretType.Shared) + .map(({ id }) => ({ + type: CommitType.DELETE, + secretVersionId: secretVersions[id].id + })); + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType || ActorType.PLATFORM, + metadata: { + id: actorId + } + }, + message: "Secret Deleted", + folderId, + changes: commitChanges + }, + tx + ); + } + return deletedSecrets; }; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index 1ef4a2d416..c4b5835c47 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -17,6 +17,7 @@ import { import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, + ProjectPermissionCommitsActions, ProjectPermissionSecretActions, ProjectPermissionSet, ProjectPermissionSub @@ -34,6 +35,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { ActorType } from "../auth/auth-type"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { KmsDataKey } from "../kms/kms-types"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; @@ -90,6 +92,7 @@ type TSecretV2BridgeServiceFactoryDep = { secretVersionTagDAL: Pick; secretTagDAL: TSecretTagDALFactory; permissionService: Pick; + folderCommitService: Pick; projectEnvDAL: Pick; folderDAL: Pick< TSecretFolderDALFactory, @@ -124,6 +127,7 @@ export const secretV2BridgeServiceFactory = ({ projectEnvDAL, secretTagDAL, secretVersionDAL, + folderCommitService, folderDAL, permissionService, snapshotService, @@ -321,12 +325,14 @@ export const secretV2BridgeServiceFactory = ({ userId: inputSecret.type === SecretType.Personal ? actorId : null, tagIds: inputSecret.tagIds, references: nestedReferences, + metadata: secretMetadata ? JSON.stringify(secretMetadata) : [], secretMetadata } ], resourceMetadataDAL, secretDAL, secretVersionDAL, + folderCommitService, secretTagDAL, secretVersionTagDAL, actor: { @@ -510,6 +516,7 @@ export const secretV2BridgeServiceFactory = ({ folderId, orgId: actorOrgId, resourceMetadataDAL, + folderCommitService, inputSecrets: [ { filter: { id: secretId }, @@ -523,6 +530,7 @@ export const secretV2BridgeServiceFactory = ({ skipMultilineEncoding: inputSecret.skipMultilineEncoding, key: inputSecret.newSecretName || secretName, tags: inputSecret.tagIds, + metadata: secretMetadata ? JSON.stringify(secretMetadata) : [], secretMetadata, ...encryptedValue } @@ -650,6 +658,9 @@ export const secretV2BridgeServiceFactory = ({ projectId, folderId, actorId, + actorType: actor, + folderCommitService, + secretVersionDAL, secretDAL, secretQueueService, inputSecrets: [ @@ -1590,6 +1601,7 @@ export const secretV2BridgeServiceFactory = ({ orgId: actorOrgId, secretDAL, resourceMetadataDAL, + folderCommitService, secretVersionDAL, secretTagDAL, secretVersionTagDAL, @@ -1859,6 +1871,7 @@ export const secretV2BridgeServiceFactory = ({ const bulkUpdatedSecrets = await fnSecretBulkUpdate({ folderId, orgId: actorOrgId, + folderCommitService, tx, inputSecrets: secretsToUpdate.map((el) => { const originalSecret = secretsToUpdateInDBGroupedByKey[el.secretKey][0]; @@ -1928,6 +1941,7 @@ export const secretV2BridgeServiceFactory = ({ secretVersionDAL, secretTagDAL, secretVersionTagDAL, + folderCommitService, actor: { type: actor, actorId @@ -2061,6 +2075,8 @@ export const secretV2BridgeServiceFactory = ({ fnSecretBulkDelete({ secretDAL, secretQueueService, + folderCommitService, + secretVersionDAL, inputSecrets: inputSecrets.map(({ type, secretKey }) => ({ secretKey, type: type || SecretType.Shared @@ -2068,6 +2084,7 @@ export const secretV2BridgeServiceFactory = ({ projectId, folderId, actorId, + actorType: actor, tx }) ); @@ -2159,15 +2176,25 @@ export const secretV2BridgeServiceFactory = ({ actorOrgId, actionProjectType: ActionProjectType.SecretManager }); - ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); + + const canRead = + permission.can(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback) || + permission.can(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); + + if (!canRead) throw new ForbiddenRequestError({ message: "You do not have permission to read secret versions" }); + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId: folder.projectId }); - const secretVersions = await secretVersionDAL.findVersionsBySecretIdWithActors(secretId, folder.projectId, { - offset, - limit, - sort: [["createdAt", "desc"]] + const secretVersions = await secretVersionDAL.findVersionsBySecretIdWithActors({ + secretId, + projectId: folder.projectId, + findOpt: { + offset, + limit, + sort: [["createdAt", "desc"]] + } }); return secretVersions.map((el) => { const secretValueHidden = !hasSecretReadValueOrDescribePermission( @@ -2469,6 +2496,7 @@ export const secretV2BridgeServiceFactory = ({ tx, secretTagDAL, resourceMetadataDAL, + folderCommitService, secretVersionTagDAL, actor: { type: actor, @@ -2495,6 +2523,7 @@ export const secretV2BridgeServiceFactory = ({ folderId: destinationFolder.id, orgId: actorOrgId, resourceMetadataDAL, + folderCommitService, secretVersionDAL, secretDAL, tx, @@ -2840,6 +2869,76 @@ export const secretV2BridgeServiceFactory = ({ }; }; + const getSecretVersionsByIds = async ({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + secretVersionNumbers, + secretPath, + envId, + projectId + }: TGetSecretVersionsDTO & { + secretVersionNumbers: string[]; + secretPath: string; + envId: string; + projectId: string; + }) => { + const environment = await projectEnvDAL.findOne({ id: envId, projectId }); + + const { permission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.SecretManager + }); + + const canRead = + permission.can(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback) || + permission.can(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); + + if (!canRead) throw new ForbiddenRequestError({ message: "You do not have permission to read secret versions" }); + + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const secretVersions = await secretVersionDAL.findVersionsBySecretIdWithActors({ + secretId, + projectId, + secretVersions: secretVersionNumbers + }); + return secretVersions.map((el) => { + const secretValueHidden = !hasSecretReadValueOrDescribePermission( + permission, + ProjectPermissionSecretActions.ReadValue, + { + environment: environment.slug, + secretPath, + secretName: el.key, + ...(el.tags?.length && { + secretTags: el.tags.map((tag) => tag.slug) + }) + } + ); + + return reshapeBridgeSecret( + projectId, + environment.slug, + secretPath, + { + ...el, + value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : "", + comment: el.encryptedComment ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() : "" + }, + secretValueHidden + ); + }); + }; + return { createSecret, deleteSecret, @@ -2858,6 +2957,7 @@ export const secretV2BridgeServiceFactory = ({ getSecretReferenceTree, getSecretsByFolderMappings, getSecretById, - getAccessibleSecrets + getAccessibleSecrets, + getSecretVersionsByIds }; }; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts index f4a27d4c56..f4171b1a7f 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -8,6 +8,7 @@ import { SecretsOrderBy } from "@app/services/secret/secret-types"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal"; @@ -178,9 +179,10 @@ export type TFnSecretBulkInsert = { secretVersionDAL: Pick; secretTagDAL: Pick; secretVersionTagDAL: Pick; + folderCommitService: Pick; actor?: { type: string; - actorId: string; + actorId?: string; }; }; @@ -206,9 +208,10 @@ export type TFnSecretBulkUpdate = { secretVersionDAL: Pick; secretTagDAL: Pick; secretVersionTagDAL: Pick; + folderCommitService: Pick; actor?: { type: string; - actorId: string; + actorId?: string; }; tx?: Knex; }; @@ -218,11 +221,14 @@ export type TFnSecretBulkDelete = { projectId: string; inputSecrets: Array<{ type: SecretType; secretKey: string }>; actorId: string; + actorType?: string; tx?: Knex; secretDAL: Pick; secretQueueService: { removeSecretReminder: (data: TRemoveSecretReminderDTO, tx?: Knex) => Promise; }; + folderCommitService: Pick; + secretVersionDAL: Pick; }; export type THandleReminderDTO = { diff --git a/backend/src/services/secret-v2-bridge/secret-version-dal.ts b/backend/src/services/secret-v2-bridge/secret-version-dal.ts index b54b073a67..9537b79e34 100644 --- a/backend/src/services/secret-v2-bridge/secret-version-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-version-dal.ts @@ -4,7 +4,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { SecretVersionsV2Schema, TableName, TSecretVersionsV2, TSecretVersionsV2Update } from "@app/db/schemas"; import { BadRequestError, DatabaseError } from "@app/lib/errors"; -import { ormify, selectAllTableCols, sqlNestRelationships, TFindOpt } from "@app/lib/knex"; +import { buildFindFilter, ormify, selectAllTableCols, sqlNestRelationships, TFindOpt } from "@app/lib/knex"; import { logger } from "@app/lib/logger"; import { QueueName } from "@app/queue"; @@ -138,7 +138,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { {} ); } catch (error) { - throw new DatabaseError({ error, name: "FindLatestVersinMany" }); + throw new DatabaseError({ error, name: "FindLatestVersionMany" }); } }; @@ -162,6 +162,8 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) .join("version_cte", "version_cte.id", `${TableName.SecretVersionV2}.id`) .whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + // Projects with version >= 3 will require to have all secret versions for PIT + .andWhere(`${TableName.Project}.version`, "<", 3) .delete(); } catch (error) { throw new DatabaseError({ @@ -172,13 +174,21 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret version v2 completed`); }; - const findVersionsBySecretIdWithActors = async ( - secretId: string, - projectId: string, - { offset, limit, sort = [["createdAt", "desc"]] }: TFindOpt = {}, - tx?: Knex - ) => { + const findVersionsBySecretIdWithActors = async ({ + secretId, + projectId, + secretVersions, + findOpt = {}, + tx + }: { + secretId: string; + projectId: string; + secretVersions?: string[]; + findOpt?: TFindOpt; + tx?: Knex; + }) => { try { + const { offset, limit, sort = [["createdAt", "desc"]] } = findOpt; const query = (tx || db)(TableName.SecretVersionV2) .leftJoin(TableName.Users, `${TableName.Users}.id`, `${TableName.SecretVersionV2}.userActorId`) .leftJoin( @@ -189,22 +199,24 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { .leftJoin(TableName.Identity, `${TableName.Identity}.id`, `${TableName.SecretVersionV2}.identityActorId`) .leftJoin(TableName.SecretV2, `${TableName.SecretVersionV2}.secretId`, `${TableName.SecretV2}.id`) .leftJoin( - TableName.SecretV2JnTag, - `${TableName.SecretV2}.id`, - `${TableName.SecretV2JnTag}.${TableName.SecretV2}Id` + TableName.SecretVersionV2Tag, + `${TableName.SecretVersionV2}.id`, + `${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id` ) .leftJoin( TableName.SecretTag, - `${TableName.SecretV2JnTag}.${TableName.SecretTag}Id`, + `${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`, `${TableName.SecretTag}.id` ) .where((qb) => { void qb.where(`${TableName.SecretVersionV2}.secretId`, secretId); void qb.where(`${TableName.ProjectMembership}.projectId`, projectId); + if (secretVersions?.length) void qb.whereIn(`${TableName.SecretVersionV2}.version`, secretVersions); }) .orWhere((qb) => { void qb.where(`${TableName.SecretVersionV2}.secretId`, secretId); void qb.whereNull(`${TableName.ProjectMembership}.projectId`); + if (secretVersions?.length) void qb.whereIn(`${TableName.SecretVersionV2}.version`, secretVersions); }) .select( selectAllTableCols(TableName.SecretVersionV2), @@ -260,6 +272,178 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { } }; + // Function to fetch latest versions by secretIds + const getLatestVersionsBySecretIds = async ( + folderId: string, + secretIds: string[], + tx?: Knex + ): Promise> => { + if (!secretIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + return knexInstance(TableName.SecretVersionV2) + .where("folderId", folderId) + .whereIn(`${TableName.SecretVersionV2}.secretId`, secretIds) + .join( + knexInstance(TableName.SecretVersionV2) + .groupBy("secretId") + .max("version") + .select("secretId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretVersionV2}.secretId`, "latestVersion.secretId").andOn( + `${TableName.SecretVersionV2}.version`, + "latestVersion.max" + ); + } + ); + }; + + // Function to fetch specific versions by versionIds + const getSpecificVersionsWithLatestInfo = async ( + folderId: string, + versionIds: string[], + tx?: Knex + ): Promise> => { + if (!versionIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + + // Get the specific versions + const specificVersions = await knexInstance(TableName.SecretVersionV2) + .where("folderId", folderId) + .whereIn("id", versionIds); + + // Get the secretIds from these versions + const specificSecretIds = [...new Set(specificVersions.map((v) => v.secretId).filter(Boolean))]; + + if (!specificSecretIds.length) return specificVersions; + + // Get max versions for these secretIds + const maxVersionsQuery = await knexInstance(TableName.SecretVersionV2) + .whereIn("secretId", specificSecretIds) + .groupBy("secretId") + .select("secretId") + .max("version", { as: "maxVersion" }); + + // Create a lookup map for max versions + const maxVersionMap = maxVersionsQuery.reduce( + (acc, item) => { + acc[item.secretId] = item.maxVersion; + return acc; + }, + {} as Record + ); + + // Update the version field with maxVersion when needed + return specificVersions.map((version) => { + // Replace version with maxVersion + return { + ...version, + version: maxVersionMap[version.secretId] || version.version + }; + }); + }; + + const findByIdsWithLatestVersion = async ( + folderId: string, + secretIds: string[], + versionIds?: string[], + tx?: Knex + ) => { + try { + if (!secretIds.length && (!versionIds || !versionIds.length)) return {}; + + const [latestVersions, specificVersionsWithLatest] = await Promise.all([ + secretIds.length ? getLatestVersionsBySecretIds(folderId, secretIds, tx) : [], + versionIds?.length ? getSpecificVersionsWithLatestInfo(folderId, versionIds, tx) : [] + ]); + + const allDocs = [...latestVersions, ...specificVersionsWithLatest]; + + // Convert array to record with secretId as key + return allDocs.reduce>( + (prev, curr) => ({ ...prev, [curr.secretId || ""]: curr }), + {} + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindByIdsWithLatestVersion" }); + } + }; + + const findByIdAndPreviousVersion = async (secretVersionId: string, tx?: Knex) => { + try { + const targetSecretVersion = await (tx || db.replicaNode())(TableName.SecretVersionV2) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ id: secretVersionId }, TableName.SecretVersionV2)) + .leftJoin( + TableName.SecretVersionV2Tag, + `${TableName.SecretVersionV2}.id`, + `${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id` + ) + .leftJoin( + TableName.SecretTag, + `${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`, + `${TableName.SecretTag}.id` + ) + .select(selectAllTableCols(TableName.SecretVersionV2)) + .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId")) + .select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor")) + .select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")) + .first(); + if (targetSecretVersion) { + const previousSecretVersion = await (tx || db.replicaNode())(TableName.SecretVersionV2) + .where( + // eslint-disable-next-line @typescript-eslint/no-misused-promises + buildFindFilter( + { version: targetSecretVersion.version - 1, secretId: targetSecretVersion.secretId }, + TableName.SecretVersionV2 + ) + ) + .leftJoin( + TableName.SecretVersionV2Tag, + `${TableName.SecretVersionV2}.id`, + `${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id` + ) + .leftJoin( + TableName.SecretTag, + `${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`, + `${TableName.SecretTag}.id` + ) + .select(selectAllTableCols(TableName.SecretVersionV2)) + .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId")) + .select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor")) + .select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")) + .first(); + if (!previousSecretVersion) return []; + const docs = [previousSecretVersion, targetSecretVersion]; + + const data = sqlNestRelationships({ + data: docs, + key: "id", + parentMapper: (el) => ({ _id: el.id, ...SecretVersionsV2Schema.parse(el) }), + childrenMapper: [ + { + key: "tagId", + label: "tags" as const, + mapper: ({ tagId: id, tagColor: color, tagSlug: slug }) => ({ + id, + color, + slug, + name: slug + }) + } + ] + }); + + return data; + } + return []; + } catch (error) { + throw new DatabaseError({ error, name: "FindByIdAndPreviousVersion" }); + } + }; + return { ...secretVersionV2Orm, pruneExcessVersions, @@ -267,6 +451,8 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { bulkUpdate, findLatestVersionByFolderId, findVersionsBySecretIdWithActors, - findBySecretId + findBySecretId, + findByIdsWithLatestVersion, + findByIdAndPreviousVersion }; }; diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index e5f3acdea4..96f89ab5f2 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -778,6 +778,7 @@ export const createManySecretsRawFnFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, + folderCommitService, kmsService, resourceMetadataDAL }: TCreateManySecretsRawFnFactory) => { @@ -850,6 +851,7 @@ export const createManySecretsRawFnFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService, tx }) ); @@ -942,6 +944,7 @@ export const updateManySecretsRawFnFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, resourceMetadataDAL, + folderCommitService, kmsService }: TUpdateManySecretsRawFnFactory) => { const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL); @@ -1032,7 +1035,8 @@ export const updateManySecretsRawFnFactory = ({ secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, - secretVersionTagDAL: secretVersionTagV2BridgeDAL + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService }) ); diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 714df0d3f8..87b2eb467d 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -35,6 +35,7 @@ import { TSecretSyncQueueFactory } from "@app/services/secret-sync/secret-sync-q import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { ActorType } from "../auth/auth-type"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TIntegrationDALFactory } from "../integration/integration-dal"; import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal"; import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service"; @@ -112,6 +113,7 @@ type TSecretQueueFactoryDep = { orgService: Pick; projectUserMembershipRoleDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; secretReminderRecipientsDAL: Pick< TSecretReminderRecipientsDALFactory, "delete" | "findUsersBySecretId" | "insertMany" | "transaction" @@ -178,7 +180,8 @@ export const secretQueueFactory = ({ projectKeyDAL, resourceMetadataDAL, secretReminderRecipientsDAL, - secretSyncQueue + secretSyncQueue, + folderCommitService }: TSecretQueueFactoryDep) => { const integrationMeter = opentelemetry.metrics.getMeter("Integrations"); const errorHistogram = integrationMeter.createHistogram("integration_secret_sync_errors", { @@ -366,7 +369,8 @@ export const secretQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const updateManySecretsRawFn = updateManySecretsRawFnFactory({ @@ -382,7 +386,8 @@ export const secretQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); /** diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 46ed7ef83e..4fa7404d34 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -44,7 +44,8 @@ import { TGetSecretsRawByFolderMappingsDTO } from "@app/services/secret-v2-bridge/secret-v2-bridge-types"; -import { ActorType } from "../auth/auth-type"; +import { ActorAuthMethod, ActorType } from "../auth/auth-type"; +import { ChangeType } from "../folder-commit/folder-commit-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; @@ -2521,6 +2522,36 @@ export const secretServiceFactory = ({ }); }; + const getSecretVersionsV2ByIds = async ({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + secretVersions, + secretPath, + envId, + projectId + }: TGetSecretVersionsDTO & { + secretVersions: string[]; + secretPath: string; + envId: string; + projectId: string; + }) => { + const secretVersionV2 = await secretV2BridgeService.getSecretVersionsByIds({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + secretVersionNumbers: secretVersions, + secretPath, + envId, + projectId + }); + return secretVersionV2; + }; + const attachTags = async ({ secretName, tagSlugs, @@ -3279,6 +3310,53 @@ export const secretServiceFactory = ({ return secrets; }; + const getChangeVersions = async ( + change: { + secretVersion: string; + secretId?: string; + id?: string; + isUpdate?: boolean; + changeType?: string; + }, + previousVersion: string, + actorId: string, + actor: ActorType, + actorOrgId: string, + actorAuthMethod: ActorAuthMethod, + envId: string, + projectId: string, + secretPath: string + ) => { + const currentVersion = change.secretVersion; + const secretId = change.secretId ? change.secretId : change.id; + if (!secretId) { + return; + } + const versions = await getSecretVersionsV2ByIds({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + // if it's update add also the previous secretversionid + secretVersions: + change.isUpdate || change.changeType === ChangeType.UPDATE + ? [currentVersion, previousVersion] + : [currentVersion], + secretPath, + envId, + projectId + }); + return versions?.map((v) => ({ + secretKey: v.secretKey, + secretComment: v.secretComment, + skipMultilineEncoding: v.skipMultilineEncoding, + tags: v.tags?.map((tag) => tag.slug), + metadata: v.metadata, + secretValue: v.secretValue + })); + }; + return { attachTags, detachTags, @@ -3309,6 +3387,8 @@ export const secretServiceFactory = ({ getSecretsRawByFolderMappings, getSecretAccessList, getSecretByIdRaw, - getAccessibleSecrets + getAccessibleSecrets, + getSecretVersionsV2ByIds, + getChangeVersions }; }; diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 30e3dfafa4..91fc2eb6a3 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -14,6 +14,7 @@ import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-fold import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { ActorType } from "../auth/auth-type"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; @@ -441,6 +442,7 @@ export type TCreateManySecretsRawFnFactory = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; }; export type TCreateManySecretsRawFn = { @@ -478,6 +480,7 @@ export type TUpdateManySecretsRawFnFactory = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; }; export type TUpdateManySecretsRawFn = { diff --git a/docs/documentation/platform/pit-recovery.mdx b/docs/documentation/platform/pit-recovery.mdx index 448faddbb7..24d0071a7c 100644 --- a/docs/documentation/platform/pit-recovery.mdx +++ b/docs/documentation/platform/pit-recovery.mdx @@ -4,38 +4,130 @@ description: "Learn how to rollback secrets and configurations to any snapshot w --- - Point-in-Time Recovery is a paid feature. - - If you're using Infisical Cloud, then it is available under the **Pro Tier**. If you're self-hosting Infisical, - then you should contact sales@infisical.com to purchase an enterprise license to use it. + Point-in-Time Recovery is a paid feature. If you're using Infisical Cloud, + then it is available under the **Pro Tier**. If you're self-hosting Infisical, + then you should contact sales@infisical.com to purchase an enterprise license + to use it. Infisical's point-in-time recovery functionality allows secrets to be rolled back to any point in time for any given [folder](./folder) or [environment](/documentation/platform/project#project-environments). -Every time a secret is updated, a new snapshot is taken – capturing the state of the folder and environment at that point of time. -## Snapshots + + + ## Understanding Commits -Similar to Git, a commit (also known as snapshot) in Infisical is the state of your project's secrets at a specific point in time scoped to -an environment and [folder](./folder) within it. + Similar to Git, a commit in Infisical represents a snapshot of changes made to your project's resources at a specific point in time. Each commit is scoped to an environment and [folder](./folder) within it. Unlike the legacy snapshot system, the new commits interface provides granular tracking of individual changes, allowing you to see exactly what was modified, added, or removed in each commit. -To view a list of snapshots for the current folder, press the **Commits** button. + ### Accessing Commits -![PIT commits](../../images/platform/pit-recovery/pit-recovery-commits.png) + From your secrets management interface, you can access the commits functionality by clicking the **Commits Button**. This button is located in the top-right area of your secrets view and shows the number of commits for the current folder (e.g., "4 Commits"). -This opens up a sidebar from which you can select to view a particular snapshot: + ![Commits Button](../../images/platform/pit-recovery/pit-recovery-revamp/pit-commits-button.png) -![PIT snapshots](../../images/platform/pit-recovery/pit-recovery-commits-drawer.png) + ### Commits List View -## Rolling back + The commits page displays a comprehensive chronological history of all changes made to your environment and folders: -After pressing on a snapshot from the sidebar, you can view it and roll back the state -of the folder to that point in time by pressing the **Rollback** button. + ![Commits List View](../../images/platform/pit-recovery/pit-recovery-revamp/pit-commits-history.png) -![PIT snapshot](../../images/platform/pit-recovery/pit-recovery-rollback.png) + - **Chronological Sorting**: Commits are grouped by date + - **Commit Information**: Each commit shows: + - Commit message + - Author information + - Relative timestamp + - Unique commit hash identifier + - **Search Functionality**: Use the search bar to quickly find specific commits + - **Sorting Options**: Sort commits by various criteria using the sort controls -Rolling back secrets to a past snapshot creates a creates a snapshot at the top of the stack and updates secret versions. + ### Detailed Commit Inspection - -Rollbacks are localized to not affect other folders within the same environment. This means each [folder](./folder) maintains its own independent history of changes, offering precise and isolated control over rollback actions. -Put differently, every [folder](./folder) possesses a distinct and separate timeline, providing granular control when managing your secrets. - \ No newline at end of file + Clicking on any commit from the list opens a detailed view showing the list of changes made in that commit. + + ![Detailed Commit Inspection](../../images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes.png) + + #### Change Categories + + The commit changes details can be grouped into the following categories: + + **Folder Changes** + - Shows folder additions, modifications, or deletions + - Displays the folder properties changes in JSON format, including: + - Folder name + - Folder description + + **Secret Changes** + - Lists all secrets that were added, updated, or removed + - Shows the complete secret configuration including: + - Secret key and value + - Comments, tags and metadata + - Encoding settings (e.g., skipMultilineEncoding) + - Values are displayed with appropriate masking for security + + **Visual Indicators** + - Green "+" indicators show additions + - Red "-" indicators show deletions + - Modified content shows both old and new states + + ### Restoration Options + + Each commit provides two distinct restoration methods accessible via the **Restore Options** dropdown: + + ![Restore Options](../../images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes-options.png) + + #### Revert changes + This option provides surgical precision for undoing specific modifications: + + - **Granular Control**: Reverts only the specific changes introduced in that individual commit + - **Selective Restoration**: Preserves all other changes made after the commit + - **Targeted Undo**: Perfect for reversing a specific problematic change without affecting other work + - **Minimal Impact**: Only affects the resources that were modified in that particular commit + - **Use Case**: Ideal when you want to undo a specific change while keeping all other modifications intact + + #### Roll back to this commit + This option performs a complete restoration to the selected point in time: + + ![Rollback](../../images/platform/pit-recovery/pit-recovery-revamp/pit-commit-restore.png) + + - **Complete State Restoration**: Returns the entire folder to its exact state at the time of this commit + - **Restore All Child Folders**: If enabled, it'll also restore all nested folders to their exact state at the time of this commit. + - **Destructive Operation**: Discards ALL changes made after the selected commit + - **New Commit Creation**: Creates a new commit representing this rollback operation + - **Use Case**: Ideal when you want to completely undo a series of changes and return to a known good state + + **Warning**: This operation will undo all modifications made after the selected commit, which may include multiple secrets and configuration changes. + + + + + The snapshots interface is deprecated and will be removed in a future version. Please use the new Commits interface for more granular point-in-time recovery operations. + + + ## Snapshots + + Similar to Git, a commit (also known as snapshot) in Infisical is the state of your project's secrets at a specific point in time scoped to + an environment and [folder](./folder) within it. + + To view a list of snapshots for the current folder, press the **Commits** button. + + ![PIT commits](../../images/platform/pit-recovery/pit-recovery-commits.png) + + This opens up a sidebar from which you can select to view a particular snapshot: + + ![PIT snapshots](../../images/platform/pit-recovery/pit-recovery-commits-drawer.png) + + ## Rolling back + + After pressing on a snapshot from the sidebar, you can view it and roll back the state + of the folder to that point in time by pressing the **Rollback** button. + + ![PIT snapshot](../../images/platform/pit-recovery/pit-recovery-rollback.png) + + Rolling back secrets to a past snapshot creates a snapshot at the top of the stack and updates secret versions. + + + Rollbacks are localized to not affect other folders within the same environment. This means each [folder](./folder) maintains its own independent history of changes, offering precise and isolated control over rollback actions. + Put differently, every [folder](./folder) possesses a distinct and separate timeline, providing granular control when managing your secrets. + + + + diff --git a/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes-options.png b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes-options.png new file mode 100644 index 0000000000..60d9dff011 Binary files /dev/null and b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes-options.png differ diff --git a/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes.png b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes.png new file mode 100644 index 0000000000..56357e9efe Binary files /dev/null and b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes.png differ diff --git a/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-restore.png b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-restore.png new file mode 100644 index 0000000000..3ed22c63b4 Binary files /dev/null and b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-restore.png differ diff --git a/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-button.png b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-button.png new file mode 100644 index 0000000000..273760ca5d Binary files /dev/null and b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-button.png differ diff --git a/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-history.png b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-history.png new file mode 100644 index 0000000000..3832209f4e Binary files /dev/null and b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-history.png differ diff --git a/docs/internals/permissions/project-permissions.mdx b/docs/internals/permissions/project-permissions.mdx index 98f3bfeb24..da9351188a 100644 --- a/docs/internals/permissions/project-permissions.mdx +++ b/docs/internals/permissions/project-permissions.mdx @@ -176,6 +176,13 @@ Supports conditions and permission inversion | `read` | View secret versions and snapshots | | `create` | Roll back secrets to snapshots | +#### Subject: `commits` + +| Action | Description | +| -------- | ---------------------------------- | +| `read` | View commits and changes across folders | +| `perform-rollback` | Roll back commits changes and restore folders to previous state| + #### Subject: `secret-approval` | Action | Description | diff --git a/docs/mint.json b/docs/mint.json index a7dd9d5c7b..2443784b9b 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -826,8 +826,7 @@ "api-reference/endpoints/workspaces/delete-workspace", "api-reference/endpoints/workspaces/get-workspace", "api-reference/endpoints/workspaces/update-workspace", - "api-reference/endpoints/workspaces/secret-snapshots", - "api-reference/endpoints/workspaces/rollback-snapshot" + "api-reference/endpoints/workspaces/secret-snapshots" ] }, { diff --git a/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx b/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx index 584a3f6c7c..31a6268662 100644 --- a/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx +++ b/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx @@ -14,13 +14,15 @@ type Props = { selectedPathSegmentIndex: number; environmentSlug: string; projectId: string; + disableCopy?: boolean; }; export const SecretDashboardPathBreadcrumb = ({ secretPathSegments, selectedPathSegmentIndex, environmentSlug, - projectId + projectId, + disableCopy }: Props) => { const [, isCopying, setIsCopying] = useTimedReset({ initialState: false @@ -32,7 +34,7 @@ export const SecretDashboardPathBreadcrumb = ({ return (
- {isLastItem ? ( + {isLastItem && !disableCopy ? (
[{ workspaceId, environment, directory }, "folder-commits-count"] as const, + + history: ({ + workspaceId, + environment, + directory + }: { + workspaceId: string; + environment: string; + directory?: string; + }) => [{ workspaceId, environment, directory }, "folder-commits"] as const, + + details: ({ workspaceId, commitId }: { workspaceId: string; commitId: string }) => + [{ workspaceId, commitId }, "commit-details"] as const, + + rollbackPreview: ({ + folderId, + commitId, + envSlug, + projectId, + deepRollback + }: { + folderId: string; + commitId: string; + envSlug: string; + projectId: string; + deepRollback: boolean; + }) => [{ folderId, commitId, envSlug, projectId, deepRollback }, "rollback-preview"] as const +}; + +const fetchFolderCommitsCount = async ({ + workspaceId, + environment, + directory +}: { + workspaceId: string; + environment: string; + directory?: string; +}) => { + const res = await apiRequest.get<{ count: number; folderId: string }>( + "/api/v1/pit/commits/count", + { + params: { + environment, + path: directory, + projectId: workspaceId + } + } + ); + return res.data; +}; + +const fetchFolderCommitHistory = async ( + workspaceId: string, + environment: string, + directory: string, + offset: number = 0, + limit: number = 20, + search?: string, + sort: "asc" | "desc" = "desc" +): Promise<{ + commits: CommitHistoryItem[]; + total: number; + hasMore: boolean; +}> => { + const res = await apiRequest.get<{ + commits: CommitHistoryItem[]; + total: number; + hasMore: boolean; + }>("/api/v1/pit/commits", { + params: { + environment, + path: directory, + projectId: workspaceId, + offset, + limit, + search, + sort + } + }); + return res.data; +}; + +export const fetchCommitDetails = async (workspaceId: string, commitId: string) => { + const { data } = await apiRequest.get( + `/api/v1/pit/commits/${commitId}/changes`, + { + params: { + projectId: workspaceId + } + } + ); + return data; +}; + +export const fetchRollbackPreview = async ( + folderId: string, + commitId: string, + envSlug: string, + workspaceId: string, + deepRollback: boolean, + secretPath: string +): Promise => { + const { data } = await apiRequest.get( + `/api/v1/pit/commits/${commitId}/compare`, + { + params: { + folderId, + environment: envSlug, + deepRollback, + secretPath, + projectId: workspaceId + } + } + ); + return data; +}; + +const fetchRollback = async ( + folderId: string, + commitId: string, + workspaceId: string, + deepRollback: boolean, + message?: string, + envSlug?: string +) => { + const { data } = await apiRequest.post<{ success: boolean }>( + `/api/v1/pit/commits/${commitId}/rollback`, + { + folderId, + deepRollback, + message, + environment: envSlug, + projectId: workspaceId + } + ); + return data; +}; + +const fetchRevert = async (commitId: string, workspaceId: string) => { + const { data } = await apiRequest.post<{ success: boolean; message: string }>( + `/api/v1/pit/commits/${commitId}/revert`, + { + projectId: workspaceId + } + ); + return data; +}; + +export const useCommitRevert = ({ + commitId, + projectId, + environment, + directory +}: { + commitId: string; + projectId: string; + environment: string; + directory: string; +}) => { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: () => fetchRevert(commitId, projectId), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: [ + commitKeys.details({ workspaceId: projectId, commitId }), + commitKeys.history({ workspaceId: projectId, environment, directory }), + commitKeys.count({ workspaceId: projectId, environment, directory }) + ] + }); + } + }); +}; + +export const useCommitRollback = ({ + workspaceId, + commitId, + folderId, + deepRollback, + environment, + directory, + envSlug +}: { + workspaceId: string; + commitId: string; + folderId: string; + deepRollback: boolean; + environment: string; + directory: string; + envSlug: string; +}) => { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (message: string) => + fetchRollback(folderId, commitId, workspaceId, deepRollback, message, envSlug), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: [ + commitKeys.details({ workspaceId, commitId }), + commitKeys.history({ workspaceId, environment, directory }), + commitKeys.count({ workspaceId, environment, directory }) + ] + }); + } + }); +}; + +export const useGetFolderCommitsCount = ({ + workspaceId, + environment, + directory, + isPaused +}: { + workspaceId: string; + environment: string; + directory: string; + isPaused?: boolean; +}) => + useQuery({ + enabled: Boolean(workspaceId && environment) && !isPaused, + queryKey: commitKeys.count({ workspaceId, environment, directory }), + queryFn: () => fetchFolderCommitsCount({ workspaceId, environment, directory }) + }); + +export const useGetFolderCommitHistory = ({ + workspaceId, + environment, + directory, + offset = 0, + limit = 20, + search, + sort = "desc" +}: { + workspaceId: string; + environment: string; + directory: string; + offset?: number; + limit?: number; + search?: string; + sort?: "asc" | "desc"; +}) => { + return useQuery({ + queryKey: [ + commitKeys.history({ workspaceId, environment, directory }), + offset, + limit, + search, + sort + ], + queryFn: () => + fetchFolderCommitHistory(workspaceId, environment, directory, offset, limit, search, sort), + enabled: Boolean(workspaceId && environment) + }); +}; + +export const useGetCommitDetails = (workspaceId: string, commitId: string) => { + return useQuery({ + queryKey: commitKeys.details({ workspaceId, commitId }), + queryFn: () => fetchCommitDetails(workspaceId, commitId), + enabled: Boolean(workspaceId) && Boolean(commitId) + }); +}; + +export const useGetRollbackPreview = ( + folderId: string, + commitId: string, + envSlug: string, + projectId: string, + deepRollback: boolean, + secretPath: string +) => { + return useQuery({ + queryKey: commitKeys.rollbackPreview({ folderId, commitId, envSlug, projectId, deepRollback }), + queryFn: () => + fetchRollbackPreview(folderId, commitId, envSlug, projectId, deepRollback, secretPath), + enabled: Boolean(folderId) && Boolean(commitId) + }); +}; diff --git a/frontend/src/hooks/api/folderCommits/types.ts b/frontend/src/hooks/api/folderCommits/types.ts new file mode 100644 index 0000000000..878e3224da --- /dev/null +++ b/frontend/src/hooks/api/folderCommits/types.ts @@ -0,0 +1,64 @@ +import { CommitType, SecretVersions } from "../types"; + +export type CommitHistoryItem = { + id: string; + commitId: string; + actorMetadata: { + id: string; + name?: string; + }; + actorType: string; + message: string; + folderId: string; + envId: string; + createdAt: string; + updatedAt: string; + isLatest: boolean; +}; + +export type TFolderCommitChanges = { + id: string; + folderCommitId: string; + changeType: CommitType; + isUpdate: boolean; + secretVersionId: string | null; + folderVersionId: string | null; + createdAt: string; + updatedAt: string; + versions: SecretVersions[]; + secretKey?: string; + folderName?: string; + secretVersion?: string; + folderVersion?: string; +}; + +export type FolderReconstructedItem = { + type: string; + id: string; + versionId: string; + folderName?: string; + folderVersion?: number; + secretKey?: string; + secretVersion?: number; +}; + +export type CommitWithChanges = { + changes: CommitHistoryItem & { + changes: TFolderCommitChanges[]; + }; +}; + +export type RollbackChange = { + type: "folder" | "secret"; + id: string; + versionId: string; + changeType: "create" | "update" | "delete"; + commitId: string; +}; + +export type RollbackPreview = { + folderId: string; + folderName: string; + folderPath: string; + changes: RollbackChange[]; +}; diff --git a/frontend/src/hooks/api/secretApprovalRequest/types.ts b/frontend/src/hooks/api/secretApprovalRequest/types.ts index 3ac2574b5b..241d3c1e29 100644 --- a/frontend/src/hooks/api/secretApprovalRequest/types.ts +++ b/frontend/src/hooks/api/secretApprovalRequest/types.ts @@ -11,7 +11,8 @@ export enum ApprovalStatus { export enum CommitType { DELETE = "delete", UPDATE = "update", - CREATE = "create" + CREATE = "create", + ADD = "add" } export type TSecretApprovalSecChangeData = { diff --git a/frontend/src/hooks/api/secretFolders/queries.tsx b/frontend/src/hooks/api/secretFolders/queries.tsx index b49fbd6cc7..9e12f329da 100644 --- a/frontend/src/hooks/api/secretFolders/queries.tsx +++ b/frontend/src/hooks/api/secretFolders/queries.tsx @@ -10,6 +10,7 @@ import { import { apiRequest } from "@app/config/request"; import { dashboardKeys } from "@app/hooks/api/dashboard/queries"; +import { commitKeys } from "../folderCommits/queries"; import { secretSnapshotKeys } from "../secretSnapshots/queries"; import { TCreateFolderDTO, @@ -166,6 +167,9 @@ export const useCreateFolder = () => { queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId: projectId, environment, directory: path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId: projectId, environment, directory: path }) + }); } }); }; @@ -200,6 +204,12 @@ export const useUpdateFolder = () => { queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId: projectId, environment, directory: path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId: projectId, environment, directory: path }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId: projectId, environment, directory: path }) + }); } }); }; @@ -234,6 +244,12 @@ export const useDeleteFolder = () => { queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId: projectId, environment, directory: path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId: projectId, environment, directory: path }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId: projectId, environment, directory: path }) + }); } }); }; @@ -279,6 +295,20 @@ export const useUpdateFolderBatch = () => { directory: folder.path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ + workspaceId: projectId, + environment: folder.environment, + directory: folder.path + }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ + workspaceId: projectId, + environment: folder.environment, + directory: folder.path + }) + }); }); } }); diff --git a/frontend/src/hooks/api/secrets/mutations.tsx b/frontend/src/hooks/api/secrets/mutations.tsx index 3862d1f8d3..82bf623e64 100644 --- a/frontend/src/hooks/api/secrets/mutations.tsx +++ b/frontend/src/hooks/api/secrets/mutations.tsx @@ -3,6 +3,7 @@ import { MutationOptions, useMutation, useQueryClient } from "@tanstack/react-qu import { apiRequest } from "@app/config/request"; import { dashboardKeys } from "@app/hooks/api/dashboard/queries"; +import { commitKeys } from "../folderCommits/queries"; import { secretApprovalRequestKeys } from "../secretApprovalRequest/queries"; import { secretSnapshotKeys } from "../secretSnapshots/queries"; import { secretKeys } from "./queries"; @@ -59,6 +60,12 @@ export const useCreateSecretV3 = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -118,6 +125,12 @@ export const useUpdateSecretV3 = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -164,6 +177,12 @@ export const useDeleteSecretV3 = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -200,6 +219,12 @@ export const useCreateSecretBatch = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -236,6 +261,12 @@ export const useUpdateSecretBatch = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -274,6 +305,12 @@ export const useDeleteSecretBatch = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -347,6 +384,20 @@ export const useMoveSecrets = ({ directory: sourceSecretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ + workspaceId: projectId, + environment: sourceEnvironment, + directory: sourceSecretPath + }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ + workspaceId: projectId, + environment: sourceEnvironment, + directory: sourceSecretPath + }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId: projectId }) }); diff --git a/frontend/src/hooks/api/workspace/queries.tsx b/frontend/src/hooks/api/workspace/queries.tsx index 2ae2bc463d..fa92401013 100644 --- a/frontend/src/hooks/api/workspace/queries.tsx +++ b/frontend/src/hooks/api/workspace/queries.tsx @@ -282,7 +282,8 @@ export const useUpdateProject = () => { newProjectName, newProjectDescription, newSlug, - secretSharing + secretSharing, + showSnapshotsLegacy }) => { const { data } = await apiRequest.patch<{ workspace: Workspace }>( `/api/v1/workspace/${projectID}`, @@ -290,7 +291,8 @@ export const useUpdateProject = () => { name: newProjectName, description: newProjectDescription, slug: newSlug, - secretSharing + secretSharing, + showSnapshotsLegacy } ); return data.workspace; diff --git a/frontend/src/hooks/api/workspace/types.ts b/frontend/src/hooks/api/workspace/types.ts index 481bdcc088..fbaea77422 100644 --- a/frontend/src/hooks/api/workspace/types.ts +++ b/frontend/src/hooks/api/workspace/types.ts @@ -39,6 +39,7 @@ export type Workspace = { roles?: TProjectRole[]; hasDeleteProtection: boolean; secretSharing: boolean; + showSnapshotsLegacy: boolean; }; export type WorkspaceEnv = { @@ -79,6 +80,7 @@ export type UpdateProjectDTO = { newProjectDescription?: string; newSlug?: string; secretSharing?: boolean; + showSnapshotsLegacy?: boolean; }; export type UpdatePitVersionLimitDTO = { projectSlug: string; pitVersionLimit: number }; diff --git a/frontend/src/pages/project/RoleDetailsBySlugPage/components/PolicySelectionModal.tsx b/frontend/src/pages/project/RoleDetailsBySlugPage/components/PolicySelectionModal.tsx index e636bdfe46..cdf6b3d296 100644 --- a/frontend/src/pages/project/RoleDetailsBySlugPage/components/PolicySelectionModal.tsx +++ b/frontend/src/pages/project/RoleDetailsBySlugPage/components/PolicySelectionModal.tsx @@ -23,6 +23,7 @@ import { useGetProjectTypeFromRoute } from "@app/hooks"; import { ProjectType } from "@app/hooks/api/workspace/types"; import { + EXCLUDED_PERMISSION_SUBS, isConditionalSubjects, PROJECT_PERMISSION_OBJECT, ProjectTypePermissionSubjects, @@ -66,6 +67,7 @@ const Content = ({ onClose }: ContentProps) => { subject as ProjectPermissionSub ] && (search ? title.toLowerCase().includes(search.toLowerCase()) : true) ) + .filter(([subject]) => !EXCLUDED_PERMISSION_SUBS.includes(subject as ProjectPermissionSub)) .sort((a, b) => a[1].title.localeCompare(b[1].title)) .map(([subject]) => subject); diff --git a/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx b/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx index c986be37b6..d7872a223b 100644 --- a/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx +++ b/frontend/src/pages/project/RoleDetailsBySlugPage/components/ProjectRoleModifySection.utils.tsx @@ -12,6 +12,7 @@ import { } from "@app/context"; import { PermissionConditionOperators, + ProjectPermissionCommitsActions, ProjectPermissionDynamicSecretActions, ProjectPermissionGroupActions, ProjectPermissionIdentityActions, @@ -92,6 +93,11 @@ const SecretSyncPolicyActionSchema = z.object({ [ProjectPermissionSecretSyncActions.RemoveSecrets]: z.boolean().optional() }); +const CommitPolicyActionSchema = z.object({ + [ProjectPermissionCommitsActions.Read]: z.boolean().optional(), + [ProjectPermissionCommitsActions.PerformRollback]: z.boolean().optional() +}); + const SecretRotationPolicyActionSchema = z.object({ [ProjectPermissionSecretRotationActions.Read]: z.boolean().optional(), [ProjectPermissionSecretRotationActions.ReadGeneratedCredentials]: z.boolean().optional(), @@ -285,6 +291,7 @@ export const projectRoleFormSchema = z.object({ }) .array() .default([]), + [ProjectPermissionSub.Commits]: CommitPolicyActionSchema.array().default([]), [ProjectPermissionSub.Member]: MemberPolicyActionSchema.array().default([]), [ProjectPermissionSub.Groups]: GroupPolicyActionSchema.array().default([]), [ProjectPermissionSub.Role]: GeneralPolicyActionSchema.array().default([]), @@ -885,6 +892,17 @@ export const rolePermission2Form = (permissions: TProjectPermission[] = []) => { return; } + if (subject === ProjectPermissionSub.Commits) { + const canRead = action.includes(ProjectPermissionCommitsActions.Read); + const canPerformRollback = action.includes(ProjectPermissionCommitsActions.PerformRollback); + + if (!formVal[subject]) formVal[subject] = [{}]; + if (canRead) formVal[subject]![0][ProjectPermissionCommitsActions.Read] = true; + if (canPerformRollback) + formVal[subject]![0][ProjectPermissionCommitsActions.PerformRollback] = true; + return; + } + if (subject === ProjectPermissionSub.PkiSubscribers) { if (!formVal[subject]) formVal[subject] = []; @@ -1032,6 +1050,8 @@ export const formRolePermission2API = (formVal: TFormSchema["permissions"]) => { return permissions; }; +export const EXCLUDED_PERMISSION_SUBS = [ProjectPermissionSub.SecretRollback]; + export type TProjectPermissionObject = { [K in ProjectPermissionSub]: { title: string; @@ -1227,6 +1247,13 @@ export const PROJECT_PERMISSION_OBJECT: TProjectPermissionObject = { { label: "Remove", value: "delete" } ] }, + [ProjectPermissionSub.Commits]: { + title: "Commits", + actions: [ + { label: "View", value: ProjectPermissionCommitsActions.Read }, + { label: "Perform Rollback", value: ProjectPermissionCommitsActions.PerformRollback } + ] + }, [ProjectPermissionSub.Tags]: { title: "Tags", actions: [ @@ -1518,7 +1545,8 @@ const SecretsManagerPermissionSubjects = (enabled = false) => ({ [ProjectPermissionSub.IpAllowList]: enabled, [ProjectPermissionSub.SecretRollback]: enabled, [ProjectPermissionSub.SecretRotation]: enabled, - [ProjectPermissionSub.ServiceTokens]: enabled + [ProjectPermissionSub.ServiceTokens]: enabled, + [ProjectPermissionSub.Commits]: enabled }); const KmsPermissionSubjects = (enabled = false) => ({ @@ -1898,6 +1926,10 @@ export const RoleTemplates: Record = { { subject: ProjectPermissionSub.SecretSyncs, actions: [ProjectPermissionSecretSyncActions.Read] + }, + { + subject: ProjectPermissionSub.Commits, + actions: [ProjectPermissionCommitsActions.Read] } ] }, @@ -1955,6 +1987,10 @@ export const RoleTemplates: Record = { { subject: ProjectPermissionSub.SecretSyncs, actions: Object.values(ProjectPermissionSecretSyncActions) + }, + { + subject: ProjectPermissionSub.Commits, + actions: Object.values(ProjectPermissionCommitsActions) } ] }, diff --git a/frontend/src/pages/project/RoleDetailsBySlugPage/components/RolePermissionsSection.tsx b/frontend/src/pages/project/RoleDetailsBySlugPage/components/RolePermissionsSection.tsx index 382cb0b559..31f63ac51b 100644 --- a/frontend/src/pages/project/RoleDetailsBySlugPage/components/RolePermissionsSection.tsx +++ b/frontend/src/pages/project/RoleDetailsBySlugPage/components/RolePermissionsSection.tsx @@ -25,6 +25,7 @@ import { PermissionEmptyState } from "./PermissionEmptyState"; import { PkiSubscriberPermissionConditions } from "./PkiSubscriberPermissionConditions"; import { PkiTemplatePermissionConditions } from "./PkiTemplatePermissionConditions"; import { + EXCLUDED_PERMISSION_SUBS, formRolePermission2API, isConditionalSubjects, PROJECT_PERMISSION_OBJECT, @@ -176,6 +177,7 @@ export const RolePermissionsSection = ({ roleSlug, isDisabled }: Props) => {
{!isPending && } {(Object.keys(PROJECT_PERMISSION_OBJECT) as ProjectPermissionSub[]) + .filter((subject) => !EXCLUDED_PERMISSION_SUBS.includes(subject)) .filter((subject) => ProjectTypePermissionSubjects[currentWorkspace.type][subject]) .map((subject) => ( { + const envSlug = useParams({ + from: ROUTE_PATHS.SecretManager.CommitDetailsPage.id, + select: (el) => el.environment + }); + const selectedCommitId = useParams({ + from: ROUTE_PATHS.SecretManager.CommitDetailsPage.id, + select: (el) => el.commitId + }); + const folderId = useParams({ + from: ROUTE_PATHS.SecretManager.CommitDetailsPage.id, + select: (el) => el.folderId + }); + const { currentWorkspace } = useWorkspace(); + + const navigate = useNavigate(); + const routerQueryParams: { secretPath?: string } = useSearch({ + from: ROUTE_PATHS.SecretManager.CommitDetailsPage.id + }); + + const secretPath = (routerQueryParams.secretPath as string) || "/"; + + const handleGoBackToHistory = () => { + navigate({ + to: `/${ProjectType.SecretManager}/$projectId/commits/$environment/$folderId` as const, + params: { + projectId: currentWorkspace.id, + folderId, + environment: envSlug + }, + search: (query) => ({ + ...query, + secretPath + }) + }); + }; + + const handleGoToRollbackPreview = () => { + navigate({ + to: `/${ProjectType.SecretManager}/$projectId/commits/$environment/$folderId/$commitId/restore` as const, + params: { + projectId: currentWorkspace.id, + folderId, + environment: envSlug, + commitId: selectedCommitId + }, + search: (query) => ({ + ...query, + secretPath + }) + }); + }; + + return ( +
+
+ + + +
+
+ ); +}; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/CommitDetailsTab.tsx b/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/CommitDetailsTab.tsx new file mode 100644 index 0000000000..c403f9f3fc --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/CommitDetailsTab.tsx @@ -0,0 +1,362 @@ +import { useEffect, useState } from "react"; +import { faAngleDown } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { DropdownMenuItem } from "@radix-ui/react-dropdown-menu"; +import { useSearch } from "@tanstack/react-router"; + +import { createNotification } from "@app/components/notifications"; +import { ProjectPermissionCan } from "@app/components/permissions"; +import { + DeleteActionModal, + DropdownMenu, + DropdownMenuContent, + DropdownMenuTrigger, + IconButton, + Spinner +} from "@app/components/v2"; +import { ROUTE_PATHS } from "@app/const/routes"; +import { + ProjectPermissionCommitsActions, + ProjectPermissionSub +} from "@app/context/ProjectPermissionContext/types"; +import { usePopUp } from "@app/hooks"; +import { CommitWithChanges } from "@app/hooks/api/folderCommits"; +import { useCommitRevert, useGetCommitDetails } from "@app/hooks/api/folderCommits/queries"; +import { CommitType } from "@app/hooks/api/types"; + +import { SecretVersionDiffView } from "../SecretVersionDiffView"; +import { MergedItem } from "./types"; + +const formatDisplayDate = (dateString: string): string => { + try { + const date = new Date(dateString); + const options: Intl.DateTimeFormatOptions = { + year: "numeric", + month: "short", + day: "numeric", + hour: "numeric", + minute: "numeric", + hour12: true + }; + return new Intl.DateTimeFormat("en-US", options).format(date); + } catch { + return dateString; + } +}; + +export const CommitDetailsTab = ({ + selectedCommitId, + workspaceId, + envSlug, + goBackToHistory, + goToRollbackPreview +}: { + selectedCommitId: string; + workspaceId: string; + envSlug: string; + goBackToHistory: () => void; + goToRollbackPreview: () => void; +}): JSX.Element => { + // State for tracking collapsed items (empty by default means all are expanded) + const [collapsedItems, setCollapsedItems] = useState>({}); + + const { handlePopUpToggle, popUp, handlePopUpOpen, handlePopUpClose } = usePopUp([ + "revertChanges" + ] as const); + + const { data: commitDetails, isLoading } = useGetCommitDetails(workspaceId, selectedCommitId); + + const routerQueryParams: { secretPath?: string } = useSearch({ + from: ROUTE_PATHS.SecretManager.CommitDetailsPage.id + }); + const secretPath = (routerQueryParams.secretPath as string) || "/"; + + const { mutateAsync: revert } = useCommitRevert({ + commitId: selectedCommitId, + projectId: workspaceId, + environment: envSlug, + directory: secretPath + }); + + useEffect(() => { + setCollapsedItems({}); + }, [selectedCommitId]); + + const toggleItemCollapsed = (itemId: string): void => { + setCollapsedItems((prev) => ({ ...prev, [itemId]: !prev[itemId] })); + }; + + const handleRevertChanges = async (): Promise => { + const response = await revert(); + if (!response.success) { + createNotification({ + type: "error", + text: response.message + }); + return; + } + createNotification({ + type: "success", + text: response.message + }); + + handlePopUpClose("revertChanges"); + + goBackToHistory(); + }; + + // If no commit is selected or data is loading, show appropriate message + if (!selectedCommitId) { + return ( +
+

Select a commit to view details

+
+ ); + } + + if (isLoading) { + return ( +
+ +
+ ); + } + + if (!commitDetails) { + return ( +
+

No details found for this commit

+
+ ); + } + + // Parse the commit details if it's a string + let parsedCommitDetails: CommitWithChanges; + try { + parsedCommitDetails = + typeof commitDetails === "string" ? JSON.parse(commitDetails) : commitDetails; + } catch (error) { + console.error("Failed to parse commit details:", error); + return ( +
+

Error parsing commit details

+
+ ); + } + + // Get all changes from the commit + const commitChanges = parsedCommitDetails.changes?.changes || []; + + // Separate changes by type + const addedChanges = commitChanges.filter((c) => c.changeType === CommitType.ADD && !c.isUpdate); + const updatedChanges = commitChanges.filter((c) => c.changeType === CommitType.ADD && c.isUpdate); + const deletedChanges = commitChanges.filter((c) => c.changeType === CommitType.DELETE); + + // Create merged item list from changes only + const changedItems: MergedItem[] = []; + + // Add items from added changes + addedChanges.forEach((change) => { + changedItems.push({ + id: change.id, + type: change.secretVersionId || change.secretKey ? "secret" : "folder", + versionId: change.secretVersionId || change.id, + folderName: change.folderName, + folderVersion: change.folderVersion, + secretKey: change.secretKey, + secretVersion: change.secretVersion, + isAdded: true, + versions: change.versions, + changeId: change.id + }); + }); + + // Add items from updated changes + updatedChanges.forEach((change) => { + changedItems.push({ + id: change.id, + type: change.secretVersionId || change.secretKey ? "secret" : "folder", + versionId: change.secretVersionId || change.id, + folderName: change.folderName, + folderVersion: change.folderVersion, + secretKey: change.secretKey, + secretVersion: change.secretVersion, + isUpdated: true, + versions: change.versions, + changeId: change.id + }); + }); + + // Add deleted items + deletedChanges.forEach((change) => { + changedItems.push({ + id: change.id, + type: change.secretVersionId || change.secretKey ? "secret" : "folder", + secretKey: change.secretKey, + folderName: change.folderName, + secretVersion: change.secretVersion, + folderVersion: change.folderVersion, + isDeleted: true, + versions: change.versions, + changeId: change.id + }); + }); + + // Sort items: deleted first, then folders, then secrets, all alphabetically + const sortedChangedItems = [...changedItems].sort((a, b) => { + // First sort deleted items to the top + if (a.isDeleted !== b.isDeleted) { + return a.isDeleted ? -1 : 1; + } + + // Then sort by type (folders before secrets) + if (a.type !== b.type) { + return a.type === "folder" ? -1 : 1; + } + + // Finally sort alphabetically by name + const aName = a.type === "folder" ? a.folderName || "" : a.secretKey || ""; + const bName = b.type === "folder" ? b.folderName || "" : b.secretKey || ""; + return aName.localeCompare(bName); + }); + + // Render an item from the merged list + const renderMergedItem = (item: MergedItem): JSX.Element => { + return ( +
+ toggleItemCollapsed(id)} + /> +
+ ); + }; + + // Format actor display + const actorDisplay = + parsedCommitDetails.changes?.actorMetadata?.name || + parsedCommitDetails.changes?.actorType || + "Unknown"; + + return ( +
+
+
+
+
+
+

+ {parsedCommitDetails.changes?.message || "No message"} +

+
+
+
+

+ Commited by + {actorDisplay} + on + + {formatDisplayDate( + parsedCommitDetails.changes?.createdAt || new Date().toISOString() + )} + + {parsedCommitDetails.changes?.isLatest && ( + (Latest) + )} +

+
+
+
+ + {(isAllowed) => ( + + + +

Restore Options

+ + + + + {!parsedCommitDetails.changes.isLatest && ( + goToRollbackPreview()} + > +
+
+ + Roll back to this commit + + + Return this folder to its exact state at the time of this commit, + discarding all other changes made after it + +
+
+ + )} + + handlePopUpOpen("revertChanges")} + > +
+
+ Revert changes + + Will restore to the previous version of affected resources + +
+
+ + + + )} + +
+
+ +
+
+
+ {sortedChangedItems.length > 0 ? ( + sortedChangedItems.map((item) => renderMergedItem(item)) + ) : ( +
+

No changed items found

+
+ )} +
+
+
+
+ + handlePopUpToggle("revertChanges", isOpen)} + onDeleteApproved={handleRevertChanges} + buttonText="Yes, revert changes" + /> +
+ ); +}; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/index.ts b/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/index.ts new file mode 100644 index 0000000000..490d67f1fb --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/index.ts @@ -0,0 +1 @@ +export { CommitDetailsTab } from "./CommitDetailsTab"; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/types.ts b/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/types.ts new file mode 100644 index 0000000000..f3c914d8f9 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/CommitDetailsTab/types.ts @@ -0,0 +1,14 @@ +export interface MergedItem { + id: string; + type: "secret" | "folder"; + versionId?: string; + folderName?: string; + folderVersion?: string; + secretKey?: string; + secretVersion?: string; + isAdded?: boolean; + isUpdated?: boolean; + isDeleted?: boolean; + versions?: any[]; + changeId: string; +} diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/RollbackPreviewTab.tsx b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/RollbackPreviewTab.tsx new file mode 100644 index 0000000000..820e0e4a26 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/RollbackPreviewTab.tsx @@ -0,0 +1,389 @@ +/* eslint-disable jsx-a11y/no-static-element-interactions */ +/* eslint-disable jsx-a11y/click-events-have-key-events */ +import { useEffect, useState } from "react"; +import { faFolder, faInfoCircle } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { useNavigate, useParams, useSearch } from "@tanstack/react-router"; + +import { createNotification } from "@app/components/notifications"; +import { ProjectPermissionCan } from "@app/components/permissions"; +import { + Button, + DeleteActionModal, + Input, + PageHeader, + Spinner, + Switch, + Tooltip +} from "@app/components/v2"; +import { ROUTE_PATHS } from "@app/const/routes"; +import { useWorkspace } from "@app/context"; +import { + ProjectPermissionCommitsActions, + ProjectPermissionSub +} from "@app/context/ProjectPermissionContext/types"; +import { usePopUp } from "@app/hooks"; +import { useCommitRollback, useGetRollbackPreview } from "@app/hooks/api/folderCommits/queries"; +import { ProjectType } from "@app/hooks/api/workspace/types"; + +import { SecretVersionDiffView } from "../SecretVersionDiffView"; + +interface Version { + // Common fields + id?: string; + version: number; + createdAt?: string; + updatedAt?: string; + + // Secret-specific fields + secretKey?: string; + secretValue?: string; + secretComment?: string; + skipMultilineEncoding?: boolean; + secretReminderRepeatDays?: number | null; + secretReminderNote?: string | null; + secretReminderRecipients?: string[]; + tags?: string[]; + metadata?: Record; + + // Folder-specific fields + name?: string; + envId?: string; + folderId?: string; + + [key: string]: any; +} + +interface RollbackChange { + type: "secret" | "folder"; + id: string; + versionId: string; + fromVersion?: number; + changeType: "create" | "update" | "delete"; + commitId: string; + secretKey?: string; + secretVersion?: number; + folderName?: string; + folderVersion?: number; + versions?: Version[]; + createdAt?: string; +} + +interface FolderChanges { + folderId: string; + folderName: string; + folderPath: string; + changes: RollbackChange[]; +} + +export const RollbackPreviewTab = (): JSX.Element => { + const [deepRollback, setDeepRollback] = useState(false); + const [message, setMessage] = useState(""); + const [selectedFolderId, setSelectedFolderId] = useState(null); + const { currentWorkspace } = useWorkspace(); + const envSlug = useParams({ + from: ROUTE_PATHS.SecretManager.RollbackPreviewPage.id, + select: (el) => el.environment + }); + const selectedCommitId = useParams({ + from: ROUTE_PATHS.SecretManager.RollbackPreviewPage.id, + select: (el) => el.commitId + }); + const folderId = useParams({ + from: ROUTE_PATHS.SecretManager.RollbackPreviewPage.id, + select: (el) => el.folderId + }); + + const navigate = useNavigate(); + const routerQueryParams = useSearch({ + from: ROUTE_PATHS.SecretManager.RollbackPreviewPage.id + }); + + const secretPath = (routerQueryParams.secretPath as string) || "/"; + + const goBackToHistory = () => { + navigate({ + to: `/${ProjectType.SecretManager}/$projectId/commits/$environment/$folderId` as const, + params: { + projectId: currentWorkspace.id, + folderId, + environment: envSlug + }, + search: (query) => ({ + ...query, + secretPath + }) + }); + }; + + const { handlePopUpToggle, popUp, handlePopUpOpen, handlePopUpClose } = usePopUp([ + "rollbackConfirm" + ] as const); + + const { mutateAsync: rollback } = useCommitRollback({ + workspaceId: currentWorkspace.id, + commitId: selectedCommitId, + folderId, + deepRollback, + environment: envSlug, + directory: secretPath, + envSlug + }); + + const { data: rollbackChangesNested, isLoading } = useGetRollbackPreview( + folderId, + selectedCommitId, + envSlug, + currentWorkspace.id, + deepRollback, + secretPath + ); + + const handleRollback = async (): Promise => { + try { + await rollback(message); + + createNotification({ + type: "success", + text: "Rollback completed successfully" + }); + + handlePopUpClose("rollbackConfirm"); + goBackToHistory(); + } catch (error) { + createNotification({ + type: "error", + text: error instanceof Error ? error.message : "Failed to rollback changes" + }); + } + }; + + const folderChanges: FolderChanges[] = rollbackChangesNested || []; + + const currentFolderChanges: FolderChanges = folderChanges.find( + (folder) => folder.folderId === folderId + ) || { + folderId: folderId || "", + folderName: "Current Folder", + folderPath: secretPath, + changes: [] + }; + const nestedFolderChanges: FolderChanges[] = folderChanges.filter( + (folder) => folder.folderId !== folderId + ); + + useEffect(() => { + // Select the current folder by default + if (folderChanges.length > 0) { + setSelectedFolderId(currentFolderChanges.folderId); + } + }, [folderChanges, currentFolderChanges.folderId]); + + if (!selectedCommitId) { + return ( +
+

Select a commit to view rollback preview

+
+ ); + } + + if (isLoading) { + return ( +
+ +
+ ); + } + + const renderSidebar = (): JSX.Element => { + return ( +
+
setSelectedFolderId(currentFolderChanges.folderId)} + > +
+
+ + + {currentFolderChanges.folderPath || currentFolderChanges.folderName} + +
+ {currentFolderChanges.changes.length > 0 && ( + + {currentFolderChanges.changes.length} + + )} +
+
+ + {deepRollback && nestedFolderChanges.length > 0 && ( + <> +
+ Child folders to be restored +
+ {nestedFolderChanges.map((folder) => ( +
setSelectedFolderId(folder.folderId)} + > +
+
+ + + {folder.folderPath || folder.folderName} + +
+ {folder.changes.length > 0 && ( + + {folder.changes.length} + + )} +
+
+ ))} + + )} +
+ ); + }; + + const getSelectedFolderChanges = (): RollbackChange[] => { + if (!selectedFolderId) return []; + + const folder = folderChanges.find((f) => f.folderId === selectedFolderId); + return folder?.changes || []; + }; + + const renderMainContent = (): JSX.Element => { + const selectedFolderChanges = getSelectedFolderChanges(); + const selectedFolder = folderChanges.find((f) => f.folderId === selectedFolderId); + + if (!selectedFolder || selectedFolderChanges.length === 0) { + return ( +
+

No changes in selected folder

+
+ ); + } + + return ( +
+
+ {selectedFolderChanges.map((change) => ( +
+ +
+ ))} +
+
+ ); + }; + + return ( +
+ +
+
+
+ + +
+ {renderSidebar()} + {renderMainContent()} +
+ +
+
+
+ + + + + Restore All Child Folders + +
+
+
+ +
+
+ setMessage(e.target.value)} + className="w-full border-mineshaft-500 bg-mineshaft-700 py-2 text-sm" + maxLength={256} + /> + +
+
+
+ + handlePopUpToggle("rollbackConfirm", isOpen)} + onDeleteApproved={handleRollback} + buttonText="Restore" + /> +
{" "} +
{" "} + +
+ ); +}; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/index.ts b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/index.ts new file mode 100644 index 0000000000..9c840b9490 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/index.ts @@ -0,0 +1 @@ +export { RollbackPreviewTab } from "./RollbackPreviewTab"; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/route.tsx b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/route.tsx new file mode 100644 index 0000000000..34b0e59bd0 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/RollbackPreviewTab/route.tsx @@ -0,0 +1,91 @@ +import { createFileRoute, linkOptions, stripSearchParams } from "@tanstack/react-router"; +import { zodValidator } from "@tanstack/zod-adapter"; +import { z } from "zod"; + +import { SecretDashboardPathBreadcrumb } from "@app/components/navigation/SecretDashboardPathBreadcrumb"; +import { BreadcrumbTypes } from "@app/components/v2"; + +import { RollbackPreviewTab } from "./RollbackPreviewTab"; + +const RollbackPreviewTabQueryParamsSchema = z.object({ + secretPath: z.string().catch("/") +}); + +export const Route = createFileRoute( + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/restore" +)({ + component: RollbackPreviewTab, + validateSearch: zodValidator(RollbackPreviewTabQueryParamsSchema), + search: { + middlewares: [stripSearchParams({ secretPath: "/" })] + }, + beforeLoad: ({ context, params, search }) => { + const secretPathSegments = search.secretPath.split("/").filter(Boolean); + + return { + breadcrumbs: [ + ...context.breadcrumbs, + { + type: BreadcrumbTypes.Dropdown, + label: + context.project.environments.find((el) => el.slug === params.environment)?.name || "", + dropdownTitle: "Environments", + links: context.project.environments.map((el) => ({ + label: el.name, + link: linkOptions({ + to: "/secret-manager/$projectId/secrets/$envSlug", + params: { + projectId: params.projectId, + envSlug: el.slug + } + }) + })) + }, + ...secretPathSegments.map((_, index) => ({ + type: BreadcrumbTypes.Component, + component: () => ( + + ) + })), + { + label: "Commits", + link: linkOptions({ + to: "/secret-manager/$projectId/commits/$environment/$folderId", + params: { + projectId: params.projectId, + environment: params.environment, + folderId: params.folderId + }, + search: { + secretPath: search.secretPath + } + }) + }, + { + label: params.commitId, + link: linkOptions({ + to: "/secret-manager/$projectId/commits/$environment/$folderId/$commitId", + params: { + projectId: params.projectId, + environment: params.environment, + folderId: params.folderId, + commitId: params.commitId + }, + search: { + secretPath: search.secretPath + } + }) + }, + { + label: "Restore" + } + ] + }; + } +}); diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/SecretVersionDiffView/SecretVersionDiffView.tsx b/frontend/src/pages/secret-manager/CommitDetailsPage/components/SecretVersionDiffView/SecretVersionDiffView.tsx new file mode 100644 index 0000000000..7744d36261 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/SecretVersionDiffView/SecretVersionDiffView.tsx @@ -0,0 +1,652 @@ +/* eslint-disable no-nested-ternary */ +import { useCallback, useRef, useState } from "react"; +import { faChevronDown, faChevronUp } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; + +export interface Version { + id?: string; + version: number; + [key: string]: any; +} + +type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; +type JsonObject = { [key: string]: JsonValue }; +type JsonArray = JsonValue[]; + +export interface DiffViewItem { + type: "secret" | "folder"; + isAdded?: boolean; + isDeleted?: boolean; + isUpdated?: boolean; + versions?: Version[]; + isRollback?: boolean; + id: string; + secretKey?: string; + folderName?: string; +} + +interface SecretVersionDiffViewProps { + item: DiffViewItem; + isCollapsed?: boolean; + onToggleCollapse?: (id: string) => void; + showHeader?: boolean; + customHeader?: JSX.Element; + excludedFieldsHighlight?: string[]; +} + +const isObject = (obj: JsonValue): obj is JsonObject => { + return obj !== null && typeof obj === "object" && !Array.isArray(obj); +}; + +const isArray = (obj: JsonValue): obj is JsonArray => { + return Array.isArray(obj); +}; + +const deepEqual = (a: JsonValue, b: JsonValue): boolean => { + if (a === b) return true; + if (a == null || b == null) return false; + if (typeof a !== typeof b) return false; + + if (isArray(a) && isArray(b)) { + if (a.length !== b.length) return false; + return a.every((item: JsonValue, index: number) => deepEqual(item, b[index])); + } + + if (isObject(a) && isObject(b)) { + const keysA = Object.keys(a); + const keysB = Object.keys(b); + if (keysA.length !== keysB.length) return false; + return keysA.every((key) => keysB.includes(key) && deepEqual(a[key], b[key])); + } + + return false; +}; + +const getDiffPaths = (oldObj: JsonValue, newObj: JsonValue, path: string = ""): Set => { + const diffPaths = new Set(); + + if (oldObj === newObj) return diffPaths; + + if (oldObj == null || newObj == null) { + diffPaths.add(path || "root"); + return diffPaths; + } + + if (typeof oldObj !== typeof newObj) { + diffPaths.add(path || "root"); + return diffPaths; + } + + if (isArray(oldObj) && isArray(newObj)) { + return diffPaths; + } + + if (isObject(oldObj) && isObject(newObj)) { + const allKeys = new Set([...Object.keys(oldObj), ...Object.keys(newObj)]); + + allKeys.forEach((key) => { + const currentPath = path ? `${path}.${key}` : key; + + if (path.includes("[")) { + return; + } + + if (!(key in oldObj) || !(key in newObj) || !deepEqual(oldObj[key], newObj[key])) { + diffPaths.add(currentPath); + + if ( + key in oldObj && + key in newObj && + (isObject(oldObj[key]) || isArray(oldObj[key])) && + (isObject(newObj[key]) || isArray(newObj[key])) + ) { + const nestedDiffs = getDiffPaths(oldObj[key], newObj[key], currentPath); + nestedDiffs.forEach((p) => diffPaths.add(p)); + } + } + }); + return diffPaths; + } + + if (oldObj !== newObj) { + diffPaths.add(path || "root"); + } + + return diffPaths; +}; + +const getNestedValue = (obj: JsonValue, path: string): JsonValue => { + if (!path) return obj; + + const parts = path.split(/[.[\]]+/).filter(Boolean); + let current: JsonValue = obj; + + parts.forEach((part) => { + if (current == null) { + return; + } + if (isObject(current)) { + current = current[part]; + } else if (isArray(current)) { + const index = parseInt(part, 10); + if (!Number.isNaN(index)) { + current = current[index]; + } + } + }); + + return current; +}; + +const isPathDifferent = (jsonPath: string, diffPaths: Set): boolean => { + if (diffPaths.has(jsonPath)) return true; + + const diffPathsArray = Array.from(diffPaths); + return diffPathsArray.some((diffPath) => { + return ( + jsonPath.startsWith(`${diffPath}.`) || + jsonPath.startsWith(`${diffPath}[`) || + diffPath.startsWith(`${jsonPath}.`) || + diffPath.startsWith(`${jsonPath}[`) + ); + }); +}; + +const isContainerActuallyChanged = ( + path: string, + diffPaths: Set, + oldObj: JsonValue, + newObj: JsonValue +): boolean => { + if (diffPaths.has(path)) { + if (oldObj == null || newObj == null) return true; + if (typeof oldObj !== typeof newObj) return true; + if (isArray(oldObj) !== isArray(newObj)) return true; + if (isObject(oldObj) !== isObject(newObj)) return true; + + if (!isObject(oldObj) && !isArray(oldObj)) return true; + + return false; + } + + if (isArray(oldObj) && isArray(newObj)) { + return false; + } + + if (isObject(oldObj) && isObject(newObj)) { + return false; + } + + return oldObj !== newObj; +}; + +const renderJsonWithDiffs = ( + obj: JsonValue, + diffPaths: Set, + isOldVersion: boolean, + path: string = "", + indentLevel: number = 0, + keyName?: string, + isLastItem: boolean = false, + excludedFieldsHighlight: string[] = [], + oldVersionObj?: JsonValue, + newVersionObj?: JsonValue +): JSX.Element => { + const indent = " ".repeat(indentLevel); + + let isDifferent = false; + + if (path.includes("[") && oldVersionObj && newVersionObj) { + const arrayMatch = path.match(/^([^[]+)\[(\d+)\]/); + if (arrayMatch) { + const arrayPath = arrayMatch[1]; + const itemIndex = parseInt(arrayMatch[2], 10); + + const oldArray = getNestedValue(oldVersionObj, arrayPath); + const newArray = getNestedValue(newVersionObj, arrayPath); + + if (isArray(oldArray) && isArray(newArray)) { + const currentItem = isOldVersion ? oldArray[itemIndex] : newArray[itemIndex]; + + if (isOldVersion) { + isDifferent = !newArray.some((newItem: JsonValue) => deepEqual(currentItem, newItem)); + } else { + isDifferent = !oldArray.some((oldItem: JsonValue) => deepEqual(currentItem, oldItem)); + } + } else { + isDifferent = isPathDifferent(path, diffPaths); + } + } else { + isDifferent = isPathDifferent(path, diffPaths); + } + } else { + isDifferent = isPathDifferent(path, diffPaths); + } + + const getLineClass = (different: boolean) => { + if (!different) return "flex"; + return isOldVersion ? "flex bg-red-950 text-red-300" : "flex bg-green-950 text-green-300"; + }; + + const getHighlightClass = (different: boolean) => { + if (!different) return ""; + return isOldVersion ? "bg-red-900 rounded px-1" : "bg-green-900 rounded px-1"; + }; + + const prefix = isDifferent ? (isOldVersion ? "-" : "+") : " "; + const keyDisplay = keyName ? `"${keyName}": ` : ""; + const comma = !isLastItem ? "," : ""; + + const reactKey = `${path || "root"}-${keyName || "value"}-${indentLevel}-${typeof obj}`; + + if ( + obj === null || + typeof obj === "string" || + typeof obj === "number" || + typeof obj === "boolean" + ) { + let valueDisplay = ""; + if (obj === null) valueDisplay = "null"; + else if (typeof obj === "string") valueDisplay = `"${obj}"`; + else valueDisplay = String(obj); + + return ( +
+
{prefix}
+
+ {indent} + {keyName && {keyDisplay}} + {valueDisplay} + {comma} +
+
+ ); + } + + if (isArray(obj) && obj.length === 0) { + return ( +
+
{prefix}
+
+ {indent} + {keyName && {keyDisplay}} + [] + {comma} +
+
+ ); + } + + if (isObject(obj) && Object.keys(obj).length === 0) { + return ( +
+
{prefix}
+
+ {indent} + {keyName && {keyDisplay}} + {"{}"} + {comma} +
+
+ ); + } + + let isContainerAddedOrRemoved = false; + + if (oldVersionObj && newVersionObj) { + const oldValue = getNestedValue(oldVersionObj, path); + const newValue = getNestedValue(newVersionObj, path); + + if (oldValue == null || newValue == null) { + isContainerAddedOrRemoved = true; + } else if (typeof oldValue !== typeof newValue) { + isContainerAddedOrRemoved = true; + } else if (isArray(oldValue) !== isArray(newValue)) { + isContainerAddedOrRemoved = true; + } else if (isObject(oldValue) !== isObject(newValue)) { + isContainerAddedOrRemoved = true; + } + } else { + isContainerAddedOrRemoved = isContainerActuallyChanged( + path, + diffPaths, + isOldVersion ? obj : oldVersionObj || null, + isOldVersion ? newVersionObj || null : obj + ); + } + + if (isArray(obj)) { + return ( +
+
+
+ {isContainerAddedOrRemoved ? (isOldVersion ? "-" : "+") : " "} +
+
+ {indent} + {keyName && ( + + {keyDisplay} + + )} + [ +
+
+ + {obj.map((item: JsonValue, index: number) => { + const itemPath = path ? `${path}[${index}]` : `[${index}]`; + const isLast = index === obj.length - 1; + + return ( +
+ {renderJsonWithDiffs( + item, + diffPaths, + isOldVersion, + itemPath, + indentLevel + 1, + undefined, + isLast, + excludedFieldsHighlight, + oldVersionObj, + newVersionObj + )} +
+ ); + })} + +
+
+ {isContainerAddedOrRemoved ? (isOldVersion ? "-" : "+") : " "} +
+
+ {indent} + ] + {comma} +
+
+
+ ); + } + + if (isObject(obj)) { + const keys = Object.keys(obj); + + return ( +
+
+
+ {isContainerAddedOrRemoved ? (isOldVersion ? "-" : "+") : " "} +
+
+ {indent} + {keyName && ( + + {keyDisplay} + + )} + {"{"} +
+
+ + {keys.map((key, index) => { + const keyPath = path ? `${path}.${key}` : key; + const isLast = index === keys.length - 1; + const propKey = `${reactKey}-prop-${key}`; + + return ( +
+ {renderJsonWithDiffs( + obj[key], + diffPaths, + isOldVersion, + keyPath, + indentLevel + 1, + key, + isLast, + excludedFieldsHighlight, + oldVersionObj, + newVersionObj + )} +
+ ); + })} + +
+
+ {isContainerAddedOrRemoved ? (isOldVersion ? "-" : "+") : " "} +
+
+ {indent} + {"}"} + {comma} +
+
+
+ ); + } + + return ( +
+
{prefix}
+
+ {indent} + {keyDisplay} + {String(obj)} + {comma} +
+
+ ); +}; + +const formatAddedJson = (json: JsonValue): JSX.Element => { + const lines = JSON.stringify(json, null, 2).split("\n"); + return ( +
+ {lines.map((line, lineIndex) => { + const lineKey = `added-${line.slice(0, 30)}-${lineIndex}`; + return ( +
+
+
+
{line}
+
+ ); + })} +
+ ); +}; + +const formatDeletedJson = (json: JsonValue): JSX.Element => { + const lines = JSON.stringify(json, null, 2).split("\n"); + return ( +
+ {lines.map((line, lineIndex) => { + const lineKey = `deleted-${line.slice(0, 30)}-${lineIndex}`; + return ( +
+
-
+
{line}
+
+ ); + })} +
+ ); +}; + +const cleanVersionForComparison = (version: Version): JsonValue => { + const { id, version: versionNumber, ...cleanVersion } = version; + return cleanVersion; +}; + +export const SecretVersionDiffView = ({ + item, + isCollapsed = false, + onToggleCollapse, + showHeader = true, + customHeader, + excludedFieldsHighlight = ["metadata", "tags"] +}: SecretVersionDiffViewProps) => { + const oldContainerRef = useRef(null); + const newContainerRef = useRef(null); + const [internalCollapsed, setInternalCollapsed] = useState(isCollapsed); + + const handleToggle = useCallback(() => { + if (onToggleCollapse && item.id) { + onToggleCollapse(item.id); + } else { + setInternalCollapsed((prev) => !prev); + } + }, [onToggleCollapse, item.id]); + + const collapsed = onToggleCollapse ? isCollapsed : internalCollapsed; + + if (!item.versions || item.versions.length === 0) { + return
No details available
; + } + + const sortedVersions = [...item.versions].sort((a, b) => b.version - a.version); + let oldVersion = null; + let newVersion = null; + let oldVersionContent = null; + let newVersionContent = null; + let diffPaths = new Set(); + + if (item.isUpdated && sortedVersions.length >= 2) { + if (item.isRollback) { + [oldVersion, newVersion] = sortedVersions; + } else { + [newVersion, oldVersion] = sortedVersions; + } + + const cleanOldVersion = cleanVersionForComparison(oldVersion); + const cleanNewVersion = cleanVersionForComparison(newVersion); + diffPaths = getDiffPaths(cleanOldVersion, cleanNewVersion); + + if (diffPaths.size === 0) { + return null; + } + + oldVersionContent = ( +
+ {renderJsonWithDiffs( + cleanOldVersion, + diffPaths, + true, + "", + 0, + undefined, + false, + excludedFieldsHighlight, + cleanOldVersion, + cleanNewVersion + )} +
+ ); + newVersionContent = ( +
+ {renderJsonWithDiffs( + cleanNewVersion, + diffPaths, + false, + "", + 0, + undefined, + false, + excludedFieldsHighlight, + cleanOldVersion, + cleanNewVersion + )} +
+ ); + } else if (item.isAdded) { + [newVersion] = sortedVersions; + const cleanNewVersion = cleanVersionForComparison(newVersion); + newVersionContent = formatAddedJson(cleanNewVersion); + } else if (item.isDeleted) { + [oldVersion] = sortedVersions; + const cleanOldVersion = cleanVersionForComparison(oldVersion); + oldVersionContent = formatDeletedJson(cleanOldVersion); + } else { + return null; + } + + const renderHeader = () => { + if (customHeader) { + return customHeader; + } + + const isSecret = item.type === "secret"; + const key = isSecret ? item.secretKey || "Unnamed Secret" : item.folderName || "Unnamed Folder"; + let textStyle = "text-white"; + let changeBadge = null; + + if (item.isDeleted) { + textStyle = "line-through text-red-300"; + changeBadge = ( + + {isSecret ? "Secret" : "Folder"} Deleted + + ); + } else if (item.isAdded) { + changeBadge = ( + + {isSecret ? "Secret" : "Folder"} Added + + ); + } else if (item.isUpdated) { + changeBadge = ( + + {isSecret ? "Secret" : "Folder"} Updated + + ); + } + + return ( +
{ + if (e.key === "Enter" || e.key === " ") { + handleToggle(); + e.preventDefault(); + } + }} + role="button" + tabIndex={0} + aria-expanded={!collapsed} + > +
+ {key} + {changeBadge} +
+ +
+ ); + }; + + return ( +
+ {showHeader && renderHeader()} + + {!collapsed && ( +
+
+
+ {oldVersionContent} +
+ +
+ {newVersionContent} +
+
+
+ )} +
+ ); +}; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/components/SecretVersionDiffView/index.ts b/frontend/src/pages/secret-manager/CommitDetailsPage/components/SecretVersionDiffView/index.ts new file mode 100644 index 0000000000..7c1cfdd794 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/components/SecretVersionDiffView/index.ts @@ -0,0 +1 @@ +export { SecretVersionDiffView } from "./SecretVersionDiffView"; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/index.tsx b/frontend/src/pages/secret-manager/CommitDetailsPage/index.tsx new file mode 100644 index 0000000000..0623db6216 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/index.tsx @@ -0,0 +1 @@ +export { CommitDetailsPage } from "./CommitDetailsPage"; diff --git a/frontend/src/pages/secret-manager/CommitDetailsPage/route.tsx b/frontend/src/pages/secret-manager/CommitDetailsPage/route.tsx new file mode 100644 index 0000000000..8cf046d2e3 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitDetailsPage/route.tsx @@ -0,0 +1,76 @@ +import { createFileRoute, linkOptions, stripSearchParams } from "@tanstack/react-router"; +import { zodValidator } from "@tanstack/zod-adapter"; +import { z } from "zod"; + +import { SecretDashboardPathBreadcrumb } from "@app/components/navigation/SecretDashboardPathBreadcrumb"; +import { BreadcrumbTypes } from "@app/components/v2"; + +import { CommitDetailsPage } from "./CommitDetailsPage"; + +const CommitDetailsPageQueryParamsSchema = z.object({ + secretPath: z.string().catch("/") +}); + +export const Route = createFileRoute( + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/" +)({ + component: CommitDetailsPage, + validateSearch: zodValidator(CommitDetailsPageQueryParamsSchema), + search: { + middlewares: [stripSearchParams({ secretPath: "/" })] + }, + beforeLoad: ({ context, params, search }) => { + const secretPathSegments = search.secretPath.split("/").filter(Boolean); + + return { + breadcrumbs: [ + ...context.breadcrumbs, + { + type: BreadcrumbTypes.Dropdown, + label: + context.project.environments.find((el) => el.slug === params.environment)?.name || "", + dropdownTitle: "Environments", + links: context.project.environments.map((el) => ({ + label: el.name, + link: linkOptions({ + to: "/secret-manager/$projectId/secrets/$envSlug", + params: { + projectId: params.projectId, + envSlug: el.slug + } + }) + })) + }, + ...secretPathSegments.map((_, index) => ({ + type: BreadcrumbTypes.Component, + component: () => ( + + ) + })), + { + label: "Commits", + link: linkOptions({ + to: "/secret-manager/$projectId/commits/$environment/$folderId", + params: { + projectId: params.projectId, + environment: params.environment, + folderId: params.folderId + }, + search: { + secretPath: search.secretPath + } + }) + }, + { + label: params.commitId + } + ] + }; + } +}); diff --git a/frontend/src/pages/secret-manager/CommitsPage/CommitsPage.tsx b/frontend/src/pages/secret-manager/CommitsPage/CommitsPage.tsx new file mode 100644 index 0000000000..5479f4e36c --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitsPage/CommitsPage.tsx @@ -0,0 +1,70 @@ +import { useNavigate, useParams, useSearch } from "@tanstack/react-router"; + +import { ProjectPermissionCan } from "@app/components/permissions"; +import { PageHeader } from "@app/components/v2"; +import { ROUTE_PATHS } from "@app/const/routes"; +import { useWorkspace } from "@app/context"; +import { + ProjectPermissionCommitsActions, + ProjectPermissionSub +} from "@app/context/ProjectPermissionContext/types"; +import { ProjectType } from "@app/hooks/api/workspace/types"; + +import { CommitHistoryTab } from "./components/CommitHistoryTab"; + +export const CommitsPage = () => { + const envSlug = useParams({ + from: ROUTE_PATHS.SecretManager.CommitsPage.id, + select: (el) => el.environment + }); + const { currentWorkspace } = useWorkspace(); + const navigate = useNavigate(); + const folderId = useParams({ + from: ROUTE_PATHS.SecretManager.CommitsPage.id, + select: (el) => el.folderId + }); + const routerQueryParams: { secretPath?: string } = useSearch({ + from: ROUTE_PATHS.SecretManager.CommitsPage.id + }); + + const secretPath = routerQueryParams?.secretPath || "/"; + + const handleSelectCommit = (commitId: string) => { + navigate({ + to: `/${ProjectType.SecretManager}/$projectId/commits/$environment/$folderId/$commitId` as const, + params: { + projectId: currentWorkspace.id, + folderId, + environment: envSlug, + commitId + }, + search: (query) => ({ + ...query, + secretPath + }) + }); + }; + + return ( +
+
+ + + + +
+
+ ); +}; diff --git a/frontend/src/pages/secret-manager/CommitsPage/components/CommitHistoryTab/CommitHistoryTab.tsx b/frontend/src/pages/secret-manager/CommitsPage/components/CommitHistoryTab/CommitHistoryTab.tsx new file mode 100644 index 0000000000..8f529ba86f --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitsPage/components/CommitHistoryTab/CommitHistoryTab.tsx @@ -0,0 +1,320 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { + faArrowDownWideShort, + faArrowUpWideShort, + faCopy, + faSearch +} from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { format, formatDistanceToNow } from "date-fns"; + +import { Button, Input, Spinner } from "@app/components/v2"; +import { CopyButton } from "@app/components/v2/CopyButton"; +import { useGetFolderCommitHistory } from "@app/hooks/api/folderCommits"; + +interface CommitActorMetadata { + email?: string; + name?: string; +} + +interface Commit { + id: string; + message: string; + createdAt: string; + actorType: string; + actorMetadata?: CommitActorMetadata; +} + +const formatTimeAgo = (timestamp: string): string => { + return formatDistanceToNow(new Date(timestamp), { addSuffix: true }); +}; + +/** + * Commit Item component for displaying a single commit + */ +const CommitItem = ({ + commit, + onSelectCommit +}: { + commit: Commit; + onSelectCommit: (commitId: string, tab: string) => void; +}) => { + return ( +
+
+
+
+
+ +
+

+ + {commit.actorMetadata?.email || commit.actorMetadata?.name || commit.actorType} +

committed

+ + +

+
+
+
+ + +
+
+
+
+
+ ); +}; + +/** + * Date Group component for displaying commits grouped by date + */ +const DateGroup = ({ + date, + commits, + onSelectCommit +}: { + date: string; + commits: Commit[]; + onSelectCommit: (commitId: string, tab: string) => void; +}) => { + return ( +
+
+
+
+
+
+

Commits on {date}

+
+ +
+
+
+ {commits.map((commit) => ( +
+
+ +
+
+ ))} +
+
+
+ ); +}; + +export const CommitHistoryTab = ({ + onSelectCommit, + projectId, + environment, + secretPath +}: { + onSelectCommit: (commitId: string, tab: string) => void; + projectId: string; + environment: string; + secretPath: string; +}) => { + const [searchTerm, setSearchTerm] = useState(""); + const [debouncedSearchTerm, setDebouncedSearchTerm] = useState(""); + const [sortDirection, setSortDirection] = useState<"asc" | "desc">("desc"); + const [offset, setOffset] = useState(0); + const [allCommits, setAllCommits] = useState([]); + const debounceTimeoutRef = useRef(); + const limit = 5; + + // Debounce search term + useEffect(() => { + if (debounceTimeoutRef.current) { + clearTimeout(debounceTimeoutRef.current); + } + + debounceTimeoutRef.current = setTimeout(() => { + setDebouncedSearchTerm(searchTerm); + }, 500); + + return () => { + if (debounceTimeoutRef.current) { + clearTimeout(debounceTimeoutRef.current); + } + }; + }, [searchTerm]); + + const { + data: response, + isLoading, + isFetching + } = useGetFolderCommitHistory({ + workspaceId: projectId, + environment, + directory: secretPath, + offset, + limit, + search: debouncedSearchTerm, + sort: sortDirection + }); + + const commits = response?.commits || []; + const hasMore = response?.hasMore || false; + + // Reset accumulated commits when search or sort changes + useEffect(() => { + setAllCommits([]); + setOffset(0); + }, [debouncedSearchTerm, sortDirection]); + + // Accumulate commits instead of replacing them + useEffect(() => { + if (commits.length > 0) { + if (offset === 0) { + // First load or after search/sort change - replace all commits + setAllCommits(commits); + } else { + // Subsequent loads - append new commits + setAllCommits((prev) => [...prev, ...commits]); + } + } + }, [commits, offset]); + + const groupedCommits = useMemo(() => { + return allCommits.reduce( + (acc, commit) => { + const date = format(new Date(commit.createdAt), "MMM d, yyyy"); + if (!acc[date]) { + acc[date] = []; + } + acc[date].push(commit); + return acc; + }, + {} as Record + ); + }, [allCommits]); + + const handleSort = useCallback(() => { + setSortDirection((prev) => (prev === "desc" ? "asc" : "desc")); + }, []); + + const handleSearch = useCallback((value: string) => { + setSearchTerm(value); + }, []); + + const loadMoreCommits = useCallback(() => { + if (hasMore && !isFetching) { + setOffset((prev) => prev + limit); + } + }, [hasMore, isFetching, limit]); + + return ( +
+
+
+
+ handleSearch(e.target.value)} + value={searchTerm} + aria-label="Search commits" + /> +
+
+
+ +
+
+ + {isLoading && offset === 0 ? ( +
+ +
+ ) : ( +
+ {Object.keys(groupedCommits).length > 0 ? ( + <> + {Object.entries(groupedCommits).map(([date, dateCommits]) => ( + + ))} + + ) : ( +
+
+ )} + + {hasMore && ( +
+ +
+ )} +
+ )} +
+ ); +}; diff --git a/frontend/src/pages/secret-manager/CommitsPage/components/CommitHistoryTab/index.tsx b/frontend/src/pages/secret-manager/CommitsPage/components/CommitHistoryTab/index.tsx new file mode 100644 index 0000000000..de271b16a9 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitsPage/components/CommitHistoryTab/index.tsx @@ -0,0 +1 @@ +export { CommitHistoryTab } from "./CommitHistoryTab"; diff --git a/frontend/src/pages/secret-manager/CommitsPage/index.tsx b/frontend/src/pages/secret-manager/CommitsPage/index.tsx new file mode 100644 index 0000000000..8203e39006 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitsPage/index.tsx @@ -0,0 +1 @@ +export { CommitsPage } from "./CommitsPage"; diff --git a/frontend/src/pages/secret-manager/CommitsPage/route.tsx b/frontend/src/pages/secret-manager/CommitsPage/route.tsx new file mode 100644 index 0000000000..6ea9159d25 --- /dev/null +++ b/frontend/src/pages/secret-manager/CommitsPage/route.tsx @@ -0,0 +1,71 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { createFileRoute, linkOptions, stripSearchParams } from "@tanstack/react-router"; +import { zodValidator } from "@tanstack/zod-adapter"; +import { z } from "zod"; + +import { SecretDashboardPathBreadcrumb } from "@app/components/navigation/SecretDashboardPathBreadcrumb"; +import { BreadcrumbTypes } from "@app/components/v2"; + +import { CommitsPage } from "./CommitsPage"; + +const CommitsPageQueryParamsSchema = z.object({ + secretPath: z.string().catch("/") +}); + +export const Route = createFileRoute( + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/" +)({ + component: CommitsPage, + validateSearch: zodValidator(CommitsPageQueryParamsSchema), + search: { + middlewares: [stripSearchParams({ secretPath: "/" })] + }, + beforeLoad: ({ context, params, search }) => { + const secretPathSegments = search.secretPath.split("/").filter(Boolean); + + return { + breadcrumbs: [ + ...context.breadcrumbs, + { + type: BreadcrumbTypes.Dropdown, + label: + context.project.environments.find((el) => el.slug === params.environment)?.name || "", + dropdownTitle: "Environments", + links: context.project.environments.map((el) => ({ + label: el.name, + link: linkOptions({ + to: "/secret-manager/$projectId/secrets/$envSlug", + params: { + projectId: params.projectId, + envSlug: el.slug + } + }) + })) + }, + ...secretPathSegments.map((_, index) => ({ + type: BreadcrumbTypes.Component, + component: () => ( + + ) + })), + { + label: "Commits", + link: linkOptions({ + to: "/secret-manager/$projectId/commits/$environment/$folderId", + params: { + projectId: params.projectId, + environment: params.environment, + folderId: params.folderId + } + }) + } + ] + }; + } +}); diff --git a/frontend/src/pages/secret-manager/SecretDashboardPage/SecretDashboardPage.tsx b/frontend/src/pages/secret-manager/SecretDashboardPage/SecretDashboardPage.tsx index 49bf1a72cd..84edeec9a9 100644 --- a/frontend/src/pages/secret-manager/SecretDashboardPage/SecretDashboardPage.tsx +++ b/frontend/src/pages/secret-manager/SecretDashboardPage/SecretDashboardPage.tsx @@ -44,6 +44,7 @@ import { } from "@app/hooks/api"; import { useGetProjectSecretsDetails } from "@app/hooks/api/dashboard"; import { DashboardSecretsOrderBy } from "@app/hooks/api/dashboard/types"; +import { useGetFolderCommitsCount } from "@app/hooks/api/folderCommits"; import { OrderByDirection } from "@app/hooks/api/generic/types"; import { ProjectType } from "@app/hooks/api/workspace/types"; import { hasSecretReadValueOrDescribePermission } from "@app/lib/fn/permission"; @@ -277,6 +278,17 @@ const Page = () => { limit: 10 }); + const { + data: { count: folderCommitsCount, folderId } = { count: 0, folderId: "" }, + isPending: isFolderCommitsCountLoading, + isFetching: isFolderCommitsCountFetching + } = useGetFolderCommitsCount({ + directory: secretPath, + workspaceId, + environment, + isPaused: !canDoReadRollback + }); + const { data: snapshotCount, isPending: isSnapshotCountLoading, @@ -288,6 +300,39 @@ const Page = () => { isPaused: !canDoReadRollback }); + const isPITEnabled = !currentWorkspace?.showSnapshotsLegacy; + + const changesCount = useMemo(() => { + return isPITEnabled ? folderCommitsCount : snapshotCount; + }, [folderCommitsCount, snapshotCount]); + + const isChangesCountPending = useMemo(() => { + return isPITEnabled ? isFolderCommitsCountLoading || isSnapshotCountLoading : false; + }, [isFolderCommitsCountLoading, isSnapshotCountLoading]); + + const isChangesCountFetching = useMemo(() => { + return isPITEnabled ? isFolderCommitsCountFetching || isSnapshotCountFetching : false; + }, [isFolderCommitsCountFetching, isSnapshotCountFetching]); + + const handleOnClickRollbackMode = () => { + if (isPITEnabled) { + navigate({ + to: `/${ProjectType.SecretManager}/$projectId/commits/$environment/$folderId` as const, + params: { + projectId: workspaceId, + folderId, + environment + }, + search: (query) => ({ + ...query, + secretPath + }) + }); + } else { + handlePopUpToggle("snapshots", true); + } + }; + const noAccessSecretCount = Math.max( (page * perPage > totalCount ? totalCount % perPage : perPage) - (imports?.length || 0) - @@ -448,13 +493,14 @@ const Page = () => { onVisibilityToggle={handleToggleVisibility} onSearchChange={handleSearchChange} onToggleTagFilter={handleTagToggle} - snapshotCount={snapshotCount || 0} - isSnapshotCountLoading={isSnapshotCountLoading && isSnapshotCountFetching} + snapshotCount={changesCount || 0} + isSnapshotCountLoading={isChangesCountPending && isChangesCountFetching} onToggleRowType={handleToggleRowType} - onClickRollbackMode={() => handlePopUpToggle("snapshots", true)} + onClickRollbackMode={handleOnClickRollbackMode} protectedBranchPolicyName={boardPolicy?.name} importedBy={importedBy} usedBySecretSyncs={usedBySecretSyncs} + isPITEnabled={isPITEnabled} />
@@ -618,7 +664,7 @@ const Page = () => { secretPath={secretPath} secrets={secrets} folders={folders} - snapshotCount={snapshotCount} + snapshotCount={changesCount} onGoBack={handleResetSnapshot} onClickListSnapshot={() => handlePopUpToggle("snapshots", true)} /> diff --git a/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/ActionBar.tsx b/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/ActionBar.tsx index 13b6d45b78..c6fdc4a9dd 100644 --- a/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/ActionBar.tsx +++ b/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/ActionBar.tsx @@ -56,7 +56,10 @@ import { useSubscription, useWorkspace } from "@app/context"; -import { ProjectPermissionSecretRotationActions } from "@app/context/ProjectPermissionContext/types"; +import { + ProjectPermissionCommitsActions, + ProjectPermissionSecretRotationActions +} from "@app/context/ProjectPermissionContext/types"; import { usePopUp } from "@app/hooks"; import { useCreateFolder, @@ -123,6 +126,7 @@ type Props = { isImported: boolean; }[]; }[]; + isPITEnabled: boolean; }; export const ActionBar = ({ @@ -142,6 +146,7 @@ export const ActionBar = ({ onToggleRowType, protectedBranchPolicyName, importedBy, + isPITEnabled = false, usedBySecretSyncs }: Props) => { const { handlePopUpOpen, handlePopUpToggle, handlePopUpClose, popUp } = usePopUp([ @@ -777,8 +782,8 @@ export const ActionBar = ({
{(isAllowed) => ( )} diff --git a/frontend/src/pages/secret-manager/SecretDashboardPage/components/PitDrawer/PitDrawer.tsx b/frontend/src/pages/secret-manager/SecretDashboardPage/components/PitDrawer/PitDrawer.tsx index 37a1188ff1..c3d2bbf66d 100644 --- a/frontend/src/pages/secret-manager/SecretDashboardPage/components/PitDrawer/PitDrawer.tsx +++ b/frontend/src/pages/secret-manager/SecretDashboardPage/components/PitDrawer/PitDrawer.tsx @@ -3,6 +3,7 @@ import { InfiniteData } from "@tanstack/react-query"; import { formatDistance } from "date-fns"; import { Button, Drawer, DrawerContent } from "@app/components/v2"; +import { NoticeBannerV2 } from "@app/components/v2/NoticeBannerV2/NoticeBannerV2"; import { TSecretSnapshot } from "@app/hooks/api/secretSnapshots/types"; type Props = { @@ -39,6 +40,20 @@ export const PitDrawer = ({ subTitle="Note: This will recover secrets for all environments in this project" >
+ +

+ Snapshots are being deprecated in favor of{" "} + + Commits + + . This feature will be officially removed in November 2025. +

+ {secretSnaphots?.pages?.map((group, i) => ( {group.map(({ id, createdAt }, index) => ( diff --git a/frontend/src/pages/secret-manager/SecretDashboardPage/components/SecretListView/SecretListView.tsx b/frontend/src/pages/secret-manager/SecretDashboardPage/components/SecretListView/SecretListView.tsx index 70d783d2b1..89c1cae25d 100644 --- a/frontend/src/pages/secret-manager/SecretDashboardPage/components/SecretListView/SecretListView.tsx +++ b/frontend/src/pages/secret-manager/SecretDashboardPage/components/SecretListView/SecretListView.tsx @@ -9,6 +9,7 @@ import { usePopUp } from "@app/hooks"; import { useCreateSecretV3, useDeleteSecretV3, useUpdateSecretV3 } from "@app/hooks/api"; import { dashboardKeys } from "@app/hooks/api/dashboard/queries"; import { UsedBySecretSyncs } from "@app/hooks/api/dashboard/types"; +import { commitKeys } from "@app/hooks/api/folderCommits/queries"; import { secretApprovalRequestKeys } from "@app/hooks/api/secretApprovalRequest/queries"; import { secretKeys } from "@app/hooks/api/secrets/queries"; import { SecretType, SecretV3RawSanitized } from "@app/hooks/api/secrets/types"; @@ -264,6 +265,12 @@ export const SecretListView = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId, environment, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); @@ -315,6 +322,12 @@ export const SecretListView = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId, environment, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); diff --git a/frontend/src/pages/secret-manager/SecretDashboardPage/components/SnapshotView/SnapshotView.tsx b/frontend/src/pages/secret-manager/SecretDashboardPage/components/SnapshotView/SnapshotView.tsx index c84e75429b..7e7a5f8c87 100644 --- a/frontend/src/pages/secret-manager/SecretDashboardPage/components/SnapshotView/SnapshotView.tsx +++ b/frontend/src/pages/secret-manager/SecretDashboardPage/components/SnapshotView/SnapshotView.tsx @@ -12,6 +12,7 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { createNotification } from "@app/components/notifications"; import { ProjectPermissionCan } from "@app/components/permissions"; import { Button, ContentLoader, Input, Tag, Tooltip } from "@app/components/v2"; +import { NoticeBannerV2 } from "@app/components/v2/NoticeBannerV2/NoticeBannerV2"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/context"; import { useGetSnapshotSecrets, usePerformSecretRollback } from "@app/hooks/api"; import { SecretV3RawSanitized, TSecretFolder } from "@app/hooks/api/types"; @@ -60,6 +61,7 @@ export const SnapshotView = ({ const rollingFolder = snapshotData?.folders || []; const rollingSecrets = snapshotData?.secrets || []; + const isAllowedRollback = false; const folderDiffView = useMemo(() => { const folderGroupById = folders.reduce>( @@ -157,6 +159,20 @@ export const SnapshotView = ({
Snapshot
{new Date(snapshotData?.createdAt || "").toLocaleString()}
+ +

+ Snapshots are being deprecated in favor of{" "} + + Commits + + . This feature will be officially removed in November 2025. +

+
-
- -
+ {isAllowedRollback && ( +
+ +
+ )}