From 3a1168c7e82c7ee79938b8c53aadcdda51f41487 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 31 May 2024 19:12:55 +0800 Subject: [PATCH 01/17] feat: added initial version pruning and result limiting --- .../20240531042507_add-pit-version-limit.ts | 21 ++++++++ backend/src/db/schemas/projects.ts | 3 +- .../secret-snapshot-service.ts | 17 +++++- .../services/secret-snapshot/snapshot-dal.ts | 52 ++++++++++++++++++- backend/src/server/routes/index.ts | 1 + .../src/services/project/project-service.ts | 3 +- backend/src/services/secret/secret-service.ts | 13 ++++- .../src/services/secret/secret-version-dal.ts | 37 +++++++++++++ 8 files changed, 140 insertions(+), 7 deletions(-) create mode 100644 backend/src/db/migrations/20240531042507_add-pit-version-limit.ts diff --git a/backend/src/db/migrations/20240531042507_add-pit-version-limit.ts b/backend/src/db/migrations/20240531042507_add-pit-version-limit.ts new file mode 100644 index 0000000000..e37c24e2c8 --- /dev/null +++ b/backend/src/db/migrations/20240531042507_add-pit-version-limit.ts @@ -0,0 +1,21 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + const hasPitVersionLimitColumn = await knex.schema.hasColumn(TableName.Project, "pitVersionLimit"); + await knex.schema.alterTable(TableName.Project, (tb) => { + if (!hasPitVersionLimitColumn) { + tb.integer("pitVersionLimit").notNullable().defaultTo(10); + } + }); +} + +export async function down(knex: Knex): Promise { + const hasPitVersionLimitColumn = await knex.schema.hasColumn(TableName.Project, "pitVersionLimit"); + await knex.schema.alterTable(TableName.Project, (tb) => { + if (hasPitVersionLimitColumn) { + tb.dropColumn("pitVersionLimit"); + } + }); +} diff --git a/backend/src/db/schemas/projects.ts b/backend/src/db/schemas/projects.ts index 3965e24c0a..9197a355d9 100644 --- a/backend/src/db/schemas/projects.ts +++ b/backend/src/db/schemas/projects.ts @@ -16,7 +16,8 @@ export const ProjectsSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), version: z.number().default(1), - upgradeStatus: z.string().nullable().optional() + upgradeStatus: z.string().nullable().optional(), + pitVersionLimit: z.number() }); export type TProjects = z.infer; diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index 0e71ad1268..6197c08256 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -4,6 +4,7 @@ import { TableName, TSecretTagJunctionInsert } from "@app/db/schemas"; import { BadRequestError, InternalServerError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; +import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; @@ -37,6 +38,7 @@ type TSecretSnapshotServiceFactoryDep = { folderDAL: Pick; permissionService: Pick; licenseService: Pick; + projectDAL: Pick; }; export type TSecretSnapshotServiceFactory = ReturnType; @@ -48,6 +50,7 @@ export const secretSnapshotServiceFactory = ({ snapshotSecretDAL, snapshotFolderDAL, folderDAL, + projectDAL, secretDAL, permissionService, licenseService, @@ -81,8 +84,9 @@ export const secretSnapshotServiceFactory = ({ const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found" }); + const project = await projectDAL.findById(projectId); const count = await snapshotDAL.countOfSnapshotsByFolderId(folder.id); - return count; + return Math.min(count, project.pitVersionLimit); }; const listSnapshots = async ({ @@ -114,7 +118,16 @@ export const secretSnapshotServiceFactory = ({ const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found" }); - const snapshots = await snapshotDAL.find({ folderId: folder.id }, { limit, offset, sort: [["createdAt", "desc"]] }); + const { pitVersionLimit } = await projectDAL.findById(projectId); + const computedQueryLimit = Math.min(pitVersionLimit - offset, limit); + if (offset > pitVersionLimit || computedQueryLimit <= 0) { + return []; + } + + const snapshots = await snapshotDAL.find( + { folderId: folder.id }, + { limit: computedQueryLimit, offset, sort: [["createdAt", "desc"]] } + ); return snapshots; }; diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index cdd5a999b8..1618024bbc 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -325,12 +325,62 @@ export const snapshotDALFactory = (db: TDbClient) => { } }; + const pruneExcessSnapshots = async (tx?: Knex) => { + try { + const folders = await (tx || db)(TableName.SecretFolder).select("id"); + const folderIds = folders.map((folder) => folder.id); + const PRUNE_FOLDER_BATCH_SIZE = 500; + + const pruneBatches = []; + for (let x = 0; x < folderIds.length; x += PRUNE_FOLDER_BATCH_SIZE) { + const batch = folderIds.slice(x, x + PRUNE_FOLDER_BATCH_SIZE); + pruneBatches.push(batch); + } + + for await (const folderBatch of pruneBatches) { + const rankedSnapshots = (tx || db)(TableName.Snapshot) + .whereIn(`${TableName.Snapshot}.folderId`, folderBatch) + .select( + "folderId", + "id", + (tx || db).raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` + ) + ) + .as("ranked_snapshots"); + + const snapshotsToKeep = (tx || db) + .select("id") + .from(rankedSnapshots) + .where( + "row_num", + "<=", + (tx || db) + .select(`${TableName.Project}.pitVersionLimit`) + .from(TableName.Project) + .join(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) + .join(TableName.Snapshot, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`) + .join(rankedSnapshots, "ranked_snapshots.folderId", `${TableName.Snapshot}.folderId`) + .limit(1) + ); + + await (tx || db)(TableName.Snapshot) + .whereIn("folderId", folderBatch) + .whereNotIn("id", snapshotsToKeep) + .delete(); + } + } catch (error) { + throw new DatabaseError({ error, name: "SnapshotPrune" }); + } + }; + return { ...secretSnapshotOrm, findById, findLatestSnapshotByFolderId, findRecursivelySnapshots, countOfSnapshotsByFolderId, - findSecretSnapshotDataById + findSecretSnapshotDataById, + pruneExcessSnapshots }; }; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 1593515ec9..b0b2db9b19 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -535,6 +535,7 @@ export const registerRoutes = async ( licenseService, folderDAL, secretDAL, + projectDAL, snapshotDAL, snapshotFolderDAL, snapshotSecretDAL, diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index f58fd77885..1e588dd08f 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -133,7 +133,8 @@ export const projectServiceFactory = ({ name: workspaceName, orgId: organization.id, slug: projectSlug || slugify(`${workspaceName}-${alphaNumericNanoId(4)}`), - version: ProjectVersion.V2 + version: ProjectVersion.V2, + pitVersionLimit: 10 }, tx ); diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 39e47a28e1..77c4024ac7 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -72,7 +72,7 @@ type TSecretServiceFactoryDep = { secretDAL: TSecretDALFactory; secretTagDAL: TSecretTagDALFactory; secretVersionDAL: TSecretVersionDALFactory; - projectDAL: Pick; + projectDAL: Pick; projectEnvDAL: Pick; folderDAL: Pick< TSecretFolderDALFactory, @@ -1354,7 +1354,16 @@ export const secretServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); - const secretVersions = await secretVersionDAL.find({ secretId }, { offset, limit, sort: [["createdAt", "desc"]] }); + const { pitVersionLimit } = await projectDAL.findById(folder.projectId); + const computedQueryLimit = Math.min(pitVersionLimit - offset, limit); + if (offset > pitVersionLimit || computedQueryLimit <= 0) { + return []; + } + + const secretVersions = await secretVersionDAL.find( + { secretId }, + { offset, limit: computedQueryLimit, sort: [["createdAt", "desc"]] } + ); return secretVersions; }; diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index 758352ed24..5d52f3c07f 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -110,8 +110,45 @@ export const secretVersionDALFactory = (db: TDbClient) => { } }; + const pruneExcessVersions = async (tx?: Knex) => { + try { + const rankedSecretVersions = (tx || db)(TableName.SecretVersion) + .select( + "id", + "secretId", + "folderId", + (tx || db).raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num` + ) + ) + .as("ranked_secret_versions"); + + const versionsToKeep = (tx || db)(rankedSecretVersions) + .select("id") + .where( + "row_num", + "<=", + (tx || db) + .select(`${TableName.Project}.pitVersionLimit`) + .from(TableName.Project) + .join(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .join(rankedSecretVersions, "ranked_secret_versions.folderId", `${TableName.SecretFolder}.id`) + .limit(1) + ); + + await (tx || db)(TableName.SecretVersion).whereNotIn("id", versionsToKeep).delete(); + } catch (error) { + throw new DatabaseError({ + error, + name: "Secret Version Prune" + }); + } + }; + return { ...secretVersionOrm, + pruneExcessVersions, findLatestVersionMany, bulkUpdate, findLatestVersionByFolderId, From 9117067ab505e08598378884c2b4d6517d5e72dc Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 31 May 2024 21:38:16 +0800 Subject: [PATCH 02/17] feat: finalized pruning logic --- .../services/secret-snapshot/snapshot-dal.ts | 21 ++++++----- .../secret-folder-version-dal.ts | 36 ++++++++++++++++++- .../src/services/secret/secret-version-dal.ts | 22 ++++++------ 3 files changed, 56 insertions(+), 23 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index 1618024bbc..a35345c17c 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -349,20 +349,19 @@ export const snapshotDALFactory = (db: TDbClient) => { ) .as("ranked_snapshots"); + const folderLimits = (tx || db)(TableName.Snapshot) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.Snapshot}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .whereIn(`${TableName.Snapshot}.folderId`, folderBatch) + .groupBy(`${TableName.Snapshot}.folderId`, `${TableName.Project}.pitVersionLimit`) + .select("folderId", "pitVersionLimit") + .as("folder_limits"); + const snapshotsToKeep = (tx || db) .select("id") .from(rankedSnapshots) - .where( - "row_num", - "<=", - (tx || db) - .select(`${TableName.Project}.pitVersionLimit`) - .from(TableName.Project) - .join(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) - .join(TableName.Snapshot, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`) - .join(rankedSnapshots, "ranked_snapshots.folderId", `${TableName.Snapshot}.folderId`) - .limit(1) - ); + .join(folderLimits, "folder_limits.folderId", "ranked_snapshots.folderId") + .whereRaw(`ranked_snapshots.row_num <= folder_limits."pitVersionLimit"`); await (tx || db)(TableName.Snapshot) .whereIn("folderId", folderBatch) diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index f133308cf4..cfc3706408 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -62,5 +62,39 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { } }; - return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId }; + const pruneExcessVersions = async (tx?: Knex) => { + try { + const rankedFolderVersions = (tx || db)(TableName.SecretFolderVersion) + .select( + "id", + "folderId", + (tx || db).raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretFolderVersion}."folderId" ORDER BY ${TableName.SecretFolderVersion}."createdAt" DESC) AS row_num` + ) + ) + .as("ranked_folder_versions"); + + const folderLimits = (tx || db)(TableName.SecretFolderVersion) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .groupBy(`${TableName.SecretFolderVersion}.folderId`, `${TableName.Project}.pitVersionLimit`) + .select("folderId", "pitVersionLimit") + .as("folder_limits"); + + const versionsToKeep = (tx || db)(rankedFolderVersions) + .select("id") + .from(rankedFolderVersions) + .join(folderLimits, "folder_limits.folderId", "ranked_folder_versions.folderId") + .whereRaw(`ranked_folder_versions.row_num <= folder_limits."pitVersionLimit"`); + + await (tx || db)(TableName.SecretFolderVersion).whereNotIn("id", versionsToKeep).delete(); + } catch (error) { + throw new DatabaseError({ + error, + name: "Secret Version Prune" + }); + } + }; + + return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId, pruneExcessVersions }; }; diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index 5d52f3c07f..374907c3d2 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -123,19 +123,19 @@ export const secretVersionDALFactory = (db: TDbClient) => { ) .as("ranked_secret_versions"); + const folderLimits = (tx || db)(TableName.SecretVersion) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .groupBy(`${TableName.SecretVersion}.folderId`, `${TableName.Project}.pitVersionLimit`) + .select("folderId", "pitVersionLimit") + .as("folder_limits"); + const versionsToKeep = (tx || db)(rankedSecretVersions) .select("id") - .where( - "row_num", - "<=", - (tx || db) - .select(`${TableName.Project}.pitVersionLimit`) - .from(TableName.Project) - .join(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) - .join(TableName.SecretFolder, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) - .join(rankedSecretVersions, "ranked_secret_versions.folderId", `${TableName.SecretFolder}.id`) - .limit(1) - ); + .from(rankedSecretVersions) + .join(folderLimits, "folder_limits.folderId", "ranked_secret_versions.folderId") + .whereRaw(`ranked_secret_versions.row_num <= folder_limits."pitVersionLimit"`); await (tx || db)(TableName.SecretVersion).whereNotIn("id", versionsToKeep).delete(); } catch (error) { From abd8d6aa8af73b0421e649e0a0733ca9ade807b2 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 31 May 2024 23:18:02 +0800 Subject: [PATCH 03/17] feat: added support for version limit update --- .../src/server/routes/v1/project-router.ts | 38 ++++++++ .../src/services/project/project-service.ts | 33 ++++++- backend/src/services/project/project-types.ts | 5 ++ frontend/src/hooks/api/workspace/queries.tsx | 16 ++++ frontend/src/hooks/api/workspace/types.ts | 4 +- .../PointInTimeVersionLimitSection.tsx | 88 +++++++++++++++++++ .../PointInTimeVersionLimitSection/index.tsx | 1 + .../ProjectGeneralTab/ProjectGeneralTab.tsx | 2 + 8 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx create mode 100644 frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/index.tsx diff --git a/backend/src/server/routes/v1/project-router.ts b/backend/src/server/routes/v1/project-router.ts index 1cf655a973..0984b66f61 100644 --- a/backend/src/server/routes/v1/project-router.ts +++ b/backend/src/server/routes/v1/project-router.ts @@ -334,6 +334,44 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { } }); + server.route({ + method: "PUT", + url: "/:workspaceSlug/version-limit", + config: { + rateLimit: writeLimit + }, + schema: { + params: z.object({ + workspaceSlug: z.string().trim() + }), + body: z.object({ + pitVersionLimit: z.number().min(1).max(100) + }), + response: { + 200: z.object({ + message: z.string(), + workspace: ProjectsSchema + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const workspace = await server.services.project.updateVersionLimit({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + pitVersionLimit: req.body.pitVersionLimit, + workspaceSlug: req.params.workspaceSlug + }); + + return { + message: "Successfully changed workspace version limit", + workspace + }; + } + }); + server.route({ method: "GET", url: "/:workspaceId/integrations", diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index 1e588dd08f..ac6cfbc003 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -39,6 +39,7 @@ import { TToggleProjectAutoCapitalizationDTO, TUpdateProjectDTO, TUpdateProjectNameDTO, + TUpdateProjectVersionLimitDTO, TUpgradeProjectDTO } from "./project-types"; @@ -407,6 +408,35 @@ export const projectServiceFactory = ({ return updatedProject; }; + const updateVersionLimit = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + pitVersionLimit, + workspaceSlug + }: TUpdateProjectVersionLimitDTO) => { + const project = await projectDAL.findProjectBySlug(workspaceSlug, actorOrgId); + if (!project) { + throw new BadRequestError({ + message: "Project not found" + }); + } + + const { hasRole } = await permissionService.getProjectPermission( + actor, + actorId, + project.id, + actorAuthMethod, + actorOrgId + ); + + if (!hasRole(ProjectMembershipRole.Admin)) + throw new BadRequestError({ message: "Only admins are allowed to take this action" }); + + return projectDAL.updateById(project.id, { pitVersionLimit }); + }; + const updateName = async ({ projectId, actor, @@ -502,6 +532,7 @@ export const projectServiceFactory = ({ getAProject, toggleAutoCapitalization, updateName, - upgradeProject + upgradeProject, + updateVersionLimit }; }; diff --git a/backend/src/services/project/project-types.ts b/backend/src/services/project/project-types.ts index dcd424e18c..fbcfd2d9a8 100644 --- a/backend/src/services/project/project-types.ts +++ b/backend/src/services/project/project-types.ts @@ -43,6 +43,11 @@ export type TToggleProjectAutoCapitalizationDTO = { autoCapitalization: boolean; } & TProjectPermission; +export type TUpdateProjectVersionLimitDTO = { + pitVersionLimit: number; + workspaceSlug: string; +} & Omit; + export type TUpdateProjectNameDTO = { name: string; } & TProjectPermission; diff --git a/frontend/src/hooks/api/workspace/queries.tsx b/frontend/src/hooks/api/workspace/queries.tsx index 71dbb8e01e..6462490bbd 100644 --- a/frontend/src/hooks/api/workspace/queries.tsx +++ b/frontend/src/hooks/api/workspace/queries.tsx @@ -20,6 +20,7 @@ import { TUpdateWorkspaceIdentityRoleDTO, TUpdateWorkspaceUserRoleDTO, UpdateEnvironmentDTO, + UpdatePitVersionLimitDTO, Workspace } from "./types"; @@ -249,6 +250,21 @@ export const useToggleAutoCapitalization = () => { }); }; +export const useUpdateWorkspaceVersionLimit = () => { + const queryClient = useQueryClient(); + + return useMutation<{}, {}, UpdatePitVersionLimitDTO>({ + mutationFn: ({ projectSlug, pitVersionLimit }) => { + return apiRequest.put(`/api/v1/workspace/${projectSlug}/version-limit`, { + pitVersionLimit + }); + }, + onSuccess: () => { + queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace); + } + }); +}; + export const useDeleteWorkspace = () => { const queryClient = useQueryClient(); diff --git a/frontend/src/hooks/api/workspace/types.ts b/frontend/src/hooks/api/workspace/types.ts index 8be9beed0d..8c28d09389 100644 --- a/frontend/src/hooks/api/workspace/types.ts +++ b/frontend/src/hooks/api/workspace/types.ts @@ -16,6 +16,7 @@ export type Workspace = { upgradeStatus: string | null; autoCapitalization: boolean; environments: WorkspaceEnv[]; + pitVersionLimit: number; slug: string; }; @@ -48,6 +49,7 @@ export type CreateWorkspaceDTO = { }; export type RenameWorkspaceDTO = { workspaceID: string; newWorkspaceName: string }; +export type UpdatePitVersionLimitDTO = { projectSlug: string; pitVersionLimit: number }; export type ToggleAutoCapitalizationDTO = { workspaceID: string; state: boolean }; export type DeleteWorkspaceDTO = { workspaceID: string }; @@ -128,4 +130,4 @@ export type TUpdateWorkspaceGroupRoleDTO = { temporaryAccessStartTime: string; } )[]; -}; \ No newline at end of file +}; diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx new file mode 100644 index 0000000000..8c820b727d --- /dev/null +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx @@ -0,0 +1,88 @@ +import { Controller, useForm } from "react-hook-form"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { z } from "zod"; + +import { createNotification } from "@app/components/notifications"; +import { Button, FormControl, Input } from "@app/components/v2"; +import { useProjectPermission, useWorkspace } from "@app/context"; +import { ProjectMembershipRole } from "@app/hooks/api/roles/types"; +import { useUpdateWorkspaceVersionLimit } from "@app/hooks/api/workspace/queries"; + +const formSchema = z.object({ + pitVersionLimit: z.coerce.number().min(1).max(100) +}); + +type TForm = z.infer; + +export const PointInTimeVersionLimitSection = () => { + const { mutateAsync: updatePitVersion } = useUpdateWorkspaceVersionLimit(); + + const { currentWorkspace } = useWorkspace(); + const { membership } = useProjectPermission(); + + const { + control, + formState: { isSubmitting }, + handleSubmit + } = useForm({ + resolver: zodResolver(formSchema), + values: { + pitVersionLimit: currentWorkspace?.pitVersionLimit || 10 + } + }); + + if (!currentWorkspace) return null; + + const handleVersionLimitSubmit = async ({ pitVersionLimit }: TForm) => { + try { + await updatePitVersion({ + pitVersionLimit, + projectSlug: currentWorkspace.slug + }); + + createNotification({ + text: "Successfully updated version limit", + type: "success" + }); + } catch (err) { + createNotification({ + text: "Failed updating project's version limit", + type: "error" + }); + } + }; + + const isAdmin = membership.roles.includes(ProjectMembershipRole.Admin); + return ( +
+
+

Point in Time Recovery

+
+

+ This defines the maximum number of folder snapshots, secret versions, and folder versions + that are retained by the system +

+
+
+ ( + + + + )} + /> +
+ +
+
+ ); +}; diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/index.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/index.tsx new file mode 100644 index 0000000000..242b8c79a1 --- /dev/null +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/index.tsx @@ -0,0 +1 @@ +export { PointInTimeVersionLimitSection } from "./PointInTimeVersionLimitSection"; diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx index 7d7c30fb0a..511dff93e1 100644 --- a/frontend/src/views/Settings/ProjectSettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx @@ -3,6 +3,7 @@ import { BackfillSecretReferenceSecretion } from "../BackfillSecretReferenceSect import { DeleteProjectSection } from "../DeleteProjectSection"; import { E2EESection } from "../E2EESection"; import { EnvironmentSection } from "../EnvironmentSection"; +import { PointInTimeVersionLimitSection } from "../PointInTimeVersionLimitSection"; import { ProjectNameChangeSection } from "../ProjectNameChangeSection"; import { SecretTagsSection } from "../SecretTagsSection"; @@ -14,6 +15,7 @@ export const ProjectGeneralTab = () => { + From 4d8f94a9dc4909d714af93b8ee5b5e08dd612db3 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 31 May 2024 23:25:56 +0800 Subject: [PATCH 04/17] feat: added version prune to daily resource queue --- backend/src/server/routes/index.ts | 3 +++ .../resource-cleanup/resource-cleanup-queue.ts | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index b0b2db9b19..8c4ff2a701 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -795,6 +795,9 @@ export const registerRoutes = async ( const dailyResourceCleanUp = dailyResourceCleanUpQueueServiceFactory({ auditLogDAL, queueService, + secretVersionDAL, + secretFolderVersionDAL: folderVersionDAL, + snapshotDAL, identityAccessTokenDAL, secretSharingDAL }); diff --git a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts index afae2677f7..2e01e35494 100644 --- a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts +++ b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts @@ -1,13 +1,19 @@ import { TAuditLogDALFactory } from "@app/ee/services/audit-log/audit-log-dal"; +import { TSnapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identity-access-token-dal"; +import { TSecretVersionDALFactory } from "../secret/secret-version-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretSharingDALFactory } from "../secret-sharing/secret-sharing-dal"; type TDailyResourceCleanUpQueueServiceFactoryDep = { auditLogDAL: Pick; identityAccessTokenDAL: Pick; + secretVersionDAL: Pick; + secretFolderVersionDAL: Pick; + snapshotDAL: Pick; secretSharingDAL: Pick; queueService: TQueueServiceFactory; }; @@ -17,6 +23,9 @@ export type TDailyResourceCleanUpQueueServiceFactory = ReturnType { @@ -25,6 +34,9 @@ export const dailyResourceCleanUpQueueServiceFactory = ({ await auditLogDAL.pruneAuditLog(); await identityAccessTokenDAL.removeExpiredTokens(); await secretSharingDAL.pruneExpiredSharedSecrets(); + await snapshotDAL.pruneExcessSnapshots(); + await secretVersionDAL.pruneExcessVersions(); + await secretFolderVersionDAL.pruneExcessVersions(); logger.info(`${QueueName.DailyResourceCleanUp}: queue task completed`); }); From d76760fa9c9fc43914e6cdf7c83694dfeb585f9b Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 31 May 2024 23:42:49 +0800 Subject: [PATCH 05/17] misc: updated schema --- backend/src/db/schemas/projects.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/db/schemas/projects.ts b/backend/src/db/schemas/projects.ts index 9197a355d9..dd7e999900 100644 --- a/backend/src/db/schemas/projects.ts +++ b/backend/src/db/schemas/projects.ts @@ -17,7 +17,7 @@ export const ProjectsSchema = z.object({ updatedAt: z.date(), version: z.number().default(1), upgradeStatus: z.string().nullable().optional(), - pitVersionLimit: z.number() + pitVersionLimit: z.number().default(10) }); export type TProjects = z.infer; From 4eb08c64d45b8fcc2a1302a3db345fb2a8ab4ed2 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Sat, 1 Jun 2024 01:07:25 +0800 Subject: [PATCH 06/17] misc: updated error message --- backend/src/services/secret-folder/secret-folder-version-dal.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index cfc3706408..2f32001dcd 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -91,7 +91,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { } catch (error) { throw new DatabaseError({ error, - name: "Secret Version Prune" + name: "Secret Folder Version Prune" }); } }; From b8e9417466cf47b52b518312639e398b4e97f7f8 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Sat, 1 Jun 2024 03:26:35 +0800 Subject: [PATCH 07/17] misc: modified pruning sql logic --- .../services/secret-snapshot/snapshot-dal.ts | 54 ++++++++----------- .../src/services/secret/secret-version-dal.ts | 48 ++++++++--------- 2 files changed, 44 insertions(+), 58 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index a35345c17c..44fcadf390 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -325,9 +325,9 @@ export const snapshotDALFactory = (db: TDbClient) => { } }; - const pruneExcessSnapshots = async (tx?: Knex) => { + const pruneExcessSnapshots = async () => { try { - const folders = await (tx || db)(TableName.SecretFolder).select("id"); + const folders = await db(TableName.SecretFolder).select("id"); const folderIds = folders.map((folder) => folder.id); const PRUNE_FOLDER_BATCH_SIZE = 500; @@ -338,35 +338,27 @@ export const snapshotDALFactory = (db: TDbClient) => { } for await (const folderBatch of pruneBatches) { - const rankedSnapshots = (tx || db)(TableName.Snapshot) - .whereIn(`${TableName.Snapshot}.folderId`, folderBatch) - .select( - "folderId", - "id", - (tx || db).raw( - `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` - ) - ) - .as("ranked_snapshots"); - - const folderLimits = (tx || db)(TableName.Snapshot) - .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.Snapshot}.envId`) - .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) - .whereIn(`${TableName.Snapshot}.folderId`, folderBatch) - .groupBy(`${TableName.Snapshot}.folderId`, `${TableName.Project}.pitVersionLimit`) - .select("folderId", "pitVersionLimit") - .as("folder_limits"); - - const snapshotsToKeep = (tx || db) - .select("id") - .from(rankedSnapshots) - .join(folderLimits, "folder_limits.folderId", "ranked_snapshots.folderId") - .whereRaw(`ranked_snapshots.row_num <= folder_limits."pitVersionLimit"`); - - await (tx || db)(TableName.Snapshot) - .whereIn("folderId", folderBatch) - .whereNotIn("id", snapshotsToKeep) - .delete(); + await secretSnapshotOrm.transaction(async (txn) => { + return txn(TableName.Snapshot) + .with("snapshot_cte", (qb) => { + void qb + .from(TableName.Snapshot) + .whereIn(`${TableName.Snapshot}.folderId`, folderBatch) + .select( + "folderId", + `${TableName.Snapshot}.id as id`, + txn.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` + ) + ); + }) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Snapshot}.folderId`) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`) + .whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); + }); } } catch (error) { throw new DatabaseError({ error, name: "SnapshotPrune" }); diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index 374907c3d2..596d0f4832 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -110,34 +110,28 @@ export const secretVersionDALFactory = (db: TDbClient) => { } }; - const pruneExcessVersions = async (tx?: Knex) => { + const pruneExcessVersions = async () => { try { - const rankedSecretVersions = (tx || db)(TableName.SecretVersion) - .select( - "id", - "secretId", - "folderId", - (tx || db).raw( - `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num` - ) - ) - .as("ranked_secret_versions"); - - const folderLimits = (tx || db)(TableName.SecretVersion) - .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`) - .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) - .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) - .groupBy(`${TableName.SecretVersion}.folderId`, `${TableName.Project}.pitVersionLimit`) - .select("folderId", "pitVersionLimit") - .as("folder_limits"); - - const versionsToKeep = (tx || db)(rankedSecretVersions) - .select("id") - .from(rankedSecretVersions) - .join(folderLimits, "folder_limits.folderId", "ranked_secret_versions.folderId") - .whereRaw(`ranked_secret_versions.row_num <= folder_limits."pitVersionLimit"`); - - await (tx || db)(TableName.SecretVersion).whereNotIn("id", versionsToKeep).delete(); + await secretVersionOrm.transaction((txn) => { + return txn(TableName.SecretVersion) + .with("version_cte", (qb) => { + void qb + .from(TableName.SecretVersion) + .select( + "id", + "folderId", + txn.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num` + ) + ); + }) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("version_cte", "version_cte.id", `${TableName.SecretVersion}.id`) + .whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); + }); } catch (error) { throw new DatabaseError({ error, From ab093dfc852c6bfa4c76db5095c45ab2b819ede9 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Mon, 3 Jun 2024 12:49:40 +0800 Subject: [PATCH 08/17] misc: simplified delete query for secret folder version --- .../secret-folder-version-dal.ts | 45 +++++++++---------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index 2f32001dcd..e9c247b522 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -62,32 +62,27 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { } }; - const pruneExcessVersions = async (tx?: Knex) => { + const pruneExcessVersions = async () => { try { - const rankedFolderVersions = (tx || db)(TableName.SecretFolderVersion) - .select( - "id", - "folderId", - (tx || db).raw( - `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretFolderVersion}."folderId" ORDER BY ${TableName.SecretFolderVersion}."createdAt" DESC) AS row_num` - ) - ) - .as("ranked_folder_versions"); - - const folderLimits = (tx || db)(TableName.SecretFolderVersion) - .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`) - .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) - .groupBy(`${TableName.SecretFolderVersion}.folderId`, `${TableName.Project}.pitVersionLimit`) - .select("folderId", "pitVersionLimit") - .as("folder_limits"); - - const versionsToKeep = (tx || db)(rankedFolderVersions) - .select("id") - .from(rankedFolderVersions) - .join(folderLimits, "folder_limits.folderId", "ranked_folder_versions.folderId") - .whereRaw(`ranked_folder_versions.row_num <= folder_limits."pitVersionLimit"`); - - await (tx || db)(TableName.SecretFolderVersion).whereNotIn("id", versionsToKeep).delete(); + await secretFolderVerOrm.transaction((txn) => { + return txn(TableName.SecretFolderVersion) + .with("folder_cte", (qb) => { + void qb + .from(TableName.SecretFolderVersion) + .select( + "id", + "folderId", + txn.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretFolderVersion}."folderId" ORDER BY ${TableName.SecretFolderVersion}."createdAt" DESC) AS row_num` + ) + ); + }) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("folder_cte", "folder_cte.id", `${TableName.SecretFolderVersion}.id`) + .whereRaw(`folder_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); + }); } catch (error) { throw new DatabaseError({ error, From cd6caab508e539aadd09e62a3f32a2d4ed4aa05d Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Mon, 3 Jun 2024 17:56:00 +0800 Subject: [PATCH 09/17] misc: migrated to using keyset pagnination --- .../services/secret-snapshot/snapshot-dal.ts | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index 44fcadf390..b49e520b8a 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-await-in-loop */ import { Knex } from "knex"; import { TDbClient } from "@app/db"; @@ -11,6 +12,7 @@ import { } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; +import { logger } from "@app/lib/logger"; export type TSnapshotDALFactory = ReturnType; @@ -326,28 +328,32 @@ export const snapshotDALFactory = (db: TDbClient) => { }; const pruneExcessSnapshots = async () => { - try { - const folders = await db(TableName.SecretFolder).select("id"); - const folderIds = folders.map((folder) => folder.id); - const PRUNE_FOLDER_BATCH_SIZE = 500; + const PRUNE_FOLDER_BATCH_SIZE = 10000; + let uuidOffset = "00000000-0000-0000-0000-000000000000"; - const pruneBatches = []; - for (let x = 0; x < folderIds.length; x += PRUNE_FOLDER_BATCH_SIZE) { - const batch = folderIds.slice(x, x + PRUNE_FOLDER_BATCH_SIZE); - pruneBatches.push(batch); - } + // eslint-disable-next-line no-constant-condition, no-unreachable-loop + while (true) { + const folderBatch = await db(TableName.SecretFolder) + .where("id", ">", uuidOffset) + .orderBy("id", "asc") + .limit(PRUNE_FOLDER_BATCH_SIZE) + .select("id"); - for await (const folderBatch of pruneBatches) { - await secretSnapshotOrm.transaction(async (txn) => { - return txn(TableName.Snapshot) + const batchEntries = folderBatch.map((folder) => folder.id); + logger.info("UUID offset:", uuidOffset); + + if (folderBatch.length) { + try { + logger.info(`Pruning snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}`); + await db(TableName.Snapshot) .with("snapshot_cte", (qb) => { void qb .from(TableName.Snapshot) - .whereIn(`${TableName.Snapshot}.folderId`, folderBatch) + .whereIn(`${TableName.Snapshot}.folderId`, batchEntries) .select( "folderId", `${TableName.Snapshot}.id as id`, - txn.raw( + db.raw( `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` ) ); @@ -358,10 +364,16 @@ export const snapshotDALFactory = (db: TDbClient) => { .join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`) .whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`) .delete(); - }); + } catch (err) { + logger.error( + `Failed to prune snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}` + ); + } finally { + uuidOffset = batchEntries[batchEntries.length - 1]; + } + } else { + return; } - } catch (error) { - throw new DatabaseError({ error, name: "SnapshotPrune" }); } }; From 4d830f1d1a494edec02ac632c1a656ad34a81c8a Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Mon, 3 Jun 2024 17:58:39 +0800 Subject: [PATCH 10/17] misc: added outer try catch block --- .../services/secret-snapshot/snapshot-dal.ts | 82 ++++++++++--------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index b49e520b8a..7c17c4f072 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -331,49 +331,53 @@ export const snapshotDALFactory = (db: TDbClient) => { const PRUNE_FOLDER_BATCH_SIZE = 10000; let uuidOffset = "00000000-0000-0000-0000-000000000000"; - // eslint-disable-next-line no-constant-condition, no-unreachable-loop - while (true) { - const folderBatch = await db(TableName.SecretFolder) - .where("id", ">", uuidOffset) - .orderBy("id", "asc") - .limit(PRUNE_FOLDER_BATCH_SIZE) - .select("id"); + try { + // eslint-disable-next-line no-constant-condition, no-unreachable-loop + while (true) { + const folderBatch = await db(TableName.SecretFolder) + .where("id", ">", uuidOffset) + .orderBy("id", "asc") + .limit(PRUNE_FOLDER_BATCH_SIZE) + .select("id"); - const batchEntries = folderBatch.map((folder) => folder.id); - logger.info("UUID offset:", uuidOffset); + const batchEntries = folderBatch.map((folder) => folder.id); + logger.info("UUID offset:", uuidOffset); - if (folderBatch.length) { - try { - logger.info(`Pruning snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}`); - await db(TableName.Snapshot) - .with("snapshot_cte", (qb) => { - void qb - .from(TableName.Snapshot) - .whereIn(`${TableName.Snapshot}.folderId`, batchEntries) - .select( - "folderId", - `${TableName.Snapshot}.id as id`, - db.raw( - `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` - ) - ); - }) - .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Snapshot}.folderId`) - .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) - .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) - .join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`) - .whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`) - .delete(); - } catch (err) { - logger.error( - `Failed to prune snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}` - ); - } finally { - uuidOffset = batchEntries[batchEntries.length - 1]; + if (folderBatch.length) { + try { + logger.info(`Pruning snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}`); + await db(TableName.Snapshot) + .with("snapshot_cte", (qb) => { + void qb + .from(TableName.Snapshot) + .whereIn(`${TableName.Snapshot}.folderId`, batchEntries) + .select( + "folderId", + `${TableName.Snapshot}.id as id`, + db.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` + ) + ); + }) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Snapshot}.folderId`) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`) + .whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); + } catch (err) { + logger.error( + `Failed to prune snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}` + ); + } finally { + uuidOffset = batchEntries[batchEntries.length - 1]; + } + } else { + return; } - } else { - return; } + } catch (error) { + throw new DatabaseError({ error, name: "SnapshotPrune" }); } }; From 68a30f4212a885a82cecae33685be41cc61d54d0 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Mon, 3 Jun 2024 22:06:32 +0800 Subject: [PATCH 11/17] misc: removed transactional --- .../secret-folder-version-dal.ts | 36 +++++++++--------- .../src/services/secret/secret-version-dal.ts | 38 +++++++++---------- 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index e9c247b522..47074321f6 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -64,25 +64,23 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { const pruneExcessVersions = async () => { try { - await secretFolderVerOrm.transaction((txn) => { - return txn(TableName.SecretFolderVersion) - .with("folder_cte", (qb) => { - void qb - .from(TableName.SecretFolderVersion) - .select( - "id", - "folderId", - txn.raw( - `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretFolderVersion}."folderId" ORDER BY ${TableName.SecretFolderVersion}."createdAt" DESC) AS row_num` - ) - ); - }) - .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`) - .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) - .join("folder_cte", "folder_cte.id", `${TableName.SecretFolderVersion}.id`) - .whereRaw(`folder_cte.row_num > ${TableName.Project}."pitVersionLimit"`) - .delete(); - }); + await db(TableName.SecretFolderVersion) + .with("folder_cte", (qb) => { + void qb + .from(TableName.SecretFolderVersion) + .select( + "id", + "folderId", + db.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretFolderVersion}."folderId" ORDER BY ${TableName.SecretFolderVersion}."createdAt" DESC) AS row_num` + ) + ); + }) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("folder_cte", "folder_cte.id", `${TableName.SecretFolderVersion}.id`) + .whereRaw(`folder_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); } catch (error) { throw new DatabaseError({ error, diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index 596d0f4832..baff6ab996 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -112,26 +112,24 @@ export const secretVersionDALFactory = (db: TDbClient) => { const pruneExcessVersions = async () => { try { - await secretVersionOrm.transaction((txn) => { - return txn(TableName.SecretVersion) - .with("version_cte", (qb) => { - void qb - .from(TableName.SecretVersion) - .select( - "id", - "folderId", - txn.raw( - `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num` - ) - ); - }) - .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`) - .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) - .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) - .join("version_cte", "version_cte.id", `${TableName.SecretVersion}.id`) - .whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`) - .delete(); - }); + await db(TableName.SecretVersion) + .with("version_cte", (qb) => { + void qb + .from(TableName.SecretVersion) + .select( + "id", + "folderId", + db.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num` + ) + ); + }) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("version_cte", "version_cte.id", `${TableName.SecretVersion}.id`) + .whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); } catch (error) { throw new DatabaseError({ error, From f21a13f38871630759ae491c8f875fe013dc443b Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 4 Jun 2024 23:46:02 +0800 Subject: [PATCH 12/17] adjustment: removed artificial limiting of pit versions --- .../secret-snapshot/secret-snapshot-service.ts | 18 ++---------------- .../services/secret-snapshot/snapshot-dal.ts | 1 - backend/src/server/routes/index.ts | 1 - backend/src/services/secret/secret-service.ts | 13 ++----------- .../PointInTimeVersionLimitSection.tsx | 9 +++++++-- 5 files changed, 11 insertions(+), 31 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index 6197c08256..5a720a9e65 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -4,7 +4,6 @@ import { TableName, TSecretTagJunctionInsert } from "@app/db/schemas"; import { BadRequestError, InternalServerError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; -import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; @@ -38,7 +37,6 @@ type TSecretSnapshotServiceFactoryDep = { folderDAL: Pick; permissionService: Pick; licenseService: Pick; - projectDAL: Pick; }; export type TSecretSnapshotServiceFactory = ReturnType; @@ -50,7 +48,6 @@ export const secretSnapshotServiceFactory = ({ snapshotSecretDAL, snapshotFolderDAL, folderDAL, - projectDAL, secretDAL, permissionService, licenseService, @@ -84,9 +81,7 @@ export const secretSnapshotServiceFactory = ({ const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found" }); - const project = await projectDAL.findById(projectId); - const count = await snapshotDAL.countOfSnapshotsByFolderId(folder.id); - return Math.min(count, project.pitVersionLimit); + return snapshotDAL.countOfSnapshotsByFolderId(folder.id); }; const listSnapshots = async ({ @@ -118,16 +113,7 @@ export const secretSnapshotServiceFactory = ({ const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found" }); - const { pitVersionLimit } = await projectDAL.findById(projectId); - const computedQueryLimit = Math.min(pitVersionLimit - offset, limit); - if (offset > pitVersionLimit || computedQueryLimit <= 0) { - return []; - } - - const snapshots = await snapshotDAL.find( - { folderId: folder.id }, - { limit: computedQueryLimit, offset, sort: [["createdAt", "desc"]] } - ); + const snapshots = await snapshotDAL.find({ folderId: folder.id }, { limit, offset, sort: [["createdAt", "desc"]] }); return snapshots; }; diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index 7c17c4f072..d36a1d00e5 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -341,7 +341,6 @@ export const snapshotDALFactory = (db: TDbClient) => { .select("id"); const batchEntries = folderBatch.map((folder) => folder.id); - logger.info("UUID offset:", uuidOffset); if (folderBatch.length) { try { diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 8c4ff2a701..4442b3bb9d 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -535,7 +535,6 @@ export const registerRoutes = async ( licenseService, folderDAL, secretDAL, - projectDAL, snapshotDAL, snapshotFolderDAL, snapshotSecretDAL, diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 77c4024ac7..39e47a28e1 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -72,7 +72,7 @@ type TSecretServiceFactoryDep = { secretDAL: TSecretDALFactory; secretTagDAL: TSecretTagDALFactory; secretVersionDAL: TSecretVersionDALFactory; - projectDAL: Pick; + projectDAL: Pick; projectEnvDAL: Pick; folderDAL: Pick< TSecretFolderDALFactory, @@ -1354,16 +1354,7 @@ export const secretServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); - const { pitVersionLimit } = await projectDAL.findById(folder.projectId); - const computedQueryLimit = Math.min(pitVersionLimit - offset, limit); - if (offset > pitVersionLimit || computedQueryLimit <= 0) { - return []; - } - - const secretVersions = await secretVersionDAL.find( - { secretId }, - { offset, limit: computedQueryLimit, sort: [["createdAt", "desc"]] } - ); + const secretVersions = await secretVersionDAL.find({ secretId }, { offset, limit, sort: [["createdAt", "desc"]] }); return secretVersions; }; diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx index 8c820b727d..4091ec06ed 100644 --- a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx @@ -22,7 +22,7 @@ export const PointInTimeVersionLimitSection = () => { const { control, - formState: { isSubmitting }, + formState: { isSubmitting, isDirty }, handleSubmit } = useForm({ resolver: zodResolver(formSchema), @@ -79,7 +79,12 @@ export const PointInTimeVersionLimitSection = () => { )} /> - From ee152f2d2044970c37885fcdbe2753aaafedf049 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 4 Jun 2024 23:50:10 +0800 Subject: [PATCH 13/17] misc: added cleanup frequency note for pit versions --- .../PointInTimeVersionLimitSection.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx index 4091ec06ed..f3035922b5 100644 --- a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx @@ -60,7 +60,8 @@ export const PointInTimeVersionLimitSection = () => {

This defines the maximum number of folder snapshots, secret versions, and folder versions - that are retained by the system + that are retained by the system. The cleanup of excess snapshots and versions happens once a + day on midnight of UTC.

From d9c4c332ea2037a57bef9df450d811379830e94c Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Mon, 10 Jun 2024 20:40:19 +0800 Subject: [PATCH 14/17] feat: added handling of versioned folders and cleanup script --- .../services/secret-snapshot/snapshot-dal.ts | 74 ++++++++++++++++++- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index d36a1d00e5..d2dab8e068 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -329,9 +329,10 @@ export const snapshotDALFactory = (db: TDbClient) => { const pruneExcessSnapshots = async () => { const PRUNE_FOLDER_BATCH_SIZE = 10000; - let uuidOffset = "00000000-0000-0000-0000-000000000000"; try { + let uuidOffset = "00000000-0000-0000-0000-000000000000"; + // cleanup snapshots from root/non-versioned folders // eslint-disable-next-line no-constant-condition, no-unreachable-loop while (true) { const folderBatch = await db(TableName.SecretFolder) @@ -366,15 +367,82 @@ export const snapshotDALFactory = (db: TDbClient) => { .delete(); } catch (err) { logger.error( - `Failed to prune snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}` + `Failed to prune snapshots from root/non-versioned folders in range ${batchEntries[0]}:${ + batchEntries[batchEntries.length - 1] + }` ); } finally { uuidOffset = batchEntries[batchEntries.length - 1]; } } else { - return; + break; } } + + // cleanup snapshots from versioned folders + uuidOffset = "00000000-0000-0000-0000-000000000000"; + // eslint-disable-next-line no-constant-condition + while (true) { + const folderBatch = await db(TableName.SecretFolderVersion) + .select("folderId") + .distinct("folderId") + .where("folderId", ">", uuidOffset) + .orderBy("folderId", "asc") + .limit(PRUNE_FOLDER_BATCH_SIZE); + + const batchEntries = folderBatch.map((folder) => folder.folderId); + + if (folderBatch.length) { + try { + logger.info(`Pruning snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}`); + await db(TableName.Snapshot) + .with("snapshot_cte", (qb) => { + void qb + .from(TableName.Snapshot) + .whereIn(`${TableName.Snapshot}.folderId`, batchEntries) + .select( + "folderId", + `${TableName.Snapshot}.id as id`, + db.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` + ) + ); + }) + .join( + TableName.SecretFolderVersion, + `${TableName.SecretFolderVersion}.folderId`, + `${TableName.Snapshot}.folderId` + ) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`) + .whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); + } catch (err) { + logger.error( + `Failed to prune snapshots from versioned folders in range ${batchEntries[0]}:${ + batchEntries[batchEntries.length - 1] + }` + ); + } finally { + uuidOffset = batchEntries[batchEntries.length - 1]; + } + } else { + break; + } + } + + // cleanup orphaned snapshots (those that don't belong to an existing folder and folder version) + await db(TableName.Snapshot) + // eslint-disable-next-line func-names + .whereNotIn("folderId", function () { + void this.select("folderId").from(TableName.SecretFolderVersion); + }) + // eslint-disable-next-line func-names + .whereNotIn("folderId", function () { + void this.select("folderId").from(TableName.SecretFolder); + }) + .delete(); } catch (error) { throw new DatabaseError({ error, name: "SnapshotPrune" }); } From e696bff00471cd8142c786cc3899bade9a0b79de Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Mon, 10 Jun 2024 21:08:14 +0800 Subject: [PATCH 15/17] misc: optimized prune orphan snapshots --- .../src/ee/services/secret-snapshot/snapshot-dal.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index d2dab8e068..12897e6b9a 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -434,13 +434,12 @@ export const snapshotDALFactory = (db: TDbClient) => { // cleanup orphaned snapshots (those that don't belong to an existing folder and folder version) await db(TableName.Snapshot) - // eslint-disable-next-line func-names .whereNotIn("folderId", function () { - void this.select("folderId").from(TableName.SecretFolderVersion); - }) - // eslint-disable-next-line func-names - .whereNotIn("folderId", function () { - void this.select("folderId").from(TableName.SecretFolder); + void this.select("folderId") + .from(TableName.SecretFolderVersion) + .union(function () { + void this.select("id").from(TableName.SecretFolder); + }); }) .delete(); } catch (error) { From 87df5a2749ca675db6e57ef4454b1895f61ca642 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 11 Jun 2024 21:54:14 +0800 Subject: [PATCH 16/17] misc: addressed PR comments --- .../src/ee/services/secret-snapshot/snapshot-dal.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index 12897e6b9a..e45098132e 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -363,6 +363,7 @@ export const snapshotDALFactory = (db: TDbClient) => { .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) .join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`) + .whereNull(`${TableName.SecretFolder}.parentId`) .whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`) .delete(); } catch (err) { @@ -434,12 +435,11 @@ export const snapshotDALFactory = (db: TDbClient) => { // cleanup orphaned snapshots (those that don't belong to an existing folder and folder version) await db(TableName.Snapshot) - .whereNotIn("folderId", function () { - void this.select("folderId") + .whereNotIn("folderId", (qb) => { + void qb + .select("folderId") .from(TableName.SecretFolderVersion) - .union(function () { - void this.select("id").from(TableName.SecretFolder); - }); + .union((qb1) => void qb1.select("id").from(TableName.SecretFolder)); }) .delete(); } catch (error) { From 30f3dac35ff780671a1987d13497845623230ae7 Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Wed, 12 Jun 2024 15:56:47 -0400 Subject: [PATCH 17/17] rephrase input and filer for resvered folder --- .../services/secret-snapshot/snapshot-dal.ts | 21 ++++++++++++++++++- .../PointInTimeVersionLimitSection.tsx | 8 +++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index e45098132e..92c6b611dd 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -327,6 +327,24 @@ export const snapshotDALFactory = (db: TDbClient) => { } }; + /** + * Prunes excess snapshots from the database to ensure only a specified number of recent snapshots are retained for each folder. + * + * This function operates in three main steps: + * 1. Pruning snapshots from root/non-versioned folders. + * 2. Pruning snapshots from versioned folders. + * 3. Removing orphaned snapshots that do not belong to any existing folder or folder version. + * + * The function processes snapshots in batches, determined by the `PRUNE_FOLDER_BATCH_SIZE` constant, + * to manage the large datasets without overwhelming the DB. + * + * Steps: + * - Fetch a batch of folder IDs. + * - For each batch, use a Common Table Expression (CTE) to rank snapshots within each folder by their creation date. + * - Identify and delete snapshots that exceed the project's point-in-time version limit (`pitVersionLimit`). + * - Repeat the process for versioned folders. + * - Finally, delete orphaned snapshots that do not have an associated folder. + */ const pruneExcessSnapshots = async () => { const PRUNE_FOLDER_BATCH_SIZE = 10000; @@ -337,6 +355,7 @@ export const snapshotDALFactory = (db: TDbClient) => { while (true) { const folderBatch = await db(TableName.SecretFolder) .where("id", ">", uuidOffset) + .where("isReserved", false) .orderBy("id", "asc") .limit(PRUNE_FOLDER_BATCH_SIZE) .select("id"); @@ -345,7 +364,7 @@ export const snapshotDALFactory = (db: TDbClient) => { if (folderBatch.length) { try { - logger.info(`Pruning snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}`); + logger.info(`Pruning snapshots in [range=${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}]`); await db(TableName.Snapshot) .with("snapshot_cte", (qb) => { void qb diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx index f3035922b5..0ce5c70585 100644 --- a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx @@ -56,12 +56,10 @@ export const PointInTimeVersionLimitSection = () => { return (
-

Point in Time Recovery

+

Version Retention

- This defines the maximum number of folder snapshots, secret versions, and folder versions - that are retained by the system. The cleanup of excess snapshots and versions happens once a - day on midnight of UTC. + This defines the maximum number of recent secret versions to keep per folder. Excess versions will be removed at midnight (UTC) each day.

@@ -73,7 +71,7 @@ export const PointInTimeVersionLimitSection = () => {