From a06dee66f8ea9091598f3ba559e2ff5645da246b Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 9 Jul 2024 15:20:58 +0800 Subject: [PATCH] feat: initial logic for moving secrets --- backend/src/server/routes/index.ts | 5 +- backend/src/server/routes/v3/secret-router.ts | 42 ++ backend/src/services/secret/secret-service.ts | 378 +++++++++++++++++- backend/src/services/secret/secret-types.ts | 11 + 4 files changed, 432 insertions(+), 4 deletions(-) diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index bc8b153e03..aec2ef3f7f 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -706,7 +706,10 @@ export const registerRoutes = async ( secretQueueService, secretImportDAL, projectEnvDAL, - projectBotService + projectBotService, + secretApprovalPolicyService, + secretApprovalRequestDAL, + secretApprovalRequestSecretDAL }); const secretSharingService = secretSharingServiceFactory({ diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index 910cc3db97..d12572272a 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -1325,6 +1325,48 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } }); + server.route({ + method: "POST", + url: "/move", + config: { + rateLimit: secretsLimit + }, + schema: { + body: z.object({ + projectSlug: z.string().trim(), + sourceEnvironment: z.string().trim(), + sourceSecretPath: z.string().trim().default("/").transform(removeTrailingSlash), + destinationEnvironment: z.string().trim(), + destinationSecretPath: z.string().trim().default("/").transform(removeTrailingSlash), + secrets: z + .object({ + id: z.string() + }) + .array() + .min(1) + }), + response: { + 200: z.union([ + z.object({ + secrets: SecretsSchema.omit({ secretBlindIndex: true }).array() + }), + z.object({ approval: SecretApprovalRequestsSchema }).describe("When secret protection policy is enabled") + ]) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + // TODO: publish audit log + return server.services.secret.moveSecrets({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + ...req.body + }); + } + }); + server.route({ method: "POST", url: "/batch", diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index cbfccba91c..bc31c675ff 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -11,6 +11,9 @@ import { } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service"; +import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; +import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service"; import { getConfig } from "@app/lib/config/env"; import { @@ -18,9 +21,10 @@ import { decryptSymmetric128BitHexKeyUTF8, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; -import { BadRequestError } from "@app/lib/errors"; +import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { groupBy, pick } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; +import { alphaNumericNanoId } from "@app/lib/nanoid"; import { ActorType } from "../auth/auth-type"; import { TProjectDALFactory } from "../project/project-dal"; @@ -44,6 +48,7 @@ import { } from "./secret-fns"; import { TSecretQueueFactory } from "./secret-queue"; import { + SecretOperations, TAttachSecretTagsDTO, TBackFillSecretReferencesDTO, TCreateBulkSecretDTO, @@ -59,6 +64,7 @@ import { TGetSecretsDTO, TGetSecretsRawDTO, TGetSecretVersionsDTO, + TMoveSecretsDTO, TUpdateBulkSecretDTO, TUpdateManySecretRawDTO, TUpdateSecretDTO, @@ -84,6 +90,12 @@ type TSecretServiceFactoryDep = { projectBotService: Pick; secretImportDAL: Pick; secretVersionTagDAL: Pick; + secretApprovalPolicyService: Pick; + secretApprovalRequestDAL: Pick; + secretApprovalRequestSecretDAL: Pick< + TSecretApprovalRequestSecretDALFactory, + "insertMany" | "insertApprovalSecretTags" + >; }; export type TSecretServiceFactory = ReturnType; @@ -100,7 +112,10 @@ export const secretServiceFactory = ({ projectDAL, projectBotService, secretImportDAL, - secretVersionTagDAL + secretVersionTagDAL, + secretApprovalPolicyService, + secretApprovalRequestDAL, + secretApprovalRequestSecretDAL }: TSecretServiceFactoryDep) => { const getSecretReference = async (projectId: string) => { // if bot key missing means e2e still exist @@ -1683,6 +1698,362 @@ export const secretServiceFactory = ({ return { message: "Successfully backfilled secret references" }; }; + const moveSecrets = async ({ + sourceEnvironment, + sourceSecretPath, + destinationEnvironment, + destinationSecretPath, + secrets, + projectSlug, + actor, + actorId, + actorAuthMethod, + actorOrgId + }: TMoveSecretsDTO) => { + const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); + if (!project) { + throw new NotFoundError({ + message: "Project not found." + }); + } + + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + project.id, + actorAuthMethod, + actorOrgId + ); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Delete, + subject(ProjectPermissionSub.Secrets, { environment: sourceEnvironment, secretPath: sourceSecretPath }) + ); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Create, + subject(ProjectPermissionSub.Secrets, { environment: destinationEnvironment, secretPath: destinationSecretPath }) + ); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Edit, + subject(ProjectPermissionSub.Secrets, { environment: destinationEnvironment, secretPath: destinationSecretPath }) + ); + + const botKey = await projectBotService.getBotKey(project.id); + if (!botKey) { + throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); + } + + const sourceFolder = await folderDAL.findBySecretPath(project.id, sourceEnvironment, sourceSecretPath); + if (!sourceFolder) { + throw new NotFoundError({ + message: "Source path does not exist." + }); + } + + const destinationFolder = await folderDAL.findBySecretPath( + project.id, + destinationEnvironment, + destinationSecretPath + ); + + if (!destinationFolder) { + throw new NotFoundError({ + message: "Destination path does not exist." + }); + } + + const sourceSecrets = await secretDAL.find({ + type: SecretType.Shared, + $in: { + id: secrets.map((secret) => secret.id) + } + }); + + if (sourceSecrets.length !== secrets.length) { + throw new BadRequestError({ + message: "Invalid secrets" + }); + } + + const decryptedSourceSecrets = sourceSecrets.map((secret) => ({ + ...secret, + secretKey: decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretKeyCiphertext, + iv: secret.secretKeyIV, + tag: secret.secretKeyTag, + key: botKey + }), + secretValue: decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretValueCiphertext, + iv: secret.secretValueIV, + tag: secret.secretValueTag, + key: botKey + }) + })); + + // Moving secrets is a two-step process. + // First step is to create/update the secret in the destination: + const destinationSecretsFromDB = await secretDAL.find({ + folderId: destinationFolder.id + }); + + const decryptedDestinationSecrets = destinationSecretsFromDB.map((secret) => { + return { + ...secret, + secretKey: decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretKeyCiphertext, + iv: secret.secretKeyIV, + tag: secret.secretKeyTag, + key: botKey + }), + secretValue: decryptSymmetric128BitHexKeyUTF8({ + ciphertext: secret.secretValueCiphertext, + iv: secret.secretValueIV, + tag: secret.secretValueTag, + key: botKey + }) + }; + }); + + const destinationSecretsGroupedByBlindIndex = groupBy( + decryptedDestinationSecrets.filter(({ secretBlindIndex }) => Boolean(secretBlindIndex)), + (i) => i.secretBlindIndex as string + ); + + const locallyCreatedSecrets = decryptedSourceSecrets + .filter(({ secretBlindIndex }) => !destinationSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0]) + .map((el) => ({ ...el, operation: SecretOperations.Create })); // rewrite update ops to create + + const locallyUpdatedSecrets = decryptedSourceSecrets + .filter( + ({ secretBlindIndex, secretKey, secretValue }) => + destinationSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0] && + // if key or value changed + (destinationSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0]?.secretKey !== secretKey || + destinationSecretsGroupedByBlindIndex[secretBlindIndex as string]?.[0]?.secretValue !== secretValue) + ) + .map((el) => ({ ...el, operation: SecretOperations.Update })); // rewrite update ops to create + + const isEmpty = locallyCreatedSecrets.length + locallyUpdatedSecrets.length === 0; + + if (isEmpty) { + throw new BadRequestError({ + message: "No changes were detected between the source and destination." + }); + } + + const destinationFolderPolicy = await secretApprovalPolicyService.getSecretApprovalPolicy( + project.id, + destinationFolder.environment.slug, + destinationFolder.path + ); + + if (destinationFolderPolicy && actor === ActorType.USER) { + // if secret approval policy exists for destination, we create the secret approval request + const localSecretsIds = decryptedDestinationSecrets.map(({ id }) => id); + const latestSecretVersions = await secretVersionDAL.findLatestVersionMany(destinationFolder.id, localSecretsIds); + + await secretApprovalRequestDAL.transaction(async (tx) => { + const approvalRequestDoc = await secretApprovalRequestDAL.create( + { + folderId: destinationFolder.id, + slug: alphaNumericNanoId(), + policyId: destinationFolderPolicy.id, + status: "open", + hasMerged: false, + committerUserId: actorId + }, + tx + ); + + const commits = locallyCreatedSecrets.concat(locallyUpdatedSecrets).map((doc) => { + const { operation } = doc; + const localSecret = destinationSecretsGroupedByBlindIndex[doc.secretBlindIndex as string]?.[0]; + + return { + op: operation, + keyEncoding: doc.keyEncoding, + algorithm: doc.algorithm, + requestId: approvalRequestDoc.id, + metadata: doc.metadata, + secretKeyIV: doc.secretKeyIV, + secretKeyTag: doc.secretKeyTag, + secretKeyCiphertext: doc.secretKeyCiphertext, + secretValueIV: doc.secretValueIV, + secretValueTag: doc.secretValueTag, + secretValueCiphertext: doc.secretValueCiphertext, + secretBlindIndex: doc.secretBlindIndex, + secretCommentIV: doc.secretCommentIV, + secretCommentTag: doc.secretCommentTag, + secretCommentCiphertext: doc.secretCommentCiphertext, + skipMultilineEncoding: doc.skipMultilineEncoding, + // except create operation other two needs the secret id and version id + ...(operation !== SecretOperations.Create + ? { secretId: localSecret.id, secretVersion: latestSecretVersions[localSecret.id].id } + : {}) + }; + }); + const approvalCommits = await secretApprovalRequestSecretDAL.insertMany(commits, tx); + return { ...approvalRequestDoc, commits: approvalCommits }; + }); + } else { + // apply changes directly + await secretDAL.transaction(async (tx) => { + if (locallyCreatedSecrets.length) { + await fnSecretBulkInsert({ + folderId: destinationFolder.id, + secretVersionDAL, + secretDAL, + tx, + secretTagDAL, + secretVersionTagDAL, + inputSecrets: locallyCreatedSecrets.map((doc) => { + return { + keyEncoding: doc.keyEncoding, + algorithm: doc.algorithm, + type: doc.type, + metadata: doc.metadata, + secretKeyIV: doc.secretKeyIV, + secretKeyTag: doc.secretKeyTag, + secretKeyCiphertext: doc.secretKeyCiphertext, + secretValueIV: doc.secretValueIV, + secretValueTag: doc.secretValueTag, + secretValueCiphertext: doc.secretValueCiphertext, + secretBlindIndex: doc.secretBlindIndex, + secretCommentIV: doc.secretCommentIV, + secretCommentTag: doc.secretCommentTag, + secretCommentCiphertext: doc.secretCommentCiphertext, + skipMultilineEncoding: doc.skipMultilineEncoding + }; + }) + }); + } + if (locallyUpdatedSecrets.length) { + await fnSecretBulkUpdate({ + projectId: project.id, + folderId: destinationFolder.id, + secretVersionDAL, + secretDAL, + tx, + secretTagDAL, + secretVersionTagDAL, + inputSecrets: locallyUpdatedSecrets.map((doc) => { + return { + filter: { + folderId: destinationFolder.id, + id: destinationSecretsGroupedByBlindIndex[doc.secretBlindIndex as string][0].id + }, + data: { + keyEncoding: doc.keyEncoding, + algorithm: doc.algorithm, + type: doc.type, + metadata: doc.metadata, + secretKeyIV: doc.secretKeyIV, + secretKeyTag: doc.secretKeyTag, + secretKeyCiphertext: doc.secretKeyCiphertext, + secretValueIV: doc.secretValueIV, + secretValueTag: doc.secretValueTag, + secretValueCiphertext: doc.secretValueCiphertext, + secretBlindIndex: doc.secretBlindIndex, + secretCommentIV: doc.secretCommentIV, + secretCommentTag: doc.secretCommentTag, + secretCommentCiphertext: doc.secretCommentCiphertext, + skipMultilineEncoding: doc.skipMultilineEncoding + } + }; + }) + }); + } + + await snapshotService.performSnapshot(destinationFolder.id); + await secretQueueService.syncSecrets({ + projectId: project.id, + secretPath: destinationFolder.path, + environmentSlug: destinationFolder.environment.slug, + actorId, + actor + }); + }); + } + + // Next step is to delete the secrets from the source folder: + const sourceSecretsGroupByBlindIndex = groupBy(sourceSecrets, (i) => i.secretBlindIndex as string); + const locallyDeletedSecrets = decryptedSourceSecrets.map((el) => ({ ...el, operation: SecretOperations.Delete })); + + const sourceFolderPolicy = await secretApprovalPolicyService.getSecretApprovalPolicy( + project.id, + sourceFolder.environment.slug, + sourceFolder.path + ); + + if (sourceFolderPolicy && actor === ActorType.USER) { + // if secret approval policy exists for source, we create the secret approval request + const localSecretsIds = decryptedSourceSecrets.map(({ id }) => id); + const latestSecretVersions = await secretVersionDAL.findLatestVersionMany(sourceFolder.id, localSecretsIds); + + await secretApprovalRequestDAL.transaction(async (tx) => { + const approvalRequestDoc = await secretApprovalRequestDAL.create( + { + folderId: sourceFolder.id, + slug: alphaNumericNanoId(), + policyId: sourceFolderPolicy.id, + status: "open", + hasMerged: false, + committerUserId: actorId + }, + tx + ); + + const commits = locallyDeletedSecrets.map((doc) => { + const { operation } = doc; + const localSecret = sourceSecretsGroupByBlindIndex[doc.secretBlindIndex as string]?.[0]; + + return { + op: operation, + keyEncoding: doc.keyEncoding, + algorithm: doc.algorithm, + requestId: approvalRequestDoc.id, + metadata: doc.metadata, + secretKeyIV: doc.secretKeyIV, + secretKeyTag: doc.secretKeyTag, + secretKeyCiphertext: doc.secretKeyCiphertext, + secretValueIV: doc.secretValueIV, + secretValueTag: doc.secretValueTag, + secretValueCiphertext: doc.secretValueCiphertext, + secretBlindIndex: doc.secretBlindIndex, + secretCommentIV: doc.secretCommentIV, + secretCommentTag: doc.secretCommentTag, + secretCommentCiphertext: doc.secretCommentCiphertext, + skipMultilineEncoding: doc.skipMultilineEncoding, + secretId: localSecret.id, + secretVersion: latestSecretVersions[localSecret.id].id + }; + }); + const approvalCommits = await secretApprovalRequestSecretDAL.insertMany(commits, tx); + return { ...approvalRequestDoc, commits: approvalCommits }; + }); + } else { + // if no secret approval policy is present, we delete directly. + await secretDAL.delete({ + $in: { + id: locallyDeletedSecrets.map(({ id }) => id) + }, + folderId: sourceFolder.id + }); + + await snapshotService.performSnapshot(sourceFolder.id); + await secretQueueService.syncSecrets({ + projectId: project.id, + secretPath: sourceFolder.path, + environmentSlug: sourceFolder.environment.slug, + actorId, + actor + }); + } + }; + return { attachTags, detachTags, @@ -1703,6 +2074,7 @@ export const secretServiceFactory = ({ updateManySecretsRaw, deleteManySecretsRaw, getSecretVersions, - backfillSecretReferences + backfillSecretReferences, + moveSecrets }; }; diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 10df2f2583..de7b6b7de5 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -397,3 +397,14 @@ export type TSyncSecretsDTO = { // used for import creation to trigger replication pickOnlyImportIds?: string[]; }); + +export type TMoveSecretsDTO = { + projectSlug: string; + sourceEnvironment: string; + sourceSecretPath: string; + destinationEnvironment: string; + destinationSecretPath: string; + secrets: { + id: string; + }[]; +} & Omit;