From 75813deb81dd52c3a12a2df44aa9b6698f4c3c49 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 20 Feb 2024 02:34:30 +0100 Subject: [PATCH] Fixed integrations & bulk update issue --- .../secret-approval-request-secret-dal.ts | 6 +- backend/src/lib/secret/index.ts | 262 ++++++++++-------- backend/src/server/routes/index.ts | 1 + .../integration-auth/integration-auth-dal.ts | 29 +- backend/src/services/project/project-queue.ts | 65 ++++- backend/src/services/secret/secret-dal.ts | 6 +- .../src/services/secret/secret-version-dal.ts | 6 +- 7 files changed, 249 insertions(+), 126 deletions(-) diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts index ff150fe67a..736cd253ea 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts @@ -31,16 +31,14 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => { throw new BadRequestError({ message: "Some of the secret approvals do not exist" }); } + if (data.length === 0) return []; + const updatedApprovalSecrets = await (tx || db)(TableName.SecretApprovalRequestSecret) .insert(data) .onConflict("id") // this will cause a conflict then merge the data .merge() // Merge the data with the existing data .returning("*"); - if (!updatedApprovalSecrets || updatedApprovalSecrets.length === 0) { - throw new BadRequestError({ message: "Failed to bulk update secret approvals" }); - } - return updatedApprovalSecrets; } catch (error) { throw new DatabaseError({ error, name: "bulk update secret" }); diff --git a/backend/src/lib/secret/index.ts b/backend/src/lib/secret/index.ts index 036b49970e..dd23c318a3 100644 --- a/backend/src/lib/secret/index.ts +++ b/backend/src/lib/secret/index.ts @@ -2,9 +2,11 @@ import crypto from "crypto"; import { z } from "zod"; import { + IntegrationAuthsSchema, SecretApprovalRequestsSecretsSchema, SecretsSchema, SecretVersionsSchema, + TIntegrationAuths, TProjectKeys, TSecretApprovalRequestsSecrets, TSecrets, @@ -25,6 +27,16 @@ const DecryptedSecretSchema = z.object({ original: SecretsSchema }); +const DecryptedIntegrationAuthsSchema = z.object({ + decrypted: z.object({ + id: z.string(), + access: z.string(), + accessId: z.string(), + refresh: z.string() + }), + original: IntegrationAuthsSchema +}); + const DecryptedSecretVersionsSchema = z.object({ decrypted: DecryptedValuesSchema, original: SecretVersionsSchema @@ -38,6 +50,13 @@ export const DecryptedSecretApprovalsSchema = z.object({ export type DecryptedSecret = z.infer; export type DecryptedSecretVersions = z.infer; export type DecryptedSecretApprovals = z.infer; +export type DecryptedIntegrationAuths = z.infer; + +type TLatestKey = TProjectKeys & { + sender: { + publicKey: string; + }; +}; const decryptCipher = ({ ciphertext, @@ -59,69 +78,20 @@ const decryptCipher = ({ return cleartext; }; -const getDecryptedValues = ({ - secretKeyCiphertext, - secretKeyIV, - secretKeyTag, - secretValueCiphertext, - secretValueIV, - secretValueTag, +const getDecryptedValues = (data: Array<{ ciphertext: string; iv: string; tag: string }>, key: string | Buffer) => { + const results = []; - secretCommentCiphertext, - secretCommentIV, - secretCommentTag, - key -}: { - secretKeyCiphertext: string; - secretKeyIV: string; - secretKeyTag: string; - secretValueCiphertext: string; - secretValueIV: string; - secretValueTag: string; - secretCommentCiphertext?: string | null; - secretCommentIV?: string | null; - secretCommentTag?: string | null; - key: string | Buffer; -}) => { - const secretKey = decryptCipher({ - ciphertext: secretKeyCiphertext, - iv: secretKeyIV, - tag: secretKeyTag, - key - }); - - const secretValue = decryptCipher({ - ciphertext: secretValueCiphertext, - iv: secretValueIV, - tag: secretValueTag, - key - }); - - const secretComment = - secretCommentCiphertext && secretCommentIV && secretCommentTag - ? decryptCipher({ - ciphertext: secretCommentCiphertext, - iv: secretCommentIV, - tag: secretCommentTag, - key - }) - : ""; - - return { - secretKey, - secretValue, - secretComment - }; -}; -export const decryptSecrets = ( - encryptedSecrets: TSecrets[], - privateKey: string, - latestKey: TProjectKeys & { - sender: { - publicKey: string; - }; + for (const { ciphertext, iv, tag } of data) { + if (!ciphertext || !iv || !tag) { + results.push(""); + } else { + results.push(decryptCipher({ ciphertext, iv, tag, key })); + } } -) => { + + return results; +}; +export const decryptSecrets = (encryptedSecrets: TSecrets[], privateKey: string, latestKey: TLatestKey) => { const key = decryptAsymmetric({ ciphertext: latestKey.encryptedKey, nonce: latestKey.nonce, @@ -132,22 +102,32 @@ export const decryptSecrets = ( const decryptedSecrets: DecryptedSecret[] = []; encryptedSecrets.forEach((encSecret) => { - const decrypted = getDecryptedValues({ - secretKeyCiphertext: encSecret.secretKeyCiphertext, - secretKeyIV: encSecret.secretKeyIV, - secretKeyTag: encSecret.secretKeyTag, - secretValueCiphertext: encSecret.secretValueCiphertext, - secretValueIV: encSecret.secretValueIV, - secretValueTag: encSecret.secretValueTag, - secretCommentCiphertext: encSecret.secretCommentCiphertext, - secretCommentIV: encSecret.secretCommentIV, - secretCommentTag: encSecret.secretCommentTag, + const [secretKey, secretValue, secretComment] = getDecryptedValues( + [ + { + ciphertext: encSecret.secretKeyCiphertext, + iv: encSecret.secretKeyIV, + tag: encSecret.secretKeyTag + }, + { + ciphertext: encSecret.secretValueCiphertext, + iv: encSecret.secretValueIV, + tag: encSecret.secretValueTag + }, + { + ciphertext: encSecret.secretCommentCiphertext || "", + iv: encSecret.secretCommentIV || "", + tag: encSecret.secretCommentTag || "" + } + ], key - }); + ); const decryptedSecret: DecryptedSecret = { decrypted: { - ...decrypted, + secretKey, + secretValue, + secretComment, id: encSecret.id }, original: encSecret @@ -162,11 +142,7 @@ export const decryptSecrets = ( export const decryptSecretVersions = ( encryptedSecretVersions: TSecretVersions[], privateKey: string, - latestKey: TProjectKeys & { - sender: { - publicKey: string; - }; - } + latestKey: TLatestKey ) => { const key = decryptAsymmetric({ ciphertext: latestKey.encryptedKey, @@ -178,22 +154,32 @@ export const decryptSecretVersions = ( const decryptedSecrets: DecryptedSecretVersions[] = []; encryptedSecretVersions.forEach((encSecret) => { - const decrypted = getDecryptedValues({ - secretKeyCiphertext: encSecret.secretKeyCiphertext, - secretKeyIV: encSecret.secretKeyIV, - secretKeyTag: encSecret.secretKeyTag, - secretValueCiphertext: encSecret.secretValueCiphertext, - secretValueIV: encSecret.secretValueIV, - secretValueTag: encSecret.secretValueTag, - secretCommentCiphertext: encSecret.secretCommentCiphertext, - secretCommentIV: encSecret.secretCommentIV, - secretCommentTag: encSecret.secretCommentTag, + const [secretKey, secretValue, secretComment] = getDecryptedValues( + [ + { + ciphertext: encSecret.secretKeyCiphertext, + iv: encSecret.secretKeyIV, + tag: encSecret.secretKeyTag + }, + { + ciphertext: encSecret.secretValueCiphertext, + iv: encSecret.secretValueIV, + tag: encSecret.secretValueTag + }, + { + ciphertext: encSecret.secretCommentCiphertext || "", + iv: encSecret.secretCommentIV || "", + tag: encSecret.secretCommentTag || "" + } + ], key - }); + ); const decryptedSecret: DecryptedSecretVersions = { decrypted: { - ...decrypted, + secretKey, + secretValue, + secretComment, id: encSecret.id }, original: encSecret @@ -208,11 +194,7 @@ export const decryptSecretVersions = ( export const decryptSecretApprovals = ( encryptedSecretApprovals: TSecretApprovalRequestsSecrets[], privateKey: string, - latestKey: TProjectKeys & { - sender: { - publicKey: string; - }; - } + latestKey: TLatestKey ) => { const key = decryptAsymmetric({ ciphertext: latestKey.encryptedKey, @@ -223,26 +205,36 @@ export const decryptSecretApprovals = ( const decryptedSecrets: DecryptedSecretApprovals[] = []; - encryptedSecretApprovals.forEach((encSecret) => { - const decrypted = getDecryptedValues({ - secretKeyCiphertext: encSecret.secretKeyCiphertext, - secretKeyIV: encSecret.secretKeyIV, - secretKeyTag: encSecret.secretKeyTag, - secretValueCiphertext: encSecret.secretValueCiphertext, - secretValueIV: encSecret.secretValueIV, - secretValueTag: encSecret.secretValueTag, - secretCommentCiphertext: encSecret.secretCommentCiphertext, - secretCommentIV: encSecret.secretCommentIV, - secretCommentTag: encSecret.secretCommentTag, + encryptedSecretApprovals.forEach((encApproval) => { + const [secretKey, secretValue, secretComment] = getDecryptedValues( + [ + { + ciphertext: encApproval.secretKeyCiphertext, + iv: encApproval.secretKeyIV, + tag: encApproval.secretKeyTag + }, + { + ciphertext: encApproval.secretValueCiphertext, + iv: encApproval.secretValueIV, + tag: encApproval.secretValueTag + }, + { + ciphertext: encApproval.secretCommentCiphertext || "", + iv: encApproval.secretCommentIV || "", + tag: encApproval.secretCommentTag || "" + } + ], key - }); + ); const decryptedSecret: DecryptedSecretApprovals = { decrypted: { - ...decrypted, - id: encSecret.id + secretKey, + secretValue, + secretComment, + id: encApproval.id }, - original: encSecret + original: encApproval }; decryptedSecrets.push(DecryptedSecretApprovalsSchema.parse(decryptedSecret)); @@ -250,3 +242,53 @@ export const decryptSecretApprovals = ( return decryptedSecrets; }; + +export const decryptIntegrationAuths = ( + encryptedIntegrationAuths: TIntegrationAuths[], + privateKey: string, + latestKey: TLatestKey +) => { + const key = decryptAsymmetric({ + ciphertext: latestKey.encryptedKey, + nonce: latestKey.nonce, + publicKey: latestKey.sender.publicKey, + privateKey + }); + + const decryptedIntegrationAuths: DecryptedIntegrationAuths[] = []; + + encryptedIntegrationAuths.forEach((encAuth) => { + const [access, accessId, refresh] = getDecryptedValues( + [ + { + ciphertext: encAuth.accessCiphertext || "", + iv: encAuth.accessIV || "", + tag: encAuth.accessTag || "" + }, + { + ciphertext: encAuth.accessIdCiphertext || "", + iv: encAuth.accessIdIV || "", + tag: encAuth.accessIdTag || "" + }, + { + ciphertext: encAuth.refreshCiphertext || "", + iv: encAuth.refreshIV || "", + tag: encAuth.refreshTag || "" + } + ], + key + ); + + decryptedIntegrationAuths.push({ + decrypted: { + id: encAuth.id, + access, + accessId, + refresh + }, + original: encAuth + }); + }); + + return decryptedIntegrationAuths; +}; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index fd1b45878d..9444aff16d 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -307,6 +307,7 @@ export const registerRoutes = async ( folderDAL, projectDAL, orgDAL, + integrationAuthDAL, orgService, projectEnvDAL, userDAL, diff --git a/backend/src/services/integration-auth/integration-auth-dal.ts b/backend/src/services/integration-auth/integration-auth-dal.ts index f2d9c9ea21..d32cd1579e 100644 --- a/backend/src/services/integration-auth/integration-auth-dal.ts +++ b/backend/src/services/integration-auth/integration-auth-dal.ts @@ -1,10 +1,35 @@ +import { Knex } from "knex"; + import { TDbClient } from "@app/db"; -import { TableName } from "@app/db/schemas"; +import { TableName, TIntegrationAuths, TIntegrationAuthsUpdate } from "@app/db/schemas"; +import { BadRequestError, DatabaseError } from "@app/lib/errors"; import { ormify } from "@app/lib/knex"; export type TIntegrationAuthDALFactory = ReturnType; export const integrationAuthDALFactory = (db: TDbClient) => { const integrationAuthOrm = ormify(db, TableName.IntegrationAuth); - return integrationAuthOrm; + + const bulkUpdate = async ( + data: Array<{ filter: Partial; data: TIntegrationAuthsUpdate }>, + tx?: Knex + ) => { + try { + const integrationAuths = await Promise.all( + data.map(async ({ filter, data: updateData }) => { + const [doc] = await (tx || db)(TableName.IntegrationAuth).where(filter).update(updateData).returning("*"); + if (!doc) throw new BadRequestError({ message: "Failed to update document" }); + return doc; + }) + ); + return integrationAuths; + } catch (error) { + throw new DatabaseError({ error, name: "bulk update secret" }); + } + }; + + return { + ...integrationAuthOrm, + bulkUpdate + }; }; diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts index bea0074d28..8582c8289c 100644 --- a/backend/src/services/project/project-queue.ts +++ b/backend/src/services/project/project-queue.ts @@ -1,5 +1,6 @@ /* eslint-disable no-await-in-loop */ import { + IntegrationAuthsSchema, ProjectMembershipRole, ProjectUpgradeStatus, ProjectVersion, @@ -7,6 +8,7 @@ import { SecretKeyEncoding, SecretsSchema, SecretVersionsSchema, + TIntegrationAuths, TSecretApprovalRequestsSecrets, TSecrets, TSecretVersions @@ -21,9 +23,15 @@ import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { logger } from "@app/lib/logger"; -import { decryptSecretApprovals, decryptSecrets, decryptSecretVersions } from "@app/lib/secret"; +import { + decryptIntegrationAuths, + decryptSecretApprovals, + decryptSecrets, + decryptSecretVersions +} from "@app/lib/secret"; import { QueueJobs, QueueName, TQueueJobTypes, TQueueServiceFactory } from "@app/queue"; +import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal"; import { TOrgDALFactory } from "../org/org-dal"; import { TOrgServiceFactory } from "../org/org-service"; import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; @@ -50,6 +58,7 @@ type TProjectQueueFactoryDep = { projectBotDAL: Pick; orgService: Pick; projectMembershipDAL: Pick; + integrationAuthDAL: TIntegrationAuthDALFactory; userDAL: Pick; projectEnvDAL: Pick; @@ -63,6 +72,7 @@ export const projectQueueFactory = ({ folderDAL, userDAL, secretVersionDAL, + integrationAuthDAL, secretApprovalRequestDAL, secretApprovalSecretDAL, projectKeyDAL, @@ -150,9 +160,14 @@ export const projectQueueFactory = ({ approvalSecrets.push(...secretApprovals); } + const projectIntegrationAuths = await integrationAuthDAL.find({ + projectId: project.id + }); + const decryptedSecrets = decryptSecrets(secrets, userPrivateKey, oldProjectKey); const decryptedSecretVersions = decryptSecretVersions(secretVersions, userPrivateKey, oldProjectKey); const decryptedApprovalSecrets = decryptSecretApprovals(approvalSecrets, userPrivateKey, oldProjectKey); + const decryptedIntegrationAuths = decryptIntegrationAuths(projectIntegrationAuths, userPrivateKey, oldProjectKey); if (secrets.length !== decryptedSecrets.length) { throw new Error("Failed to decrypt some secret versions"); @@ -304,6 +319,7 @@ export const projectQueueFactory = ({ const updatedSecrets: TSecrets[] = []; const updatedSecretVersions: TSecretVersions[] = []; const updatedSecretApprovals: TSecretApprovalRequestsSecrets[] = []; + const updatedIntegrationAuths: TIntegrationAuths[] = []; for (const rawSecret of decryptedSecrets) { const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.decrypted.secretKey, botKey); const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.decrypted.secretValue || "", botKey); @@ -348,6 +364,7 @@ export const projectQueueFactory = ({ const payload: TSecretVersions = { ...rawSecretVersion.original, + keyEncoding: SecretKeyEncoding.UTF8, secretKeyCiphertext: secretKeyEncrypted.ciphertext, secretKeyIV: secretKeyEncrypted.iv, @@ -382,6 +399,7 @@ export const projectQueueFactory = ({ const payload: TSecretApprovalRequestsSecrets = { ...rawSecretApproval.original, + keyEncoding: SecretKeyEncoding.UTF8, secretKeyCiphertext: secretKeyEncrypted.ciphertext, secretKeyIV: secretKeyEncrypted.iv, @@ -403,6 +421,35 @@ export const projectQueueFactory = ({ updatedSecretApprovals.push(payload); } + for (const integrationAuth of decryptedIntegrationAuths) { + const access = encryptSymmetric128BitHexKeyUTF8(integrationAuth.decrypted.access, botKey); + const accessId = encryptSymmetric128BitHexKeyUTF8(integrationAuth.decrypted.accessId, botKey); + const refresh = encryptSymmetric128BitHexKeyUTF8(integrationAuth.decrypted.refresh, botKey); + + const payload: TIntegrationAuths = { + ...integrationAuth.original, + keyEncoding: SecretKeyEncoding.UTF8, + + accessCiphertext: access.ciphertext, + accessIV: access.iv, + accessTag: access.tag, + + accessIdCiphertext: accessId.ciphertext, + accessIdIV: accessId.iv, + accessIdTag: accessId.tag, + + refreshCiphertext: refresh.ciphertext, + refreshIV: refresh.iv, + refreshTag: refresh.tag + } as const; + + if (!IntegrationAuthsSchema.safeParse(payload).success) { + throw new Error(`Invalid integration auth payload: ${JSON.stringify(payload)}`); + } + + updatedIntegrationAuths.push(payload); + } + if (updatedSecrets.length !== secrets.length) { throw new Error("Failed to update some secrets"); } @@ -412,6 +459,9 @@ export const projectQueueFactory = ({ if (updatedSecretApprovals.length !== approvalSecrets.length) { throw new Error("Failed to update some secret approvals"); } + if (updatedIntegrationAuths.length !== projectIntegrationAuths.length) { + throw new Error("Failed to update some integration auths"); + } const secretUpdates = await secretDAL.bulkUpdateNoVersionIncrement(updatedSecrets, tx); const secretVersionUpdates = await secretVersionDAL.bulkUpdateNoVersionIncrement(updatedSecretVersions, tx); @@ -419,11 +469,22 @@ export const projectQueueFactory = ({ updatedSecretApprovals, tx ); + const integrationAuthUpdates = await integrationAuthDAL.bulkUpdate( + updatedIntegrationAuths.map((el) => ({ + filter: { id: el.id }, + data: { + ...el, + id: undefined + } + })), + tx + ); if ( secretUpdates.length !== updatedSecrets.length || secretVersionUpdates.length !== updatedSecretVersions.length || - secretApprovalUpdates.length !== updatedSecretApprovals.length + secretApprovalUpdates.length !== updatedSecretApprovals.length || + integrationAuthUpdates.length !== updatedIntegrationAuths.length ) { throw new Error("Parts of the upgrade failed. Some secrets were not updated"); } diff --git a/backend/src/services/secret/secret-dal.ts b/backend/src/services/secret/secret-dal.ts index 9c880938cc..11cd522ca2 100644 --- a/backend/src/services/secret/secret-dal.ts +++ b/backend/src/services/secret/secret-dal.ts @@ -60,16 +60,14 @@ export const secretDALFactory = (db: TDbClient) => { throw new BadRequestError({ message: "Some of the secrets do not exist" }); } + if (data.length === 0) return []; + const updatedSecrets = await (tx || db)(TableName.Secret) .insert(data) .onConflict("id") // this will cause a conflict then merge the data .merge() // Merge the data with the existing data .returning("*"); - if (!updatedSecrets || updatedSecrets.length === 0) { - throw new BadRequestError({ message: "Failed to bulk update secret approvals" }); - } - return updatedSecrets; } catch (error) { throw new DatabaseError({ error, name: "bulk update secret" }); diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index c9d9cff91f..758352ed24 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -73,16 +73,14 @@ export const secretVersionDALFactory = (db: TDbClient) => { throw new BadRequestError({ message: "Some of the secret versions do not exist" }); } + if (data.length === 0) return []; + const updatedSecretVersions = await (tx || db)(TableName.SecretVersion) .insert(data) .onConflict("id") // this will cause a conflict then merge the data .merge() // Merge the data with the existing data .returning("*"); - if (!updatedSecretVersions || updatedSecretVersions.length === 0) { - throw new BadRequestError({ message: "Failed to bulk update secret versions" }); - } - return updatedSecretVersions; } catch (error) { throw new DatabaseError({ error, name: "bulk update secret" });