github/infisical
1
1
Fork 0
mirror of https://github.com/Infisical/infisical.git synced 2026-05-02 03:02:03 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fixed integrations & bulk update issue

Browse Source
This commit is contained in:
Daniel Hougaard
2024-02-20 02:34:30 +01:00
parent 66e57d5d11
commit 75813deb81
7 changed files with 249 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts
View File

@@ -31,16 +31,14 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
throw new BadRequestError({ message: "Some of the secret approvals do not exist" });
}
if (data.length === 0) return [];
const updatedApprovalSecrets = await (tx || db)(TableName.SecretApprovalRequestSecret)
.insert(data)
.onConflict("id") // this will cause a conflict then merge the data
.merge() // Merge the data with the existing data
.returning("*");
if (!updatedApprovalSecrets || updatedApprovalSecrets.length === 0) {
throw new BadRequestError({ message: "Failed to bulk update secret approvals" });
}
return updatedApprovalSecrets;
} catch (error) {
throw new DatabaseError({ error, name: "bulk update secret" });

262
backend/src/lib/secret/index.ts
View File

@@ -2,9 +2,11 @@ import crypto from "crypto";
import { z } from "zod";
import {
IntegrationAuthsSchema,
SecretApprovalRequestsSecretsSchema,
SecretsSchema,
SecretVersionsSchema,
TIntegrationAuths,
TProjectKeys,
TSecretApprovalRequestsSecrets,
TSecrets,
@@ -25,6 +27,16 @@ const DecryptedSecretSchema = z.object({
original: SecretsSchema
});
const DecryptedIntegrationAuthsSchema = z.object({
decrypted: z.object({
id: z.string(),
access: z.string(),
accessId: z.string(),
refresh: z.string()
}),
original: IntegrationAuthsSchema
});
const DecryptedSecretVersionsSchema = z.object({
decrypted: DecryptedValuesSchema,
original: SecretVersionsSchema
@@ -38,6 +50,13 @@ export const DecryptedSecretApprovalsSchema = z.object({
export type DecryptedSecret = z.infer<typeof DecryptedSecretSchema>;
export type DecryptedSecretVersions = z.infer<typeof DecryptedSecretVersionsSchema>;
export type DecryptedSecretApprovals = z.infer<typeof DecryptedSecretApprovalsSchema>;
export type DecryptedIntegrationAuths = z.infer<typeof DecryptedIntegrationAuthsSchema>;
type TLatestKey = TProjectKeys & {
sender: {
publicKey: string;
};
};
const decryptCipher = ({
ciphertext,
@@ -59,69 +78,20 @@ const decryptCipher = ({
return cleartext;
};
const getDecryptedValues = ({
secretKeyCiphertext,
secretKeyIV,
secretKeyTag,
secretValueCiphertext,
secretValueIV,
secretValueTag,
const getDecryptedValues = (data: Array<{ ciphertext: string; iv: string; tag: string }>, key: string | Buffer) => {
const results = [];
secretCommentCiphertext,
secretCommentIV,
secretCommentTag,
key
}: {
secretKeyCiphertext: string;
secretKeyIV: string;
secretKeyTag: string;
secretValueCiphertext: string;
secretValueIV: string;
secretValueTag: string;
secretCommentCiphertext?: string | null;
secretCommentIV?: string | null;
secretCommentTag?: string | null;
key: string | Buffer;
}) => {
const secretKey = decryptCipher({
ciphertext: secretKeyCiphertext,
iv: secretKeyIV,
tag: secretKeyTag,
key
});
const secretValue = decryptCipher({
ciphertext: secretValueCiphertext,
iv: secretValueIV,
tag: secretValueTag,
key
});
const secretComment =
secretCommentCiphertext && secretCommentIV && secretCommentTag
? decryptCipher({
ciphertext: secretCommentCiphertext,
iv: secretCommentIV,
tag: secretCommentTag,
key
})
: "";
return {
secretKey,
secretValue,
secretComment
};
};
export const decryptSecrets = (
encryptedSecrets: TSecrets[],
privateKey: string,
latestKey: TProjectKeys & {
sender: {
publicKey: string;
};
for (const { ciphertext, iv, tag } of data) {
if (!ciphertext || !iv || !tag) {
results.push("");
} else {
results.push(decryptCipher({ ciphertext, iv, tag, key }));
}
}
) => {
return results;
};
export const decryptSecrets = (encryptedSecrets: TSecrets[], privateKey: string, latestKey: TLatestKey) => {
const key = decryptAsymmetric({
ciphertext: latestKey.encryptedKey,
nonce: latestKey.nonce,
@@ -132,22 +102,32 @@ export const decryptSecrets = (
const decryptedSecrets: DecryptedSecret[] = [];
encryptedSecrets.forEach((encSecret) => {
const decrypted = getDecryptedValues({
secretKeyCiphertext: encSecret.secretKeyCiphertext,
secretKeyIV: encSecret.secretKeyIV,
secretKeyTag: encSecret.secretKeyTag,
secretValueCiphertext: encSecret.secretValueCiphertext,
secretValueIV: encSecret.secretValueIV,
secretValueTag: encSecret.secretValueTag,
secretCommentCiphertext: encSecret.secretCommentCiphertext,
secretCommentIV: encSecret.secretCommentIV,
secretCommentTag: encSecret.secretCommentTag,
const [secretKey, secretValue, secretComment] = getDecryptedValues(
[
{
ciphertext: encSecret.secretKeyCiphertext,
iv: encSecret.secretKeyIV,
tag: encSecret.secretKeyTag
},
{
ciphertext: encSecret.secretValueCiphertext,
iv: encSecret.secretValueIV,
tag: encSecret.secretValueTag
},
{
ciphertext: encSecret.secretCommentCiphertext || "",
iv: encSecret.secretCommentIV || "",
tag: encSecret.secretCommentTag || ""
}
],
key
});
);
const decryptedSecret: DecryptedSecret = {
decrypted: {
...decrypted,
secretKey,
secretValue,
secretComment,
id: encSecret.id
},
original: encSecret
@@ -162,11 +142,7 @@ export const decryptSecrets = (
export const decryptSecretVersions = (
encryptedSecretVersions: TSecretVersions[],
privateKey: string,
latestKey: TProjectKeys & {
sender: {
publicKey: string;
};
}
latestKey: TLatestKey
) => {
const key = decryptAsymmetric({
ciphertext: latestKey.encryptedKey,
@@ -178,22 +154,32 @@ export const decryptSecretVersions = (
const decryptedSecrets: DecryptedSecretVersions[] = [];
encryptedSecretVersions.forEach((encSecret) => {
const decrypted = getDecryptedValues({
secretKeyCiphertext: encSecret.secretKeyCiphertext,
secretKeyIV: encSecret.secretKeyIV,
secretKeyTag: encSecret.secretKeyTag,
secretValueCiphertext: encSecret.secretValueCiphertext,
secretValueIV: encSecret.secretValueIV,
secretValueTag: encSecret.secretValueTag,
secretCommentCiphertext: encSecret.secretCommentCiphertext,
secretCommentIV: encSecret.secretCommentIV,
secretCommentTag: encSecret.secretCommentTag,
const [secretKey, secretValue, secretComment] = getDecryptedValues(
[
{
ciphertext: encSecret.secretKeyCiphertext,
iv: encSecret.secretKeyIV,
tag: encSecret.secretKeyTag
},
{
ciphertext: encSecret.secretValueCiphertext,
iv: encSecret.secretValueIV,
tag: encSecret.secretValueTag
},
{
ciphertext: encSecret.secretCommentCiphertext || "",
iv: encSecret.secretCommentIV || "",
tag: encSecret.secretCommentTag || ""
}
],
key
});
);
const decryptedSecret: DecryptedSecretVersions = {
decrypted: {
...decrypted,
secretKey,
secretValue,
secretComment,
id: encSecret.id
},
original: encSecret
@@ -208,11 +194,7 @@ export const decryptSecretVersions = (
export const decryptSecretApprovals = (
encryptedSecretApprovals: TSecretApprovalRequestsSecrets[],
privateKey: string,
latestKey: TProjectKeys & {
sender: {
publicKey: string;
};
}
latestKey: TLatestKey
) => {
const key = decryptAsymmetric({
ciphertext: latestKey.encryptedKey,
@@ -223,26 +205,36 @@ export const decryptSecretApprovals = (
const decryptedSecrets: DecryptedSecretApprovals[] = [];
encryptedSecretApprovals.forEach((encSecret) => {
const decrypted = getDecryptedValues({
secretKeyCiphertext: encSecret.secretKeyCiphertext,
secretKeyIV: encSecret.secretKeyIV,
secretKeyTag: encSecret.secretKeyTag,
secretValueCiphertext: encSecret.secretValueCiphertext,
secretValueIV: encSecret.secretValueIV,
secretValueTag: encSecret.secretValueTag,
secretCommentCiphertext: encSecret.secretCommentCiphertext,
secretCommentIV: encSecret.secretCommentIV,
secretCommentTag: encSecret.secretCommentTag,
encryptedSecretApprovals.forEach((encApproval) => {
const [secretKey, secretValue, secretComment] = getDecryptedValues(
[
{
ciphertext: encApproval.secretKeyCiphertext,
iv: encApproval.secretKeyIV,
tag: encApproval.secretKeyTag
},
{
ciphertext: encApproval.secretValueCiphertext,
iv: encApproval.secretValueIV,
tag: encApproval.secretValueTag
},
{
ciphertext: encApproval.secretCommentCiphertext || "",
iv: encApproval.secretCommentIV || "",
tag: encApproval.secretCommentTag || ""
}
],
key
});
);
const decryptedSecret: DecryptedSecretApprovals = {
decrypted: {
...decrypted,
id: encSecret.id
secretKey,
secretValue,
secretComment,
id: encApproval.id
},
original: encSecret
original: encApproval
};
decryptedSecrets.push(DecryptedSecretApprovalsSchema.parse(decryptedSecret));
@@ -250,3 +242,53 @@ export const decryptSecretApprovals = (
return decryptedSecrets;
};
export const decryptIntegrationAuths = (
encryptedIntegrationAuths: TIntegrationAuths[],
privateKey: string,
latestKey: TLatestKey
) => {
const key = decryptAsymmetric({
ciphertext: latestKey.encryptedKey,
nonce: latestKey.nonce,
publicKey: latestKey.sender.publicKey,
privateKey
});
const decryptedIntegrationAuths: DecryptedIntegrationAuths[] = [];
encryptedIntegrationAuths.forEach((encAuth) => {
const [access, accessId, refresh] = getDecryptedValues(
[
{
ciphertext: encAuth.accessCiphertext || "",
iv: encAuth.accessIV || "",
tag: encAuth.accessTag || ""
},
{
ciphertext: encAuth.accessIdCiphertext || "",
iv: encAuth.accessIdIV || "",
tag: encAuth.accessIdTag || ""
},
{
ciphertext: encAuth.refreshCiphertext || "",
iv: encAuth.refreshIV || "",
tag: encAuth.refreshTag || ""
}
],
key
);
decryptedIntegrationAuths.push({
decrypted: {
id: encAuth.id,
access,
accessId,
refresh
},
original: encAuth
});
});
return decryptedIntegrationAuths;
};

1
backend/src/server/routes/index.ts
View File

@@ -307,6 +307,7 @@ export const registerRoutes = async (
folderDAL,
projectDAL,
orgDAL,
integrationAuthDAL,
orgService,
projectEnvDAL,
userDAL,

29
backend/src/services/integration-auth/integration-auth-dal.ts
View File

@@ -1,10 +1,35 @@
import { Knex } from "knex";
import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
import { TableName, TIntegrationAuths, TIntegrationAuthsUpdate } from "@app/db/schemas";
import { BadRequestError, DatabaseError } from "@app/lib/errors";
import { ormify } from "@app/lib/knex";
export type TIntegrationAuthDALFactory = ReturnType<typeof integrationAuthDALFactory>;
export const integrationAuthDALFactory = (db: TDbClient) => {
const integrationAuthOrm = ormify(db, TableName.IntegrationAuth);
return integrationAuthOrm;
const bulkUpdate = async (
data: Array<{ filter: Partial<TIntegrationAuths>; data: TIntegrationAuthsUpdate }>,
tx?: Knex
) => {
try {
const integrationAuths = await Promise.all(
data.map(async ({ filter, data: updateData }) => {
const [doc] = await (tx || db)(TableName.IntegrationAuth).where(filter).update(updateData).returning("*");
if (!doc) throw new BadRequestError({ message: "Failed to update document" });
return doc;
})
);
return integrationAuths;
} catch (error) {
throw new DatabaseError({ error, name: "bulk update secret" });
}
};
return {
...integrationAuthOrm,
bulkUpdate
};
};

65
backend/src/services/project/project-queue.ts
View File

@@ -1,5 +1,6 @@
/* eslint-disable no-await-in-loop */
import {
IntegrationAuthsSchema,
ProjectMembershipRole,
ProjectUpgradeStatus,
ProjectVersion,
@@ -7,6 +8,7 @@ import {
SecretKeyEncoding,
SecretsSchema,
SecretVersionsSchema,
TIntegrationAuths,
TSecretApprovalRequestsSecrets,
TSecrets,
TSecretVersions
@@ -21,9 +23,15 @@ import {
infisicalSymmetricEncypt
} from "@app/lib/crypto/encryption";
import { logger } from "@app/lib/logger";
import { decryptSecretApprovals, decryptSecrets, decryptSecretVersions } from "@app/lib/secret";
import {
decryptIntegrationAuths,
decryptSecretApprovals,
decryptSecrets,
decryptSecretVersions
} from "@app/lib/secret";
import { QueueJobs, QueueName, TQueueJobTypes, TQueueServiceFactory } from "@app/queue";
import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal";
import { TOrgDALFactory } from "../org/org-dal";
import { TOrgServiceFactory } from "../org/org-service";
import { TProjectBotDALFactory } from "../project-bot/project-bot-dal";
@@ -50,6 +58,7 @@ type TProjectQueueFactoryDep = {
projectBotDAL: Pick<TProjectBotDALFactory, "findOne" | "delete" | "create">;
orgService: Pick<TOrgServiceFactory, "addGhostUser">;
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "create">;
integrationAuthDAL: TIntegrationAuthDALFactory;
userDAL: Pick<TUserDALFactory, "findUserEncKeyByUserId">;
projectEnvDAL: Pick<TProjectEnvDALFactory, "find">;
@@ -63,6 +72,7 @@ export const projectQueueFactory = ({
folderDAL,
userDAL,
secretVersionDAL,
integrationAuthDAL,
secretApprovalRequestDAL,
secretApprovalSecretDAL,
projectKeyDAL,
@@ -150,9 +160,14 @@ export const projectQueueFactory = ({
approvalSecrets.push(...secretApprovals);
}
const projectIntegrationAuths = await integrationAuthDAL.find({
projectId: project.id
});
const decryptedSecrets = decryptSecrets(secrets, userPrivateKey, oldProjectKey);
const decryptedSecretVersions = decryptSecretVersions(secretVersions, userPrivateKey, oldProjectKey);
const decryptedApprovalSecrets = decryptSecretApprovals(approvalSecrets, userPrivateKey, oldProjectKey);
const decryptedIntegrationAuths = decryptIntegrationAuths(projectIntegrationAuths, userPrivateKey, oldProjectKey);
if (secrets.length !== decryptedSecrets.length) {
throw new Error("Failed to decrypt some secret versions");
@@ -304,6 +319,7 @@ export const projectQueueFactory = ({
const updatedSecrets: TSecrets[] = [];
const updatedSecretVersions: TSecretVersions[] = [];
const updatedSecretApprovals: TSecretApprovalRequestsSecrets[] = [];
const updatedIntegrationAuths: TIntegrationAuths[] = [];
for (const rawSecret of decryptedSecrets) {
const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.decrypted.secretKey, botKey);
const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.decrypted.secretValue || "", botKey);
@@ -348,6 +364,7 @@ export const projectQueueFactory = ({
const payload: TSecretVersions = {
...rawSecretVersion.original,
keyEncoding: SecretKeyEncoding.UTF8,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
@@ -382,6 +399,7 @@ export const projectQueueFactory = ({
const payload: TSecretApprovalRequestsSecrets = {
...rawSecretApproval.original,
keyEncoding: SecretKeyEncoding.UTF8,
secretKeyCiphertext: secretKeyEncrypted.ciphertext,
secretKeyIV: secretKeyEncrypted.iv,
@@ -403,6 +421,35 @@ export const projectQueueFactory = ({
updatedSecretApprovals.push(payload);
}
for (const integrationAuth of decryptedIntegrationAuths) {
const access = encryptSymmetric128BitHexKeyUTF8(integrationAuth.decrypted.access, botKey);
const accessId = encryptSymmetric128BitHexKeyUTF8(integrationAuth.decrypted.accessId, botKey);
const refresh = encryptSymmetric128BitHexKeyUTF8(integrationAuth.decrypted.refresh, botKey);
const payload: TIntegrationAuths = {
...integrationAuth.original,
keyEncoding: SecretKeyEncoding.UTF8,
accessCiphertext: access.ciphertext,
accessIV: access.iv,
accessTag: access.tag,
accessIdCiphertext: accessId.ciphertext,
accessIdIV: accessId.iv,
accessIdTag: accessId.tag,
refreshCiphertext: refresh.ciphertext,
refreshIV: refresh.iv,
refreshTag: refresh.tag
} as const;
if (!IntegrationAuthsSchema.safeParse(payload).success) {
throw new Error(`Invalid integration auth payload: ${JSON.stringify(payload)}`);
}
updatedIntegrationAuths.push(payload);
}
if (updatedSecrets.length !== secrets.length) {
throw new Error("Failed to update some secrets");
}
@@ -412,6 +459,9 @@ export const projectQueueFactory = ({
if (updatedSecretApprovals.length !== approvalSecrets.length) {
throw new Error("Failed to update some secret approvals");
}
if (updatedIntegrationAuths.length !== projectIntegrationAuths.length) {
throw new Error("Failed to update some integration auths");
}
const secretUpdates = await secretDAL.bulkUpdateNoVersionIncrement(updatedSecrets, tx);
const secretVersionUpdates = await secretVersionDAL.bulkUpdateNoVersionIncrement(updatedSecretVersions, tx);
@@ -419,11 +469,22 @@ export const projectQueueFactory = ({
updatedSecretApprovals,
tx
);
const integrationAuthUpdates = await integrationAuthDAL.bulkUpdate(
updatedIntegrationAuths.map((el) => ({
filter: { id: el.id },
data: {
...el,
id: undefined
}
})),
tx
);
if (
secretUpdates.length !== updatedSecrets.length ||
secretVersionUpdates.length !== updatedSecretVersions.length ||
secretApprovalUpdates.length !== updatedSecretApprovals.length
secretApprovalUpdates.length !== updatedSecretApprovals.length ||
integrationAuthUpdates.length !== updatedIntegrationAuths.length
) {
throw new Error("Parts of the upgrade failed. Some secrets were not updated");
}

6
backend/src/services/secret/secret-dal.ts
View File

@@ -60,16 +60,14 @@ export const secretDALFactory = (db: TDbClient) => {
throw new BadRequestError({ message: "Some of the secrets do not exist" });
}
if (data.length === 0) return [];
const updatedSecrets = await (tx || db)(TableName.Secret)
.insert(data)
.onConflict("id") // this will cause a conflict then merge the data
.merge() // Merge the data with the existing data
.returning("*");
if (!updatedSecrets || updatedSecrets.length === 0) {
throw new BadRequestError({ message: "Failed to bulk update secret approvals" });
}
return updatedSecrets;
} catch (error) {
throw new DatabaseError({ error, name: "bulk update secret" });

6
backend/src/services/secret/secret-version-dal.ts
View File

@@ -73,16 +73,14 @@ export const secretVersionDALFactory = (db: TDbClient) => {
throw new BadRequestError({ message: "Some of the secret versions do not exist" });
}
if (data.length === 0) return [];
const updatedSecretVersions = await (tx || db)(TableName.SecretVersion)
.insert(data)
.onConflict("id") // this will cause a conflict then merge the data
.merge() // Merge the data with the existing data
.returning("*");
if (!updatedSecretVersions || updatedSecretVersions.length === 0) {
throw new BadRequestError({ message: "Failed to bulk update secret versions" });
}
return updatedSecretVersions;
} catch (error) {
throw new DatabaseError({ error, name: "bulk update secret" });