diff --git a/backend/src/services/integration-auth/integration-sync-secret.ts b/backend/src/services/integration-auth/integration-sync-secret.ts index 6351b4d824..70e3435a2a 100644 --- a/backend/src/services/integration-auth/integration-sync-secret.ts +++ b/backend/src/services/integration-auth/integration-sync-secret.ts @@ -18,7 +18,7 @@ import { UpdateSecretCommand } from "@aws-sdk/client-secrets-manager"; import { Octokit } from "@octokit/rest"; -import AWS from "aws-sdk"; +import AWS, { AWSError } from "aws-sdk"; import { AxiosError } from "axios"; import sodium from "libsodium-wrappers"; import isEqual from "lodash.isequal"; @@ -452,7 +452,11 @@ const syncSecretsAWSParameterStore = async ({ accessId: string | null; accessToken: string; }) => { - if (!accessId) return; + let response: { isSynced: boolean; syncMessage: string } | null = null; + + if (!accessId) { + throw new Error("AWS access ID is required"); + } const config = new AWS.Config({ region: integration.region as string, @@ -557,6 +561,11 @@ const syncSecretsAWSParameterStore = async ({ `AWS Parameter Store Error [integration=${integration.id}]: double check AWS account permissions (refer to the Infisical docs)` ); } + + response = { + isSynced: false, + syncMessage: (err as AWSError)?.message || "Error syncing with AWS Parameter Store" + }; } } } @@ -585,6 +594,8 @@ const syncSecretsAWSParameterStore = async ({ } } } + + return response; }; /** @@ -603,7 +614,9 @@ const syncSecretsAWSSecretManager = async ({ }) => { const metadata = z.record(z.any()).parse(integration.metadata || {}); - if (!accessId) return; + if (!accessId) { + throw new Error("AWS access ID is required"); + } const secretsManager = new SecretsManagerClient({ region: integration.region as string, @@ -722,7 +735,7 @@ const syncSecretsAWSSecretManager = async ({ } } } catch (err) { - // case when AWS manager can't find the specified secret + // case 1: when AWS manager can't find the specified secret if (err instanceof ResourceNotFoundException && secretsManager) { await secretsManager.send( new CreateSecretCommand({ @@ -734,6 +747,9 @@ const syncSecretsAWSSecretManager = async ({ : [] }) ); + // case 2: something unexpected went wrong, so we'll throw the error to reflect the error in the integration sync status + } else { + throw err; } } }; @@ -753,14 +769,12 @@ const syncSecretsAWSSecretManager = async ({ const syncSecretsHeroku = async ({ createManySecretsRawFn, updateManySecretsRawFn, - integrationDAL, integration, secrets, accessToken }: { createManySecretsRawFn: (params: TCreateManySecretsRawFn) => Promise>; updateManySecretsRawFn: (params: TUpdateManySecretsRawFn) => Promise>; - integrationDAL: Pick; integration: TIntegrations & { projectId: string; environment: { @@ -862,10 +876,6 @@ const syncSecretsHeroku = async ({ } } ); - - await integrationDAL.updateById(integration.id, { - lastUsed: new Date() - }); }; /** @@ -2656,7 +2666,9 @@ const syncSecretsHashiCorpVault = async ({ accessId: string | null; accessToken: string; }) => { - if (!accessId) return; + if (!accessId) { + throw new Error("Access ID is required"); + } interface LoginAppRoleRes { auth: { @@ -3486,6 +3498,8 @@ export const syncIntegrationSecrets = async ({ accessToken: string; appendices?: { prefix: string; suffix: string }; }) => { + let response: { isSynced: boolean; syncMessage: string } | null = null; + switch (integration.integration) { case Integrations.GCP_SECRET_MANAGER: await syncSecretsGCPSecretManager({ @@ -3502,7 +3516,7 @@ export const syncIntegrationSecrets = async ({ }); break; case Integrations.AWS_PARAMETER_STORE: - await syncSecretsAWSParameterStore({ + response = await syncSecretsAWSParameterStore({ integration, secrets, accessId, @@ -3521,7 +3535,6 @@ export const syncIntegrationSecrets = async ({ await syncSecretsHeroku({ createManySecretsRawFn, updateManySecretsRawFn, - integrationDAL, integration, secrets, accessToken @@ -3727,4 +3740,6 @@ export const syncIntegrationSecrets = async ({ default: throw new BadRequestError({ message: "Invalid integration" }); } + + return response; }; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 42e13b4456..ac27d912f7 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -421,94 +421,88 @@ export const secretQueueFactory = ({ const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) { - logger.error(new Error("Secret path not found")); - return; + throw new Error("Secret path not found"); } - // start syncing all linked imports also - if (depth < MAX_SYNC_SECRET_DEPTH) { - // find all imports made with the given environment and secret path - const linkSourceDto = { - projectId, - importEnv: folder.environment.id, - importPath: secretPath, - isReplication: false - }; - const imports = await secretImportDAL.find(linkSourceDto); + // find all imports made with the given environment and secret path + const linkSourceDto = { + projectId, + importEnv: folder.environment.id, + importPath: secretPath, + isReplication: false + }; + const imports = await secretImportDAL.find(linkSourceDto); - if (imports.length) { - // keep calling sync secret for all the imports made - const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId); - const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds); - const foldersGroupedById = groupBy(importedFolders.filter(Boolean), (i) => i?.id as string); - logger.info( - `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` - ); - await Promise.all( - imports - .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0]?.path as string)) - // filter out already synced ones - .filter( - ({ folderId }) => - !deDupeQueue[ - uniqueSecretQueueKey( - foldersGroupedById[folderId][0]?.environmentSlug as string, - foldersGroupedById[folderId][0]?.path as string - ) - ] - ) - .map(({ folderId }) => - syncSecrets({ - projectId, - secretPath: foldersGroupedById[folderId][0]?.path as string, - environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string, - _deDupeQueue: deDupeQueue, - _depth: depth + 1, - excludeReplication: true - }) - ) - ); - } - - const secretReferences = await secretDAL.findReferencedSecretReferences( - projectId, - folder.environment.slug, - secretPath + if (imports.length) { + // keep calling sync secret for all the imports made + const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId); + const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds); + const foldersGroupedById = groupBy(importedFolders.filter(Boolean), (i) => i?.id as string); + logger.info( + `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` + ); + await Promise.all( + imports + .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0]?.path as string)) + // filter out already synced ones + .filter( + ({ folderId }) => + !deDupeQueue[ + uniqueSecretQueueKey( + foldersGroupedById[folderId][0]?.environmentSlug as string, + foldersGroupedById[folderId][0]?.path as string + ) + ] + ) + .map(({ folderId }) => + syncSecrets({ + projectId, + secretPath: foldersGroupedById[folderId][0]?.path as string, + environmentSlug: foldersGroupedById[folderId][0]?.environmentSlug as string, + _deDupeQueue: deDupeQueue, + _depth: depth + 1, + excludeReplication: true + }) + ) + ); + } + + const secretReferences = await secretDAL.findReferencedSecretReferences( + projectId, + folder.environment.slug, + secretPath + ); + if (secretReferences.length) { + const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId); + const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds); + const referencedFoldersGroupedById = groupBy(referencedFolders.filter(Boolean), (i) => i?.id as string); + logger.info( + `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` + ); + await Promise.all( + secretReferences + .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0]?.path)) + // filter out already synced ones + .filter( + ({ folderId }) => + !deDupeQueue[ + uniqueSecretQueueKey( + referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, + referencedFoldersGroupedById[folderId][0]?.path as string + ) + ] + ) + .map(({ folderId }) => + syncSecrets({ + projectId, + secretPath: referencedFoldersGroupedById[folderId][0]?.path as string, + environmentSlug: referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, + _deDupeQueue: deDupeQueue, + _depth: depth + 1, + excludeReplication: true + }) + ) ); - if (secretReferences.length) { - const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId); - const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds); - const referencedFoldersGroupedById = groupBy(referencedFolders.filter(Boolean), (i) => i?.id as string); - logger.info( - `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}] [secretPath=${job.data.secretPath}] [depth=${depth}]` - ); - await Promise.all( - secretReferences - .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0]?.path)) - // filter out already synced ones - .filter( - ({ folderId }) => - !deDupeQueue[ - uniqueSecretQueueKey( - referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, - referencedFoldersGroupedById[folderId][0]?.path as string - ) - ] - ) - .map(({ folderId }) => - syncSecrets({ - projectId, - secretPath: referencedFoldersGroupedById[folderId][0]?.path as string, - environmentSlug: referencedFoldersGroupedById[folderId][0]?.environmentSlug as string, - _deDupeQueue: deDupeQueue, - _depth: depth + 1, - excludeReplication: true - }) - ) - ); - } - } else { - logger.info(`getIntegrationSecrets: Secret depth exceeded for [projectId=${projectId}] [folderId=${folder.id}]`); } const integrations = await integrationDAL.findByProjectIdV2(projectId, environment); // note: returns array of integrations + integration auths in this environment @@ -550,7 +544,7 @@ export const secretQueueFactory = ({ } try { - await syncIntegrationSecrets({ + const response = await syncIntegrationSecrets({ createManySecretsRawFn, updateManySecretsRawFn, integrationDAL, @@ -568,13 +562,15 @@ export const secretQueueFactory = ({ await integrationDAL.updateById(integration.id, { lastSyncJobId: job.id, lastUsed: new Date(), - syncMessage: "", - isSynced: true + syncMessage: response?.syncMessage ?? "", + isSynced: response?.isSynced ?? true }); - } catch (err: unknown) { + } catch (err) { logger.info("Secret integration sync error: %o", err); + const message = - err instanceof AxiosError ? JSON.stringify((err as AxiosError)?.response?.data) : (err as Error)?.message; + (err instanceof AxiosError ? JSON.stringify(err?.response?.data) : (err as Error)?.message) || + "Unknown error occurred."; await integrationDAL.updateById(integration.id, { lastSyncJobId: job.id,