From 35e38c23dd3713533976d7e7c16a9eec85aef815 Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Fri, 26 Apr 2024 00:26:18 +0530 Subject: [PATCH 1/3] feat(server): recursive imported secret fetch for api --- .../secret-import/secret-import-fns.ts | 102 +++++++++++++----- .../secret-import/secret-import-service.ts | 2 +- backend/src/services/secret/secret-service.ts | 6 +- 3 files changed, 83 insertions(+), 27 deletions(-) diff --git a/backend/src/services/secret-import/secret-import-fns.ts b/backend/src/services/secret-import/secret-import-fns.ts index 00b33a13c8..df127ef567 100644 --- a/backend/src/services/secret-import/secret-import-fns.ts +++ b/backend/src/services/secret-import/secret-import-fns.ts @@ -1,33 +1,59 @@ -import { SecretType, TSecretImports } from "@app/db/schemas"; +import { SecretType, TSecretImports, TSecrets } from "@app/db/schemas"; import { groupBy } from "@app/lib/fn"; import { TSecretDALFactory } from "../secret/secret-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretImportDALFactory } from "./secret-import-dal"; +type TSecretImportSecrets = { + secretPath: string; + environment: string; + environmentInfo: { + id: string; + slug: string; + name: string; + }; + folderId: string | undefined; + importFolderId: string; + secrets: (TSecrets & { workspace: string; environment: string; _id: string })[]; +}; + +const LEVEL_BREAK = 10; export const fnSecretsFromImports = async ({ allowedImports, folderDAL, - secretDAL + secretDAL, + secretImportDAL, + depth = 0 }: { allowedImports: (Omit & { importEnv: { id: string; slug: string; name: string }; })[]; folderDAL: Pick; secretDAL: Pick; + secretImportDAL: Pick; + depth?: number; }) => { - const importedFolders = await folderDAL.findByManySecretPath( - allowedImports.map(({ importEnv, importPath }) => ({ - envId: importEnv.id, - secretPath: importPath - })) - ); - const folderIds = importedFolders.map((el) => el?.id).filter(Boolean) as string[]; - if (!folderIds.length) { + // avoid going more than a depth + if (depth >= LEVEL_BREAK) return []; + + const importedFolders = ( + await folderDAL.findByManySecretPath( + allowedImports.map(({ importEnv, importPath }) => ({ + envId: importEnv.id, + secretPath: importPath + })) + ) + ).filter(Boolean); // remove undefined ones + if (!importedFolders.length) { return []; } + + const importedFolderIds = importedFolders.map((el) => el?.id) as string[]; + const importedFolderGroupBySourceImport = groupBy(importedFolders, (i) => `${i?.envId}-${i?.path}`); const importedSecrets = await secretDAL.find( { - $in: { folderId: folderIds }, + $in: { folderId: importedFolderIds }, type: SecretType.Shared }, { @@ -35,18 +61,46 @@ export const fnSecretsFromImports = async ({ } ); - const importedSecsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId); - return allowedImports.map(({ importPath, importEnv }, i) => ({ - secretPath: importPath, - environment: importEnv.slug, - environmentInfo: importEnv, - folderId: importedFolders?.[i]?.id, - // this will ensure for cases when secrets are empty. Could be due to missing folder for a path or when emtpy secrets inside a given path - secrets: (importedSecsGroupByFolderId?.[importedFolders?.[i]?.id as string] || []).map((item) => ({ - ...item, + const importedSecretsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId); + + // now we need to check recursively deeper imports made inside other imports + // we go level wise meaning we take all imports of a tree level and then go deeper ones level by level + const deeperImports = await secretImportDAL.findByFolderIds(importedFolderIds); + let secretsFromDeeperImports: TSecretImportSecrets[] = []; + if (deeperImports.length) { + secretsFromDeeperImports = await fnSecretsFromImports({ + allowedImports: deeperImports, + secretImportDAL, + folderDAL, + secretDAL, + depth: depth + 1 + }); + } + const secretsFromdeeperImportGroupedByFolderId = groupBy(secretsFromDeeperImports, (i) => i.importFolderId); + + const secrets = allowedImports.map(({ importPath, importEnv, id, folderId }, i) => { + const sourceImportFolder = importedFolderGroupBySourceImport[`${importEnv.id}-${importPath}`][0]; + const folderDeeperImportSecrets = + secretsFromdeeperImportGroupedByFolderId?.[sourceImportFolder?.id || ""]?.[0]?.secrets || []; + + return { + secretPath: importPath, environment: importEnv.slug, - workspace: "", // This field should not be used, it's only here to keep the older Python SDK versions backwards compatible with the new Postgres backend. - _id: item.id // The old Python SDK depends on the _id field being returned. We return this to keep the older Python SDK versions backwards compatible with the new Postgres backend. - })) - })); + environmentInfo: importEnv, + folderId: importedFolders?.[i]?.id, + id, + importFolderId: folderId, + // this will ensure for cases when secrets are empty. Could be due to missing folder for a path or when emtpy secrets inside a given path + secrets: (importedSecretsGroupByFolderId?.[importedFolders?.[i]?.id as string] || []) + .map((item) => ({ + ...item, + environment: importEnv.slug, + workspace: "", // This field should not be used, it's only here to keep the older Python SDK versions backwards compatible with the new Postgres backend. + _id: item.id // The old Python SDK depends on the _id field being returned. We return this to keep the older Python SDK versions backwards compatible with the new Postgres backend. + })) + .concat(folderDeeperImportSecrets) + }; + }); + + return secrets; }; diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 2d59284d4a..43676ba04c 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -290,7 +290,7 @@ export const secretImportServiceFactory = ({ }) ) ); - return fnSecretsFromImports({ allowedImports, folderDAL, secretDAL }); + return fnSecretsFromImports({ allowedImports, folderDAL, secretDAL, secretImportDAL }); }; return { diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 584395c542..5c7b9bef9a 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -525,7 +525,8 @@ export const secretServiceFactory = ({ const importedSecrets = await fnSecretsFromImports({ allowedImports, secretDAL, - folderDAL + folderDAL, + secretImportDAL }); return { @@ -630,7 +631,8 @@ export const secretServiceFactory = ({ const importedSecrets = await fnSecretsFromImports({ allowedImports, secretDAL, - folderDAL + folderDAL, + secretImportDAL }); for (let i = importedSecrets.length - 1; i >= 0; i -= 1) { for (let j = 0; j < importedSecrets[i].secrets.length; j += 1) { From 2d2d9a5987377ffe2a1750564af51553d3075e83 Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Fri, 26 Apr 2024 01:00:42 +0530 Subject: [PATCH 2/3] feat(server): added cyclic detector --- .../services/secret-import/secret-import-fns.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/backend/src/services/secret-import/secret-import-fns.ts b/backend/src/services/secret-import/secret-import-fns.ts index df127ef567..28372c280d 100644 --- a/backend/src/services/secret-import/secret-import-fns.ts +++ b/backend/src/services/secret-import/secret-import-fns.ts @@ -19,12 +19,14 @@ type TSecretImportSecrets = { }; const LEVEL_BREAK = 10; +const getImportUniqKey = (envSlug: string, path: string) => `${envSlug}=${path}`; export const fnSecretsFromImports = async ({ - allowedImports, + allowedImports: nonCyclicImports, folderDAL, secretDAL, secretImportDAL, - depth = 0 + depth = 0, + cyclicDetector = new Set() }: { allowedImports: (Omit & { importEnv: { id: string; slug: string; name: string }; @@ -33,10 +35,15 @@ export const fnSecretsFromImports = async ({ secretDAL: Pick; secretImportDAL: Pick; depth?: number; + cyclicDetector?: Set; }) => { // avoid going more than a depth if (depth >= LEVEL_BREAK) return []; + const allowedImports = nonCyclicImports.filter( + ({ importPath, importEnv }) => !cyclicDetector.has(getImportUniqKey(importEnv.slug, importPath)) + ); + const importedFolders = ( await folderDAL.findByManySecretPath( allowedImports.map(({ importEnv, importPath }) => ({ @@ -63,6 +70,9 @@ export const fnSecretsFromImports = async ({ const importedSecretsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId); + allowedImports.forEach(({ importPath, importEnv }) => { + cyclicDetector.add(getImportUniqKey(importEnv.slug, importPath)); + }); // now we need to check recursively deeper imports made inside other imports // we go level wise meaning we take all imports of a tree level and then go deeper ones level by level const deeperImports = await secretImportDAL.findByFolderIds(importedFolderIds); @@ -73,7 +83,8 @@ export const fnSecretsFromImports = async ({ secretImportDAL, folderDAL, secretDAL, - depth: depth + 1 + depth: depth + 1, + cyclicDetector }); } const secretsFromdeeperImportGroupedByFolderId = groupBy(secretsFromDeeperImports, (i) => i.importFolderId); From 8b8baf1ef29aa45c09094fc1e80f895362545aa5 Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Thu, 25 Apr 2024 15:40:55 -0400 Subject: [PATCH 3/3] nit: variable rename --- backend/src/services/secret-import/secret-import-fns.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/services/secret-import/secret-import-fns.ts b/backend/src/services/secret-import/secret-import-fns.ts index 28372c280d..fffc22a993 100644 --- a/backend/src/services/secret-import/secret-import-fns.ts +++ b/backend/src/services/secret-import/secret-import-fns.ts @@ -21,7 +21,7 @@ type TSecretImportSecrets = { const LEVEL_BREAK = 10; const getImportUniqKey = (envSlug: string, path: string) => `${envSlug}=${path}`; export const fnSecretsFromImports = async ({ - allowedImports: nonCyclicImports, + allowedImports: possibleCyclicImports, folderDAL, secretDAL, secretImportDAL, @@ -40,7 +40,7 @@ export const fnSecretsFromImports = async ({ // avoid going more than a depth if (depth >= LEVEL_BREAK) return []; - const allowedImports = nonCyclicImports.filter( + const allowedImports = possibleCyclicImports.filter( ({ importPath, importEnv }) => !cyclicDetector.has(getImportUniqKey(importEnv.slug, importPath)) );