Merge pull request #4440 from Infisical/daniel/vault-migration

feat(vault-migration): gateway support & kv v1 support
2026-01-10 07:58:15 -05:00 · 2025-08-30 01:02:46 +02:00
parent 82faf3a797 ece0af7787
commit acf3bdc5a3
8 changed files with 294 additions and 98 deletions
--- a/backend/src/server/routes/index.ts
+++ b/backend/src/server/routes/index.ts
@@ -1746,7 +1746,8 @@ export const registerRoutes = async (
  const migrationService = externalMigrationServiceFactory({
    externalMigrationQueue,
    userDAL,
-    permissionService
+    permissionService,
+    gatewayService
  });

  const externalGroupOrgRoleMappingService = externalGroupOrgRoleMappingServiceFactory({
--- a/backend/src/server/routes/v3/external-migration-router.ts
+++ b/backend/src/server/routes/v3/external-migration-router.ts
@@ -66,7 +66,8 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider
        vaultAccessToken: z.string(),
        vaultNamespace: z.string().trim().optional(),
        vaultUrl: z.string(),
-        mappingType: z.nativeEnum(VaultMappingType)
+        mappingType: z.nativeEnum(VaultMappingType),
+        gatewayId: z.string().optional()
      })
    },
    onRequest: verifyAuth([AuthMode.JWT]),
--- a/backend/src/services/external-migration/external-migration-fns/vault.ts
+++ b/backend/src/services/external-migration/external-migration-fns/vault.ts
@@ -1,12 +1,21 @@
+import https from "node:https";
+
 import axios, { AxiosInstance } from "axios";
 import { v4 as uuidv4 } from "uuid";

+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { BadRequestError } from "@app/lib/errors";
+import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
 import { logger } from "@app/lib/logger";
 import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";

 import { InfisicalImportData, VaultMappingType } from "../external-migration-types";

+enum KvVersion {
+  V1 = "1",
+  V2 = "2"
+}
+
 type VaultData = {
  namespace: string;
  mount: string;
@@ -14,7 +23,42 @@ type VaultData = {
  secretData: Record<string, string>;
 };

-const vaultFactory = () => {
+const vaultFactory = (gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">) => {
+  const $gatewayProxyWrapper = async <T>(
+    inputs: {
+      gatewayId: string;
+      targetHost?: string;
+      targetPort?: number;
+    },
+    gatewayCallback: (host: string, port: number, httpsAgent?: https.Agent) => Promise<T>
+  ): Promise<T> => {
+    const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(inputs.gatewayId);
+    const [relayHost, relayPort] = relayDetails.relayAddress.split(":");
+
+    const callbackResult = await withGatewayProxy(
+      async (port, httpsAgent) => {
+        const res = await gatewayCallback("http://localhost", port, httpsAgent);
+        return res;
+      },
+      {
+        protocol: GatewayProxyProtocol.Http,
+        targetHost: inputs.targetHost,
+        targetPort: inputs.targetPort,
+        relayHost,
+        relayPort: Number(relayPort),
+        identityId: relayDetails.identityId,
+        orgId: relayDetails.orgId,
+        tlsOptions: {
+          ca: relayDetails.certChain,
+          cert: relayDetails.certificate,
+          key: relayDetails.privateKey.toString()
+        }
+      }
+    );
+
+    return callbackResult;
+  };
+
  const getMounts = async (request: AxiosInstance) => {
    const response = await request
      .get<{
@@ -31,11 +75,24 @@ const vaultFactory = () => {

  const getPaths = async (
    request: AxiosInstance,
-    { mountPath, secretPath = "" }: { mountPath: string; secretPath?: string }
+    { mountPath, secretPath = "" }: { mountPath: string; secretPath?: string },
+    kvVersion: KvVersion
  ) => {
    try {
-      // For KV v2: /v1/{mount}/metadata/{path}?list=true
-      const path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`;
+      if (kvVersion === KvVersion.V2) {
+        // For KV v2: /v1/{mount}/metadata/{path}?list=true
+        const path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`;
+        const response = await request.get<{
+          data: {
+            keys: string[];
+          };
+        }>(`/v1/${path}?list=true`);
+
+        return response.data.data.keys;
+      }
+
+      // kv version v1: /v1/{mount}?list=true
+      const path = secretPath ? `${mountPath}/${secretPath}` : mountPath;
      const response = await request.get<{
        data: {
          keys: string[];
@@ -56,21 +113,42 @@ const vaultFactory = () => {

  const getSecrets = async (
    request: AxiosInstance,
-    { mountPath, secretPath }: { mountPath: string; secretPath: string }
+    { mountPath, secretPath }: { mountPath: string; secretPath: string },
+    kvVersion: KvVersion
  ) => {
-    // For KV v2: /v1/{mount}/data/{path}
+    if (kvVersion === KvVersion.V2) {
+      // For KV v2: /v1/{mount}/data/{path}
+      const response = await request
+        .get<{
+          data: {
+            data: Record<string, string>; // KV v2 has nested data structure
+            metadata: {
+              created_time: string;
+              deletion_time: string;
+              destroyed: boolean;
+              version: number;
+            };
+          };
+        }>(`/v1/${mountPath}/data/${secretPath}`)
+        .catch((err) => {
+          if (axios.isAxiosError(err)) {
+            logger.error(err.response?.data, "External migration: Failed to get Vault secret");
+          }
+          throw err;
+        });
+
+      return response.data.data.data;
+    }
+
+    // kv version v1
+
    const response = await request
      .get<{
-        data: {
-          data: Record<string, string>; // KV v2 has nested data structure
-          metadata: {
-            created_time: string;
-            deletion_time: string;
-            destroyed: boolean;
-            version: number;
-          };
-        };
-      }>(`/v1/${mountPath}/data/${secretPath}`)
+        data: Record<string, string>; // KV v1 has flat data structure
+        lease_duration: number;
+        lease_id: string;
+        renewable: boolean;
+      }>(`/v1/${mountPath}/${secretPath}`)
      .catch((err) => {
        if (axios.isAxiosError(err)) {
          logger.error(err.response?.data, "External migration: Failed to get Vault secret");
@@ -78,7 +156,7 @@ const vaultFactory = () => {
        throw err;
      });

-    return response.data.data.data;
+    return response.data.data;
  };

  // helper function to check if a mount is KV v2 (will be useful if we add support for Vault KV v1)
@@ -89,9 +167,10 @@ const vaultFactory = () => {
  const recursivelyGetAllPaths = async (
    request: AxiosInstance,
    mountPath: string,
+    kvVersion: KvVersion,
    currentPath: string = ""
  ): Promise<string[]> => {
-    const paths = await getPaths(request, { mountPath, secretPath: currentPath });
+    const paths = await getPaths(request, { mountPath, secretPath: currentPath }, kvVersion);

    if (paths === null || paths.length === 0) {
      return [];
@@ -105,7 +184,7 @@ const vaultFactory = () => {

      if (path.endsWith("/")) {
        // it's a folder so we recurse into it
-        const subSecrets = await recursivelyGetAllPaths(request, mountPath, fullItemPath);
+        const subSecrets = await recursivelyGetAllPaths(request, mountPath, kvVersion, fullItemPath);
        allSecrets.push(...subSecrets);
      } else {
        // it's a secret so we add it to our results
@@ -119,60 +198,93 @@ const vaultFactory = () => {
  async function collectVaultData({
    baseUrl,
    namespace,
-    accessToken
+    accessToken,
+    gatewayId
  }: {
    baseUrl: string;
    namespace?: string;
    accessToken: string;
+    gatewayId?: string;
  }): Promise<VaultData[]> {
-    const request = axios.create({
-      baseURL: baseUrl,
-      headers: {
-        "X-Vault-Token": accessToken,
-        ...(namespace ? { "X-Vault-Namespace": namespace } : {})
+    const getData = async (host: string, port?: number, httpsAgent?: https.Agent) => {
+      const allData: VaultData[] = [];
+
+      const request = axios.create({
+        baseURL: port ? `${host}:${port}` : host,
+        headers: {
+          "X-Vault-Token": accessToken,
+          ...(namespace ? { "X-Vault-Namespace": namespace } : {})
+        },
+        httpsAgent
+      });
+
+      // Get all mounts in this namespace
+      const mounts = await getMounts(request);
+
+      for (const mount of Object.keys(mounts)) {
+        if (!mount.endsWith("/")) {
+          delete mounts[mount];
+        }
      }
-    });

-    const allData: VaultData[] = [];
+      for await (const [mountPath, mountInfo] of Object.entries(mounts)) {
+        // skip non-KV mounts
+        if (!mountInfo.type.startsWith("kv")) {
+          // eslint-disable-next-line no-continue
+          continue;
+        }

-    // Get all mounts in this namespace
-    const mounts = await getMounts(request);
+        const kvVersion = mountInfo.options?.version === "2" ? KvVersion.V2 : KvVersion.V1;

-    for (const mount of Object.keys(mounts)) {
-      if (!mount.endsWith("/")) {
-        delete mounts[mount];
+        // get all paths in this mount
+        const paths = await recursivelyGetAllPaths(request, `${mountPath.replace(/\/$/, "")}`, kvVersion);
+
+        const cleanMountPath = mountPath.replace(/\/$/, "");
+
+        for await (const secretPath of paths) {
+          // get the actual secret data
+          const secretData = await getSecrets(
+            request,
+            {
+              mountPath: cleanMountPath,
+              secretPath: secretPath.replace(`${cleanMountPath}/`, "")
+            },
+            kvVersion
+          );
+
+          allData.push({
+            namespace: namespace || "",
+            mount: mountPath.replace(/\/$/, ""),
+            path: secretPath.replace(`${cleanMountPath}/`, ""),
+            secretData
+          });
+        }
      }
+
+      return allData;
+    };
+
+    let data;
+
+    if (gatewayId) {
+      const url = new URL(baseUrl);
+
+      const { port, protocol, hostname } = url;
+      const cleanedProtocol = protocol.slice(0, -1);
+
+      data = await $gatewayProxyWrapper(
+        {
+          gatewayId,
+          targetHost: `${cleanedProtocol}://${hostname}`,
+          targetPort: port ? Number(port) : 8200 // 8200, default port for Vault self-hosted/dedicated
+        },
+        getData
+      );
+    } else {
+      data = await getData(baseUrl);
    }

-    for await (const [mountPath, mountInfo] of Object.entries(mounts)) {
-      // skip non-KV mounts
-      if (!mountInfo.type.startsWith("kv")) {
-        // eslint-disable-next-line no-continue
-        continue;
-      }
-
-      // get all paths in this mount
-      const paths = await recursivelyGetAllPaths(request, `${mountPath.replace(/\/$/, "")}`);
-
-      const cleanMountPath = mountPath.replace(/\/$/, "");
-
-      for await (const secretPath of paths) {
-        // get the actual secret data
-        const secretData = await getSecrets(request, {
-          mountPath: cleanMountPath,
-          secretPath: secretPath.replace(`${cleanMountPath}/`, "")
-        });
-
-        allData.push({
-          namespace: namespace || "",
-          mount: mountPath.replace(/\/$/, ""),
-          path: secretPath.replace(`${cleanMountPath}/`, ""),
-          secretData
-        });
-      }
-    }
-
-    return allData;
+    return data;
  }

  return {
@@ -296,17 +408,22 @@ export const transformToInfisicalFormatNamespaceToProjects = (
  };
 };

-export const importVaultDataFn = async ({
-  vaultAccessToken,
-  vaultNamespace,
-  vaultUrl,
-  mappingType
-}: {
-  vaultAccessToken: string;
-  vaultNamespace?: string;
-  vaultUrl: string;
-  mappingType: VaultMappingType;
-}) => {
+export const importVaultDataFn = async (
+  {
+    vaultAccessToken,
+    vaultNamespace,
+    vaultUrl,
+    mappingType,
+    gatewayId
+  }: {
+    vaultAccessToken: string;
+    vaultNamespace?: string;
+    vaultUrl: string;
+    mappingType: VaultMappingType;
+    gatewayId?: string;
+  },
+  { gatewayService }: { gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId"> }
+) => {
  await blockLocalAndPrivateIpAddresses(vaultUrl);

  if (mappingType === VaultMappingType.Namespace && !vaultNamespace) {
@@ -315,12 +432,13 @@ export const importVaultDataFn = async ({
    });
  }

-  const vaultApi = vaultFactory();
+  const vaultApi = vaultFactory(gatewayService);

  const vaultData = await vaultApi.collectVaultData({
    accessToken: vaultAccessToken,
    baseUrl: vaultUrl,
-    namespace: vaultNamespace
+    namespace: vaultNamespace,
+    gatewayId
  });

  const infisicalData = transformToInfisicalFormatNamespaceToProjects(vaultData, mappingType);
--- a/backend/src/services/external-migration/external-migration-service.ts
+++ b/backend/src/services/external-migration/external-migration-service.ts
@@ -1,4 +1,5 @@
 import { OrgMembershipRole } from "@app/db/schemas";
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
 import { crypto } from "@app/lib/crypto/cryptography";
 import { BadRequestError, ForbiddenRequestError } from "@app/lib/errors";
@@ -12,6 +13,7 @@ type TExternalMigrationServiceFactoryDep = {
  permissionService: TPermissionServiceFactory;
  externalMigrationQueue: TExternalMigrationQueueFactory;
  userDAL: Pick<TUserDALFactory, "findById">;
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
 };

 export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrationServiceFactory>;
@@ -19,7 +21,8 @@ export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrati
 export const externalMigrationServiceFactory = ({
  permissionService,
  externalMigrationQueue,
-  userDAL
+  userDAL,
+  gatewayService
 }: TExternalMigrationServiceFactoryDep) => {
  const importEnvKeyData = async ({
    decryptionKey,
@@ -72,6 +75,7 @@ export const externalMigrationServiceFactory = ({
    vaultNamespace,
    mappingType,
    vaultUrl,
+    gatewayId,
    actor,
    actorId,
    actorOrgId,
@@ -91,12 +95,18 @@ export const externalMigrationServiceFactory = ({

    const user = await userDAL.findById(actorId);

-    const vaultData = await importVaultDataFn({
-      vaultAccessToken,
-      vaultNamespace,
-      vaultUrl,
-      mappingType
-    });
+    const vaultData = await importVaultDataFn(
+      {
+        vaultAccessToken,
+        vaultNamespace,
+        vaultUrl,
+        mappingType,
+        gatewayId
+      },
+      {
+        gatewayService
+      }
+    );

    const stringifiedJson = JSON.stringify({
      data: vaultData,
--- a/backend/src/services/external-migration/external-migration-types.ts
+++ b/backend/src/services/external-migration/external-migration-types.ts
@@ -31,6 +31,7 @@ export type TImportVaultDataDTO = {
  vaultNamespace?: string;
  mappingType: VaultMappingType;
  vaultUrl: string;
+  gatewayId?: string;
 } & Omit<TOrgPermission, "orgId">;

 export type TImportInfisicalDataCreate = {
--- a/docs/documentation/platform/external-migrations/vault.mdx
+++ b/docs/documentation/platform/external-migrations/vault.mdx
@@ -8,12 +8,12 @@ description: "Learn how to migrate secrets from Vault to Infisical."

 Migrating from Vault Self-Hosted or Dedicated Vault is a straight forward process with our inbuilt migration option. In order to migrate from Vault, you'll need to provide Infisical an access token to your Vault instance.

-Currently the Vault migration only supports migrating secrets from the KV v2 secrets engine. If you're using a different secrets engine, please open an issue on our [GitHub repository](https://github.com/infisical/infisical/issues).
+Currently the Vault migration only supports migrating secrets from the KV V2 and V1 secrets engine. If you're using a different secrets engine, please open an issue on our [GitHub repository](https://github.com/infisical/infisical/issues).


 ### Prerequisites

- A Vault instance with the KV v2 secrets engine enabled.
+- A Vault instance with the KV secret engine enabled.
 - An access token to your Vault instance.


--- a/frontend/src/hooks/api/migration/mutations.tsx
+++ b/frontend/src/hooks/api/migration/mutations.tsx
@@ -46,18 +46,21 @@ export const useImportVault = () => {
      vaultAccessToken,
      vaultNamespace,
      vaultUrl,
-      mappingType
+      mappingType,
+      gatewayId
    }: {
      vaultAccessToken: string;
      vaultNamespace?: string;
      vaultUrl: string;
      mappingType: string;
+      gatewayId?: string;
    }) => {
      await apiRequest.post("/api/v3/external-migration/vault/", {
        vaultAccessToken,
        vaultNamespace,
        vaultUrl,
-        mappingType
+        mappingType,
+        gatewayId
      });
    }
  });
--- a/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/VaultPlatformModal.tsx
+++ b/frontend/src/pages/organization/SettingsPage/components/ExternalMigrationsTab/components/VaultPlatformModal.tsx
@@ -6,9 +6,16 @@ import { twMerge } from "tailwind-merge";
 import { z } from "zod";

 import { createNotification } from "@app/components/notifications";
-import { Button, FormControl, Input, Tooltip } from "@app/components/v2";
+import { Button, FormControl, Input, Select, SelectItem, Tooltip } from "@app/components/v2";
 import { NoticeBannerV2 } from "@app/components/v2/NoticeBannerV2/NoticeBannerV2";
 import { useImportVault } from "@app/hooks/api/migration/mutations";
+import { OrgPermissionCan } from "@app/components/permissions";
+import {
+  OrgGatewayPermissionActions,
+  OrgPermissionSubjects
+} from "@app/context/OrgPermissionContext/types";
+import { useQuery } from "@tanstack/react-query";
+import { gatewaysQueryKeys } from "@app/hooks/api";

 type Props = {
  id?: string;
@@ -62,36 +69,32 @@ const MAPPING_TYPE_MENU_ITEMS = [
 export const VaultPlatformModal = ({ onClose }: Props) => {
  const formSchema = z.object({
    vaultUrl: z.string().min(1),
+    gatewayId: z.string().optional(),
    vaultNamespace: z.string().trim().optional(),
    vaultAccessToken: z.string().min(1),
    mappingType: z.nativeEnum(VaultMappingType).default(VaultMappingType.KeyVault)
  });
  type TFormData = z.infer<typeof formSchema>;

+  const { data: gateways, isPending: isGatewayLoading } = useQuery(gatewaysQueryKeys.list());
  const { mutateAsync: importVault } = useImportVault();

  const {
    control,
    handleSubmit,
    reset,
-    formState: { isLoading, isDirty, isSubmitting, isValid, errors }
+    formState: { isLoading, isDirty, isSubmitting, isValid }
  } = useForm<TFormData>({
    resolver: zodResolver(formSchema)
  });

-  console.log({
-    isSubmitting,
-    isLoading,
-    isValid,
-    errors
-  });
-
  const onSubmit = async (data: TFormData) => {
    await importVault({
      vaultAccessToken: data.vaultAccessToken,
      vaultNamespace: data.vaultNamespace,
      vaultUrl: data.vaultUrl,
-      mappingType: data.mappingType
+      mappingType: data.mappingType,
+      ...(data.gatewayId && { gatewayId: data.gatewayId })
    });
    createNotification({
      title: "Import started",
@@ -110,11 +113,70 @@ export const VaultPlatformModal = ({ onClose }: Props) => {
          The Vault migration currently supports importing static secrets from Vault
          Dedicated/Self-Hosted.
          <div className="mt-2 text-xs opacity-80">
-            Currently only KV Secret Engine V2 is supported for Vault migrations.
+            Currently only KV Secret Engine is supported for Vault migrations.
          </div>
        </p>
      </NoticeBannerV2>
      <form onSubmit={handleSubmit(onSubmit)} autoComplete="off">
+        <div className="w-full flex-1">
+          <OrgPermissionCan
+            I={OrgGatewayPermissionActions.AttachGateways}
+            a={OrgPermissionSubjects.Gateway}
+          >
+            {(isAllowed) => (
+              <Controller
+                control={control}
+                name="gatewayId"
+                defaultValue=""
+                render={({ field: { value, onChange }, fieldState: { error } }) => (
+                  <FormControl
+                    isError={Boolean(error?.message)}
+                    errorText={error?.message}
+                    label="Gateway"
+                    isOptional
+                  >
+                    <Tooltip
+                      isDisabled={isAllowed}
+                      content="Restricted access. You don't have permission to attach gateways to resources."
+                    >
+                      <div>
+                        <Select
+                          isDisabled={!isAllowed}
+                          value={value as string}
+                          onValueChange={(v) => {
+                            if (v !== "") {
+                              onChange(v);
+                            }
+                          }}
+                          className="w-full border border-mineshaft-500"
+                          dropdownContainerClassName="max-w-none"
+                          isLoading={isGatewayLoading}
+                          placeholder="Default: Internet Gateway"
+                          position="popper"
+                        >
+                          <SelectItem
+                            value=""
+                            onClick={() => {
+                              onChange(undefined);
+                            }}
+                          >
+                            Internet Gateway
+                          </SelectItem>
+                          {gateways?.map((el) => (
+                            <SelectItem value={el.id} key={el.id}>
+                              {el.name}
+                            </SelectItem>
+                          ))}
+                        </Select>
+                      </div>
+                    </Tooltip>
+                  </FormControl>
+                )}
+              />
+            )}
+          </OrgPermissionCan>
+        </div>
+
        <Controller
          control={control}
          name="vaultUrl"