Merge branch 'substitute_envs' into main

cli/packages/cmd/run.go

@@ -33,6 +33,13 @@ var runCmd = &cobra.Command{
 			return
 		}
 
+		substitute, err := cmd.Flags().GetBool("substitute")
+		if err != nil {
+			log.Errorln("Unable to parse the substitute flag")
+			log.Debugln(err)
+			return
+		}
+
 		projectId, err := cmd.Flags().GetString("projectId")
 		if err != nil {
 			log.Errorln("Unable to parse the project id flag")
@@ -82,7 +89,13 @@ var runCmd = &cobra.Command{
 			}
 		}
 
-		execCmd(args[0], args[1:], envsFromApi)
+		if substitute {
+			substitutions := util.SubstituteSecrets(envsFromApi)
+			execCmd(args[0], args[1:], substitutions)
+		} else {
+			execCmd(args[0], args[1:], envsFromApi)
+		}
+
 	},
 }
 
@@ -90,6 +103,7 @@ func init() {
 	rootCmd.AddCommand(runCmd)
 	runCmd.Flags().StringP("env", "e", "dev", "Set the environment (dev, prod, etc.) from which your secrets should be pulled from")
 	runCmd.Flags().String("projectId", "", "The project ID from which your secrets should be pulled from")
+	runCmd.Flags().Bool("substitute", true, "Parse shell variable substitutions in your secrets")
 }
 
 // Credit: inspired by AWS Valut

cli/packages/models/error.go

@@ -0,0 +1,14 @@
+package models
+
+import log "github.com/sirupsen/logrus"
+
+// Custom error type so that we can give helpful messages in CLI
+type Error struct {
+	Err             error
+	DebugMessage    string
+	FriendlyMessage string
+}
+
+func (e *Error) printFriendlyMessage() {
+	log.Infoln(e.FriendlyMessage)
+}

cli/packages/util/secrets.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"regexp"
 	"strings"
 
 	"github.com/Infisical/infisical-merge/packages/models"
@@ -205,3 +206,73 @@ func GetWorkSpacesFromAPI(userCreds models.UserCredentials) (workspaces []models
 
 	return getWorkSpacesResponse.Workspaces, nil
 }
+
+func getExpandedEnvVariable(secrets []models.SingleEnvironmentVariable, variableWeAreLookingFor string, hashMapOfCompleteVariables map[string]string, hashMapOfSelfRefs map[string]string) string {
+	if value, found := hashMapOfCompleteVariables[variableWeAreLookingFor]; found {
+		return value
+	}
+
+	for _, secret := range secrets {
+		if secret.Key == variableWeAreLookingFor {
+			regex := regexp.MustCompile(`\${([^\}]*)}`)
+			variablesToPopulate := regex.FindAllString(secret.Value, -1)
+
+			// case: variable is a constant so return its value
+			if len(variablesToPopulate) == 0 {
+				return secret.Value
+			}
+
+			valueToEdit := secret.Value
+			for _, variableWithSign := range variablesToPopulate {
+				variableWithoutSign := strings.Trim(variableWithSign, "}")
+				variableWithoutSign = strings.Trim(variableWithoutSign, "${")
+
+				// case: reference to self
+				if variableWithoutSign == secret.Key {
+					hashMapOfSelfRefs[variableWithoutSign] = variableWithoutSign
+					continue
+				} else {
+					var expandedVariableValue string
+
+					if preComputedVariable, found := hashMapOfCompleteVariables[variableWithoutSign]; found {
+						expandedVariableValue = preComputedVariable
+					} else {
+						expandedVariableValue = getExpandedEnvVariable(secrets, variableWithoutSign, hashMapOfCompleteVariables, hashMapOfSelfRefs)
+						hashMapOfCompleteVariables[variableWithoutSign] = expandedVariableValue
+					}
+
+					// If after expanding all the vars above, is the current var a self ref? if so no replacement needed for it
+					if _, found := hashMapOfSelfRefs[variableWithoutSign]; found {
+						continue
+					} else {
+						valueToEdit = strings.ReplaceAll(valueToEdit, variableWithSign, expandedVariableValue)
+					}
+				}
+			}
+
+			return valueToEdit
+
+		} else {
+			continue
+		}
+	}
+
+	return "${" + variableWeAreLookingFor + "}"
+}
+
+func SubstituteSecrets(secrets []models.SingleEnvironmentVariable) []models.SingleEnvironmentVariable {
+	hashMapOfCompleteVariables := make(map[string]string)
+	hashMapOfSelfRefs := make(map[string]string)
+	expandedSecrets := []models.SingleEnvironmentVariable{}
+
+	for _, secret := range secrets {
+		expandedVariable := getExpandedEnvVariable(secrets, secret.Key, hashMapOfCompleteVariables, hashMapOfSelfRefs)
+		expandedSecrets = append(expandedSecrets, models.SingleEnvironmentVariable{
+			Key:   secret.Key,
+			Value: expandedVariable,
+		})
+
+	}
+
+	return expandedSecrets
+}

cli/packages/util/secrets_test.go

@@ -0,0 +1,160 @@
+package util
+
+import (
+	"testing"
+
+	"github.com/Infisical/infisical-merge/packages/models"
+)
+
+// References to self should return the value unaltered
+func Test_SubstituteSecrets_When_ReferenceToSelf(t *testing.T) {
+
+	var tests = []struct {
+		Key           string
+		Value         string
+		ExpectedValue string
+	}{
+		{Key: "A", Value: "${A}", ExpectedValue: "${A}"},
+		{Key: "A", Value: "${A} ${A}", ExpectedValue: "${A} ${A}"},
+		{Key: "A", Value: "${A}${A}", ExpectedValue: "${A}${A}"},
+	}
+
+	for _, test := range tests {
+		secret := models.SingleEnvironmentVariable{
+			Key:   test.Key,
+			Value: test.Value,
+		}
+
+		secrets := []models.SingleEnvironmentVariable{secret}
+		result := SubstituteSecrets(secrets)
+
+		if result[0].Value != test.ExpectedValue {
+			t.Errorf("Test_SubstituteSecrets_When_ReferenceToSelf: expected %s but got %s for input %s", test.ExpectedValue, result[0].Value, test.Value)
+		}
+
+	}
+}
+
+func Test_SubstituteSecrets_When_ReferenceDoesNotExist(t *testing.T) {
+
+	var tests = []struct {
+		Key           string
+		Value         string
+		ExpectedValue string
+	}{
+		{Key: "A", Value: "${X}", ExpectedValue: "${X}"},
+		{Key: "A", Value: "${H}HELLO", ExpectedValue: "${H}HELLO"},
+		{Key: "A", Value: "${L}${S}", ExpectedValue: "${L}${S}"},
+	}
+
+	for _, test := range tests {
+		secret := models.SingleEnvironmentVariable{
+			Key:   test.Key,
+			Value: test.Value,
+		}
+
+		secrets := []models.SingleEnvironmentVariable{secret}
+		result := SubstituteSecrets(secrets)
+
+		if result[0].Value != test.ExpectedValue {
+			t.Errorf("Test_SubstituteSecrets_When_ReferenceToSelf: expected %s but got %s for input %s", test.ExpectedValue, result[0].Value, test.Value)
+		}
+
+	}
+}
+
+func Test_SubstituteSecrets_When_ReferenceDoesNotExist_And_Self_Referencing(t *testing.T) {
+
+	tests := []struct {
+		Key           string
+		Value         string
+		ExpectedValue string
+	}{
+		{
+			Key:           "O",
+			Value:         "${P} ==$$ ${X} ${UNKNOWN} ${A}",
+			ExpectedValue: "DOMAIN === ${A} DOMAIN >>> ==$$ DOMAIN ${UNKNOWN} ${A}",
+		},
+		{
+			Key:           "X",
+			Value:         "DOMAIN",
+			ExpectedValue: "DOMAIN",
+		},
+		{
+			Key:           "A",
+			Value:         "*${A}* ${X}",
+			ExpectedValue: "*${A}* DOMAIN",
+		},
+		{
+			Key:           "H",
+			Value:         "${X} >>>",
+			ExpectedValue: "DOMAIN >>>",
+		},
+		{
+			Key:           "P",
+			Value:         "DOMAIN === ${A} ${H}",
+			ExpectedValue: "DOMAIN === ${A} DOMAIN >>>",
+		},
+		{
+			Key:           "T",
+			Value:         "${P} ==$$ ${X} ${UNKNOWN} ${A} ${P} ==$$ ${X} ${UNKNOWN} ${A}",
+			ExpectedValue: "DOMAIN === ${A} DOMAIN >>> ==$$ DOMAIN ${UNKNOWN} ${A} DOMAIN === ${A} DOMAIN >>> ==$$ DOMAIN ${UNKNOWN} ${A}",
+		},
+		{
+			Key:           "S",
+			Value:         "${ SSS$$ ${HEY}",
+			ExpectedValue: "${ SSS$$ ${HEY}",
+		},
+	}
+
+	secrets := []models.SingleEnvironmentVariable{}
+	for _, test := range tests {
+		secrets = append(secrets, models.SingleEnvironmentVariable{Key: test.Key, Value: test.Value})
+	}
+
+	results := SubstituteSecrets(secrets)
+
+	for index, expanded := range results {
+		if expanded.Value != tests[index].ExpectedValue {
+			t.Errorf("Test_SubstituteSecrets_When_ReferenceToSelf: expected [%s] but got [%s] for input [%s]", tests[index].ExpectedValue, expanded.Value, tests[index].Value)
+		}
+	}
+}
+
+func Test_SubstituteSecrets_When_No_SubstituteNeeded(t *testing.T) {
+
+	tests := []struct {
+		Key           string
+		Value         string
+		ExpectedValue string
+	}{
+		{
+			Key:           "DOMAIN",
+			Value:         "infisical.com",
+			ExpectedValue: "infisical.com",
+		},
+		{
+			Key:           "API_KEY",
+			Value:         "hdgsvjshcgkdckhevdkd",
+			ExpectedValue: "hdgsvjshcgkdckhevdkd",
+		},
+		{
+			Key:           "ENV",
+			Value:         "PROD",
+			ExpectedValue: "PROD",
+		},
+	}
+
+	secrets := []models.SingleEnvironmentVariable{}
+	for _, test := range tests {
+		secrets = append(secrets, models.SingleEnvironmentVariable{Key: test.Key, Value: test.Value})
+	}
+
+	results := SubstituteSecrets(secrets)
+
+	for index, expanded := range results {
+		if expanded.Value != tests[index].ExpectedValue {
+			t.Errorf("Test_SubstituteSecrets_When_ReferenceToSelf: expected [%s] but got [%s] for input [%s]", tests[index].ExpectedValue, expanded.Value, tests[index].Value)
+		}
+	}
+}