From 104a91647c3814c13cec8049970b59fe70753b6e Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Mon, 29 Apr 2024 21:18:13 +0800 Subject: [PATCH 01/91] fix: resolved cli offline mode get --- cli/packages/util/secrets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/packages/util/secrets.go b/cli/packages/util/secrets.go index 27f0636a90..c3690b83db 100644 --- a/cli/packages/util/secrets.go +++ b/cli/packages/util/secrets.go @@ -332,7 +332,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo return nil, err } - if loggedInUserDetails.LoginExpired { + if isConnected && loggedInUserDetails.LoginExpired { PrintErrorMessageAndExit("Your login session has expired, please run [infisical login] and try again") } From 877b9a409e8fcd765a039d8df3690e8240e44279 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 30 Apr 2024 21:00:34 +0800 Subject: [PATCH 02/91] adjustment: modified isConnected check to query linked infisical URL --- cli/packages/util/common.go | 6 ++++-- cli/packages/util/secrets.go | 23 ++++++++++++----------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/cli/packages/util/common.go b/cli/packages/util/common.go index 2b57383eff..42ee74c77a 100644 --- a/cli/packages/util/common.go +++ b/cli/packages/util/common.go @@ -4,6 +4,8 @@ import ( "fmt" "net/http" "os" + + "github.com/Infisical/infisical-merge/packages/config" ) func GetHomeDir() (string, error) { @@ -21,7 +23,7 @@ func WriteToFile(fileName string, dataToWrite []byte, filePerm os.FileMode) erro return nil } -func CheckIsConnectedToInternet() (ok bool) { - _, err := http.Get("http://clients3.google.com/generate_204") +func CheckIsConnectedToInfisicalAPI() (ok bool) { + _, err := http.Get(fmt.Sprintf("%v/status", config.INFISICAL_URL)) return err == nil } diff --git a/cli/packages/util/secrets.go b/cli/packages/util/secrets.go index c3690b83db..0ca66cc4fb 100644 --- a/cli/packages/util/secrets.go +++ b/cli/packages/util/secrets.go @@ -307,27 +307,28 @@ func FilterSecretsByTag(plainTextSecrets []models.SingleEnvironmentVariable, tag } func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectConfigFilePath string) ([]models.SingleEnvironmentVariable, error) { - isConnected := CheckIsConnectedToInternet() var secretsToReturn []models.SingleEnvironmentVariable // var serviceTokenDetails api.GetServiceTokenDetailsResponse var errorToReturn error if params.InfisicalToken == "" && params.UniversalAuthAccessToken == "" { - if isConnected { - log.Debug().Msg("GetAllEnvironmentVariables: Connected to internet, checking logged in creds") - - if projectConfigFilePath == "" { - RequireLocalWorkspaceFile() - } else { - ValidateWorkspaceFile(projectConfigFilePath) - } - - RequireLogin() + if projectConfigFilePath == "" { + RequireLocalWorkspaceFile() + } else { + ValidateWorkspaceFile(projectConfigFilePath) } + RequireLogin() + log.Debug().Msg("GetAllEnvironmentVariables: Trying to fetch secrets using logged in details") loggedInUserDetails, err := GetCurrentLoggedInUserDetails() + isConnected := CheckIsConnectedToInfisicalAPI() + + if isConnected { + log.Debug().Msg("GetAllEnvironmentVariables: Connected to Infisical instance, checking logged in creds") + } + if err != nil { return nil, err } From 772dd464f564f00eda605d765e81e4d082dc33cc Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 30 Apr 2024 21:11:29 +0800 Subject: [PATCH 03/91] test: added integration test for secrets get all and secrets get all without connection --- .../test-TestUserAuth_SecretsGetAll | 7 +++ ...estUserAuth_SecretsGetAllWithoutConnection | 8 ++++ cli/test/secrets_test.go | 44 +++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 cli/test/.snapshots/test-TestUserAuth_SecretsGetAll create mode 100644 cli/test/.snapshots/test-TestUserAuth_SecretsGetAllWithoutConnection diff --git a/cli/test/.snapshots/test-TestUserAuth_SecretsGetAll b/cli/test/.snapshots/test-TestUserAuth_SecretsGetAll new file mode 100644 index 0000000000..260607e976 --- /dev/null +++ b/cli/test/.snapshots/test-TestUserAuth_SecretsGetAll @@ -0,0 +1,7 @@ +┌───────────────┬──────────────┬─────────────┐ +│ SECRET NAME │ SECRET VALUE │ SECRET TYPE │ +├───────────────┼──────────────┼─────────────┤ +│ TEST-SECRET-1 │ test-value-1 │ shared │ +│ TEST-SECRET-2 │ test-value-2 │ shared │ +│ TEST-SECRET-3 │ test-value-3 │ shared │ +└───────────────┴──────────────┴─────────────┘ diff --git a/cli/test/.snapshots/test-TestUserAuth_SecretsGetAllWithoutConnection b/cli/test/.snapshots/test-TestUserAuth_SecretsGetAllWithoutConnection new file mode 100644 index 0000000000..c48627f73f --- /dev/null +++ b/cli/test/.snapshots/test-TestUserAuth_SecretsGetAllWithoutConnection @@ -0,0 +1,8 @@ +Warning: Unable to fetch latest secret(s) due to connection error, serving secrets from last successful fetch. For more info, run with --debug +┌───────────────┬──────────────┬─────────────┐ +│ SECRET NAME │ SECRET VALUE │ SECRET TYPE │ +├───────────────┼──────────────┼─────────────┤ +│ TEST-SECRET-1 │ test-value-1 │ shared │ +│ TEST-SECRET-2 │ test-value-2 │ shared │ +│ TEST-SECRET-3 │ test-value-3 │ shared │ +└───────────────┴──────────────┴─────────────┘ diff --git a/cli/test/secrets_test.go b/cli/test/secrets_test.go index 453666406d..b9c8fa85d9 100644 --- a/cli/test/secrets_test.go +++ b/cli/test/secrets_test.go @@ -3,9 +3,11 @@ package tests import ( "testing" + "github.com/Infisical/infisical-merge/packages/util" "github.com/bradleyjkemp/cupaloy/v2" ) + func TestServiceToken_SecretsGetWithImportsAndRecursiveCmd(t *testing.T) { SetupCli(t) @@ -85,3 +87,45 @@ func TestUniversalAuth_SecretsGetWrongEnvironment(t *testing.T) { } } + +func TestUserAuth_SecretsGetAll(t *testing.T) { + SetupCli(t) + output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "secrets", "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--include-imports=false", "--silent") + if err != nil { + t.Fatalf("error running CLI command: %v", err) + } + + // Use cupaloy to snapshot test the output + err = cupaloy.Snapshot(output) + if err != nil { + t.Fatalf("snapshot failed: %v", err) + } +} + +func TestUserAuth_SecretsGetAllWithoutConnection(t *testing.T) { + SetupCli(t) + + originalConfigFile, err := util.GetConfigFile() + if err != nil { + t.Fatalf("error getting config file") + } + newConfigFile := originalConfigFile + + // set it to a URL that will always be unreachable + newConfigFile.LoggedInUserDomain = "http://localhost:4999" + util.WriteConfigFile(&newConfigFile) + + // restore config file + defer util.WriteConfigFile(&originalConfigFile) + + output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "secrets", "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--include-imports=false", "--silent") + if err != nil { + t.Fatalf("error running CLI command: %v", err) + } + + // Use cupaloy to snapshot test the output + err = cupaloy.Snapshot(output) + if err != nil { + t.Fatalf("snapshot failed: %v", err) + } +} \ No newline at end of file From 531fa634a26a47d3e19dcb39a0975dc8d7bda168 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 30 Apr 2024 22:02:22 +0800 Subject: [PATCH 04/91] feature: add logs for cli execution error --- cli/test/helper.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/test/helper.go b/cli/test/helper.go index 995367c4b1..4704733687 100644 --- a/cli/test/helper.go +++ b/cli/test/helper.go @@ -38,6 +38,7 @@ func ExecuteCliCommand(command string, args ...string) (string, error) { cmd := exec.Command(command, args...) output, err := cmd.CombinedOutput() if err != nil { + fmt.Println(fmt.Sprint(err) + ": " + string(output)) return strings.TrimSpace(string(output)), err } return strings.TrimSpace(string(output)), nil From 85f024c814cca3ef2939215cb6e3aa74cfb5a581 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 1 May 2024 01:45:24 +0800 Subject: [PATCH 05/91] test: added scripting for user login --- cli/go.mod | 1 + cli/go.sum | 2 + ...stUserAuth_SecretsGetAllWithoutConnection} | 0 cli/test/helper.go | 4 + cli/test/login_test.go | 78 +++++++++++++++++++ cli/test/secrets_test.go | 9 ++- 6 files changed, 91 insertions(+), 3 deletions(-) rename cli/test/.snapshots/{test-TestUserAuth_SecretsGetAllWithoutConnection => test-testUserAuth_SecretsGetAllWithoutConnection} (100%) diff --git a/cli/go.mod b/cli/go.mod index 833745effc..52c7375332 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -31,6 +31,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/bradleyjkemp/cupaloy/v2 v2.8.0 // indirect github.com/chzyer/readline v1.5.1 // indirect + github.com/creack/pty v1.1.21 // indirect github.com/danieljoos/wincred v1.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dvsekhvalnov/jose2go v1.5.0 // indirect diff --git a/cli/go.sum b/cli/go.sum index 3535791366..ff3030a9cc 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -74,6 +74,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= +github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE= github.com/danieljoos/wincred v1.2.0/go.mod h1:FzQLLMKBFdvu+osBrnFODiv32YGwCfx0SkRa/eYHgec= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/cli/test/.snapshots/test-TestUserAuth_SecretsGetAllWithoutConnection b/cli/test/.snapshots/test-testUserAuth_SecretsGetAllWithoutConnection similarity index 100% rename from cli/test/.snapshots/test-TestUserAuth_SecretsGetAllWithoutConnection rename to cli/test/.snapshots/test-testUserAuth_SecretsGetAllWithoutConnection diff --git a/cli/test/helper.go b/cli/test/helper.go index 4704733687..b773d138e5 100644 --- a/cli/test/helper.go +++ b/cli/test/helper.go @@ -23,6 +23,8 @@ type Credentials struct { ServiceToken string ProjectID string EnvSlug string + UserEmail string + UserPassword string } var creds = Credentials{ @@ -32,6 +34,8 @@ var creds = Credentials{ ServiceToken: os.Getenv("CLI_TESTS_SERVICE_TOKEN"), ProjectID: os.Getenv("CLI_TESTS_PROJECT_ID"), EnvSlug: os.Getenv("CLI_TESTS_ENV_SLUG"), + UserEmail: os.Getenv("CLI_TESTS_USER_EMAIL"), + UserPassword: os.Getenv("CLI_TESTS_USER_PASSWORD"), } func ExecuteCliCommand(command string, args ...string) (string, error) { diff --git a/cli/test/login_test.go b/cli/test/login_test.go index 0f45914132..6857c00fba 100644 --- a/cli/test/login_test.go +++ b/cli/test/login_test.go @@ -1,11 +1,89 @@ package tests import ( + "fmt" + "os/exec" + "strings" "testing" + "github.com/creack/pty" "github.com/stretchr/testify/assert" ) +func UserLoginCmd(t *testing.T) { + SetupCli(t) + + // set vault to file because CI has no keyring + vaultCmd := exec.Command(FORMATTED_CLI_NAME, "vault", "set", "file") + _, err := vaultCmd.Output() + if err != nil { + t.Fatalf("error setting vault: %v", err) + } + + // Start programmatic interaction with CLI + c := exec.Command(FORMATTED_CLI_NAME, "login", "--interactive") + ptmx, err := pty.Start(c) + if err != nil { + t.Fatalf("error running CLI command: %v", err) + } + defer func() { _ = ptmx.Close() }() + + stepChan := make(chan int, 10) + + go func() { + buf := make([]byte, 1024) + step := -1 + for { + n, err := ptmx.Read(buf) + if n > 0 { + terminalOut := string(buf) + if strings.Contains(terminalOut, "Add a new account") && step < 0 { + step += 1 + stepChan <- step + } else if strings.Contains(terminalOut, "Infisical Cloud") && step < 1 { + step += 1; + stepChan <- step + } else if strings.Contains(terminalOut, "Email") && step < 2 { + step += 1; + stepChan <- step + } else if strings.Contains(terminalOut, "Password") && step < 3 { + step += 1; + stepChan <- step + } else if strings.Contains(terminalOut, "Infisical organization") && step < 4 { + step += 1; + stepChan <- step + } else if strings.Contains(terminalOut, "Enter passphrase") && step < 5 { + step += 1; + stepChan <- step + } + } + if err != nil { + close(stepChan) + return + } + fmt.Print(string(buf[:n])) + } + }() + + for i := range stepChan { + switch i { + case 0: + ptmx.Write([]byte("\n")) + case 1: + ptmx.Write([]byte("\n")) + case 2: + ptmx.Write([]byte(creds.UserEmail)) + ptmx.Write([]byte("\n")) + case 3: + ptmx.Write([]byte(creds.UserPassword)) + ptmx.Write([]byte("\n")) + case 4: + ptmx.Write([]byte("\n")) + } + } + +} + func MachineIdentityLoginCmd(t *testing.T) { SetupCli(t) diff --git a/cli/test/secrets_test.go b/cli/test/secrets_test.go index b9c8fa85d9..294434f5f3 100644 --- a/cli/test/secrets_test.go +++ b/cli/test/secrets_test.go @@ -90,6 +90,8 @@ func TestUniversalAuth_SecretsGetWrongEnvironment(t *testing.T) { func TestUserAuth_SecretsGetAll(t *testing.T) { SetupCli(t) + UserLoginCmd(t); + output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "secrets", "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--include-imports=false", "--silent") if err != nil { t.Fatalf("error running CLI command: %v", err) @@ -100,11 +102,12 @@ func TestUserAuth_SecretsGetAll(t *testing.T) { if err != nil { t.Fatalf("snapshot failed: %v", err) } + + // intentionally invoked this here because it should directly follow secretsGetAll + testUserAuth_SecretsGetAllWithoutConnection(t) } -func TestUserAuth_SecretsGetAllWithoutConnection(t *testing.T) { - SetupCli(t) - +func testUserAuth_SecretsGetAllWithoutConnection(t *testing.T) { originalConfigFile, err := util.GetConfigFile() if err != nil { t.Fatalf("error getting config file") From 6640b55504abab40fecec9a3d3c1847508636a79 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 1 May 2024 01:49:06 +0800 Subject: [PATCH 06/91] misc: added envs required for cli test of infisical login --- .github/workflows/run-cli-tests.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-cli-tests.yml b/.github/workflows/run-cli-tests.yml index e814f9143f..f55581a0f9 100644 --- a/.github/workflows/run-cli-tests.yml +++ b/.github/workflows/run-cli-tests.yml @@ -20,7 +20,12 @@ on: required: true CLI_TESTS_ENV_SLUG: required: true - + CLI_TESTS_USER_EMAIL: + required: true + CLI_TESTS_USER_PASSWORD: + required: true + INFISICAL_VAULT_FILE_PASSPHRASE: + required: true jobs: test: defaults: @@ -43,5 +48,8 @@ jobs: CLI_TESTS_SERVICE_TOKEN: ${{ secrets.CLI_TESTS_SERVICE_TOKEN }} CLI_TESTS_PROJECT_ID: ${{ secrets.CLI_TESTS_PROJECT_ID }} CLI_TESTS_ENV_SLUG: ${{ secrets.CLI_TESTS_ENV_SLUG }} + CLI_TESTS_USER_EMAIL: ${{ secrets.CLI_TESTS_USER_EMAIL }} + CLI_TESTS_USER_PASSWORD: ${{ secrets.CLI_TESTS_USER_PASSWORD }} + INFISICAL_VAULT_FILE_PASSPHRASE: ${{ secrets.INFISICAL_VAULT_FILE_PASSPHRASE }} run: go test -v -count=1 ./test From 4479e626c7f4f5c52c8c51c5835c8be5450676ff Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 1 May 2024 01:56:10 +0800 Subject: [PATCH 07/91] adjustment: renamed cli vault file phrase env --- .github/workflows/run-cli-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-cli-tests.yml b/.github/workflows/run-cli-tests.yml index f55581a0f9..f8e9d77978 100644 --- a/.github/workflows/run-cli-tests.yml +++ b/.github/workflows/run-cli-tests.yml @@ -24,7 +24,7 @@ on: required: true CLI_TESTS_USER_PASSWORD: required: true - INFISICAL_VAULT_FILE_PASSPHRASE: + CLI_TESTS_INFISICAL_VAULT_FILE_PASSPHRASE: required: true jobs: test: @@ -50,6 +50,6 @@ jobs: CLI_TESTS_ENV_SLUG: ${{ secrets.CLI_TESTS_ENV_SLUG }} CLI_TESTS_USER_EMAIL: ${{ secrets.CLI_TESTS_USER_EMAIL }} CLI_TESTS_USER_PASSWORD: ${{ secrets.CLI_TESTS_USER_PASSWORD }} - INFISICAL_VAULT_FILE_PASSPHRASE: ${{ secrets.INFISICAL_VAULT_FILE_PASSPHRASE }} + INFISICAL_VAULT_FILE_PASSPHRASE: ${{ secrets.CLI_TESTS_INFISICAL_VAULT_FILE_PASSPHRASE }} run: go test -v -count=1 ./test From 3897bd70fa107824f5c471d33a136f4747cb0159 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 1 May 2024 11:08:58 +0800 Subject: [PATCH 08/91] adjustment: removed cli display for pty --- cli/test/login_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cli/test/login_test.go b/cli/test/login_test.go index 6857c00fba..d376d05db3 100644 --- a/cli/test/login_test.go +++ b/cli/test/login_test.go @@ -1,7 +1,6 @@ package tests import ( - "fmt" "os/exec" "strings" "testing" @@ -61,7 +60,6 @@ func UserLoginCmd(t *testing.T) { close(stepChan) return } - fmt.Print(string(buf[:n])) } }() From 505ccdf8eaa1e3d50a4898825ee1879d5597b66c Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 1 May 2024 21:37:18 +0800 Subject: [PATCH 09/91] misc: added script for cli-tests env setup --- cli/.gitignore | 1 + cli/scripts/export_test_env.sh | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 cli/scripts/export_test_env.sh diff --git a/cli/.gitignore b/cli/.gitignore index 5fa3e39c55..8eb54d72b9 100644 --- a/cli/.gitignore +++ b/cli/.gitignore @@ -1,3 +1,4 @@ .infisical.json dist/ agent-config.test.yaml +.test.env \ No newline at end of file diff --git a/cli/scripts/export_test_env.sh b/cli/scripts/export_test_env.sh new file mode 100644 index 0000000000..08b5ad41b1 --- /dev/null +++ b/cli/scripts/export_test_env.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +TEST_ENV_FILE=".test.env" + +# Check if the .env file exists +if [ ! -f "$TEST_ENV_FILE" ]; then + echo "$TEST_ENV_FILE does not exist." + exit 1 +fi + +# Export the variables +while IFS='=' read -r key value +do + # Skip empty lines and lines starting with # + if [[ -z "$key" || "$key" =~ ^\# ]]; then + continue + fi + # Use eval to correctly handle values with spaces + eval export $key='$value' +done < "$TEST_ENV_FILE" + +echo "Test environment variables set." From a5ca96f2df59e2f2247bb32cfed24b2858931398 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 1 May 2024 21:39:20 +0800 Subject: [PATCH 10/91] test: restructed setup and added scripting for infisical init --- cli/go.mod | 4 +-- cli/test/export_test.go | 7 ----- cli/test/helper.go | 6 ++-- cli/test/login_test.go | 53 ++++++++++++++++++++++++++++---- cli/test/main_test.go | 23 ++++++++++++++ cli/test/run_test.go | 12 -------- cli/test/secrets_by_name_test.go | 12 -------- cli/test/secrets_test.go | 10 ------ 8 files changed, 75 insertions(+), 52 deletions(-) create mode 100644 cli/test/main_test.go diff --git a/cli/go.mod b/cli/go.mod index 52c7375332..6a1da8c6d8 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -3,7 +3,9 @@ module github.com/Infisical/infisical-merge go 1.21 require ( + github.com/bradleyjkemp/cupaloy/v2 v2.8.0 github.com/charmbracelet/lipgloss v0.5.0 + github.com/creack/pty v1.1.21 github.com/denisbrodbeck/machineid v1.0.1 github.com/fatih/semgroup v1.2.0 github.com/gitleaks/go-gitdiff v0.8.0 @@ -29,9 +31,7 @@ require ( require ( github.com/alessio/shellescape v1.4.1 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect - github.com/bradleyjkemp/cupaloy/v2 v2.8.0 // indirect github.com/chzyer/readline v1.5.1 // indirect - github.com/creack/pty v1.1.21 // indirect github.com/danieljoos/wincred v1.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dvsekhvalnov/jose2go v1.5.0 // indirect diff --git a/cli/test/export_test.go b/cli/test/export_test.go index 9a936871dc..c44bf20af0 100644 --- a/cli/test/export_test.go +++ b/cli/test/export_test.go @@ -8,7 +8,6 @@ import ( func TestUniversalAuth_ExportSecretsWithImports(t *testing.T) { MachineIdentityLoginCmd(t) - SetupCli(t) output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "export", "--token", creds.UAAccessToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--silent") @@ -24,8 +23,6 @@ func TestUniversalAuth_ExportSecretsWithImports(t *testing.T) { } func TestServiceToken_ExportSecretsWithImports(t *testing.T) { - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "export", "--token", creds.ServiceToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--silent") if err != nil { @@ -41,8 +38,6 @@ func TestServiceToken_ExportSecretsWithImports(t *testing.T) { func TestUniversalAuth_ExportSecretsWithoutImports(t *testing.T) { MachineIdentityLoginCmd(t) - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "export", "--token", creds.UAAccessToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--silent", "--include-imports=false") if err != nil { @@ -57,8 +52,6 @@ func TestUniversalAuth_ExportSecretsWithoutImports(t *testing.T) { } func TestServiceToken_ExportSecretsWithoutImports(t *testing.T) { - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "export", "--token", creds.ServiceToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--silent", "--include-imports=false") if err != nil { diff --git a/cli/test/helper.go b/cli/test/helper.go index b773d138e5..819f4c4c9d 100644 --- a/cli/test/helper.go +++ b/cli/test/helper.go @@ -2,10 +2,10 @@ package tests import ( "fmt" + "log" "os" "os/exec" "strings" - "testing" ) const ( @@ -48,7 +48,7 @@ func ExecuteCliCommand(command string, args ...string) (string, error) { return strings.TrimSpace(string(output)), nil } -func SetupCli(t *testing.T) { +func SetupCli() { if creds.ClientID == "" || creds.ClientSecret == "" || creds.ServiceToken == "" || creds.ProjectID == "" || creds.EnvSlug == "" { panic("Missing required environment variables") @@ -62,7 +62,7 @@ func SetupCli(t *testing.T) { if !alreadyBuilt { if err := exec.Command("go", "build", "../.").Run(); err != nil { - t.Fatal(err) + log.Fatal(err) } } diff --git a/cli/test/login_test.go b/cli/test/login_test.go index d376d05db3..2bc356855e 100644 --- a/cli/test/login_test.go +++ b/cli/test/login_test.go @@ -1,6 +1,8 @@ package tests import ( + "fmt" + "log" "os/exec" "strings" "testing" @@ -9,21 +11,62 @@ import ( "github.com/stretchr/testify/assert" ) -func UserLoginCmd(t *testing.T) { - SetupCli(t) +func UserInitCmd() { + c := exec.Command(FORMATTED_CLI_NAME, "init") + ptmx, err := pty.Start(c) + if err != nil { + log.Fatalf("error running CLI command: %v", err) + } + defer func() { _ = ptmx.Close() }() + stepChan := make(chan int, 10) + + go func() { + buf := make([]byte, 1024) + step := -1 + for { + n, err := ptmx.Read(buf) + if n > 0 { + terminalOut := string(buf) + fmt.Println("Terminal out is", terminalOut) + if strings.Contains(terminalOut, "Which Infisical organization would you like to select a project from?") && step < 0 { + step += 1 + stepChan <- step + } else if strings.Contains(terminalOut, "Which of your Infisical projects would you like to connect this project to?") && step < 1 { + step += 1; + stepChan <- step + } + } + if err != nil { + close(stepChan) + return + } + } + }() + + for i := range stepChan { + switch i { + case 0: + ptmx.Write([]byte("\n")) + case 1: + ptmx.Write([]byte("\n")) + } + } +} + +func UserLoginCmd() { // set vault to file because CI has no keyring vaultCmd := exec.Command(FORMATTED_CLI_NAME, "vault", "set", "file") _, err := vaultCmd.Output() if err != nil { - t.Fatalf("error setting vault: %v", err) + log.Fatalf("error setting vault: %v", err) } // Start programmatic interaction with CLI c := exec.Command(FORMATTED_CLI_NAME, "login", "--interactive") ptmx, err := pty.Start(c) if err != nil { - t.Fatalf("error running CLI command: %v", err) + log.Fatalf("error running CLI command: %v", err) } defer func() { _ = ptmx.Close() }() @@ -83,8 +126,6 @@ func UserLoginCmd(t *testing.T) { } func MachineIdentityLoginCmd(t *testing.T) { - SetupCli(t) - if creds.UAAccessToken != "" { return } diff --git a/cli/test/main_test.go b/cli/test/main_test.go new file mode 100644 index 0000000000..e14893aec0 --- /dev/null +++ b/cli/test/main_test.go @@ -0,0 +1,23 @@ +package tests + +import ( + "fmt" + "os" + "testing" +) + +func TestMain(m *testing.M) { + // Setup + fmt.Println("Setting up CLI...") + SetupCli() + fmt.Println("Performing user login...") + UserLoginCmd() + fmt.Println("Performing infisical init...") + UserInitCmd() + + // Run the tests + code := m.Run() + + // Exit + os.Exit(code) +} diff --git a/cli/test/run_test.go b/cli/test/run_test.go index 808f4f14ff..d2c6021cc2 100644 --- a/cli/test/run_test.go +++ b/cli/test/run_test.go @@ -8,8 +8,6 @@ import ( ) func TestServiceToken_RunCmdRecursiveAndImports(t *testing.T) { - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "run", "--token", creds.ServiceToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--recursive", "--silent", "--", "echo", "hello world") if err != nil { @@ -25,8 +23,6 @@ func TestServiceToken_RunCmdRecursiveAndImports(t *testing.T) { } } func TestServiceToken_RunCmdWithImports(t *testing.T) { - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "run", "--token", creds.ServiceToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--silent", "--", "echo", "hello world") if err != nil { @@ -44,8 +40,6 @@ func TestServiceToken_RunCmdWithImports(t *testing.T) { func TestUniversalAuth_RunCmdRecursiveAndImports(t *testing.T) { MachineIdentityLoginCmd(t) - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "run", "--token", creds.UAAccessToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--recursive", "--silent", "--", "echo", "hello world") if err != nil { @@ -63,8 +57,6 @@ func TestUniversalAuth_RunCmdRecursiveAndImports(t *testing.T) { func TestUniversalAuth_RunCmdWithImports(t *testing.T) { MachineIdentityLoginCmd(t) - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "run", "--token", creds.UAAccessToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--silent", "--", "echo", "hello world") if err != nil { @@ -83,8 +75,6 @@ func TestUniversalAuth_RunCmdWithImports(t *testing.T) { func TestUniversalAuth_RunCmdWithoutImports(t *testing.T) { MachineIdentityLoginCmd(t) - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "run", "--token", creds.UAAccessToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--silent", "--include-imports=false", "--", "echo", "hello world") if err != nil { @@ -101,8 +91,6 @@ func TestUniversalAuth_RunCmdWithoutImports(t *testing.T) { } func TestServiceToken_RunCmdWithoutImports(t *testing.T) { - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "run", "--token", creds.ServiceToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--silent", "--include-imports=false", "--", "echo", "hello world") if err != nil { diff --git a/cli/test/secrets_by_name_test.go b/cli/test/secrets_by_name_test.go index 440324e1ac..26a8314bb4 100644 --- a/cli/test/secrets_by_name_test.go +++ b/cli/test/secrets_by_name_test.go @@ -7,8 +7,6 @@ import ( ) func TestServiceToken_GetSecretsByNameRecursive(t *testing.T) { - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "secrets", "get", "TEST-SECRET-1", "TEST-SECRET-2", "FOLDER-SECRET-1", "--token", creds.ServiceToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--recursive", "--silent") if err != nil { @@ -23,8 +21,6 @@ func TestServiceToken_GetSecretsByNameRecursive(t *testing.T) { } func TestServiceToken_GetSecretsByNameWithNotFoundSecret(t *testing.T) { - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "secrets", "get", "TEST-SECRET-1", "TEST-SECRET-2", "FOLDER-SECRET-1", "DOES-NOT-EXIST", "--token", creds.ServiceToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--recursive", "--silent") if err != nil { @@ -39,8 +35,6 @@ func TestServiceToken_GetSecretsByNameWithNotFoundSecret(t *testing.T) { } func TestServiceToken_GetSecretsByNameWithImports(t *testing.T) { - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "secrets", "get", "TEST-SECRET-1", "STAGING-SECRET-2", "FOLDER-SECRET-1", "--token", creds.ServiceToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--recursive", "--silent") if err != nil { @@ -56,8 +50,6 @@ func TestServiceToken_GetSecretsByNameWithImports(t *testing.T) { func TestUniversalAuth_GetSecretsByNameRecursive(t *testing.T) { MachineIdentityLoginCmd(t) - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "secrets", "get", "TEST-SECRET-1", "TEST-SECRET-2", "FOLDER-SECRET-1", "--token", creds.UAAccessToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--recursive", "--silent") if err != nil { @@ -73,8 +65,6 @@ func TestUniversalAuth_GetSecretsByNameRecursive(t *testing.T) { func TestUniversalAuth_GetSecretsByNameWithNotFoundSecret(t *testing.T) { MachineIdentityLoginCmd(t) - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "secrets", "get", "TEST-SECRET-1", "TEST-SECRET-2", "FOLDER-SECRET-1", "DOES-NOT-EXIST", "--token", creds.UAAccessToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--recursive", "--silent") if err != nil { @@ -90,8 +80,6 @@ func TestUniversalAuth_GetSecretsByNameWithNotFoundSecret(t *testing.T) { func TestUniversalAuth_GetSecretsByNameWithImports(t *testing.T) { MachineIdentityLoginCmd(t) - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "secrets", "get", "TEST-SECRET-1", "STAGING-SECRET-2", "FOLDER-SECRET-1", "--token", creds.UAAccessToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--recursive", "--silent") if err != nil { diff --git a/cli/test/secrets_test.go b/cli/test/secrets_test.go index 294434f5f3..7af64eab88 100644 --- a/cli/test/secrets_test.go +++ b/cli/test/secrets_test.go @@ -9,8 +9,6 @@ import ( func TestServiceToken_SecretsGetWithImportsAndRecursiveCmd(t *testing.T) { - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "secrets", "--token", creds.ServiceToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--recursive", "--silent") if err != nil { @@ -25,8 +23,6 @@ func TestServiceToken_SecretsGetWithImportsAndRecursiveCmd(t *testing.T) { } func TestServiceToken_SecretsGetWithoutImportsAndWithoutRecursiveCmd(t *testing.T) { - SetupCli(t) - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "secrets", "--token", creds.ServiceToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--include-imports=false", "--silent") if err != nil { @@ -41,7 +37,6 @@ func TestServiceToken_SecretsGetWithoutImportsAndWithoutRecursiveCmd(t *testing. } func TestUniversalAuth_SecretsGetWithImportsAndRecursiveCmd(t *testing.T) { - SetupCli(t) MachineIdentityLoginCmd(t) output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "secrets", "--token", creds.UAAccessToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--recursive", "--silent") @@ -58,7 +53,6 @@ func TestUniversalAuth_SecretsGetWithImportsAndRecursiveCmd(t *testing.T) { } func TestUniversalAuth_SecretsGetWithoutImportsAndWithoutRecursiveCmd(t *testing.T) { - SetupCli(t) MachineIdentityLoginCmd(t) output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "secrets", "--token", creds.UAAccessToken, "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--include-imports=false", "--silent") @@ -75,7 +69,6 @@ func TestUniversalAuth_SecretsGetWithoutImportsAndWithoutRecursiveCmd(t *testing } func TestUniversalAuth_SecretsGetWrongEnvironment(t *testing.T) { - SetupCli(t) MachineIdentityLoginCmd(t) output, _ := ExecuteCliCommand(FORMATTED_CLI_NAME, "secrets", "--token", creds.UAAccessToken, "--projectId", creds.ProjectID, "--env", "invalid-env", "--recursive", "--silent") @@ -89,9 +82,6 @@ func TestUniversalAuth_SecretsGetWrongEnvironment(t *testing.T) { } func TestUserAuth_SecretsGetAll(t *testing.T) { - SetupCli(t) - UserLoginCmd(t); - output, err := ExecuteCliCommand(FORMATTED_CLI_NAME, "secrets", "--projectId", creds.ProjectID, "--env", creds.EnvSlug, "--include-imports=false", "--silent") if err != nil { t.Fatalf("error running CLI command: %v", err) From 24f7ecc548e5d8336324acf296f3653c99532d59 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 1 May 2024 21:41:07 +0800 Subject: [PATCH 11/91] misc: removed infisical init logs --- cli/test/login_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cli/test/login_test.go b/cli/test/login_test.go index 2bc356855e..1ee9d50a4f 100644 --- a/cli/test/login_test.go +++ b/cli/test/login_test.go @@ -1,7 +1,6 @@ package tests import ( - "fmt" "log" "os/exec" "strings" @@ -28,7 +27,6 @@ func UserInitCmd() { n, err := ptmx.Read(buf) if n > 0 { terminalOut := string(buf) - fmt.Println("Terminal out is", terminalOut) if strings.Contains(terminalOut, "Which Infisical organization would you like to select a project from?") && step < 0 { step += 1 stepChan <- step From 8fc4fd64f84d4ff780f0bde1a755313d356e795a Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 2 May 2024 00:49:29 +0800 Subject: [PATCH 12/91] adjustment: moved backup logic to cmd layer --- cli/packages/cmd/run.go | 91 +++++++++++++++++++++++++++--------- cli/packages/util/secrets.go | 25 +--------- 2 files changed, 71 insertions(+), 45 deletions(-) diff --git a/cli/packages/cmd/run.go b/cli/packages/cmd/run.go index 04fe2588b6..58088f5d32 100644 --- a/cli/packages/cmd/run.go +++ b/cli/packages/cmd/run.go @@ -116,35 +116,82 @@ var runCmd = &cobra.Command{ Recursive: recursive, } + var secrets []models.SingleEnvironmentVariable + var isUserSession bool + var infisicalDotJson models.WorkspaceConfigFile + var userBackupSecretsEncryptionKey []byte + if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { request.InfisicalToken = token.Token } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { request.UniversalAuthAccessToken = token.Token - } - - secrets, err := util.GetAllEnvironmentVariables(request, projectConfigDir) - - if err != nil { - util.HandleError(err, "Could not fetch secrets", "If you are using a service token to fetch secrets, please ensure it is valid") - } - - if secretOverriding { - secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_PERSONAL) } else { - secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) - } - - if shouldExpandSecrets { - - authParams := models.ExpandSecretsAuthentication{} - - if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { - authParams.InfisicalToken = token.Token - } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { - authParams.UniversalAuthAccessToken = token.Token + // user session + isUserSession = true + loggedInUserDetails, err := util.GetCurrentLoggedInUserDetails() + if err != nil { + util.HandleError(err) } - secrets = util.ExpandSecrets(secrets, authParams, projectConfigDir) + isConnected := util.CheckIsConnectedToInfisicalAPI() + + if projectConfigDir == "" { + projectConfig, err := util.GetWorkSpaceFromFile() + if err != nil { + util.HandleError(err) + } + + infisicalDotJson = projectConfig + } else { + projectConfig, err := util.GetWorkSpaceFromFilePath(projectConfigDir) + if err != nil { + util.HandleError(err) + } + + infisicalDotJson = projectConfig + } + + userBackupSecretsEncryptionKey = []byte(loggedInUserDetails.UserCredentials.PrivateKey)[0:32] + + if !isConnected { + secrets, err = util.ReadBackupSecrets(infisicalDotJson.WorkspaceId, environmentName, userBackupSecretsEncryptionKey) + if err != nil { + util.HandleError(err) + } + if len(secrets) > 0 { + util.PrintWarning("Unable to fetch latest secret(s) due to connection error, serving secrets from last successful fetch. For more info, run with --debug") + } + } + } + + if len(secrets) == 0 { + secrets, err = util.GetAllEnvironmentVariables(request, projectConfigDir) + if err != nil { + util.HandleError(err, "Could not fetch secrets", "If you are using a service token to fetch secrets, please ensure it is valid") + } + + if secretOverriding { + secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_PERSONAL) + } else { + secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) + } + + if shouldExpandSecrets { + + authParams := models.ExpandSecretsAuthentication{} + + if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { + authParams.InfisicalToken = token.Token + } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { + authParams.UniversalAuthAccessToken = token.Token + } + + secrets = util.ExpandSecrets(secrets, authParams, projectConfigDir) + } + + if isUserSession { + util.WriteBackupSecrets(infisicalDotJson.WorkspaceId, environmentName, userBackupSecretsEncryptionKey, secrets) + } } secretsByKey := getSecretsByKeys(secrets) diff --git a/cli/packages/util/secrets.go b/cli/packages/util/secrets.go index 0ca66cc4fb..2e9b0ffd28 100644 --- a/cli/packages/util/secrets.go +++ b/cli/packages/util/secrets.go @@ -319,21 +319,16 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo } RequireLogin() - log.Debug().Msg("GetAllEnvironmentVariables: Trying to fetch secrets using logged in details") loggedInUserDetails, err := GetCurrentLoggedInUserDetails() - isConnected := CheckIsConnectedToInfisicalAPI() - - if isConnected { - log.Debug().Msg("GetAllEnvironmentVariables: Connected to Infisical instance, checking logged in creds") - } + log.Debug().Msg("GetAllEnvironmentVariables: Connected to Infisical instance, checking logged in creds") if err != nil { return nil, err } - if isConnected && loggedInUserDetails.LoginExpired { + if loggedInUserDetails.LoginExpired { PrintErrorMessageAndExit("Your login session has expired, please run [infisical login] and try again") } @@ -362,22 +357,6 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo secretsToReturn, errorToReturn = GetPlainTextSecretsViaJTW(loggedInUserDetails.UserCredentials.JTWToken, loggedInUserDetails.UserCredentials.PrivateKey, infisicalDotJson.WorkspaceId, params.Environment, params.TagSlugs, params.SecretsPath, params.IncludeImport, params.Recursive) log.Debug().Msgf("GetAllEnvironmentVariables: Trying to fetch secrets JTW token [err=%s]", errorToReturn) - - backupSecretsEncryptionKey := []byte(loggedInUserDetails.UserCredentials.PrivateKey)[0:32] - if errorToReturn == nil { - WriteBackupSecrets(infisicalDotJson.WorkspaceId, params.Environment, backupSecretsEncryptionKey, secretsToReturn) - } - - // only attempt to serve cached secrets if no internet connection and if at least one secret cached - if !isConnected { - backedSecrets, err := ReadBackupSecrets(infisicalDotJson.WorkspaceId, params.Environment, backupSecretsEncryptionKey) - if len(backedSecrets) > 0 { - PrintWarning("Unable to fetch latest secret(s) due to connection error, serving secrets from last successful fetch. For more info, run with --debug") - secretsToReturn = backedSecrets - errorToReturn = err - } - } - } else { if params.InfisicalToken != "" { log.Debug().Msg("Trying to fetch secrets using service token") From 920b9a7dfae115431ed8dc5a5f00f320a8d28585 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 2 May 2024 00:59:17 +0800 Subject: [PATCH 13/91] adjustment: moved secret backup logic to cmd layer --- cli/packages/cmd/secrets.go | 79 +++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 21 deletions(-) diff --git a/cli/packages/cmd/secrets.go b/cli/packages/cmd/secrets.go index 305a1f0fd2..0a800580f7 100644 --- a/cli/packages/cmd/secrets.go +++ b/cli/packages/cmd/secrets.go @@ -88,37 +88,74 @@ var secretsCmd = &cobra.Command{ Recursive: recursive, } + var secrets []models.SingleEnvironmentVariable + var isUserSession bool + var infisicalDotJson models.WorkspaceConfigFile + var userBackupSecretsEncryptionKey []byte + if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { request.InfisicalToken = token.Token } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { request.UniversalAuthAccessToken = token.Token - } - - secrets, err := util.GetAllEnvironmentVariables(request, "") - if err != nil { - util.HandleError(err) - } - - if secretOverriding { - secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_PERSONAL) } else { - secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) - } + // user session + isUserSession = true + loggedInUserDetails, err := util.GetCurrentLoggedInUserDetails() + if err != nil { + util.HandleError(err) + } + + isConnected := util.CheckIsConnectedToInfisicalAPI() - if shouldExpandSecrets { - - authParams := models.ExpandSecretsAuthentication{} - if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { - authParams.InfisicalToken = token.Token - } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { - authParams.UniversalAuthAccessToken = token.Token + projectConfig, err := util.GetWorkSpaceFromFile() + if err != nil { + util.HandleError(err) } - secrets = util.ExpandSecrets(secrets, authParams, "") + infisicalDotJson = projectConfig + userBackupSecretsEncryptionKey = []byte(loggedInUserDetails.UserCredentials.PrivateKey)[0:32] + + if !isConnected { + secrets, err = util.ReadBackupSecrets(infisicalDotJson.WorkspaceId, environmentName, userBackupSecretsEncryptionKey) + if err != nil { + util.HandleError(err) + } + if len(secrets) > 0 { + util.PrintWarning("Unable to fetch latest secret(s) due to connection error, serving secrets from last successful fetch. For more info, run with --debug") + } + } } - // Sort the secrets by key so we can create a consistent output - secrets = util.SortSecretsByKeys(secrets) + if len(secrets) == 0 { + secrets, err = util.GetAllEnvironmentVariables(request, "") + if err != nil { + util.HandleError(err) + } + + if secretOverriding { + secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_PERSONAL) + } else { + secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) + } + + if shouldExpandSecrets { + + authParams := models.ExpandSecretsAuthentication{} + if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { + authParams.InfisicalToken = token.Token + } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { + authParams.UniversalAuthAccessToken = token.Token + } + + secrets = util.ExpandSecrets(secrets, authParams, "") + } + + // Sort the secrets by key so we can create a consistent output + secrets = util.SortSecretsByKeys(secrets) + if isUserSession { + util.WriteBackupSecrets(infisicalDotJson.WorkspaceId, environmentName, userBackupSecretsEncryptionKey, secrets) + } + } visualize.PrintAllSecretDetails(secrets) Telemetry.CaptureEvent("cli-command:secrets", posthog.NewProperties().Set("secretCount", len(secrets)).Set("version", util.CLI_VERSION)) From 04dca9432ddaabc8f3d2d762fd765d6a39c55e30 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 2 May 2024 01:09:12 +0800 Subject: [PATCH 14/91] misc: updated test comment --- cli/test/secrets_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/test/secrets_test.go b/cli/test/secrets_test.go index 7af64eab88..f5d5a7b1f3 100644 --- a/cli/test/secrets_test.go +++ b/cli/test/secrets_test.go @@ -93,7 +93,7 @@ func TestUserAuth_SecretsGetAll(t *testing.T) { t.Fatalf("snapshot failed: %v", err) } - // intentionally invoked this here because it should directly follow secretsGetAll + // explicitly called here because it should happen directly after successful secretsGetAll testUserAuth_SecretsGetAllWithoutConnection(t) } From a1e8f45a8692587a0a80571af883690a60f628b0 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 2 May 2024 01:35:19 +0800 Subject: [PATCH 15/91] misc: added new cli secrets to release build gh action --- .github/workflows/release_build_infisical_cli.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release_build_infisical_cli.yml b/.github/workflows/release_build_infisical_cli.yml index e4a5945e04..af8e28f4d5 100644 --- a/.github/workflows/release_build_infisical_cli.yml +++ b/.github/workflows/release_build_infisical_cli.yml @@ -22,6 +22,9 @@ jobs: CLI_TESTS_SERVICE_TOKEN: ${{ secrets.CLI_TESTS_SERVICE_TOKEN }} CLI_TESTS_PROJECT_ID: ${{ secrets.CLI_TESTS_PROJECT_ID }} CLI_TESTS_ENV_SLUG: ${{ secrets.CLI_TESTS_ENV_SLUG }} + CLI_TESTS_USER_EMAIL: ${{ secrets.CLI_TESTS_USER_EMAIL }} + CLI_TESTS_USER_PASSWORD: ${{ secrets.CLI_TESTS_USER_PASSWORD }} + INFISICAL_VAULT_FILE_PASSPHRASE: ${{ secrets.CLI_TESTS_INFISICAL_VAULT_FILE_PASSPHRASE }} goreleaser: runs-on: ubuntu-20.04 From 88a4fb84e66c5a68d7e6c9819f59c48713a060a5 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 2 May 2024 03:21:20 +0800 Subject: [PATCH 16/91] feature: added offline support for infisical export --- cli/packages/cmd/export.go | 83 +++++++++++++++++++++++------------- cli/packages/cmd/run.go | 13 +----- cli/packages/cmd/secrets.go | 13 +----- cli/packages/util/secrets.go | 18 ++++++++ 4 files changed, 73 insertions(+), 54 deletions(-) diff --git a/cli/packages/cmd/export.go b/cli/packages/cmd/export.go index 983c192551..271509b0cd 100644 --- a/cli/packages/cmd/export.go +++ b/cli/packages/cmd/export.go @@ -93,10 +93,33 @@ var exportCmd = &cobra.Command{ IncludeImport: includeImports, } + + var secrets []models.SingleEnvironmentVariable + var isUserSession bool + var infisicalDotJson models.WorkspaceConfigFile + var userBackupSecretsEncryptionKey []byte + var loggedInUserDetails util.LoggedInUserDetails + if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { request.InfisicalToken = token.Token } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { request.UniversalAuthAccessToken = token.Token + } else { + isUserSession = true + loggedInUserDetails, err = util.GetCurrentLoggedInUserDetails() + if err != nil { + util.HandleError(err) + } + + projectConfig, err := util.GetWorkSpaceFromFile() + if err != nil { + util.HandleError(err) + } + + infisicalDotJson = projectConfig + + userBackupSecretsEncryptionKey = []byte(loggedInUserDetails.UserCredentials.PrivateKey)[0:32] + secrets = util.GetBackupSecretsIfDisconnected(infisicalDotJson.WorkspaceId, environmentName, userBackupSecretsEncryptionKey) } if templatePath != "" { @@ -109,10 +132,6 @@ var exportCmd = &cobra.Command{ accessToken = token.Token } else { log.Debug().Msg("GetAllEnvironmentVariables: Trying to fetch secrets using logged in details") - loggedInUserDetails, err := util.GetCurrentLoggedInUserDetails() - if err != nil { - util.HandleError(err) - } accessToken = loggedInUserDetails.UserCredentials.JTWToken } @@ -124,40 +143,44 @@ var exportCmd = &cobra.Command{ return } - secrets, err := util.GetAllEnvironmentVariables(request, "") - if err != nil { - util.HandleError(err, "Unable to fetch secrets") - } - - if secretOverriding { - secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_PERSONAL) - } else { - secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) - } - - var output string - if shouldExpandSecrets { - - authParams := models.ExpandSecretsAuthentication{} - - if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { - authParams.InfisicalToken = token.Token - } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { - authParams.UniversalAuthAccessToken = token.Token + if len(secrets) == 0 { + secrets, err = util.GetAllEnvironmentVariables(request, "") + if err != nil { + util.HandleError(err, "Unable to fetch secrets") } - secrets = util.ExpandSecrets(secrets, authParams, "") - } - secrets = util.FilterSecretsByTag(secrets, tagSlugs) - secrets = util.SortSecretsByKeys(secrets) + if secretOverriding { + secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_PERSONAL) + } else { + secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) + } - output, err = formatEnvs(secrets, format) + if shouldExpandSecrets { + + authParams := models.ExpandSecretsAuthentication{} + + if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { + authParams.InfisicalToken = token.Token + } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { + authParams.UniversalAuthAccessToken = token.Token + } + + secrets = util.ExpandSecrets(secrets, authParams, "") + } + secrets = util.FilterSecretsByTag(secrets, tagSlugs) + secrets = util.SortSecretsByKeys(secrets) + + if isUserSession { + util.WriteBackupSecrets(infisicalDotJson.WorkspaceId, environmentName, userBackupSecretsEncryptionKey, secrets) + } + } + + output, err := formatEnvs(secrets, format) if err != nil { util.HandleError(err) } fmt.Print(output) - // Telemetry.CaptureEvent("cli-command:export", posthog.NewProperties().Set("secretsCount", len(secrets)).Set("version", util.CLI_VERSION)) }, } diff --git a/cli/packages/cmd/run.go b/cli/packages/cmd/run.go index 58088f5d32..0770a85564 100644 --- a/cli/packages/cmd/run.go +++ b/cli/packages/cmd/run.go @@ -133,8 +133,6 @@ var runCmd = &cobra.Command{ util.HandleError(err) } - isConnected := util.CheckIsConnectedToInfisicalAPI() - if projectConfigDir == "" { projectConfig, err := util.GetWorkSpaceFromFile() if err != nil { @@ -152,16 +150,7 @@ var runCmd = &cobra.Command{ } userBackupSecretsEncryptionKey = []byte(loggedInUserDetails.UserCredentials.PrivateKey)[0:32] - - if !isConnected { - secrets, err = util.ReadBackupSecrets(infisicalDotJson.WorkspaceId, environmentName, userBackupSecretsEncryptionKey) - if err != nil { - util.HandleError(err) - } - if len(secrets) > 0 { - util.PrintWarning("Unable to fetch latest secret(s) due to connection error, serving secrets from last successful fetch. For more info, run with --debug") - } - } + secrets = util.GetBackupSecretsIfDisconnected(infisicalDotJson.WorkspaceId, environmentName, userBackupSecretsEncryptionKey) } if len(secrets) == 0 { diff --git a/cli/packages/cmd/secrets.go b/cli/packages/cmd/secrets.go index 0a800580f7..cd31d05e92 100644 --- a/cli/packages/cmd/secrets.go +++ b/cli/packages/cmd/secrets.go @@ -105,8 +105,6 @@ var secretsCmd = &cobra.Command{ util.HandleError(err) } - isConnected := util.CheckIsConnectedToInfisicalAPI() - projectConfig, err := util.GetWorkSpaceFromFile() if err != nil { util.HandleError(err) @@ -114,16 +112,7 @@ var secretsCmd = &cobra.Command{ infisicalDotJson = projectConfig userBackupSecretsEncryptionKey = []byte(loggedInUserDetails.UserCredentials.PrivateKey)[0:32] - - if !isConnected { - secrets, err = util.ReadBackupSecrets(infisicalDotJson.WorkspaceId, environmentName, userBackupSecretsEncryptionKey) - if err != nil { - util.HandleError(err) - } - if len(secrets) > 0 { - util.PrintWarning("Unable to fetch latest secret(s) due to connection error, serving secrets from last successful fetch. For more info, run with --debug") - } - } + secrets = util.GetBackupSecretsIfDisconnected(infisicalDotJson.WorkspaceId, environmentName, userBackupSecretsEncryptionKey) } if len(secrets) == 0 { diff --git a/cli/packages/util/secrets.go b/cli/packages/util/secrets.go index 2e9b0ffd28..e97dda9b09 100644 --- a/cli/packages/util/secrets.go +++ b/cli/packages/util/secrets.go @@ -426,6 +426,24 @@ func getSecretsByKeys(secrets []models.SingleEnvironmentVariable) map[string]mod return secretMapByName } +func GetBackupSecretsIfDisconnected(workspaceId string, environment string, encryptionKey []byte) ([]models.SingleEnvironmentVariable) { + isConnected := CheckIsConnectedToInfisicalAPI() + + if !isConnected { + secrets, err := ReadBackupSecrets(workspaceId, environment, encryptionKey) + if err != nil { + HandleError(err) + } + if len(secrets) > 0 { + PrintWarning("Unable to fetch latest secret(s) due to connection error, serving secrets from last successful fetch. For more info, run with --debug") + } + + return secrets + } + + return nil +} + func ExpandSecrets(secrets []models.SingleEnvironmentVariable, auth models.ExpandSecretsAuthentication, projectConfigPathDir string) []models.SingleEnvironmentVariable { expandedSecs := make(map[string]string) interpolatedSecs := make(map[string]string) From 3a1168c7e82c7ee79938b8c53aadcdda51f41487 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 31 May 2024 19:12:55 +0800 Subject: [PATCH 17/91] feat: added initial version pruning and result limiting --- .../20240531042507_add-pit-version-limit.ts | 21 ++++++++ backend/src/db/schemas/projects.ts | 3 +- .../secret-snapshot-service.ts | 17 +++++- .../services/secret-snapshot/snapshot-dal.ts | 52 ++++++++++++++++++- backend/src/server/routes/index.ts | 1 + .../src/services/project/project-service.ts | 3 +- backend/src/services/secret/secret-service.ts | 13 ++++- .../src/services/secret/secret-version-dal.ts | 37 +++++++++++++ 8 files changed, 140 insertions(+), 7 deletions(-) create mode 100644 backend/src/db/migrations/20240531042507_add-pit-version-limit.ts diff --git a/backend/src/db/migrations/20240531042507_add-pit-version-limit.ts b/backend/src/db/migrations/20240531042507_add-pit-version-limit.ts new file mode 100644 index 0000000000..e37c24e2c8 --- /dev/null +++ b/backend/src/db/migrations/20240531042507_add-pit-version-limit.ts @@ -0,0 +1,21 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + const hasPitVersionLimitColumn = await knex.schema.hasColumn(TableName.Project, "pitVersionLimit"); + await knex.schema.alterTable(TableName.Project, (tb) => { + if (!hasPitVersionLimitColumn) { + tb.integer("pitVersionLimit").notNullable().defaultTo(10); + } + }); +} + +export async function down(knex: Knex): Promise { + const hasPitVersionLimitColumn = await knex.schema.hasColumn(TableName.Project, "pitVersionLimit"); + await knex.schema.alterTable(TableName.Project, (tb) => { + if (hasPitVersionLimitColumn) { + tb.dropColumn("pitVersionLimit"); + } + }); +} diff --git a/backend/src/db/schemas/projects.ts b/backend/src/db/schemas/projects.ts index 3965e24c0a..9197a355d9 100644 --- a/backend/src/db/schemas/projects.ts +++ b/backend/src/db/schemas/projects.ts @@ -16,7 +16,8 @@ export const ProjectsSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), version: z.number().default(1), - upgradeStatus: z.string().nullable().optional() + upgradeStatus: z.string().nullable().optional(), + pitVersionLimit: z.number() }); export type TProjects = z.infer; diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index 0e71ad1268..6197c08256 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -4,6 +4,7 @@ import { TableName, TSecretTagJunctionInsert } from "@app/db/schemas"; import { BadRequestError, InternalServerError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; +import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; @@ -37,6 +38,7 @@ type TSecretSnapshotServiceFactoryDep = { folderDAL: Pick; permissionService: Pick; licenseService: Pick; + projectDAL: Pick; }; export type TSecretSnapshotServiceFactory = ReturnType; @@ -48,6 +50,7 @@ export const secretSnapshotServiceFactory = ({ snapshotSecretDAL, snapshotFolderDAL, folderDAL, + projectDAL, secretDAL, permissionService, licenseService, @@ -81,8 +84,9 @@ export const secretSnapshotServiceFactory = ({ const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found" }); + const project = await projectDAL.findById(projectId); const count = await snapshotDAL.countOfSnapshotsByFolderId(folder.id); - return count; + return Math.min(count, project.pitVersionLimit); }; const listSnapshots = async ({ @@ -114,7 +118,16 @@ export const secretSnapshotServiceFactory = ({ const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found" }); - const snapshots = await snapshotDAL.find({ folderId: folder.id }, { limit, offset, sort: [["createdAt", "desc"]] }); + const { pitVersionLimit } = await projectDAL.findById(projectId); + const computedQueryLimit = Math.min(pitVersionLimit - offset, limit); + if (offset > pitVersionLimit || computedQueryLimit <= 0) { + return []; + } + + const snapshots = await snapshotDAL.find( + { folderId: folder.id }, + { limit: computedQueryLimit, offset, sort: [["createdAt", "desc"]] } + ); return snapshots; }; diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index cdd5a999b8..1618024bbc 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -325,12 +325,62 @@ export const snapshotDALFactory = (db: TDbClient) => { } }; + const pruneExcessSnapshots = async (tx?: Knex) => { + try { + const folders = await (tx || db)(TableName.SecretFolder).select("id"); + const folderIds = folders.map((folder) => folder.id); + const PRUNE_FOLDER_BATCH_SIZE = 500; + + const pruneBatches = []; + for (let x = 0; x < folderIds.length; x += PRUNE_FOLDER_BATCH_SIZE) { + const batch = folderIds.slice(x, x + PRUNE_FOLDER_BATCH_SIZE); + pruneBatches.push(batch); + } + + for await (const folderBatch of pruneBatches) { + const rankedSnapshots = (tx || db)(TableName.Snapshot) + .whereIn(`${TableName.Snapshot}.folderId`, folderBatch) + .select( + "folderId", + "id", + (tx || db).raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` + ) + ) + .as("ranked_snapshots"); + + const snapshotsToKeep = (tx || db) + .select("id") + .from(rankedSnapshots) + .where( + "row_num", + "<=", + (tx || db) + .select(`${TableName.Project}.pitVersionLimit`) + .from(TableName.Project) + .join(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) + .join(TableName.Snapshot, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`) + .join(rankedSnapshots, "ranked_snapshots.folderId", `${TableName.Snapshot}.folderId`) + .limit(1) + ); + + await (tx || db)(TableName.Snapshot) + .whereIn("folderId", folderBatch) + .whereNotIn("id", snapshotsToKeep) + .delete(); + } + } catch (error) { + throw new DatabaseError({ error, name: "SnapshotPrune" }); + } + }; + return { ...secretSnapshotOrm, findById, findLatestSnapshotByFolderId, findRecursivelySnapshots, countOfSnapshotsByFolderId, - findSecretSnapshotDataById + findSecretSnapshotDataById, + pruneExcessSnapshots }; }; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 1593515ec9..b0b2db9b19 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -535,6 +535,7 @@ export const registerRoutes = async ( licenseService, folderDAL, secretDAL, + projectDAL, snapshotDAL, snapshotFolderDAL, snapshotSecretDAL, diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index f58fd77885..1e588dd08f 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -133,7 +133,8 @@ export const projectServiceFactory = ({ name: workspaceName, orgId: organization.id, slug: projectSlug || slugify(`${workspaceName}-${alphaNumericNanoId(4)}`), - version: ProjectVersion.V2 + version: ProjectVersion.V2, + pitVersionLimit: 10 }, tx ); diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 39e47a28e1..77c4024ac7 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -72,7 +72,7 @@ type TSecretServiceFactoryDep = { secretDAL: TSecretDALFactory; secretTagDAL: TSecretTagDALFactory; secretVersionDAL: TSecretVersionDALFactory; - projectDAL: Pick; + projectDAL: Pick; projectEnvDAL: Pick; folderDAL: Pick< TSecretFolderDALFactory, @@ -1354,7 +1354,16 @@ export const secretServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); - const secretVersions = await secretVersionDAL.find({ secretId }, { offset, limit, sort: [["createdAt", "desc"]] }); + const { pitVersionLimit } = await projectDAL.findById(folder.projectId); + const computedQueryLimit = Math.min(pitVersionLimit - offset, limit); + if (offset > pitVersionLimit || computedQueryLimit <= 0) { + return []; + } + + const secretVersions = await secretVersionDAL.find( + { secretId }, + { offset, limit: computedQueryLimit, sort: [["createdAt", "desc"]] } + ); return secretVersions; }; diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index 758352ed24..5d52f3c07f 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -110,8 +110,45 @@ export const secretVersionDALFactory = (db: TDbClient) => { } }; + const pruneExcessVersions = async (tx?: Knex) => { + try { + const rankedSecretVersions = (tx || db)(TableName.SecretVersion) + .select( + "id", + "secretId", + "folderId", + (tx || db).raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num` + ) + ) + .as("ranked_secret_versions"); + + const versionsToKeep = (tx || db)(rankedSecretVersions) + .select("id") + .where( + "row_num", + "<=", + (tx || db) + .select(`${TableName.Project}.pitVersionLimit`) + .from(TableName.Project) + .join(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .join(rankedSecretVersions, "ranked_secret_versions.folderId", `${TableName.SecretFolder}.id`) + .limit(1) + ); + + await (tx || db)(TableName.SecretVersion).whereNotIn("id", versionsToKeep).delete(); + } catch (error) { + throw new DatabaseError({ + error, + name: "Secret Version Prune" + }); + } + }; + return { ...secretVersionOrm, + pruneExcessVersions, findLatestVersionMany, bulkUpdate, findLatestVersionByFolderId, From 9117067ab505e08598378884c2b4d6517d5e72dc Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 31 May 2024 21:38:16 +0800 Subject: [PATCH 18/91] feat: finalized pruning logic --- .../services/secret-snapshot/snapshot-dal.ts | 21 ++++++----- .../secret-folder-version-dal.ts | 36 ++++++++++++++++++- .../src/services/secret/secret-version-dal.ts | 22 ++++++------ 3 files changed, 56 insertions(+), 23 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index 1618024bbc..a35345c17c 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -349,20 +349,19 @@ export const snapshotDALFactory = (db: TDbClient) => { ) .as("ranked_snapshots"); + const folderLimits = (tx || db)(TableName.Snapshot) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.Snapshot}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .whereIn(`${TableName.Snapshot}.folderId`, folderBatch) + .groupBy(`${TableName.Snapshot}.folderId`, `${TableName.Project}.pitVersionLimit`) + .select("folderId", "pitVersionLimit") + .as("folder_limits"); + const snapshotsToKeep = (tx || db) .select("id") .from(rankedSnapshots) - .where( - "row_num", - "<=", - (tx || db) - .select(`${TableName.Project}.pitVersionLimit`) - .from(TableName.Project) - .join(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) - .join(TableName.Snapshot, `${TableName.Snapshot}.envId`, `${TableName.Environment}.id`) - .join(rankedSnapshots, "ranked_snapshots.folderId", `${TableName.Snapshot}.folderId`) - .limit(1) - ); + .join(folderLimits, "folder_limits.folderId", "ranked_snapshots.folderId") + .whereRaw(`ranked_snapshots.row_num <= folder_limits."pitVersionLimit"`); await (tx || db)(TableName.Snapshot) .whereIn("folderId", folderBatch) diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index f133308cf4..cfc3706408 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -62,5 +62,39 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { } }; - return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId }; + const pruneExcessVersions = async (tx?: Knex) => { + try { + const rankedFolderVersions = (tx || db)(TableName.SecretFolderVersion) + .select( + "id", + "folderId", + (tx || db).raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretFolderVersion}."folderId" ORDER BY ${TableName.SecretFolderVersion}."createdAt" DESC) AS row_num` + ) + ) + .as("ranked_folder_versions"); + + const folderLimits = (tx || db)(TableName.SecretFolderVersion) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .groupBy(`${TableName.SecretFolderVersion}.folderId`, `${TableName.Project}.pitVersionLimit`) + .select("folderId", "pitVersionLimit") + .as("folder_limits"); + + const versionsToKeep = (tx || db)(rankedFolderVersions) + .select("id") + .from(rankedFolderVersions) + .join(folderLimits, "folder_limits.folderId", "ranked_folder_versions.folderId") + .whereRaw(`ranked_folder_versions.row_num <= folder_limits."pitVersionLimit"`); + + await (tx || db)(TableName.SecretFolderVersion).whereNotIn("id", versionsToKeep).delete(); + } catch (error) { + throw new DatabaseError({ + error, + name: "Secret Version Prune" + }); + } + }; + + return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId, pruneExcessVersions }; }; diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index 5d52f3c07f..374907c3d2 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -123,19 +123,19 @@ export const secretVersionDALFactory = (db: TDbClient) => { ) .as("ranked_secret_versions"); + const folderLimits = (tx || db)(TableName.SecretVersion) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .groupBy(`${TableName.SecretVersion}.folderId`, `${TableName.Project}.pitVersionLimit`) + .select("folderId", "pitVersionLimit") + .as("folder_limits"); + const versionsToKeep = (tx || db)(rankedSecretVersions) .select("id") - .where( - "row_num", - "<=", - (tx || db) - .select(`${TableName.Project}.pitVersionLimit`) - .from(TableName.Project) - .join(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) - .join(TableName.SecretFolder, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) - .join(rankedSecretVersions, "ranked_secret_versions.folderId", `${TableName.SecretFolder}.id`) - .limit(1) - ); + .from(rankedSecretVersions) + .join(folderLimits, "folder_limits.folderId", "ranked_secret_versions.folderId") + .whereRaw(`ranked_secret_versions.row_num <= folder_limits."pitVersionLimit"`); await (tx || db)(TableName.SecretVersion).whereNotIn("id", versionsToKeep).delete(); } catch (error) { From abd8d6aa8af73b0421e649e0a0733ca9ade807b2 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 31 May 2024 23:18:02 +0800 Subject: [PATCH 19/91] feat: added support for version limit update --- .../src/server/routes/v1/project-router.ts | 38 ++++++++ .../src/services/project/project-service.ts | 33 ++++++- backend/src/services/project/project-types.ts | 5 ++ frontend/src/hooks/api/workspace/queries.tsx | 16 ++++ frontend/src/hooks/api/workspace/types.ts | 4 +- .../PointInTimeVersionLimitSection.tsx | 88 +++++++++++++++++++ .../PointInTimeVersionLimitSection/index.tsx | 1 + .../ProjectGeneralTab/ProjectGeneralTab.tsx | 2 + 8 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx create mode 100644 frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/index.tsx diff --git a/backend/src/server/routes/v1/project-router.ts b/backend/src/server/routes/v1/project-router.ts index 1cf655a973..0984b66f61 100644 --- a/backend/src/server/routes/v1/project-router.ts +++ b/backend/src/server/routes/v1/project-router.ts @@ -334,6 +334,44 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { } }); + server.route({ + method: "PUT", + url: "/:workspaceSlug/version-limit", + config: { + rateLimit: writeLimit + }, + schema: { + params: z.object({ + workspaceSlug: z.string().trim() + }), + body: z.object({ + pitVersionLimit: z.number().min(1).max(100) + }), + response: { + 200: z.object({ + message: z.string(), + workspace: ProjectsSchema + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const workspace = await server.services.project.updateVersionLimit({ + actorId: req.permission.id, + actor: req.permission.type, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + pitVersionLimit: req.body.pitVersionLimit, + workspaceSlug: req.params.workspaceSlug + }); + + return { + message: "Successfully changed workspace version limit", + workspace + }; + } + }); + server.route({ method: "GET", url: "/:workspaceId/integrations", diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index 1e588dd08f..ac6cfbc003 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -39,6 +39,7 @@ import { TToggleProjectAutoCapitalizationDTO, TUpdateProjectDTO, TUpdateProjectNameDTO, + TUpdateProjectVersionLimitDTO, TUpgradeProjectDTO } from "./project-types"; @@ -407,6 +408,35 @@ export const projectServiceFactory = ({ return updatedProject; }; + const updateVersionLimit = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + pitVersionLimit, + workspaceSlug + }: TUpdateProjectVersionLimitDTO) => { + const project = await projectDAL.findProjectBySlug(workspaceSlug, actorOrgId); + if (!project) { + throw new BadRequestError({ + message: "Project not found" + }); + } + + const { hasRole } = await permissionService.getProjectPermission( + actor, + actorId, + project.id, + actorAuthMethod, + actorOrgId + ); + + if (!hasRole(ProjectMembershipRole.Admin)) + throw new BadRequestError({ message: "Only admins are allowed to take this action" }); + + return projectDAL.updateById(project.id, { pitVersionLimit }); + }; + const updateName = async ({ projectId, actor, @@ -502,6 +532,7 @@ export const projectServiceFactory = ({ getAProject, toggleAutoCapitalization, updateName, - upgradeProject + upgradeProject, + updateVersionLimit }; }; diff --git a/backend/src/services/project/project-types.ts b/backend/src/services/project/project-types.ts index dcd424e18c..fbcfd2d9a8 100644 --- a/backend/src/services/project/project-types.ts +++ b/backend/src/services/project/project-types.ts @@ -43,6 +43,11 @@ export type TToggleProjectAutoCapitalizationDTO = { autoCapitalization: boolean; } & TProjectPermission; +export type TUpdateProjectVersionLimitDTO = { + pitVersionLimit: number; + workspaceSlug: string; +} & Omit; + export type TUpdateProjectNameDTO = { name: string; } & TProjectPermission; diff --git a/frontend/src/hooks/api/workspace/queries.tsx b/frontend/src/hooks/api/workspace/queries.tsx index 71dbb8e01e..6462490bbd 100644 --- a/frontend/src/hooks/api/workspace/queries.tsx +++ b/frontend/src/hooks/api/workspace/queries.tsx @@ -20,6 +20,7 @@ import { TUpdateWorkspaceIdentityRoleDTO, TUpdateWorkspaceUserRoleDTO, UpdateEnvironmentDTO, + UpdatePitVersionLimitDTO, Workspace } from "./types"; @@ -249,6 +250,21 @@ export const useToggleAutoCapitalization = () => { }); }; +export const useUpdateWorkspaceVersionLimit = () => { + const queryClient = useQueryClient(); + + return useMutation<{}, {}, UpdatePitVersionLimitDTO>({ + mutationFn: ({ projectSlug, pitVersionLimit }) => { + return apiRequest.put(`/api/v1/workspace/${projectSlug}/version-limit`, { + pitVersionLimit + }); + }, + onSuccess: () => { + queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace); + } + }); +}; + export const useDeleteWorkspace = () => { const queryClient = useQueryClient(); diff --git a/frontend/src/hooks/api/workspace/types.ts b/frontend/src/hooks/api/workspace/types.ts index 8be9beed0d..8c28d09389 100644 --- a/frontend/src/hooks/api/workspace/types.ts +++ b/frontend/src/hooks/api/workspace/types.ts @@ -16,6 +16,7 @@ export type Workspace = { upgradeStatus: string | null; autoCapitalization: boolean; environments: WorkspaceEnv[]; + pitVersionLimit: number; slug: string; }; @@ -48,6 +49,7 @@ export type CreateWorkspaceDTO = { }; export type RenameWorkspaceDTO = { workspaceID: string; newWorkspaceName: string }; +export type UpdatePitVersionLimitDTO = { projectSlug: string; pitVersionLimit: number }; export type ToggleAutoCapitalizationDTO = { workspaceID: string; state: boolean }; export type DeleteWorkspaceDTO = { workspaceID: string }; @@ -128,4 +130,4 @@ export type TUpdateWorkspaceGroupRoleDTO = { temporaryAccessStartTime: string; } )[]; -}; \ No newline at end of file +}; diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx new file mode 100644 index 0000000000..8c820b727d --- /dev/null +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx @@ -0,0 +1,88 @@ +import { Controller, useForm } from "react-hook-form"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { z } from "zod"; + +import { createNotification } from "@app/components/notifications"; +import { Button, FormControl, Input } from "@app/components/v2"; +import { useProjectPermission, useWorkspace } from "@app/context"; +import { ProjectMembershipRole } from "@app/hooks/api/roles/types"; +import { useUpdateWorkspaceVersionLimit } from "@app/hooks/api/workspace/queries"; + +const formSchema = z.object({ + pitVersionLimit: z.coerce.number().min(1).max(100) +}); + +type TForm = z.infer; + +export const PointInTimeVersionLimitSection = () => { + const { mutateAsync: updatePitVersion } = useUpdateWorkspaceVersionLimit(); + + const { currentWorkspace } = useWorkspace(); + const { membership } = useProjectPermission(); + + const { + control, + formState: { isSubmitting }, + handleSubmit + } = useForm({ + resolver: zodResolver(formSchema), + values: { + pitVersionLimit: currentWorkspace?.pitVersionLimit || 10 + } + }); + + if (!currentWorkspace) return null; + + const handleVersionLimitSubmit = async ({ pitVersionLimit }: TForm) => { + try { + await updatePitVersion({ + pitVersionLimit, + projectSlug: currentWorkspace.slug + }); + + createNotification({ + text: "Successfully updated version limit", + type: "success" + }); + } catch (err) { + createNotification({ + text: "Failed updating project's version limit", + type: "error" + }); + } + }; + + const isAdmin = membership.roles.includes(ProjectMembershipRole.Admin); + return ( +
+
+

Point in Time Recovery

+
+

+ This defines the maximum number of folder snapshots, secret versions, and folder versions + that are retained by the system +

+
+
+ ( + + + + )} + /> +
+ +
+
+ ); +}; diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/index.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/index.tsx new file mode 100644 index 0000000000..242b8c79a1 --- /dev/null +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/index.tsx @@ -0,0 +1 @@ +export { PointInTimeVersionLimitSection } from "./PointInTimeVersionLimitSection"; diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx index 7d7c30fb0a..511dff93e1 100644 --- a/frontend/src/views/Settings/ProjectSettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx @@ -3,6 +3,7 @@ import { BackfillSecretReferenceSecretion } from "../BackfillSecretReferenceSect import { DeleteProjectSection } from "../DeleteProjectSection"; import { E2EESection } from "../E2EESection"; import { EnvironmentSection } from "../EnvironmentSection"; +import { PointInTimeVersionLimitSection } from "../PointInTimeVersionLimitSection"; import { ProjectNameChangeSection } from "../ProjectNameChangeSection"; import { SecretTagsSection } from "../SecretTagsSection"; @@ -14,6 +15,7 @@ export const ProjectGeneralTab = () => { + From 4d8f94a9dc4909d714af93b8ee5b5e08dd612db3 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 31 May 2024 23:25:56 +0800 Subject: [PATCH 20/91] feat: added version prune to daily resource queue --- backend/src/server/routes/index.ts | 3 +++ .../resource-cleanup/resource-cleanup-queue.ts | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index b0b2db9b19..8c4ff2a701 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -795,6 +795,9 @@ export const registerRoutes = async ( const dailyResourceCleanUp = dailyResourceCleanUpQueueServiceFactory({ auditLogDAL, queueService, + secretVersionDAL, + secretFolderVersionDAL: folderVersionDAL, + snapshotDAL, identityAccessTokenDAL, secretSharingDAL }); diff --git a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts index afae2677f7..2e01e35494 100644 --- a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts +++ b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts @@ -1,13 +1,19 @@ import { TAuditLogDALFactory } from "@app/ee/services/audit-log/audit-log-dal"; +import { TSnapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identity-access-token-dal"; +import { TSecretVersionDALFactory } from "../secret/secret-version-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretSharingDALFactory } from "../secret-sharing/secret-sharing-dal"; type TDailyResourceCleanUpQueueServiceFactoryDep = { auditLogDAL: Pick; identityAccessTokenDAL: Pick; + secretVersionDAL: Pick; + secretFolderVersionDAL: Pick; + snapshotDAL: Pick; secretSharingDAL: Pick; queueService: TQueueServiceFactory; }; @@ -17,6 +23,9 @@ export type TDailyResourceCleanUpQueueServiceFactory = ReturnType { @@ -25,6 +34,9 @@ export const dailyResourceCleanUpQueueServiceFactory = ({ await auditLogDAL.pruneAuditLog(); await identityAccessTokenDAL.removeExpiredTokens(); await secretSharingDAL.pruneExpiredSharedSecrets(); + await snapshotDAL.pruneExcessSnapshots(); + await secretVersionDAL.pruneExcessVersions(); + await secretFolderVersionDAL.pruneExcessVersions(); logger.info(`${QueueName.DailyResourceCleanUp}: queue task completed`); }); From d76760fa9c9fc43914e6cdf7c83694dfeb585f9b Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 31 May 2024 23:42:49 +0800 Subject: [PATCH 21/91] misc: updated schema --- backend/src/db/schemas/projects.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/db/schemas/projects.ts b/backend/src/db/schemas/projects.ts index 9197a355d9..dd7e999900 100644 --- a/backend/src/db/schemas/projects.ts +++ b/backend/src/db/schemas/projects.ts @@ -17,7 +17,7 @@ export const ProjectsSchema = z.object({ updatedAt: z.date(), version: z.number().default(1), upgradeStatus: z.string().nullable().optional(), - pitVersionLimit: z.number() + pitVersionLimit: z.number().default(10) }); export type TProjects = z.infer; From 4eb08c64d45b8fcc2a1302a3db345fb2a8ab4ed2 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Sat, 1 Jun 2024 01:07:25 +0800 Subject: [PATCH 22/91] misc: updated error message --- backend/src/services/secret-folder/secret-folder-version-dal.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index cfc3706408..2f32001dcd 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -91,7 +91,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { } catch (error) { throw new DatabaseError({ error, - name: "Secret Version Prune" + name: "Secret Folder Version Prune" }); } }; From b8e9417466cf47b52b518312639e398b4e97f7f8 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Sat, 1 Jun 2024 03:26:35 +0800 Subject: [PATCH 23/91] misc: modified pruning sql logic --- .../services/secret-snapshot/snapshot-dal.ts | 54 ++++++++----------- .../src/services/secret/secret-version-dal.ts | 48 ++++++++--------- 2 files changed, 44 insertions(+), 58 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index a35345c17c..44fcadf390 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -325,9 +325,9 @@ export const snapshotDALFactory = (db: TDbClient) => { } }; - const pruneExcessSnapshots = async (tx?: Knex) => { + const pruneExcessSnapshots = async () => { try { - const folders = await (tx || db)(TableName.SecretFolder).select("id"); + const folders = await db(TableName.SecretFolder).select("id"); const folderIds = folders.map((folder) => folder.id); const PRUNE_FOLDER_BATCH_SIZE = 500; @@ -338,35 +338,27 @@ export const snapshotDALFactory = (db: TDbClient) => { } for await (const folderBatch of pruneBatches) { - const rankedSnapshots = (tx || db)(TableName.Snapshot) - .whereIn(`${TableName.Snapshot}.folderId`, folderBatch) - .select( - "folderId", - "id", - (tx || db).raw( - `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` - ) - ) - .as("ranked_snapshots"); - - const folderLimits = (tx || db)(TableName.Snapshot) - .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.Snapshot}.envId`) - .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) - .whereIn(`${TableName.Snapshot}.folderId`, folderBatch) - .groupBy(`${TableName.Snapshot}.folderId`, `${TableName.Project}.pitVersionLimit`) - .select("folderId", "pitVersionLimit") - .as("folder_limits"); - - const snapshotsToKeep = (tx || db) - .select("id") - .from(rankedSnapshots) - .join(folderLimits, "folder_limits.folderId", "ranked_snapshots.folderId") - .whereRaw(`ranked_snapshots.row_num <= folder_limits."pitVersionLimit"`); - - await (tx || db)(TableName.Snapshot) - .whereIn("folderId", folderBatch) - .whereNotIn("id", snapshotsToKeep) - .delete(); + await secretSnapshotOrm.transaction(async (txn) => { + return txn(TableName.Snapshot) + .with("snapshot_cte", (qb) => { + void qb + .from(TableName.Snapshot) + .whereIn(`${TableName.Snapshot}.folderId`, folderBatch) + .select( + "folderId", + `${TableName.Snapshot}.id as id`, + txn.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` + ) + ); + }) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Snapshot}.folderId`) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`) + .whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); + }); } } catch (error) { throw new DatabaseError({ error, name: "SnapshotPrune" }); diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index 374907c3d2..596d0f4832 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -110,34 +110,28 @@ export const secretVersionDALFactory = (db: TDbClient) => { } }; - const pruneExcessVersions = async (tx?: Knex) => { + const pruneExcessVersions = async () => { try { - const rankedSecretVersions = (tx || db)(TableName.SecretVersion) - .select( - "id", - "secretId", - "folderId", - (tx || db).raw( - `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num` - ) - ) - .as("ranked_secret_versions"); - - const folderLimits = (tx || db)(TableName.SecretVersion) - .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`) - .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) - .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) - .groupBy(`${TableName.SecretVersion}.folderId`, `${TableName.Project}.pitVersionLimit`) - .select("folderId", "pitVersionLimit") - .as("folder_limits"); - - const versionsToKeep = (tx || db)(rankedSecretVersions) - .select("id") - .from(rankedSecretVersions) - .join(folderLimits, "folder_limits.folderId", "ranked_secret_versions.folderId") - .whereRaw(`ranked_secret_versions.row_num <= folder_limits."pitVersionLimit"`); - - await (tx || db)(TableName.SecretVersion).whereNotIn("id", versionsToKeep).delete(); + await secretVersionOrm.transaction((txn) => { + return txn(TableName.SecretVersion) + .with("version_cte", (qb) => { + void qb + .from(TableName.SecretVersion) + .select( + "id", + "folderId", + txn.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num` + ) + ); + }) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("version_cte", "version_cte.id", `${TableName.SecretVersion}.id`) + .whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); + }); } catch (error) { throw new DatabaseError({ error, From ab093dfc852c6bfa4c76db5095c45ab2b819ede9 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Mon, 3 Jun 2024 12:49:40 +0800 Subject: [PATCH 24/91] misc: simplified delete query for secret folder version --- .../secret-folder-version-dal.ts | 45 +++++++++---------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index 2f32001dcd..e9c247b522 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -62,32 +62,27 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { } }; - const pruneExcessVersions = async (tx?: Knex) => { + const pruneExcessVersions = async () => { try { - const rankedFolderVersions = (tx || db)(TableName.SecretFolderVersion) - .select( - "id", - "folderId", - (tx || db).raw( - `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretFolderVersion}."folderId" ORDER BY ${TableName.SecretFolderVersion}."createdAt" DESC) AS row_num` - ) - ) - .as("ranked_folder_versions"); - - const folderLimits = (tx || db)(TableName.SecretFolderVersion) - .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`) - .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) - .groupBy(`${TableName.SecretFolderVersion}.folderId`, `${TableName.Project}.pitVersionLimit`) - .select("folderId", "pitVersionLimit") - .as("folder_limits"); - - const versionsToKeep = (tx || db)(rankedFolderVersions) - .select("id") - .from(rankedFolderVersions) - .join(folderLimits, "folder_limits.folderId", "ranked_folder_versions.folderId") - .whereRaw(`ranked_folder_versions.row_num <= folder_limits."pitVersionLimit"`); - - await (tx || db)(TableName.SecretFolderVersion).whereNotIn("id", versionsToKeep).delete(); + await secretFolderVerOrm.transaction((txn) => { + return txn(TableName.SecretFolderVersion) + .with("folder_cte", (qb) => { + void qb + .from(TableName.SecretFolderVersion) + .select( + "id", + "folderId", + txn.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretFolderVersion}."folderId" ORDER BY ${TableName.SecretFolderVersion}."createdAt" DESC) AS row_num` + ) + ); + }) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("folder_cte", "folder_cte.id", `${TableName.SecretFolderVersion}.id`) + .whereRaw(`folder_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); + }); } catch (error) { throw new DatabaseError({ error, From cd6caab508e539aadd09e62a3f32a2d4ed4aa05d Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Mon, 3 Jun 2024 17:56:00 +0800 Subject: [PATCH 25/91] misc: migrated to using keyset pagnination --- .../services/secret-snapshot/snapshot-dal.ts | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index 44fcadf390..b49e520b8a 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-await-in-loop */ import { Knex } from "knex"; import { TDbClient } from "@app/db"; @@ -11,6 +12,7 @@ import { } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; +import { logger } from "@app/lib/logger"; export type TSnapshotDALFactory = ReturnType; @@ -326,28 +328,32 @@ export const snapshotDALFactory = (db: TDbClient) => { }; const pruneExcessSnapshots = async () => { - try { - const folders = await db(TableName.SecretFolder).select("id"); - const folderIds = folders.map((folder) => folder.id); - const PRUNE_FOLDER_BATCH_SIZE = 500; + const PRUNE_FOLDER_BATCH_SIZE = 10000; + let uuidOffset = "00000000-0000-0000-0000-000000000000"; - const pruneBatches = []; - for (let x = 0; x < folderIds.length; x += PRUNE_FOLDER_BATCH_SIZE) { - const batch = folderIds.slice(x, x + PRUNE_FOLDER_BATCH_SIZE); - pruneBatches.push(batch); - } + // eslint-disable-next-line no-constant-condition, no-unreachable-loop + while (true) { + const folderBatch = await db(TableName.SecretFolder) + .where("id", ">", uuidOffset) + .orderBy("id", "asc") + .limit(PRUNE_FOLDER_BATCH_SIZE) + .select("id"); - for await (const folderBatch of pruneBatches) { - await secretSnapshotOrm.transaction(async (txn) => { - return txn(TableName.Snapshot) + const batchEntries = folderBatch.map((folder) => folder.id); + logger.info("UUID offset:", uuidOffset); + + if (folderBatch.length) { + try { + logger.info(`Pruning snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}`); + await db(TableName.Snapshot) .with("snapshot_cte", (qb) => { void qb .from(TableName.Snapshot) - .whereIn(`${TableName.Snapshot}.folderId`, folderBatch) + .whereIn(`${TableName.Snapshot}.folderId`, batchEntries) .select( "folderId", `${TableName.Snapshot}.id as id`, - txn.raw( + db.raw( `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` ) ); @@ -358,10 +364,16 @@ export const snapshotDALFactory = (db: TDbClient) => { .join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`) .whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`) .delete(); - }); + } catch (err) { + logger.error( + `Failed to prune snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}` + ); + } finally { + uuidOffset = batchEntries[batchEntries.length - 1]; + } + } else { + return; } - } catch (error) { - throw new DatabaseError({ error, name: "SnapshotPrune" }); } }; From 4d830f1d1a494edec02ac632c1a656ad34a81c8a Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Mon, 3 Jun 2024 17:58:39 +0800 Subject: [PATCH 26/91] misc: added outer try catch block --- .../services/secret-snapshot/snapshot-dal.ts | 82 ++++++++++--------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index b49e520b8a..7c17c4f072 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -331,49 +331,53 @@ export const snapshotDALFactory = (db: TDbClient) => { const PRUNE_FOLDER_BATCH_SIZE = 10000; let uuidOffset = "00000000-0000-0000-0000-000000000000"; - // eslint-disable-next-line no-constant-condition, no-unreachable-loop - while (true) { - const folderBatch = await db(TableName.SecretFolder) - .where("id", ">", uuidOffset) - .orderBy("id", "asc") - .limit(PRUNE_FOLDER_BATCH_SIZE) - .select("id"); + try { + // eslint-disable-next-line no-constant-condition, no-unreachable-loop + while (true) { + const folderBatch = await db(TableName.SecretFolder) + .where("id", ">", uuidOffset) + .orderBy("id", "asc") + .limit(PRUNE_FOLDER_BATCH_SIZE) + .select("id"); - const batchEntries = folderBatch.map((folder) => folder.id); - logger.info("UUID offset:", uuidOffset); + const batchEntries = folderBatch.map((folder) => folder.id); + logger.info("UUID offset:", uuidOffset); - if (folderBatch.length) { - try { - logger.info(`Pruning snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}`); - await db(TableName.Snapshot) - .with("snapshot_cte", (qb) => { - void qb - .from(TableName.Snapshot) - .whereIn(`${TableName.Snapshot}.folderId`, batchEntries) - .select( - "folderId", - `${TableName.Snapshot}.id as id`, - db.raw( - `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` - ) - ); - }) - .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Snapshot}.folderId`) - .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) - .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) - .join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`) - .whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`) - .delete(); - } catch (err) { - logger.error( - `Failed to prune snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}` - ); - } finally { - uuidOffset = batchEntries[batchEntries.length - 1]; + if (folderBatch.length) { + try { + logger.info(`Pruning snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}`); + await db(TableName.Snapshot) + .with("snapshot_cte", (qb) => { + void qb + .from(TableName.Snapshot) + .whereIn(`${TableName.Snapshot}.folderId`, batchEntries) + .select( + "folderId", + `${TableName.Snapshot}.id as id`, + db.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.Snapshot}."folderId" ORDER BY ${TableName.Snapshot}."createdAt" DESC) AS row_num` + ) + ); + }) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.Snapshot}.folderId`) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("snapshot_cte", "snapshot_cte.id", `${TableName.Snapshot}.id`) + .whereRaw(`snapshot_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); + } catch (err) { + logger.error( + `Failed to prune snapshots in range ${batchEntries[0]}:${batchEntries[batchEntries.length - 1]}` + ); + } finally { + uuidOffset = batchEntries[batchEntries.length - 1]; + } + } else { + return; } - } else { - return; } + } catch (error) { + throw new DatabaseError({ error, name: "SnapshotPrune" }); } }; From 68a30f4212a885a82cecae33685be41cc61d54d0 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Mon, 3 Jun 2024 22:06:32 +0800 Subject: [PATCH 27/91] misc: removed transactional --- .../secret-folder-version-dal.ts | 36 +++++++++--------- .../src/services/secret/secret-version-dal.ts | 38 +++++++++---------- 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index e9c247b522..47074321f6 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -64,25 +64,23 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { const pruneExcessVersions = async () => { try { - await secretFolderVerOrm.transaction((txn) => { - return txn(TableName.SecretFolderVersion) - .with("folder_cte", (qb) => { - void qb - .from(TableName.SecretFolderVersion) - .select( - "id", - "folderId", - txn.raw( - `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretFolderVersion}."folderId" ORDER BY ${TableName.SecretFolderVersion}."createdAt" DESC) AS row_num` - ) - ); - }) - .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`) - .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) - .join("folder_cte", "folder_cte.id", `${TableName.SecretFolderVersion}.id`) - .whereRaw(`folder_cte.row_num > ${TableName.Project}."pitVersionLimit"`) - .delete(); - }); + await db(TableName.SecretFolderVersion) + .with("folder_cte", (qb) => { + void qb + .from(TableName.SecretFolderVersion) + .select( + "id", + "folderId", + db.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretFolderVersion}."folderId" ORDER BY ${TableName.SecretFolderVersion}."createdAt" DESC) AS row_num` + ) + ); + }) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolderVersion}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("folder_cte", "folder_cte.id", `${TableName.SecretFolderVersion}.id`) + .whereRaw(`folder_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); } catch (error) { throw new DatabaseError({ error, diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index 596d0f4832..baff6ab996 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -112,26 +112,24 @@ export const secretVersionDALFactory = (db: TDbClient) => { const pruneExcessVersions = async () => { try { - await secretVersionOrm.transaction((txn) => { - return txn(TableName.SecretVersion) - .with("version_cte", (qb) => { - void qb - .from(TableName.SecretVersion) - .select( - "id", - "folderId", - txn.raw( - `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num` - ) - ); - }) - .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`) - .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) - .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) - .join("version_cte", "version_cte.id", `${TableName.SecretVersion}.id`) - .whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`) - .delete(); - }); + await db(TableName.SecretVersion) + .with("version_cte", (qb) => { + void qb + .from(TableName.SecretVersion) + .select( + "id", + "folderId", + db.raw( + `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersion}."secretId" ORDER BY ${TableName.SecretVersion}."createdAt" DESC) AS row_num` + ) + ); + }) + .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersion}.folderId`) + .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`) + .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) + .join("version_cte", "version_cte.id", `${TableName.SecretVersion}.id`) + .whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + .delete(); } catch (error) { throw new DatabaseError({ error, From f21a13f38871630759ae491c8f875fe013dc443b Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 4 Jun 2024 23:46:02 +0800 Subject: [PATCH 28/91] adjustment: removed artificial limiting of pit versions --- .../secret-snapshot/secret-snapshot-service.ts | 18 ++---------------- .../services/secret-snapshot/snapshot-dal.ts | 1 - backend/src/server/routes/index.ts | 1 - backend/src/services/secret/secret-service.ts | 13 ++----------- .../PointInTimeVersionLimitSection.tsx | 9 +++++++-- 5 files changed, 11 insertions(+), 31 deletions(-) diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index 6197c08256..5a720a9e65 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -4,7 +4,6 @@ import { TableName, TSecretTagJunctionInsert } from "@app/db/schemas"; import { BadRequestError, InternalServerError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; -import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TSecretDALFactory } from "@app/services/secret/secret-dal"; import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; @@ -38,7 +37,6 @@ type TSecretSnapshotServiceFactoryDep = { folderDAL: Pick; permissionService: Pick; licenseService: Pick; - projectDAL: Pick; }; export type TSecretSnapshotServiceFactory = ReturnType; @@ -50,7 +48,6 @@ export const secretSnapshotServiceFactory = ({ snapshotSecretDAL, snapshotFolderDAL, folderDAL, - projectDAL, secretDAL, permissionService, licenseService, @@ -84,9 +81,7 @@ export const secretSnapshotServiceFactory = ({ const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found" }); - const project = await projectDAL.findById(projectId); - const count = await snapshotDAL.countOfSnapshotsByFolderId(folder.id); - return Math.min(count, project.pitVersionLimit); + return snapshotDAL.countOfSnapshotsByFolderId(folder.id); }; const listSnapshots = async ({ @@ -118,16 +113,7 @@ export const secretSnapshotServiceFactory = ({ const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found" }); - const { pitVersionLimit } = await projectDAL.findById(projectId); - const computedQueryLimit = Math.min(pitVersionLimit - offset, limit); - if (offset > pitVersionLimit || computedQueryLimit <= 0) { - return []; - } - - const snapshots = await snapshotDAL.find( - { folderId: folder.id }, - { limit: computedQueryLimit, offset, sort: [["createdAt", "desc"]] } - ); + const snapshots = await snapshotDAL.find({ folderId: folder.id }, { limit, offset, sort: [["createdAt", "desc"]] }); return snapshots; }; diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index 7c17c4f072..d36a1d00e5 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -341,7 +341,6 @@ export const snapshotDALFactory = (db: TDbClient) => { .select("id"); const batchEntries = folderBatch.map((folder) => folder.id); - logger.info("UUID offset:", uuidOffset); if (folderBatch.length) { try { diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 8c4ff2a701..4442b3bb9d 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -535,7 +535,6 @@ export const registerRoutes = async ( licenseService, folderDAL, secretDAL, - projectDAL, snapshotDAL, snapshotFolderDAL, snapshotSecretDAL, diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 77c4024ac7..39e47a28e1 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -72,7 +72,7 @@ type TSecretServiceFactoryDep = { secretDAL: TSecretDALFactory; secretTagDAL: TSecretTagDALFactory; secretVersionDAL: TSecretVersionDALFactory; - projectDAL: Pick; + projectDAL: Pick; projectEnvDAL: Pick; folderDAL: Pick< TSecretFolderDALFactory, @@ -1354,16 +1354,7 @@ export const secretServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); - const { pitVersionLimit } = await projectDAL.findById(folder.projectId); - const computedQueryLimit = Math.min(pitVersionLimit - offset, limit); - if (offset > pitVersionLimit || computedQueryLimit <= 0) { - return []; - } - - const secretVersions = await secretVersionDAL.find( - { secretId }, - { offset, limit: computedQueryLimit, sort: [["createdAt", "desc"]] } - ); + const secretVersions = await secretVersionDAL.find({ secretId }, { offset, limit, sort: [["createdAt", "desc"]] }); return secretVersions; }; diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx index 8c820b727d..4091ec06ed 100644 --- a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx @@ -22,7 +22,7 @@ export const PointInTimeVersionLimitSection = () => { const { control, - formState: { isSubmitting }, + formState: { isSubmitting, isDirty }, handleSubmit } = useForm({ resolver: zodResolver(formSchema), @@ -79,7 +79,12 @@ export const PointInTimeVersionLimitSection = () => { )} /> - From ee152f2d2044970c37885fcdbe2753aaafedf049 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 4 Jun 2024 23:50:10 +0800 Subject: [PATCH 29/91] misc: added cleanup frequency note for pit versions --- .../PointInTimeVersionLimitSection.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx index 4091ec06ed..f3035922b5 100644 --- a/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx +++ b/frontend/src/views/Settings/ProjectSettingsPage/components/PointInTimeVersionLimitSection/PointInTimeVersionLimitSection.tsx @@ -60,7 +60,8 @@ export const PointInTimeVersionLimitSection = () => {

This defines the maximum number of folder snapshots, secret versions, and folder versions - that are retained by the system + that are retained by the system. The cleanup of excess snapshots and versions happens once a + day on midnight of UTC.

From 81e4129e510eaa76248003b934b74d736ee147ec Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 6 Jun 2024 20:42:54 +0800 Subject: [PATCH 30/91] feat: added base captcha implementation --- .env.example | 5 ++ ...nsecutive-failed-password-attempts-user.ts | 29 ++++++++++++ backend/src/db/schemas/projects.ts | 3 +- backend/src/db/schemas/users.ts | 3 +- backend/src/lib/config/env.ts | 6 ++- backend/src/server/routes/v3/login-router.ts | 4 +- .../src/services/auth/auth-login-service.ts | 46 +++++++++++++++++-- backend/src/services/auth/auth-login-type.ts | 1 + frontend/next.config.js | 9 ++-- frontend/package-lock.json | 20 +++++++- frontend/package.json | 1 + .../components/utilities/attemptCliLogin.ts | 7 ++- .../src/components/utilities/attemptLogin.ts | 5 +- .../src/components/utilities/config/index.ts | 3 +- frontend/src/hooks/api/auth/types.ts | 1 + .../components/InitialStep/InitialStep.tsx | 33 +++++++++++-- .../components/PasswordStep/PasswordStep.tsx | 33 +++++++++++-- 17 files changed, 186 insertions(+), 23 deletions(-) create mode 100644 backend/src/db/migrations/20240606073539_add-consecutive-failed-password-attempts-user.ts diff --git a/.env.example b/.env.example index bdb3e536d0..33754d55d4 100644 --- a/.env.example +++ b/.env.example @@ -63,3 +63,8 @@ CLIENT_SECRET_GITHUB_LOGIN= CLIENT_ID_GITLAB_LOGIN= CLIENT_SECRET_GITLAB_LOGIN= + +CAPTCHA_SECRET= +CAPTCHA_ENABLED= + +NEXT_PUBLIC_CAPTCHA_SITE_KEY= diff --git a/backend/src/db/migrations/20240606073539_add-consecutive-failed-password-attempts-user.ts b/backend/src/db/migrations/20240606073539_add-consecutive-failed-password-attempts-user.ts new file mode 100644 index 0000000000..66fa031821 --- /dev/null +++ b/backend/src/db/migrations/20240606073539_add-consecutive-failed-password-attempts-user.ts @@ -0,0 +1,29 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + const hasConsecutiveFailedPasswordAttempts = await knex.schema.hasColumn( + TableName.Users, + "consecutiveFailedPasswordAttempts" + ); + + await knex.schema.alterTable(TableName.Users, (tb) => { + if (!hasConsecutiveFailedPasswordAttempts) { + tb.integer("consecutiveFailedPasswordAttempts").defaultTo(0); + } + }); +} + +export async function down(knex: Knex): Promise { + const hasConsecutiveFailedPasswordAttempts = await knex.schema.hasColumn( + TableName.Users, + "consecutiveFailedPasswordAttempts" + ); + + await knex.schema.alterTable(TableName.Users, (tb) => { + if (hasConsecutiveFailedPasswordAttempts) { + tb.dropColumn("consecutiveFailedPasswordAttempts"); + } + }); +} diff --git a/backend/src/db/schemas/projects.ts b/backend/src/db/schemas/projects.ts index 3965e24c0a..dd7e999900 100644 --- a/backend/src/db/schemas/projects.ts +++ b/backend/src/db/schemas/projects.ts @@ -16,7 +16,8 @@ export const ProjectsSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), version: z.number().default(1), - upgradeStatus: z.string().nullable().optional() + upgradeStatus: z.string().nullable().optional(), + pitVersionLimit: z.number().default(10) }); export type TProjects = z.infer; diff --git a/backend/src/db/schemas/users.ts b/backend/src/db/schemas/users.ts index 9e0b9a3b51..5134f3ee60 100644 --- a/backend/src/db/schemas/users.ts +++ b/backend/src/db/schemas/users.ts @@ -25,7 +25,8 @@ export const UsersSchema = z.object({ isEmailVerified: z.boolean().default(false).nullable().optional(), consecutiveFailedMfaAttempts: z.number().default(0).nullable().optional(), isLocked: z.boolean().default(false).nullable().optional(), - temporaryLockDateEnd: z.date().nullable().optional() + temporaryLockDateEnd: z.date().nullable().optional(), + consecutiveFailedPasswordAttempts: z.number().default(0).nullable().optional() }); export type TUsers = z.infer; diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 6ae8bf02d4..d9438d77d2 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -119,7 +119,11 @@ const envSchema = z .transform((val) => val === "true") .optional(), INFISICAL_CLOUD: zodStrBool.default("false"), - MAINTENANCE_MODE: zodStrBool.default("false") + MAINTENANCE_MODE: zodStrBool.default("false"), + + // CAPTCHA + CAPTCHA_SECRET: zpStr(z.string().optional()), + CAPTCHA_ENABLED: zodStrBool.default("false") }) .transform((data) => ({ ...data, diff --git a/backend/src/server/routes/v3/login-router.ts b/backend/src/server/routes/v3/login-router.ts index 900ad56d27..4c7df5612b 100644 --- a/backend/src/server/routes/v3/login-router.ts +++ b/backend/src/server/routes/v3/login-router.ts @@ -80,7 +80,8 @@ export const registerLoginRouter = async (server: FastifyZodProvider) => { body: z.object({ email: z.string().trim(), providerAuthToken: z.string().trim().optional(), - clientProof: z.string().trim() + clientProof: z.string().trim(), + captchaToken: z.string().trim().optional() }), response: { 200: z.discriminatedUnion("mfaEnabled", [ @@ -106,6 +107,7 @@ export const registerLoginRouter = async (server: FastifyZodProvider) => { const appCfg = getConfig(); const data = await server.services.login.loginExchangeClientProof({ + captchaToken: req.body.captchaToken, email: req.body.email, ip: req.realIp, userAgent, diff --git a/backend/src/services/auth/auth-login-service.ts b/backend/src/services/auth/auth-login-service.ts index cbf43b2454..065e304d45 100644 --- a/backend/src/services/auth/auth-login-service.ts +++ b/backend/src/services/auth/auth-login-service.ts @@ -3,6 +3,7 @@ import jwt from "jsonwebtoken"; import { TUsers, UserDeviceSchema } from "@app/db/schemas"; import { isAuthMethodSaml } from "@app/ee/services/permission/permission-fns"; import { getConfig } from "@app/lib/config/env"; +import { request } from "@app/lib/config/request"; import { generateSrpServerKey, srpCheckClientProof } from "@app/lib/crypto"; import { BadRequestError, DatabaseError, UnauthorizedError } from "@app/lib/errors"; import { getServerCfg } from "@app/services/super-admin/super-admin-service"; @@ -176,12 +177,16 @@ export const authLoginServiceFactory = ({ clientProof, ip, userAgent, - providerAuthToken + providerAuthToken, + captchaToken }: TLoginClientProofDTO) => { + const appCfg = getConfig(); + const userEnc = await userDAL.findUserEncKeyByUsername({ username: email }); if (!userEnc) throw new Error("Failed to find user"); + const user = await userDAL.findById(userEnc.userId); const cfg = getConfig(); let authMethod = AuthMethod.EMAIL; @@ -196,6 +201,30 @@ export const authLoginServiceFactory = ({ } } + if ( + user.consecutiveFailedPasswordAttempts && + user.consecutiveFailedPasswordAttempts > 10 && + appCfg.CAPTCHA_ENABLED + ) { + if (!captchaToken) { + throw new BadRequestError({ + name: "Captcha Required" + }); + } + + // validate captcha token + const response = await request.postForm<{ success: boolean }>("https://api.hcaptcha.com/siteverify", { + response: captchaToken, + secret: appCfg.CAPTCHA_SECRET + }); + + if (!response.data.success) { + throw new BadRequestError({ + name: "Invalid Captcha" + }); + } + } + if (!userEnc.serverPrivateKey || !userEnc.clientPublicKey) throw new Error("Failed to authenticate. Try again?"); const isValidClientProof = await srpCheckClientProof( userEnc.salt, @@ -204,7 +233,19 @@ export const authLoginServiceFactory = ({ userEnc.clientPublicKey, clientProof ); - if (!isValidClientProof) throw new Error("Failed to authenticate. Try again?"); + + if (!isValidClientProof) { + await userDAL.update( + { id: userEnc.id }, + { + $incr: { + consecutiveFailedPasswordAttempts: 1 + } + } + ); + + throw new Error("Failed to authenticate. Try again?"); + } await userDAL.updateUserEncryptionByUserId(userEnc.userId, { serverPrivateKey: null, @@ -212,7 +253,6 @@ export const authLoginServiceFactory = ({ }); // send multi factor auth token if they it enabled if (userEnc.isMfaEnabled && userEnc.email) { - const user = await userDAL.findById(userEnc.userId); enforceUserLockStatus(Boolean(user.isLocked), user.temporaryLockDateEnd); const mfaToken = jwt.sign( diff --git a/backend/src/services/auth/auth-login-type.ts b/backend/src/services/auth/auth-login-type.ts index 37b90f548b..4f73ec9961 100644 --- a/backend/src/services/auth/auth-login-type.ts +++ b/backend/src/services/auth/auth-login-type.ts @@ -12,6 +12,7 @@ export type TLoginClientProofDTO = { providerAuthToken?: string; ip: string; userAgent: string; + captchaToken?: string; }; export type TVerifyMfaTokenDTO = { diff --git a/frontend/next.config.js b/frontend/next.config.js index 9d894694fe..5e48e70da7 100644 --- a/frontend/next.config.js +++ b/frontend/next.config.js @@ -1,13 +1,12 @@ - const path = require("path"); const ContentSecurityPolicy = ` default-src 'self'; - script-src 'self' https://app.posthog.com https://js.stripe.com https://api.stripe.com https://widget.intercom.io https://js.intercomcdn.com 'unsafe-inline' 'unsafe-eval'; - style-src 'self' https://rsms.me 'unsafe-inline'; + script-src 'self' https://app.posthog.com https://js.stripe.com https://api.stripe.com https://widget.intercom.io https://js.intercomcdn.com https://hcaptcha.com https://*.hcaptcha.com 'unsafe-inline' 'unsafe-eval'; + style-src 'self' https://rsms.me 'unsafe-inline' https://hcaptcha.com https://*.hcaptcha.com; child-src https://api.stripe.com; - frame-src https://js.stripe.com/ https://api.stripe.com https://www.youtube.com/; - connect-src 'self' wss://nexus-websocket-a.intercom.io https://api-iam.intercom.io https://api.heroku.com/ https://id.heroku.com/oauth/authorize https://id.heroku.com/oauth/token https://checkout.stripe.com https://app.posthog.com https://api.stripe.com https://api.pwnedpasswords.com http://127.0.0.1:*; + frame-src https://js.stripe.com/ https://api.stripe.com https://www.youtube.com/ https://hcaptcha.com https://*.hcaptcha.com; + connect-src 'self' wss://nexus-websocket-a.intercom.io https://api-iam.intercom.io https://api.heroku.com/ https://id.heroku.com/oauth/authorize https://id.heroku.com/oauth/token https://checkout.stripe.com https://app.posthog.com https://api.stripe.com https://api.pwnedpasswords.com http://127.0.0.1:* https://hcaptcha.com https://*.hcaptcha.com; img-src 'self' https://static.intercomassets.com https://js.intercomcdn.com https://downloads.intercomcdn.com https://*.stripe.com https://i.ytimg.com/ data:; media-src https://js.intercomcdn.com; font-src 'self' https://fonts.intercomcdn.com/ https://maxcdn.bootstrapcdn.com https://rsms.me https://fonts.gstatic.com; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c33c9dc360..489df0ea18 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -4,7 +4,6 @@ "requires": true, "packages": { "": { - "name": "frontend", "dependencies": { "@casl/ability": "^6.5.0", "@casl/react": "^3.1.0", @@ -19,6 +18,7 @@ "@fortawesome/free-regular-svg-icons": "^6.1.1", "@fortawesome/free-solid-svg-icons": "^6.1.2", "@fortawesome/react-fontawesome": "^0.2.0", + "@hcaptcha/react-hcaptcha": "^1.10.1", "@headlessui/react": "^1.7.7", "@hookform/resolvers": "^2.9.10", "@octokit/rest": "^19.0.7", @@ -3200,6 +3200,24 @@ "react": ">=16.3" } }, + "node_modules/@hcaptcha/loader": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@hcaptcha/loader/-/loader-1.2.4.tgz", + "integrity": "sha512-3MNrIy/nWBfyVVvMPBKdKrX7BeadgiimW0AL/a/8TohNtJqxoySKgTJEXOQvYwlHemQpUzFrIsK74ody7JiMYw==" + }, + "node_modules/@hcaptcha/react-hcaptcha": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@hcaptcha/react-hcaptcha/-/react-hcaptcha-1.10.1.tgz", + "integrity": "sha512-P0en4gEZAecah7Pt3WIaJO2gFlaLZKkI0+Tfdg8fNqsDxqT9VytZWSkH4WAkiPRULK1QcGgUZK+J56MXYmPifw==", + "dependencies": { + "@babel/runtime": "^7.17.9", + "@hcaptcha/loader": "^1.2.1" + }, + "peerDependencies": { + "react": ">= 16.3.0", + "react-dom": ">= 16.3.0" + } + }, "node_modules/@headlessui/react": { "version": "1.7.18", "resolved": "https://registry.npmjs.org/@headlessui/react/-/react-1.7.18.tgz", diff --git a/frontend/package.json b/frontend/package.json index e01ef945e6..a4acb57382 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -26,6 +26,7 @@ "@fortawesome/free-regular-svg-icons": "^6.1.1", "@fortawesome/free-solid-svg-icons": "^6.1.2", "@fortawesome/react-fontawesome": "^0.2.0", + "@hcaptcha/react-hcaptcha": "^1.10.1", "@headlessui/react": "^1.7.7", "@hookform/resolvers": "^2.9.10", "@octokit/rest": "^19.0.7", diff --git a/frontend/src/components/utilities/attemptCliLogin.ts b/frontend/src/components/utilities/attemptCliLogin.ts index e95f5bf885..8f7c4bb9f5 100644 --- a/frontend/src/components/utilities/attemptCliLogin.ts +++ b/frontend/src/components/utilities/attemptCliLogin.ts @@ -30,11 +30,13 @@ export interface IsCliLoginSuccessful { const attemptLogin = async ({ email, password, - providerAuthToken + providerAuthToken, + captchaToken }: { email: string; password: string; providerAuthToken?: string; + captchaToken?: string; }): Promise => { const telemetry = new Telemetry().getInstance(); return new Promise((resolve, reject) => { @@ -70,7 +72,8 @@ const attemptLogin = async ({ } = await login2({ email, clientProof, - providerAuthToken + providerAuthToken, + captchaToken }); if (mfaEnabled) { // case: MFA is enabled diff --git a/frontend/src/components/utilities/attemptLogin.ts b/frontend/src/components/utilities/attemptLogin.ts index 195cf9b9a2..b909b1ba7c 100644 --- a/frontend/src/components/utilities/attemptLogin.ts +++ b/frontend/src/components/utilities/attemptLogin.ts @@ -22,11 +22,13 @@ interface IsLoginSuccessful { const attemptLogin = async ({ email, password, - providerAuthToken + providerAuthToken, + captchaToken }: { email: string; password: string; providerAuthToken?: string; + captchaToken?: string; }): Promise => { const telemetry = new Telemetry().getInstance(); // eslint-disable-next-line new-cap @@ -58,6 +60,7 @@ const attemptLogin = async ({ iv, tag } = await login2({ + captchaToken, email, clientProof, providerAuthToken diff --git a/frontend/src/components/utilities/config/index.ts b/frontend/src/components/utilities/config/index.ts index 10d4856c06..9b3bf37f1a 100644 --- a/frontend/src/components/utilities/config/index.ts +++ b/frontend/src/components/utilities/config/index.ts @@ -2,5 +2,6 @@ const ENV = process.env.NEXT_PUBLIC_ENV! || "development"; // investigate const POSTHOG_API_KEY = process.env.NEXT_PUBLIC_POSTHOG_API_KEY!; const POSTHOG_HOST = process.env.NEXT_PUBLIC_POSTHOG_HOST! || "https://app.posthog.com"; const INTERCOMid = process.env.NEXT_PUBLIC_INTERCOMid!; +const CAPTCHA_SITE_KEY = process.env.NEXT_PUBLIC_CAPTCHA_SITE_KEY!; -export { ENV, INTERCOMid, POSTHOG_API_KEY, POSTHOG_HOST }; +export { CAPTCHA_SITE_KEY, ENV, INTERCOMid, POSTHOG_API_KEY, POSTHOG_HOST }; diff --git a/frontend/src/hooks/api/auth/types.ts b/frontend/src/hooks/api/auth/types.ts index 41c324bffe..ce1b18bc83 100644 --- a/frontend/src/hooks/api/auth/types.ts +++ b/frontend/src/hooks/api/auth/types.ts @@ -30,6 +30,7 @@ export type Login1DTO = { }; export type Login2DTO = { + captchaToken?: string; email: string; clientProof: string; providerAuthToken?: string; diff --git a/frontend/src/views/Login/components/InitialStep/InitialStep.tsx b/frontend/src/views/Login/components/InitialStep/InitialStep.tsx index a4e5e89f5f..0ae253d03b 100644 --- a/frontend/src/views/Login/components/InitialStep/InitialStep.tsx +++ b/frontend/src/views/Login/components/InitialStep/InitialStep.tsx @@ -1,15 +1,17 @@ -import { FormEvent, useEffect, useState } from "react"; +import { FormEvent, useEffect, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import Link from "next/link"; import { useRouter } from "next/router"; import { faGithub, faGitlab, faGoogle } from "@fortawesome/free-brands-svg-icons"; import { faLock } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import HCaptcha from "@hcaptcha/react-hcaptcha"; import Error from "@app/components/basic/Error"; import { createNotification } from "@app/components/notifications"; import attemptCliLogin from "@app/components/utilities/attemptCliLogin"; import attemptLogin from "@app/components/utilities/attemptLogin"; +import { CAPTCHA_SITE_KEY } from "@app/components/utilities/config"; import { Button, Input } from "@app/components/v2"; import { useServerConfig } from "@app/context"; @@ -31,6 +33,9 @@ export const InitialStep = ({ setStep, email, setEmail, password, setPassword }: const [loginError, setLoginError] = useState(false); const { config } = useServerConfig(); const queryParams = new URLSearchParams(window.location.search); + const [captchaToken, setCaptchaToken] = useState(""); + const [shouldShowCaptcha, setShouldShowCaptcha] = useState(false); + const captchaRef = useRef(null); useEffect(() => { if ( @@ -61,7 +66,8 @@ export const InitialStep = ({ setStep, email, setEmail, password, setPassword }: // attemptCliLogin const isCliLoginSuccessful = await attemptCliLogin({ email: email.toLowerCase(), - password + password, + captchaToken }); if (isCliLoginSuccessful && isCliLoginSuccessful.success) { @@ -83,7 +89,8 @@ export const InitialStep = ({ setStep, email, setEmail, password, setPassword }: } else { const isLoginSuccessful = await attemptLogin({ email: email.toLowerCase(), - password + password, + captchaToken }); if (isLoginSuccessful && isLoginSuccessful.success) { @@ -117,6 +124,12 @@ export const InitialStep = ({ setStep, email, setEmail, password, setPassword }: return; } + if (err.response.data.error === "Captcha Required") { + setShouldShowCaptcha(true); + setIsLoading(false); + return; + } + setLoginError(true); createNotification({ text: "Login unsuccessful. Double-check your credentials and try again.", @@ -124,6 +137,10 @@ export const InitialStep = ({ setStep, email, setEmail, password, setPassword }: }); } + if (captchaRef.current) { + captchaRef.current.resetCaptcha(); + } + setIsLoading(false); }; @@ -245,6 +262,16 @@ export const InitialStep = ({ setStep, email, setEmail, password, setPassword }: className="select:-webkit-autofill:focus h-10" />
+ {shouldShowCaptcha && ( +
+ setCaptchaToken(token)} + ref={captchaRef} + /> +
+ )}
+ {shouldShowCaptcha && ( +
+ setCaptchaToken(token)} + ref={captchaRef} + /> +
+ )}