diff --git a/go.mod b/go.mod index 3dbd346d26..69f942d747 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,6 @@ require ( github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/deckarep/golang-set v1.7.1 // indirect github.com/dgraph-io/ristretto v0.0.3 - github.com/dustin/go-humanize v1.0.0 github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0 github.com/edsrzf/mmap-go v1.0.0 // indirect github.com/elastic/gosigar v0.10.5 // indirect diff --git a/shared/promptutil/prompt.go b/shared/promptutil/prompt.go index 198a96e16f..1190dd800f 100644 --- a/shared/promptutil/prompt.go +++ b/shared/promptutil/prompt.go @@ -84,7 +84,7 @@ func PasswordPrompt(promptText string, validateFunc func(string) error) (string, var responseValid bool var response string for !responseValid { - fmt.Printf("\n%s: ", au.Bold(promptText)) + fmt.Printf("%s: \n", au.Bold(promptText)) bytePassword, err := terminal.ReadPassword(int(os.Stdin.Fd())) if err != nil { return "", err diff --git a/validator/accounts/v2/BUILD.bazel b/validator/accounts/v2/BUILD.bazel index 5748923317..cf33189543 100644 --- a/validator/accounts/v2/BUILD.bazel +++ b/validator/accounts/v2/BUILD.bazel @@ -33,7 +33,6 @@ go_library( "//validator/keymanager/v2/derived:go_default_library", "//validator/keymanager/v2/direct:go_default_library", "//validator/keymanager/v2/remote:go_default_library", - "@com_github_dustin_go_humanize//:go_default_library", "@com_github_dustinkirkland_golang_petname//:go_default_library", "@com_github_k0kubun_go_ansi//:go_default_library", "@com_github_logrusorgru_aurora//:go_default_library", @@ -74,7 +73,6 @@ go_test( "//validator/keymanager/v2/derived:go_default_library", "//validator/keymanager/v2/direct:go_default_library", "//validator/keymanager/v2/remote:go_default_library", - "@com_github_dustin_go_humanize//:go_default_library", "@com_github_google_uuid//:go_default_library", "@com_github_pkg_errors//:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", diff --git a/validator/accounts/v2/accounts_create.go b/validator/accounts/v2/accounts_create.go index 06fb92cc3f..4eddd20964 100644 --- a/validator/accounts/v2/accounts_create.go +++ b/validator/accounts/v2/accounts_create.go @@ -6,6 +6,7 @@ import ( "github.com/manifoldco/promptui" "github.com/pkg/errors" + "github.com/prysmaticlabs/prysm/shared/promptutil" "github.com/prysmaticlabs/prysm/validator/flags" v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" "github.com/prysmaticlabs/prysm/validator/keymanager/v2/derived" @@ -38,7 +39,13 @@ func CreateAccount(cliCtx *cli.Context) error { if !ok { return errors.New("not a direct keymanager") } - password, err := inputPassword(cliCtx, flags.AccountPasswordFileFlag, newAccountPasswordPromptText, confirmPass) + password, err := inputPassword( + cliCtx, + flags.AccountPasswordFileFlag, + newAccountPasswordPromptText, + confirmPass, + promptutil.ValidatePasswordInput, + ) if err != nil { return errors.Wrap(err, "could not input new account password") } diff --git a/validator/accounts/v2/accounts_export.go b/validator/accounts/v2/accounts_export.go index b9a68ef138..f38972f5bf 100644 --- a/validator/accounts/v2/accounts_export.go +++ b/validator/accounts/v2/accounts_export.go @@ -14,7 +14,6 @@ import ( "github.com/pkg/errors" "github.com/prysmaticlabs/prysm/shared/params" "github.com/prysmaticlabs/prysm/validator/flags" - "github.com/prysmaticlabs/prysm/validator/keymanager/v2/direct" "github.com/urfave/cli/v2" ) @@ -178,29 +177,3 @@ func copyFileFromZip(archive *zip.Writer, sourcePath string, info os.FileInfo, p _, err = io.Copy(writer, file) return err } - -func logAccountsExported(wallet *Wallet, keymanager *direct.Keymanager, accountNames []string) error { - au := aurora.NewAurora(true) - - numAccounts := au.BrightYellow(len(accountNames)) - fmt.Println("") - if len(accountNames) == 1 { - fmt.Printf("Exported %d validator account\n", numAccounts) - } else { - fmt.Printf("Exported %d validator accounts\n", numAccounts) - } - for _, accountName := range accountNames { - fmt.Println("") - fmt.Printf("%s\n", au.BrightGreen(accountName).Bold()) - - publicKey, err := keymanager.PublicKeyForAccount(accountName) - if err != nil { - return errors.Wrap(err, "could not get public key") - } - fmt.Printf("%s %#x\n", au.BrightMagenta("[public key]").Bold(), publicKey) - - dirPath := au.BrightCyan("(wallet dir)") - fmt.Printf("%s %s\n", dirPath, filepath.Join(wallet.AccountsDir(), accountName)) - } - return nil -} diff --git a/validator/accounts/v2/accounts_import.go b/validator/accounts/v2/accounts_import.go index 1ef9119f96..3d905abcbc 100644 --- a/validator/accounts/v2/accounts_import.go +++ b/validator/accounts/v2/accounts_import.go @@ -2,7 +2,6 @@ package v2 import ( "context" - "encoding/hex" "encoding/json" "fmt" "io/ioutil" @@ -10,11 +9,8 @@ import ( "strconv" "strings" - "github.com/dustin/go-humanize" "github.com/logrusorgru/aurora" "github.com/pkg/errors" - "github.com/prysmaticlabs/prysm/shared/bytesutil" - "github.com/prysmaticlabs/prysm/shared/petnames" "github.com/prysmaticlabs/prysm/validator/flags" v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" "github.com/prysmaticlabs/prysm/validator/keymanager/v2/direct" @@ -46,6 +42,18 @@ func ImportAccount(cliCtx *cli.Context) error { "only non-HD wallets can import accounts, try creating a new wallet with wallet-v2 create", ) } + cfg, err := wallet.ReadKeymanagerConfigFromDisk(ctx) + if err != nil { + return err + } + directCfg, err := direct.UnmarshalConfigFile(cfg) + if err != nil { + return err + } + km, err := direct.NewKeymanager(ctx, wallet, directCfg) + if err != nil { + return err + } keysDir, err := inputDirectory(cliCtx, importKeysDirPromptText, flags.KeysDirFlag) if err != nil { return errors.Wrap(err, "could not parse keys directory") @@ -53,13 +61,13 @@ func ImportAccount(cliCtx *cli.Context) error { if err := wallet.SaveWallet(); err != nil { return errors.Wrap(err, "could not save wallet") } - accountsImported := make([]string, 0) - pubKeysImported := make([][]byte, 0) isDir, err := hasDir(keysDir) if err != nil { return errors.Wrap(err, "could not determine if path is a directory") } + keystoresImported := make([]*v2keymanager.Keystore, 0) + // Consider that the keysDir might be a path to a specific file and handle accordingly. if isDir { files, err := ioutil.ReadDir(keysDir) @@ -73,90 +81,39 @@ func ImportAccount(cliCtx *cli.Context) error { if !strings.HasPrefix(files[i].Name(), "keystore") { continue } - accountName, pubKey, err := wallet.importKeystore(ctx, filepath.Join(keysDir, files[i].Name())) + keystore, err := wallet.readKeystoreFile(ctx, filepath.Join(keysDir, files[i].Name())) if err != nil { return errors.Wrap(err, "could not import keystore") } - accountsImported = append(accountsImported, accountName) - pubKeysImported = append(pubKeysImported, pubKey) + keystoresImported = append(keystoresImported, keystore) } } else { - accountName, pubKey, err := wallet.importKeystore(ctx, keysDir) + keystore, err := wallet.readKeystoreFile(ctx, keysDir) if err != nil { return errors.Wrap(err, "could not import keystore") } - accountsImported = append(accountsImported, accountName) - pubKeysImported = append(pubKeysImported, pubKey) + keystoresImported = append(keystoresImported, keystore) } au := aurora.NewAurora(true) - formattedPubkeys := make([]string, len(pubKeysImported)) - for i, pk := range pubKeysImported { - formattedPubkeys[i] = fmt.Sprintf("%#x", bytesutil.Trunc(pk)) - } - fmt.Printf("Importing accounts: %s\n", au.BrightGreen(strings.Join(formattedPubkeys, ", "))) - if err := wallet.enterPasswordForAllAccounts(cliCtx, accountsImported, pubKeysImported); err != nil { - return errors.Wrap(err, "could not verify password for keystore") + if err := km.ImportKeystores(cliCtx, keystoresImported); err != nil { + return errors.Wrap(err, "could not import all keystores") } fmt.Printf( "Successfully imported %s accounts, view all of them by running accounts-v2 list\n", - au.BrightMagenta(strconv.Itoa(len(pubKeysImported))), + au.BrightMagenta(strconv.Itoa(len(keystoresImported))), ) return nil } -func (w *Wallet) importKeystore(ctx context.Context, keystoreFilePath string) (string, []byte, error) { +func (w *Wallet) readKeystoreFile(ctx context.Context, keystoreFilePath string) (*v2keymanager.Keystore, error) { keystoreBytes, err := ioutil.ReadFile(keystoreFilePath) if err != nil { - return "", nil, errors.Wrap(err, "could not read keystore file") + return nil, errors.Wrap(err, "could not read keystore file") } keystoreFile := &v2keymanager.Keystore{} if err := json.Unmarshal(keystoreBytes, keystoreFile); err != nil { - return "", nil, errors.Wrap(err, "could not decode keystore json") + return nil, errors.Wrap(err, "could not decode keystore json") } - pubKeyBytes, err := hex.DecodeString(keystoreFile.Pubkey) - if err != nil { - return "", nil, errors.Wrap(err, "could not decode public key string in keystore") - } - accountName := petnames.DeterministicName(pubKeyBytes, "-") - keystoreFileName := filepath.Base(keystoreFilePath) - if err := w.WriteFileAtPath(ctx, accountName, keystoreFileName, keystoreBytes); err != nil { - return "", nil, errors.Wrap(err, "could not write keystore to account dir") - } - return accountName, pubKeyBytes, nil -} - -func logAccountsImported(ctx context.Context, wallet *Wallet, keymanager *direct.Keymanager, accountNames []string) error { - au := aurora.NewAurora(true) - - numAccounts := au.BrightYellow(len(accountNames)) - fmt.Println("") - if len(accountNames) == 1 { - fmt.Printf("Imported %d validator account\n", numAccounts) - } else { - fmt.Printf("Imported %d validator accounts\n", numAccounts) - } - for _, accountName := range accountNames { - fmt.Println("") - // Retrieve the account creation timestamp. - keystoreFileName, err := wallet.FileNameAtPath(ctx, accountName, direct.KeystoreFileName) - if err != nil { - return errors.Wrapf(err, "could not get keystore file name for account: %s", accountName) - } - unixTimestamp, err := AccountTimestamp(keystoreFileName) - if err != nil { - return errors.Wrap(err, "could not get timestamp from keystore file name") - } - fmt.Printf("%s | Created %s\n", au.BrightGreen(accountName).Bold(), humanize.Time(unixTimestamp)) - - publicKey, err := keymanager.PublicKeyForAccount(accountName) - if err != nil { - return errors.Wrap(err, "could not get public key") - } - fmt.Printf("%s %#x\n", au.BrightMagenta("[validating public key]").Bold(), publicKey) - - dirPath := au.BrightCyan("(wallet dir)") - fmt.Printf("%s %s\n", dirPath, filepath.Join(wallet.AccountsDir(), accountName)) - } - return nil + return keystoreFile, nil } diff --git a/validator/accounts/v2/accounts_import_test.go b/validator/accounts/v2/accounts_import_test.go index 6accaf76d9..816ee23555 100644 --- a/validator/accounts/v2/accounts_import_test.go +++ b/validator/accounts/v2/accounts_import_test.go @@ -45,15 +45,13 @@ func TestImport_Noninteractive(t *testing.T) { require.NoError(t, err) require.NoError(t, wallet.SaveWallet()) ctx := context.Background() - keymanagerCfg := direct.DefaultConfig() - keymanagerCfg.AccountPasswordsDirectory = passwordsDir - encodedCfg, err := direct.MarshalConfigFile(ctx, keymanagerCfg) + encodedCfg, err := direct.MarshalConfigFile(ctx, direct.DefaultConfig()) require.NoError(t, err) require.NoError(t, wallet.WriteKeymanagerConfigToDisk(ctx, encodedCfg)) keymanager, err := direct.NewKeymanager( ctx, wallet, - keymanagerCfg, + direct.DefaultConfig(), ) require.NoError(t, err) diff --git a/validator/accounts/v2/accounts_list.go b/validator/accounts/v2/accounts_list.go index ca0069e027..95ea74f13c 100644 --- a/validator/accounts/v2/accounts_list.go +++ b/validator/accounts/v2/accounts_list.go @@ -5,7 +5,6 @@ import ( "fmt" "path/filepath" - "github.com/dustin/go-humanize" "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/prysmaticlabs/prysm/shared/petnames" @@ -75,6 +74,7 @@ func listDirectKeymanagerAccounts( } au := aurora.NewAurora(true) numAccounts := au.BrightYellow(len(accountNames)) + fmt.Printf("(keymanager kind) %s\n", au.BrightGreen("non-HD wallet").Bold()) fmt.Println("") if len(accountNames) == 1 { fmt.Printf("Showing %d validator account\n", numAccounts) @@ -93,41 +93,16 @@ func listDirectKeymanagerAccounts( } for i := 0; i < len(accountNames); i++ { fmt.Println("") - - // Retrieve the account creation timestamp. - keystoreFileName, err := wallet.FileNameAtPath(ctx, accountNames[i], direct.KeystoreFileName) - if err != nil { - return errors.Wrapf(err, "could not get keystore file name for account: %s", accountNames[i]) - } - unixTimestamp, err := AccountTimestamp(keystoreFileName) - if err != nil { - return errors.Wrap(err, "could not get timestamp from keystore file name") - } - fmt.Printf("%s | %s | Created %s\n", au.BrightBlue(fmt.Sprintf("Account %d", i)).Bold(), au.BrightGreen(accountNames[i]).Bold(), humanize.Time(unixTimestamp)) + fmt.Printf("%s | %s\n", au.BrightBlue(fmt.Sprintf("Account %d", i)).Bold(), au.BrightGreen(accountNames[i]).Bold()) fmt.Printf("%s %#x\n", au.BrightMagenta("[validating public key]").Bold(), pubKeys[i]) if !showDepositData { continue } - enc, err := wallet.ReadFileAtPath(ctx, accountNames[i], direct.DepositDataFileName) - if err != nil { - fmt.Printf( - "%s\n", - au.BrightRed("If you imported your account coming from the eth2 launchpad, you will find your "+ - "deposit_data.json in the eth2.0-deposit-cli's validator_keys folder"), - ) - continue - } fmt.Printf( - "%s %s\n", - "(deposit_data.ssz file)", - filepath.Join(wallet.AccountsDir(), accountNames[i], direct.DepositDataFileName), + "%s\n", + au.BrightRed("If you imported your account coming from the eth2 launchpad, you will find your "+ + "deposit_data.json in the eth2.0-deposit-cli's validator_keys folder"), ) - fmt.Printf(` -======================SSZ Deposit Data===================== - -%#x - -===================================================================`, enc) fmt.Println("") } fmt.Println("") @@ -191,7 +166,7 @@ func listDerivedKeymanagerAccounts( } enc, err := keymanager.DepositDataForAccount(i) if err != nil { - return errors.Wrapf(err, "could not read file for account: %s", direct.DepositDataFileName) + return errors.Wrapf(err, "could not deposit data for account: %s", accountNames[i]) } fmt.Printf(` ======================SSZ Deposit Data===================== diff --git a/validator/accounts/v2/accounts_list_test.go b/validator/accounts/v2/accounts_list_test.go index 6e731be1da..0909a6d858 100644 --- a/validator/accounts/v2/accounts_list_test.go +++ b/validator/accounts/v2/accounts_list_test.go @@ -8,14 +8,11 @@ import ( "strconv" "strings" "testing" - "time" - "github.com/dustin/go-humanize" validatorpb "github.com/prysmaticlabs/prysm/proto/validator/accounts/v2" "github.com/prysmaticlabs/prysm/shared/bls" "github.com/prysmaticlabs/prysm/shared/bytesutil" "github.com/prysmaticlabs/prysm/shared/petnames" - "github.com/prysmaticlabs/prysm/shared/testutil/assert" "github.com/prysmaticlabs/prysm/shared/testutil/require" v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" "github.com/prysmaticlabs/prysm/validator/keymanager/v2/derived" @@ -36,11 +33,12 @@ func (m *mockKeymanager) Sign(context.Context, *validatorpb.SignRequest) (bls.Si } func TestListAccounts_DirectKeymanager(t *testing.T) { - walletDir, passwordsDir, _ := setupWalletAndPasswordsDir(t) + walletDir, passwordsDir, walletPasswordFile := setupWalletAndPasswordsDir(t) cliCtx := setupWalletCtx(t, &testWalletConfig{ - walletDir: walletDir, - passwordsDir: passwordsDir, - keymanagerKind: v2keymanager.Direct, + walletDir: walletDir, + passwordsDir: passwordsDir, + keymanagerKind: v2keymanager.Direct, + walletPasswordFile: walletPasswordFile, }) wallet, err := NewWallet(cliCtx, v2keymanager.Direct) require.NoError(t, err) @@ -54,19 +52,9 @@ func TestListAccounts_DirectKeymanager(t *testing.T) { require.NoError(t, err) numAccounts := 5 - depositDataForAccounts := make([][]byte, numAccounts) - accountCreationTimestamps := make([][]byte, numAccounts) for i := 0; i < numAccounts; i++ { - accountName, err := keymanager.CreateAccount(ctx, "hello world") + _, err := keymanager.CreateAccount(ctx, "hello world") require.NoError(t, err) - depositData, err := wallet.ReadFileAtPath(ctx, accountName, direct.DepositDataFileName) - require.NoError(t, err) - depositDataForAccounts[i] = depositData - keystoreFileName, err := wallet.FileNameAtPath(ctx, accountName, direct.KeystoreFileName) - require.NoError(t, err) - timestampStart := strings.LastIndex(keystoreFileName, "-") + 1 - timestampEnd := strings.LastIndex(keystoreFileName, ".") - accountCreationTimestamps[i] = []byte(keystoreFileName[timestampStart:timestampEnd]) } rescueStdout := os.Stdout r, w, err := os.Pipe() @@ -83,13 +71,8 @@ func TestListAccounts_DirectKeymanager(t *testing.T) { // Assert the keymanager kind is printed to stdout. stringOutput := string(out) - if !strings.Contains(stringOutput, wallet.KeymanagerKind().String()) { - t.Error("Did not find Keymanager kind in output") - } - - // Assert the wallet and passwords paths are in stdout. - if !strings.Contains(stringOutput, wallet.accountsPath) { - t.Errorf("Did not find accounts path %s in output", wallet.accountsPath) + if !strings.Contains(stringOutput, "non-HD") { + t.Error("Did not find keymanager kind in output") } accountNames, err := keymanager.ValidatingAccountNames() @@ -104,23 +87,11 @@ func TestListAccounts_DirectKeymanager(t *testing.T) { t.Errorf("Did not find account %s in output", accountName) } key := pubKeys[i] - depositData := depositDataForAccounts[i] // Assert every public key is printed to stdout. if !strings.Contains(stringOutput, fmt.Sprintf("%#x", key)) { t.Errorf("Did not find pubkey %#x in output", key) } - - // Assert the deposit data for the account is printed to stdout. - if !strings.Contains(stringOutput, fmt.Sprintf("%#x", depositData)) { - t.Errorf("Did not find deposit data %#x in output", depositData) - } - - // Assert the account creation time is displayed - unixTimestampStr, err := strconv.ParseInt(string(accountCreationTimestamps[i]), 10, 64) - require.NoError(t, err) - unixTimestamp := time.Unix(unixTimestampStr, 0) - assert.Equal(t, strings.Contains(stringOutput, humanize.Time(unixTimestamp)), true) } } diff --git a/validator/accounts/v2/cmd_accounts.go b/validator/accounts/v2/cmd_accounts.go index 661fb007ef..bb6f9d7b82 100644 --- a/validator/accounts/v2/cmd_accounts.go +++ b/validator/accounts/v2/cmd_accounts.go @@ -75,6 +75,7 @@ this command outputs a deposit data string which is required to become a validat flags.WalletPasswordsDirFlag, flags.KeysDirFlag, flags.WalletPasswordFileFlag, + flags.AccountPasswordFileFlag, featureconfig.AltonaTestnet, featureconfig.OnyxTestnet, }, diff --git a/validator/accounts/v2/iface/wallet.go b/validator/accounts/v2/iface/wallet.go index 798730001f..11e0facba7 100644 --- a/validator/accounts/v2/iface/wallet.go +++ b/validator/accounts/v2/iface/wallet.go @@ -12,6 +12,7 @@ type Wallet interface { // Methods to retrieve wallet and accounts metadata. AccountsDir() string ListDirs() ([]string, error) + Password() string // Read methods for important wallet and accounts-related files. ReadEncryptedSeedFromDisk(ctx context.Context) (io.ReadCloser, error) ReadFileAtPath(ctx context.Context, filePath string, fileName string) ([]byte, error) diff --git a/validator/accounts/v2/prompt.go b/validator/accounts/v2/prompt.go index b889559804..7bdb2465bd 100644 --- a/validator/accounts/v2/prompt.go +++ b/validator/accounts/v2/prompt.go @@ -82,6 +82,7 @@ func inputPassword( passwordFileFlag *cli.StringFlag, promptText string, confirmPassword passwordConfirm, + passwordValidator func(input string) error, ) (string, error) { if cliCtx.IsSet(passwordFileFlag.Name) { passwordFilePathInput := cliCtx.String(passwordFileFlag.Name) @@ -94,7 +95,7 @@ func inputPassword( return "", errors.Wrap(err, "could not read password file") } enteredPassword := strings.TrimRight(string(data), "\r\n") - if err := promptutil.ValidatePasswordInput(enteredPassword); err != nil { + if err := passwordValidator(enteredPassword); err != nil { return "", errors.Wrap(err, "password did not pass validation") } return enteredPassword, nil @@ -103,13 +104,13 @@ func inputPassword( var walletPassword string var err error for !hasValidPassword { - walletPassword, err = promptutil.PasswordPrompt(promptText, promptutil.ValidatePasswordInput) + walletPassword, err = promptutil.PasswordPrompt(promptText, passwordValidator) if err != nil { return "", fmt.Errorf("could not read account password: %v", err) } if confirmPassword == confirmPass { - passwordConfirmation, err := promptutil.PasswordPrompt(confirmPasswordPromptText, promptutil.ValidatePasswordInput) + passwordConfirmation, err := promptutil.PasswordPrompt(confirmPasswordPromptText, passwordValidator) if err != nil { return "", fmt.Errorf("could not read password confirmation: %v", err) } @@ -138,7 +139,6 @@ func inputWeakPassword(cliCtx *cli.Context, passwordFileFlag *cli.StringFlag, pr } return strings.TrimRight(string(data), "\r\n"), nil } - walletPassword, err := promptutil.PasswordPrompt(promptText, promptutil.NotEmpty) if err != nil { return "", fmt.Errorf("could not read account password: %v", err) diff --git a/validator/accounts/v2/testing/mock.go b/validator/accounts/v2/testing/mock.go index 7d928d5c3e..3d4f514729 100644 --- a/validator/accounts/v2/testing/mock.go +++ b/validator/accounts/v2/testing/mock.go @@ -18,6 +18,7 @@ type Wallet struct { EncryptedSeedFile []byte AccountPasswords map[string]string UnlockAccounts bool + WalletPassword string lock sync.RWMutex } @@ -37,6 +38,11 @@ func (m *Wallet) AccountsDir() string { return m.InnerAccountsDir } +// Password -- +func (m *Wallet) Password() string { + return m.WalletPassword +} + // ListDirs -- func (m *Wallet) ListDirs() ([]string, error) { return m.Directories, nil @@ -82,7 +88,7 @@ func (m *Wallet) ReadFileAtPath(ctx context.Context, pathName string, fileName s return v, nil } } - return nil, errors.New("file not found") + return nil, errors.New("no files found") } // ReadEncryptedSeedFromDisk -- diff --git a/validator/accounts/v2/wallet.go b/validator/accounts/v2/wallet.go index 79a78d0f73..62844c7193 100644 --- a/validator/accounts/v2/wallet.go +++ b/validator/accounts/v2/wallet.go @@ -17,6 +17,7 @@ import ( "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/prysmaticlabs/prysm/shared/bytesutil" + "github.com/prysmaticlabs/prysm/shared/promptutil" "github.com/prysmaticlabs/prysm/validator/flags" v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" "github.com/prysmaticlabs/prysm/validator/keymanager/v2/derived" @@ -96,25 +97,19 @@ func NewWallet( keymanagerKind: keymanagerKind, walletDir: walletDir, } - if keymanagerKind == v2keymanager.Derived { + if keymanagerKind == v2keymanager.Derived || keymanagerKind == v2keymanager.Direct { walletPassword, err := inputPassword( cliCtx, flags.WalletPasswordFileFlag, newWalletPasswordPromptText, confirmPass, + promptutil.ValidatePasswordInput, ) if err != nil { return nil, errors.Wrap(err, "could not get password") } w.walletPassword = walletPassword } - if keymanagerKind == v2keymanager.Direct { - passwordsDir, err := inputDirectory(cliCtx, passwordsDirPromptText, flags.WalletPasswordsDirFlag) - if err != nil { - return nil, errors.Wrap(err, "could not get password directory") - } - w.passwordsDir = passwordsDir - } return w, nil } @@ -152,20 +147,48 @@ func OpenWallet(cliCtx *cli.Context) (*Wallet, error) { accountsPath: walletPath, keymanagerKind: keymanagerKind, } + // Check if the wallet is using the new, fast keystore format. + hasNewFormat, err := hasDir(filepath.Join(walletPath, direct.AccountsPath)) + if err != nil { + return nil, errors.Wrap(err, "could not read wallet dir") + } + log.Infof("Has new format: %v", hasNewFormat) log.Infof("%s %s", au.BrightMagenta("(wallet directory)"), w.walletDir) - if keymanagerKind == v2keymanager.Derived { - walletPassword, err := inputPassword( - cliCtx, - flags.WalletPasswordFileFlag, - walletPasswordPromptText, - noConfirmPass, - ) + if keymanagerKind == v2keymanager.Derived || keymanagerKind == v2keymanager.Direct { + var walletPassword string + if hasNewFormat { + validateExistingPass := func(input string) error { + if input == "" { + return errors.New("password input cannot be empty") + } + return nil + } + walletPassword, err = inputPassword( + cliCtx, + flags.WalletPasswordFileFlag, + walletPasswordPromptText, + noConfirmPass, + validateExistingPass, + ) + } else { + fmt.Println("\nWe have revamped how imported accounts work, improving speed significantly for your " + + "validators as well as reducing memory and CPU requirements. This unifies all your existing accounts " + + "into a single format protected by a strong password. You'll need to set a new password for this " + + "updated wallet format") + walletPassword, err = inputPassword( + cliCtx, + flags.WalletPasswordFileFlag, + newWalletPasswordPromptText, + confirmPass, + promptutil.ValidatePasswordInput, + ) + } if err != nil { return nil, err } w.walletPassword = walletPassword } - if keymanagerKind == v2keymanager.Direct { + if keymanagerKind == v2keymanager.Direct && !hasNewFormat { keymanagerCfg, err := w.ReadKeymanagerConfigFromDisk(context.Background()) if err != nil { return nil, err @@ -178,17 +201,22 @@ func OpenWallet(cliCtx *cli.Context) (*Wallet, error) { // If the user provided a flag and for the password directory, and that value does not match // the wallet's configuration then log a warning to the user. // See https://github.com/prysmaticlabs/prysm/issues/6794. - if cliCtx.IsSet(flags.WalletPasswordsDirFlag.Name) && cliCtx.String(flags.WalletPasswordsDirFlag.Name) != w.passwordsDir { + if cliCtx.IsSet(flags.WalletPasswordsDirFlag.Name) && + cliCtx.String(flags.WalletPasswordsDirFlag.Name) != w.passwordsDir { log.Warnf("The provided value for --%s does not match the wallet configuration. "+ "Please edit your wallet password directory using wallet-v2 edit-config.", flags.WalletPasswordsDirFlag.Name, ) w.passwordsDir = cliCtx.String(flags.WalletPasswordsDirFlag.Name) // Override config value. } + passwordsPath, err := expandPath(w.passwordsDir) + if err != nil { + return nil, err + } + w.passwordsDir = passwordsPath au := aurora.NewAurora(true) log.Infof("%s %s", au.BrightMagenta("(account passwords path)"), w.passwordsDir) } - log.Info("Successfully opened wallet") return w, nil } @@ -197,7 +225,7 @@ func (w *Wallet) SaveWallet() error { if err := os.MkdirAll(w.accountsPath, DirectoryPermissions); err != nil { return errors.Wrap(err, "could not create wallet directory") } - if w.keymanagerKind == v2keymanager.Direct { + if w.keymanagerKind == v2keymanager.Direct && w.passwordsDir != "" { if err := os.MkdirAll(w.passwordsDir, DirectoryPermissions); err != nil { return errors.Wrap(err, "could not create passwords directory") } @@ -215,6 +243,11 @@ func (w *Wallet) AccountsDir() string { return w.accountsPath } +// Password for the wallet. +func (w *Wallet) Password() string { + return w.walletPassword +} + // InitializeKeymanager reads a keymanager config from disk at the wallet path, // unmarshals it based on the wallet's keymanager kind, and returns its value. func (w *Wallet) InitializeKeymanager( diff --git a/validator/accounts/v2/wallet_create.go b/validator/accounts/v2/wallet_create.go index 47279a335d..87f141d43f 100644 --- a/validator/accounts/v2/wallet_create.go +++ b/validator/accounts/v2/wallet_create.go @@ -62,7 +62,6 @@ func createDirectKeymanagerWallet(cliCtx *cli.Context, wallet *Wallet) error { return errors.Wrap(err, "could not save wallet to disk") } defaultConfig := direct.DefaultConfig() - defaultConfig.AccountPasswordsDirectory = wallet.passwordsDir keymanagerConfig, err := direct.MarshalConfigFile(context.Background(), defaultConfig) if err != nil { return errors.Wrap(err, "could not marshal keymanager config file") diff --git a/validator/accounts/v2/wallet_create_test.go b/validator/accounts/v2/wallet_create_test.go index 1ee60b4adb..76777af8fe 100644 --- a/validator/accounts/v2/wallet_create_test.go +++ b/validator/accounts/v2/wallet_create_test.go @@ -21,11 +21,12 @@ import ( func TestCreateOrOpenWallet(t *testing.T) { hook := logTest.NewGlobal() - walletDir, passwordsDir, _ := setupWalletAndPasswordsDir(t) + walletDir, passwordsDir, walletPasswordFile := setupWalletAndPasswordsDir(t) cliCtx := setupWalletCtx(t, &testWalletConfig{ - walletDir: walletDir, - passwordsDir: passwordsDir, - keymanagerKind: v2keymanager.Direct, + walletDir: walletDir, + passwordsDir: passwordsDir, + keymanagerKind: v2keymanager.Direct, + walletPasswordFile: walletPasswordFile, }) createDirectWallet := func(cliCtx *cli.Context) (*Wallet, error) { w, err := NewWallet(cliCtx, v2keymanager.Direct) @@ -43,21 +44,20 @@ func TestCreateOrOpenWallet(t *testing.T) { createdWallet, err := createOrOpenWallet(cliCtx, createDirectWallet) require.NoError(t, err) testutil.AssertLogsContain(t, hook, "Successfully created new wallet") - testutil.AssertLogsDoNotContain(t, hook, "Successfully opened wallet") openedWallet, err := createOrOpenWallet(cliCtx, createDirectWallet) require.NoError(t, err) - testutil.AssertLogsContain(t, hook, "Successfully opened wallet") assert.Equal(t, createdWallet.KeymanagerKind(), openedWallet.KeymanagerKind()) assert.Equal(t, createdWallet.AccountsDir(), openedWallet.AccountsDir()) } func TestCreateWallet_Direct(t *testing.T) { - walletDir, passwordsDir, _ := setupWalletAndPasswordsDir(t) + walletDir, passwordsDir, walletPasswordFile := setupWalletAndPasswordsDir(t) cliCtx := setupWalletCtx(t, &testWalletConfig{ - walletDir: walletDir, - passwordsDir: passwordsDir, - keymanagerKind: v2keymanager.Direct, + walletDir: walletDir, + passwordsDir: passwordsDir, + keymanagerKind: v2keymanager.Direct, + walletPasswordFile: walletPasswordFile, }) // We attempt to create the wallet. @@ -77,7 +77,6 @@ func TestCreateWallet_Direct(t *testing.T) { // We assert the created configuration was as desired. wantedCfg := direct.DefaultConfig() - wantedCfg.AccountPasswordsDirectory = passwordsDir assert.DeepEqual(t, wantedCfg, cfg) } diff --git a/validator/accounts/v2/wallet_edit.go b/validator/accounts/v2/wallet_edit.go index 41e3a613d6..d90020b8b9 100644 --- a/validator/accounts/v2/wallet_edit.go +++ b/validator/accounts/v2/wallet_edit.go @@ -5,9 +5,7 @@ import ( "fmt" "github.com/pkg/errors" - "github.com/prysmaticlabs/prysm/validator/flags" v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" - "github.com/prysmaticlabs/prysm/validator/keymanager/v2/direct" "github.com/prysmaticlabs/prysm/validator/keymanager/v2/remote" "github.com/urfave/cli/v2" ) @@ -23,30 +21,7 @@ func EditWalletConfiguration(cliCtx *cli.Context) error { } switch wallet.KeymanagerKind() { case v2keymanager.Direct: - enc, err := wallet.ReadKeymanagerConfigFromDisk(ctx) - if err != nil { - return errors.Wrap(err, "could not read config") - } - cfg, err := direct.UnmarshalConfigFile(enc) - if err != nil { - return errors.Wrap(err, "could not unmarshal config") - } - log.Info("Current configuration") - // Prints the current configuration to stdout. - fmt.Println(cfg) - passwordsDir, err := inputDirectory(cliCtx, passwordsDirPromptText, flags.WalletPasswordsDirFlag) - if err != nil { - return errors.Wrap(err, "could not get password directory") - } - defaultCfg := direct.DefaultConfig() - defaultCfg.AccountPasswordsDirectory = passwordsDir - encodedCfg, err := direct.MarshalConfigFile(ctx, defaultCfg) - if err != nil { - return errors.Wrap(err, "could not marshal config file") - } - if err := wallet.WriteKeymanagerConfigToDisk(ctx, encodedCfg); err != nil { - return errors.Wrap(err, "could not write config to disk") - } + return errors.New("not possible to edit direct keymanager configuration") case v2keymanager.Derived: return errors.New("derived keymanager is not yet supported") case v2keymanager.Remote: diff --git a/validator/accounts/v2/wallet_test.go b/validator/accounts/v2/wallet_test.go index 00c1228553..ff6857cdbc 100644 --- a/validator/accounts/v2/wallet_test.go +++ b/validator/accounts/v2/wallet_test.go @@ -88,21 +88,6 @@ func setupWalletAndPasswordsDir(t testing.TB) (string, string, string) { return walletDir, passwordsDir, passwordFilePath } -func TestCreateAndReadWallet(t *testing.T) { - walletDir, passwordsDir, _ := setupWalletAndPasswordsDir(t) - cliCtx := setupWalletCtx(t, &testWalletConfig{ - walletDir: walletDir, - passwordsDir: passwordsDir, - keymanagerKind: v2keymanager.Direct, - }) - wallet, err := NewWallet(cliCtx, v2keymanager.Direct) - require.NoError(t, err) - require.NoError(t, createDirectKeymanagerWallet(cliCtx, wallet)) - // We should be able to now read the wallet as well. - _, err = OpenWallet(cliCtx) - require.NoError(t, err) -} - func TestAccountTimestamp(t *testing.T) { tests := []struct { name string diff --git a/validator/keymanager/v2/direct/BUILD.bazel b/validator/keymanager/v2/direct/BUILD.bazel index a0a084fcf9..a52de9739a 100644 --- a/validator/keymanager/v2/direct/BUILD.bazel +++ b/validator/keymanager/v2/direct/BUILD.bazel @@ -6,6 +6,8 @@ go_library( srcs = [ "direct.go", "doc.go", + "import.go", + "migrate.go", ], importpath = "github.com/prysmaticlabs/prysm/validator/keymanager/v2/direct", visibility = [ @@ -17,8 +19,8 @@ go_library( "//shared/bls:go_default_library", "//shared/bytesutil:go_default_library", "//shared/depositutil:go_default_library", - "//shared/mputil:go_default_library", "//shared/petnames:go_default_library", + "//shared/promptutil:go_default_library", "//shared/roughtime:go_default_library", "//validator/accounts/v2/iface:go_default_library", "//validator/flags:go_default_library", @@ -30,26 +32,33 @@ go_library( "@com_github_prysmaticlabs_go_ssz//:go_default_library", "@com_github_schollz_progressbar_v3//:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", + "@com_github_urfave_cli_v2//:go_default_library", "@com_github_wealdtech_go_eth2_wallet_encryptor_keystorev4//:go_default_library", ], ) go_test( name = "go_default_test", - srcs = ["direct_test.go"], + srcs = [ + "direct_test.go", + "import_test.go", + "migrate_test.go", + ], embed = [":go_default_library"], deps = [ "//proto/validator/accounts/v2:go_default_library", "//shared/bls:go_default_library", "//shared/bytesutil:go_default_library", + "//shared/petnames:go_default_library", "//shared/testutil:go_default_library", "//shared/testutil/assert:go_default_library", "//shared/testutil/require:go_default_library", "//validator/accounts/v2/testing:go_default_library", + "//validator/flags:go_default_library", "//validator/keymanager/v2:go_default_library", - "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library", - "@com_github_prysmaticlabs_go_ssz//:go_default_library", + "@com_github_google_uuid//:go_default_library", "@com_github_sirupsen_logrus//hooks/test:go_default_library", + "@com_github_urfave_cli_v2//:go_default_library", "@com_github_wealdtech_go_eth2_wallet_encryptor_keystorev4//:go_default_library", ], ) diff --git a/validator/keymanager/v2/direct/direct.go b/validator/keymanager/v2/direct/direct.go index df603f6db3..ddb75ab113 100644 --- a/validator/keymanager/v2/direct/direct.go +++ b/validator/keymanager/v2/direct/direct.go @@ -2,18 +2,14 @@ package direct import ( "context" - "encoding/hex" "encoding/json" "fmt" "io" "io/ioutil" - "os" - "path/filepath" "strings" "sync" "github.com/google/uuid" - "github.com/k0kubun/go-ansi" "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/prysmaticlabs/go-ssz" @@ -21,13 +17,12 @@ import ( "github.com/prysmaticlabs/prysm/shared/bls" "github.com/prysmaticlabs/prysm/shared/bytesutil" "github.com/prysmaticlabs/prysm/shared/depositutil" - "github.com/prysmaticlabs/prysm/shared/mputil" "github.com/prysmaticlabs/prysm/shared/petnames" + "github.com/prysmaticlabs/prysm/shared/promptutil" "github.com/prysmaticlabs/prysm/shared/roughtime" "github.com/prysmaticlabs/prysm/validator/accounts/v2/iface" "github.com/prysmaticlabs/prysm/validator/flags" v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" - "github.com/schollz/progressbar/v3" "github.com/sirupsen/logrus" keystorev4 "github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4" ) @@ -35,18 +30,17 @@ import ( var log = logrus.WithField("prefix", "direct-keymanager-v2") const ( - // TimestampFileName stores a timestamp for account creation as a - // file for a direct keymanager account. - TimestampFileName = "created_at.txt" // KeystoreFileName exposes the expected filename for the keystore file for an account. KeystoreFileName = "keystore-*.json" // KeystoreFileNameFormat exposes the filename the keystore should be formatted in. KeystoreFileNameFormat = "keystore-%d.json" // PasswordFileSuffix for passwords persisted as text to disk. PasswordFileSuffix = ".pass" - // DepositDataFileName for the ssz-encoded deposit. - DepositDataFileName = "deposit_data.ssz" - eipVersion = "EIP-2335" + // AccountsPath where all direct keymanager keystores are kept. + AccountsPath = "accounts" + accountsKeystoreFileName = "all-accounts.keystore-*.json" + accountsKeystoreFileNameFormat = "all-accounts.keystore-%d.json" + eipVersion = "EIP-2335" ) // Config for a direct keymanager. @@ -57,10 +51,17 @@ type Config struct { // Keymanager implementation for direct keystores utilizing EIP-2335. type Keymanager struct { - wallet iface.Wallet - cfg *Config - keysCache map[[48]byte]bls.SecretKey - lock sync.RWMutex + wallet iface.Wallet + cfg *Config + keysCache map[[48]byte]bls.SecretKey + accountsStore *AccountStore + lock sync.RWMutex +} + +// AccountStore -- +type AccountStore struct { + PrivateKeys [][]byte `json:"private_keys"` + PublicKeys [][]byte `json:"public_keys"` } // DefaultConfig for a direct keymanager implementation. @@ -74,10 +75,18 @@ func DefaultConfig() *Config { // NewKeymanager instantiates a new direct keymanager from configuration options. func NewKeymanager(ctx context.Context, wallet iface.Wallet, cfg *Config) (*Keymanager, error) { k := &Keymanager{ - wallet: wallet, - cfg: cfg, - keysCache: make(map[[48]byte]bls.SecretKey), + wallet: wallet, + cfg: cfg, + keysCache: make(map[[48]byte]bls.SecretKey), + accountsStore: &AccountStore{}, } + // If the user has previously created a direct keymanaged wallet, we perform + // a "silent migration" into this more effective format of storing a single keystore + // file containing all accounts. + if err := k.migrateToSingleKeystore(ctx); err != nil { + return nil, errors.Wrap(err, "could not migrate to single keystore format") + } + // If the wallet has the capability of unlocking accounts using // passphrases, then we initialize a cache of public key -> secret keys // used to retrieve secrets keys for the accounts via password unlock. @@ -138,7 +147,11 @@ func (c *Config) String() string { // ValidatingAccountNames for a direct keymanager. func (dr *Keymanager) ValidatingAccountNames() ([]string, error) { - return dr.wallet.ListDirs() + names := make([]string, len(dr.accountsStore.PublicKeys)) + for i, pubKey := range dr.accountsStore.PublicKeys { + names[i] = petnames.DeterministicName(pubKey, "-") + } + return names, nil } // CreateAccount for a direct keymanager implementation. This utilizes @@ -149,18 +162,12 @@ func (dr *Keymanager) ValidatingAccountNames() ([]string, error) { func (dr *Keymanager) CreateAccount(ctx context.Context, password string) (string, error) { // Create a petname for an account from its public key and write its password to disk. validatingKey := bls.RandKey() - accountName, err := dr.generateAccountName(validatingKey.PublicKey().Marshal()) + accountName := petnames.DeterministicName(validatingKey.PublicKey().Marshal(), "-") + dr.accountsStore.PrivateKeys = append(dr.accountsStore.PrivateKeys, validatingKey.Marshal()) + dr.accountsStore.PublicKeys = append(dr.accountsStore.PublicKeys, validatingKey.PublicKey().Marshal()) + newStore, err := dr.createAccountsKeystore(ctx, dr.accountsStore.PrivateKeys, dr.accountsStore.PublicKeys) if err != nil { - return "", errors.Wrap(err, "could not generate unique account name") - } - if err := dr.wallet.WritePasswordToDisk(ctx, accountName+".pass", password); err != nil { - return "", errors.Wrap(err, "could not write password to disk") - } - // Generates a new EIP-2335 compliant keystore file - // from a BLS private key and marshals it as JSON. - encoded, err := dr.generateKeystoreFile(validatingKey, password) - if err != nil { - return "", err + return "", errors.Wrap(err, "could not create accounts keystore") } // Generate a withdrawal key and confirm user @@ -191,9 +198,6 @@ func (dr *Keymanager) CreateAccount(ctx context.Context, password string) (strin if err != nil { return "", errors.Wrap(err, "could not marshal deposit data") } - if err := dr.wallet.WriteFileAtPath(ctx, accountName, DepositDataFileName, encodedDepositData); err != nil { - return "", errors.Wrapf(err, "could not write for account %s: %s", accountName, encodedDepositData) - } // Log the deposit transaction data to the user. fmt.Printf(` @@ -204,15 +208,21 @@ func (dr *Keymanager) CreateAccount(ctx context.Context, password string) (strin ===================================================================`, encodedDepositData) // Write the encoded keystore to disk with the timestamp appended - createdAt := roughtime.Now().Unix() - if err := dr.wallet.WriteFileAtPath(ctx, accountName, fmt.Sprintf(KeystoreFileNameFormat, createdAt), encoded); err != nil { - return "", errors.Wrapf(err, "could not write keystore file for account %s", accountName) + fileName := fmt.Sprintf(accountsKeystoreFileNameFormat, roughtime.Now().Unix()) + encoded, err := json.MarshalIndent(newStore, "", "\t") + if err != nil { + return "", err + } + if err := dr.wallet.WriteFileAtPath(ctx, AccountsPath, fileName, encoded); err != nil { + return "", errors.Wrap(err, "could not write keystore file for accounts") } log.WithFields(logrus.Fields{ "name": accountName, - "path": dr.wallet.AccountsDir(), }).Info("Successfully created new validator account") + dr.lock.Lock() + dr.keysCache[bytesutil.ToBytes48(validatingKey.PublicKey().Marshal())] = validatingKey + dr.lock.Unlock() return accountName, nil } @@ -236,23 +246,7 @@ func (dr *Keymanager) FetchValidatingPublicKeys(ctx context.Context) ([][48]byte } return publicKeys, nil } - - for i, name := range accountNames { - encoded, err := dr.wallet.ReadFileAtPath(ctx, name, KeystoreFileName) - if err != nil { - return nil, errors.Wrapf(err, "could not read keystore file for account %s", name) - } - keystoreFile := &v2keymanager.Keystore{} - if err := json.Unmarshal(encoded, keystoreFile); err != nil { - return nil, errors.Wrapf(err, "could not decode keystore json for account: %s", name) - } - pubKeyBytes, err := hex.DecodeString(keystoreFile.Pubkey) - if err != nil { - return nil, errors.Wrapf(err, "could not decode pubkey bytes: %#x", keystoreFile.Pubkey) - } - publicKeys[i] = bytesutil.ToBytes48(pubKeyBytes) - } - return publicKeys, nil + return nil, nil } // Sign signs a message using a validator key. @@ -270,167 +264,109 @@ func (dr *Keymanager) Sign(ctx context.Context, req *validatorpb.SignRequest) (b return secretKey.Sign(req.SigningRoot), nil } -// PublicKeyForAccount returns the associated public key for an account name. -func (dr *Keymanager) PublicKeyForAccount(accountName string) ([48]byte, error) { - accountKeystore, err := dr.keystoreForAccount(accountName) - if err != nil { - return [48]byte{}, errors.Wrap(err, "could not get keystore") - } - pubKey, err := hex.DecodeString(accountKeystore.Pubkey) - if err != nil { - return [48]byte{}, errors.Wrap(err, "could decode pubkey string") - } - return bytesutil.ToBytes48(pubKey), nil -} - -func (dr *Keymanager) keystoreForAccount(accountName string) (*v2keymanager.Keystore, error) { - encoded, err := dr.wallet.ReadFileAtPath(context.Background(), accountName, KeystoreFileName) - if err != nil { - return nil, errors.Wrap(err, "could not read keystore file") - } - keystoreJSON := &v2keymanager.Keystore{} - if err := json.Unmarshal(encoded, &keystoreJSON); err != nil { - return nil, errors.Wrap(err, "could not decode json") - } - return keystoreJSON, nil -} - func (dr *Keymanager) initializeSecretKeysCache(ctx context.Context) error { - accountNames, err := dr.ValidatingAccountNames() - if err != nil { + encoded, err := dr.wallet.ReadFileAtPath(ctx, AccountsPath, accountsKeystoreFileName) + if err != nil && strings.Contains(err.Error(), "no files found") { + // If there are no keys to initialize at all, just exit. + return nil + } else if err != nil { + return errors.Wrapf(err, "could not read keystore file for accounts %s", accountsKeystoreFileName) + } + keystoreFile := &v2keymanager.Keystore{} + if err := json.Unmarshal(encoded, keystoreFile); err != nil { + return errors.Wrapf(err, "could not decode keystore file for accounts %s", accountsKeystoreFileName) + } + // We extract the validator signing private key from the keystore + // by utilizing the password and initialize a new BLS secret key from + // its raw bytes. + decryptor := keystorev4.New() + enc, err := decryptor.Decrypt(keystoreFile.Crypto, dr.wallet.Password()) + if err != nil && strings.Contains(err.Error(), "invalid checksum") { + // If the password fails for an individual account, we ask the user to input + // that individual account's password until it succeeds. + enc, err = dr.askUntilPasswordConfirms(decryptor, keystoreFile) + if err != nil { + return errors.Wrap(err, "could not confirm password via prompt") + } + } else if err != nil { + return errors.Wrap(err, "could not decrypt keystore") + } + + store := &AccountStore{} + if err := json.Unmarshal(enc, store); err != nil { return err } - if len(accountNames) == 0 { + if len(store.PublicKeys) != len(store.PrivateKeys) { + return errors.New("unequal number of public keys and private keys") + } + if len(store.PublicKeys) == 0 { return nil } - // We initialize a nice progress bar to offer the user feedback - // during this slow operation. - bar := initializeProgressBar(len(accountNames)) - progressChan := make(chan struct{}, len(accountNames)) - go func() { - defer close(progressChan) - var itemsReceived int - for range progressChan { - itemsReceived++ - if err := bar.Add(1); err != nil { - log.WithError(err).Debug("Could not increase progress bar") - } - if itemsReceived == len(accountNames) { - return - } - } - }() dr.lock.Lock() defer dr.lock.Unlock() - _, err = mputil.Scatter(len(accountNames), func(offset int, entries int, _ *sync.RWMutex) (interface{}, error) { - for i := 0; i < len(accountNames[offset:offset+entries]); i++ { - name := accountNames[i] - password, err := dr.wallet.ReadPasswordFromDisk(ctx, name+PasswordFileSuffix) - if err != nil { - return nil, errors.Wrapf(err, "could not read password for account %s", name) - } - encoded, err := dr.wallet.ReadFileAtPath(ctx, name, KeystoreFileName) - if err != nil { - return nil, errors.Wrapf(err, "could not read keystore file for account %s", name) - } - keystoreFile := &v2keymanager.Keystore{} - if err := json.Unmarshal(encoded, keystoreFile); err != nil { - return nil, errors.Wrapf(err, "could not decode keystore file for account %s", name) - } - // We extract the validator signing private key from the keystore - // by utilizing the password and initialize a new BLS secret key from - // its raw bytes. - decryptor := keystorev4.New() - rawSigningKey, err := decryptor.Decrypt(keystoreFile.Crypto, password) - if err != nil { - return nil, errors.Wrapf(err, "could not decrypt signing key for account %s", name) - } - validatorSigningKey, err := bls.SecretKeyFromBytes(rawSigningKey) - if err != nil { - return nil, errors.Wrapf(err, "could not determine signing key for account %s", name) - } - // Update a simple cache of public key -> secret key utilized - // for fast signing access in the direct keymanager. - dr.keysCache[bytesutil.ToBytes48(validatorSigningKey.PublicKey().Marshal())] = validatorSigningKey - progressChan <- struct{}{} + for i := 0; i < len(store.PublicKeys); i++ { + privKey, err := bls.SecretKeyFromBytes(store.PrivateKeys[i]) + if err != nil { + return err } - return nil, nil - }) + dr.keysCache[bytesutil.ToBytes48(store.PublicKeys[i])] = privKey + } + dr.accountsStore = store return err } -func (dr *Keymanager) generateKeystoreFile(validatingKey bls.SecretKey, password string) ([]byte, error) { +func (dr *Keymanager) createAccountsKeystore( + ctx context.Context, + privateKeys [][]byte, + publicKeys [][]byte, +) (*v2keymanager.Keystore, error) { encryptor := keystorev4.New() - cryptoFields, err := encryptor.Encrypt(validatingKey.Marshal(), password) - if err != nil { - return nil, errors.Wrap(err, "could not encrypt validating key into keystore") - } id, err := uuid.NewRandom() if err != nil { return nil, err } - keystoreFile := &v2keymanager.Keystore{ + store := &AccountStore{ + PrivateKeys: privateKeys, + PublicKeys: publicKeys, + } + encodedStore, err := json.MarshalIndent(store, "", "\t") + if err != nil { + return nil, err + } + cryptoFields, err := encryptor.Encrypt(encodedStore, dr.wallet.Password()) + if err != nil { + return nil, errors.Wrap(err, "could not encrypt accounts") + } + return &v2keymanager.Keystore{ Crypto: cryptoFields, ID: id.String(), - Pubkey: fmt.Sprintf("%x", validatingKey.PublicKey().Marshal()), Version: encryptor.Version(), Name: encryptor.Name(), - } - return json.MarshalIndent(keystoreFile, "", "\t") + }, nil } -func (dr *Keymanager) generateAccountName(pubKey []byte) (string, error) { - var accountExists bool - var accountName string - for !accountExists { - accountName = petnames.DeterministicName(pubKey, "-") - exists, err := hasDir(filepath.Join(dr.wallet.AccountsDir(), accountName)) +func (dr *Keymanager) askUntilPasswordConfirms( + decryptor *keystorev4.Encryptor, keystore *v2keymanager.Keystore, +) ([]byte, error) { + au := aurora.NewAurora(true) + // Loop asking for the password until the user enters it correctly. + var secretKey []byte + for { + password, err := promptutil.PasswordPrompt( + "Wrong password entered, try again", promptutil.NotEmpty, + ) if err != nil { - return "", errors.Wrapf(err, "could not check if account exists in dir: %s", dr.wallet.AccountsDir()) + return nil, fmt.Errorf("could not read account password: %v", err) } - if !exists { - break + secretKey, err = decryptor.Decrypt(keystore.Crypto, password) + if err != nil && strings.Contains(err.Error(), "invalid checksum") { + fmt.Println(au.Red("Incorrect password entered, please try again")) + continue } + if err != nil { + return nil, err + } + break } - return accountName, nil -} - -func (dr *Keymanager) checkPasswordForAccount(accountName string, password string) error { - accountKeystore, err := dr.keystoreForAccount(accountName) - if err != nil { - return errors.Wrap(err, "could not get keystore") - } - decryptor := keystorev4.New() - _, err = decryptor.Decrypt(accountKeystore.Crypto, password) - if err != nil { - return errors.Wrap(err, "could not decrypt keystore") - } - return nil -} - -func initializeProgressBar(numItems int) *progressbar.ProgressBar { - return progressbar.NewOptions( - numItems, - progressbar.OptionFullWidth(), - progressbar.OptionSetWriter(ansi.NewAnsiStdout()), - progressbar.OptionEnableColorCodes(true), - progressbar.OptionSetTheme(progressbar.Theme{ - Saucer: "[green]=[reset]", - SaucerHead: "[green]>[reset]", - SaucerPadding: " ", - BarStart: "[", - BarEnd: "]", - }), - progressbar.OptionOnCompletion(func() { fmt.Println() }), - progressbar.OptionSetDescription("Loading validator accounts"), - ) -} - -// Checks if a directory indeed exists at the specified path. -func hasDir(dirPath string) (bool, error) { - info, err := os.Stat(dirPath) - if os.IsNotExist(err) { - return false, nil - } - return info.IsDir(), err + return secretKey, nil } diff --git a/validator/keymanager/v2/direct/direct_test.go b/validator/keymanager/v2/direct/direct_test.go index 55883c28ef..05ed54c76c 100644 --- a/validator/keymanager/v2/direct/direct_test.go +++ b/validator/keymanager/v2/direct/direct_test.go @@ -1,15 +1,11 @@ package direct import ( - "bytes" "context" "encoding/json" - "strconv" "strings" "testing" - ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" - "github.com/prysmaticlabs/go-ssz" validatorpb "github.com/prysmaticlabs/prysm/proto/validator/accounts/v2" "github.com/prysmaticlabs/prysm/shared/bls" "github.com/prysmaticlabs/prysm/shared/bytesutil" @@ -24,22 +20,24 @@ import ( func TestDirectKeymanager_CreateAccount(t *testing.T) { hook := logTest.NewGlobal() + password := "secretPassw0rd$1999" wallet := &mock.Wallet{ - Files: make(map[string]map[string][]byte), - AccountPasswords: make(map[string]string), + Files: make(map[string]map[string][]byte), + WalletPassword: password, } dr := &Keymanager{ - wallet: wallet, + keysCache: make(map[[48]byte]bls.SecretKey), + wallet: wallet, + accountsStore: &AccountStore{}, } ctx := context.Background() - password := "secretPassw0rd$1999" accountName, err := dr.CreateAccount(ctx, password) require.NoError(t, err) // Ensure the keystore file was written to the wallet // and ensure we can decrypt it using the EIP-2335 standard. var encodedKeystore []byte - for k, v := range wallet.Files[accountName] { + for k, v := range wallet.Files[AccountsPath] { if strings.Contains(k, "keystore") { encodedKeystore = v } @@ -48,50 +46,47 @@ func TestDirectKeymanager_CreateAccount(t *testing.T) { keystoreFile := &v2keymanager.Keystore{} require.NoError(t, json.Unmarshal(encodedKeystore, keystoreFile)) - // We extract the validator signing private key from the keystore - // by utilizing the password and initialize a new BLS secret key from - // its raw bytes. + // We extract the accounts from the keystore. decryptor := keystorev4.New() - rawSigningKey, err := decryptor.Decrypt(keystoreFile.Crypto, password) - require.NoError(t, err, "Could not decrypt validator signing key") - validatorSigningKey, err := bls.SecretKeyFromBytes(rawSigningKey) - require.NoError(t, err, "Could not instantiate bls secret key from bytes") - - // Decode the deposit_data.ssz file and confirm - // the public key matches the public key from the - // account's decrypted keystore. - encodedDepositData, ok := wallet.Files[accountName][DepositDataFileName] - require.Equal(t, true, ok, "Expected to have stored %s in wallet", DepositDataFileName) - depositData := ðpb.Deposit_Data{} - require.NoError(t, ssz.Unmarshal(encodedDepositData, depositData)) - - depositPublicKey := depositData.PublicKey - publicKey := validatorSigningKey.PublicKey().Marshal() - if !bytes.Equal(depositPublicKey, publicKey) { - t.Errorf( - "Expected deposit data public key %#x to match public key from keystore %#x", - depositPublicKey, - publicKey, - ) - } + encodedAccounts, err := decryptor.Decrypt(keystoreFile.Crypto, password) + require.NoError(t, err, "Could not decrypt validator accounts") + store := &AccountStore{} + require.NoError(t, json.Unmarshal(encodedAccounts, store)) + require.Equal(t, 1, len(store.PublicKeys)) + require.Equal(t, 1, len(store.PrivateKeys)) + privKey, err := bls.SecretKeyFromBytes(store.PrivateKeys[0]) + require.NoError(t, err) + pubKey := privKey.PublicKey().Marshal() + assert.DeepEqual(t, pubKey, store.PublicKeys[0]) + testutil.AssertLogsContain(t, hook, accountName) testutil.AssertLogsContain(t, hook, "Successfully created new validator account") } func TestDirectKeymanager_FetchValidatingPublicKeys(t *testing.T) { + password := "secretPassw0rd$1999" wallet := &mock.Wallet{ - Files: make(map[string]map[string][]byte), - AccountPasswords: make(map[string]string), + Files: make(map[string]map[string][]byte), + WalletPassword: password, } dr := &Keymanager{ - wallet: wallet, - keysCache: make(map[[48]byte]bls.SecretKey), + wallet: wallet, + keysCache: make(map[[48]byte]bls.SecretKey), + accountsStore: &AccountStore{}, } // First, generate accounts and their keystore.json files. ctx := context.Background() - numAccounts := 1 - accountNames, wantedPublicKeys := generateAccounts(t, numAccounts, dr) - wallet.Directories = accountNames + numAccounts := 10 + wantedPubKeys := make([][48]byte, numAccounts) + for i := 0; i < numAccounts; i++ { + privKey := bls.RandKey() + pubKey := bytesutil.ToBytes48(privKey.PublicKey().Marshal()) + dr.keysCache[pubKey] = privKey + wantedPubKeys[i] = pubKey + dr.accountsStore.PublicKeys = append(dr.accountsStore.PublicKeys, pubKey[:]) + dr.accountsStore.PrivateKeys = append(dr.accountsStore.PrivateKeys, privKey.Marshal()) + } + publicKeys, err := dr.FetchValidatingPublicKeys(ctx) require.NoError(t, err) // The results are not guaranteed to be ordered, so we ensure each @@ -100,7 +95,7 @@ func TestDirectKeymanager_FetchValidatingPublicKeys(t *testing.T) { for _, key := range publicKeys { keysMap[key] = true } - for _, wanted := range wantedPublicKeys { + for _, wanted := range wantedPubKeys { if _, ok := keysMap[wanted]; !ok { t.Errorf("Could not find expected public key %#x in results", wanted) } @@ -108,24 +103,56 @@ func TestDirectKeymanager_FetchValidatingPublicKeys(t *testing.T) { } func TestDirectKeymanager_Sign(t *testing.T) { + password := "secretPassw0rd$1999" wallet := &mock.Wallet{ Files: make(map[string]map[string][]byte), AccountPasswords: make(map[string]string), + WalletPassword: password, } dr := &Keymanager{ - wallet: wallet, - keysCache: make(map[[48]byte]bls.SecretKey), + wallet: wallet, + accountsStore: &AccountStore{}, + keysCache: make(map[[48]byte]bls.SecretKey), } // First, generate accounts and their keystore.json files. - numAccounts := 2 - accountNames, _ := generateAccounts(t, numAccounts, dr) - wallet.Directories = accountNames - ctx := context.Background() - require.NoError(t, dr.initializeSecretKeysCache(ctx)) + numAccounts := 10 + for i := 0; i < numAccounts; i++ { + _, err := dr.CreateAccount(ctx, password) + require.NoError(t, err) + } + + var encodedKeystore []byte + for k, v := range wallet.Files[AccountsPath] { + if strings.Contains(k, "keystore") { + encodedKeystore = v + } + } + keystoreFile := &v2keymanager.Keystore{} + require.NoError(t, json.Unmarshal(encodedKeystore, keystoreFile)) + + // We extract the validator signing private key from the keystore + // by utilizing the password and initialize a new BLS secret key from + // its raw bytes. + decryptor := keystorev4.New() + enc, err := decryptor.Decrypt(keystoreFile.Crypto, dr.wallet.Password()) + require.NoError(t, err) + store := &AccountStore{} + require.NoError(t, json.Unmarshal(enc, store)) + require.Equal(t, len(store.PublicKeys), len(store.PrivateKeys)) + require.NotEqual(t, 0, len(store.PublicKeys)) + + for i := 0; i < len(store.PublicKeys); i++ { + privKey, err := bls.SecretKeyFromBytes(store.PrivateKeys[i]) + require.NoError(t, err) + dr.keysCache[bytesutil.ToBytes48(store.PublicKeys[i])] = privKey + } + dr.accountsStore = store + publicKeys, err := dr.FetchValidatingPublicKeys(ctx) require.NoError(t, err) + require.Equal(t, len(publicKeys), len(store.PublicKeys)) // We prepare naive data to sign. data := []byte("hello world") @@ -146,6 +173,7 @@ func TestDirectKeymanager_Sign(t *testing.T) { t.Fatalf("Expected sig not to verify for pubkey %#x and data %v", wrongPubKey.Marshal(), data) } } + func TestDirectKeymanager_Sign_NoPublicKeySpecified(t *testing.T) { req := &validatorpb.SignRequest{ PublicKey: nil, @@ -165,43 +193,3 @@ func TestDirectKeymanager_Sign_NoPublicKeyInCache(t *testing.T) { _, err := dr.Sign(context.Background(), req) assert.ErrorContains(t, "no signing key found in keys cache", err) } - -func BenchmarkKeymanager_FetchValidatingPublicKeys(b *testing.B) { - b.StopTimer() - wallet := &mock.Wallet{ - Files: make(map[string]map[string][]byte), - AccountPasswords: make(map[string]string), - } - dr := &Keymanager{ - wallet: wallet, - keysCache: make(map[[48]byte]bls.SecretKey), - } - // First, generate accounts and their keystore.json files. - numAccounts := 1000 - generateAccounts(b, numAccounts, dr) - ctx := context.Background() - b.StartTimer() - for i := 0; i < b.N; i++ { - _, err := dr.FetchValidatingPublicKeys(ctx) - require.NoError(b, err) - } -} - -func generateAccounts(t testing.TB, numAccounts int, dr *Keymanager) ([]string, [][48]byte) { - ctx := context.Background() - accountNames := make([]string, numAccounts) - wantedPublicKeys := make([][48]byte, numAccounts) - for i := 0; i < numAccounts; i++ { - validatingKey := bls.RandKey() - wantedPublicKeys[i] = bytesutil.ToBytes48(validatingKey.PublicKey().Marshal()) - password := strconv.Itoa(i) - encoded, err := dr.generateKeystoreFile(validatingKey, password) - require.NoError(t, err) - accountName, err := dr.generateAccountName(validatingKey.PublicKey().Marshal()) - require.NoError(t, err) - assert.NoError(t, err, dr.wallet.WriteFileAtPath(ctx, accountName, KeystoreFileName, encoded)) - assert.NoError(t, err, dr.wallet.WritePasswordToDisk(ctx, accountName+PasswordFileSuffix, password)) - accountNames[i] = accountName - } - return accountNames, wantedPublicKeys -} diff --git a/validator/keymanager/v2/direct/import.go b/validator/keymanager/v2/direct/import.go new file mode 100644 index 0000000000..ada30f7ffe --- /dev/null +++ b/validator/keymanager/v2/direct/import.go @@ -0,0 +1,120 @@ +package direct + +import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" + "io/ioutil" + "strings" + + "github.com/k0kubun/go-ansi" + "github.com/logrusorgru/aurora" + "github.com/pkg/errors" + "github.com/prysmaticlabs/prysm/shared/bytesutil" + "github.com/prysmaticlabs/prysm/shared/promptutil" + "github.com/prysmaticlabs/prysm/shared/roughtime" + "github.com/prysmaticlabs/prysm/validator/flags" + v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" + "github.com/schollz/progressbar/v3" + "github.com/urfave/cli/v2" + keystorev4 "github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4" +) + +// ImportKeystores into the direct keymanager from an external source. +func (dr *Keymanager) ImportKeystores(cliCtx *cli.Context, keystores []*v2keymanager.Keystore) error { + decryptor := keystorev4.New() + privKeys := make([][]byte, len(keystores)) + pubKeys := make([][]byte, len(keystores)) + bar := initializeProgressBar(len(keystores), "Importing accounts...") + var password string + var err error + if cliCtx.IsSet(flags.AccountPasswordFileFlag.Name) { + passwordFilePath := cliCtx.String(flags.AccountPasswordFileFlag.Name) + data, err := ioutil.ReadFile(passwordFilePath) + if err != nil { + return err + } + password = string(data) + } else { + password, err = promptutil.PasswordPrompt( + "Enter the password for your imported accounts", promptutil.NotEmpty, + ) + if err != nil { + return fmt.Errorf("could not read account password: %v", err) + } + } + fmt.Println("Importing accounts, this may take a while...") + for i := 0; i < len(keystores); i++ { + privKeyBytes, pubKeyBytes, err := dr.attemptDecryptKeystore(decryptor, keystores[i], password) + if err != nil { + return err + } + privKeys[i] = privKeyBytes + pubKeys[i] = pubKeyBytes + if err := bar.Add(1); err != nil { + return errors.Wrap(err, "could not add to progress bar") + } + fmt.Printf( + "Successfully imported account with public key %#x\n", + aurora.BrightMagenta(bytesutil.Trunc(pubKeyBytes)), + ) + } + // Write the accounts to disk into a single keystore. + ctx := context.Background() + accountsKeystore, err := dr.createAccountsKeystore(ctx, privKeys, pubKeys) + if err != nil { + return err + } + encodedAccounts, err := json.MarshalIndent(accountsKeystore, "", "\t") + if err != nil { + return err + } + fileName := fmt.Sprintf(accountsKeystoreFileNameFormat, roughtime.Now().Unix()) + return dr.wallet.WriteFileAtPath(ctx, AccountsPath, fileName, encodedAccounts) +} + +// Retrieves the private key and public key from an EIP-2335 keystore file +// by decrypting using a specified password. If the password fails, +// it prompts the user for the correct password until it confirms. +func (dr *Keymanager) attemptDecryptKeystore( + enc *keystorev4.Encryptor, keystore *v2keymanager.Keystore, password string, +) ([]byte, []byte, error) { + // Attempt to decrypt the keystore with the specifies password. + var privKeyBytes []byte + var err error + privKeyBytes, err = enc.Decrypt(keystore.Crypto, password) + if err != nil && strings.Contains(err.Error(), "invalid checksum") { + // If the password fails for an individual account, we ask the user to input + // that individual account's password until it succeeds. + privKeyBytes, err = dr.askUntilPasswordConfirms(enc, keystore) + if err != nil { + return nil, nil, errors.Wrap(err, "could not confirm password via prompt") + } + } else if err != nil { + return nil, nil, errors.Wrap(err, "could not decrypt keystore") + } + pubKeyBytes, err := hex.DecodeString(keystore.Pubkey) + if err != nil { + return nil, nil, errors.Wrap(err, "could not decode pubkey from keystore") + } + return privKeyBytes, pubKeyBytes, nil +} + +func initializeProgressBar(numItems int, msg string) *progressbar.ProgressBar { + return progressbar.NewOptions( + numItems, + progressbar.OptionFullWidth(), + progressbar.OptionSetWriter(ansi.NewAnsiStdout()), + progressbar.OptionEnableColorCodes(true), + progressbar.OptionSetTheme(progressbar.Theme{ + Saucer: "[green]=[reset]", + SaucerHead: "[green]>[reset]", + SaucerPadding: " ", + BarStart: "[", + BarEnd: "]", + }), + progressbar.OptionOnCompletion(func() { fmt.Println() }), + progressbar.OptionSetDescription(msg), + ) +} diff --git a/validator/keymanager/v2/direct/import_test.go b/validator/keymanager/v2/direct/import_test.go new file mode 100644 index 0000000000..3c1877f3e6 --- /dev/null +++ b/validator/keymanager/v2/direct/import_test.go @@ -0,0 +1,110 @@ +package direct + +import ( + "crypto/rand" + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "math/big" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/prysmaticlabs/prysm/shared/bls" + "github.com/prysmaticlabs/prysm/shared/testutil" + "github.com/prysmaticlabs/prysm/shared/testutil/assert" + "github.com/prysmaticlabs/prysm/shared/testutil/require" + mock "github.com/prysmaticlabs/prysm/validator/accounts/v2/testing" + "github.com/prysmaticlabs/prysm/validator/flags" + v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" + "github.com/urfave/cli/v2" + keystorev4 "github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4" +) + +func setupCli( + tb testing.TB, + passwordFilePath string, +) *cli.Context { + app := cli.App{} + set := flag.NewFlagSet("test", 0) + set.String(flags.AccountPasswordFileFlag.Name, passwordFilePath, "") + assert.NoError(tb, set.Set(flags.AccountPasswordFileFlag.Name, passwordFilePath)) + return cli.NewContext(&app, set, nil) +} + +func createRandomKeystore(t testing.TB, password string) *v2keymanager.Keystore { + encryptor := keystorev4.New() + id, err := uuid.NewRandom() + require.NoError(t, err) + validatingKey := bls.RandKey() + pubKey := validatingKey.PublicKey().Marshal() + cryptoFields, err := encryptor.Encrypt(validatingKey.Marshal(), password) + require.NoError(t, err) + return &v2keymanager.Keystore{ + Crypto: cryptoFields, + Pubkey: fmt.Sprintf("%x", pubKey), + ID: id.String(), + Version: encryptor.Version(), + Name: encryptor.Name(), + } +} + +func TestDirectKeymanager_ImportKeystores(t *testing.T) { + randPath, err := rand.Int(rand.Reader, big.NewInt(1000000)) + require.NoError(t, err) + passwordFileDir := filepath.Join(testutil.TempDir(), fmt.Sprintf("/%d", randPath), "passwordFile") + require.NoError(t, os.MkdirAll(passwordFileDir, os.ModePerm)) + passwordFilePath := filepath.Join(passwordFileDir, "password.txt") + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(passwordFileDir), "Failed to remove directory") + }) + password := "secretPassw0rd$1999" + require.NoError(t, ioutil.WriteFile(passwordFilePath, []byte(password), os.ModePerm)) + + // Setup the keymanager. + wallet := &mock.Wallet{ + Files: make(map[string]map[string][]byte), + WalletPassword: password, + } + dr := &Keymanager{ + wallet: wallet, + accountsStore: &AccountStore{}, + } + + // Create several keystores and attempt to import them. + numAccounts := 5 + keystores := make([]*v2keymanager.Keystore, numAccounts) + for i := 0; i < numAccounts; i++ { + keystores[i] = createRandomKeystore(t, password) + } + cliCtx := setupCli(t, passwordFilePath) + require.NoError(t, dr.ImportKeystores(cliCtx, keystores)) + + // Ensure the single, all-encompassing accounts keystore was written + // to the wallet and ensure we can decrypt it using the EIP-2335 standard. + var encodedKeystore []byte + for k, v := range wallet.Files[AccountsPath] { + if strings.Contains(k, "keystore") { + encodedKeystore = v + } + } + require.NotNil(t, encodedKeystore, "could not find keystore file") + keystoreFile := &v2keymanager.Keystore{} + require.NoError(t, json.Unmarshal(encodedKeystore, keystoreFile)) + + // We decrypt the crypto fields of the accounts keystore. + decryptor := keystorev4.New() + encodedAccounts, err := decryptor.Decrypt(keystoreFile.Crypto, password) + require.NoError(t, err, "Could not decrypt validator accounts") + store := &AccountStore{} + require.NoError(t, json.Unmarshal(encodedAccounts, store)) + + // We should have successfully imported all accounts + // from external sources into a single AccountsStore + // struct preserved within a single keystore file. + assert.Equal(t, numAccounts, len(store.PublicKeys)) + assert.Equal(t, numAccounts, len(store.PrivateKeys)) +} diff --git a/validator/keymanager/v2/direct/migrate.go b/validator/keymanager/v2/direct/migrate.go new file mode 100644 index 0000000000..fe63f17295 --- /dev/null +++ b/validator/keymanager/v2/direct/migrate.go @@ -0,0 +1,105 @@ +package direct + +import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + + "github.com/logrusorgru/aurora" + "github.com/pkg/errors" + "github.com/prysmaticlabs/prysm/shared/roughtime" + v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" + keystorev4 "github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4" +) + +// Migrates the old format for validator direct-keymanaged accounts into a new, more +// efficient format which stores only a single keystore all accounts, encrypted using +// a high-entropy password. This allows for incredibly fast startup-time, requiring only +// a single decryption operation to obtain all validator accounts. This migration process +// is meant to happen only once, ensuring all future restarts of the validator client utilize +// the fast, efficient format. +// +// Old format: +// wallet/ +// direct/ +// perfectly-intense-mosquito/ +// keystore-2909299.json +// personally-conscious-echidna/ +// keystore-20390922.json +// passwords/ +// perfectly-intense-mosquito.pass +// personally-conscious-echidna.pass +// +// New format: +// wallet/ +// direct/ +// accounts/ +// all-accounts.keystore-2983823.json +func (dr *Keymanager) migrateToSingleKeystore(ctx context.Context) error { + accountNames, err := dr.wallet.ListDirs() + if err != nil { + return err + } + if len(accountNames) == 0 { + return nil + } + for _, name := range accountNames { + // If the user is already using the single keystore format, + // we have no need to migrate and we exit normally. + if strings.Contains(name, AccountsPath) { + return nil + } + } + au := aurora.NewAurora(true) + log.Infof( + "Now migrating accounts to a more efficient format, this is a %s setup\n", + au.BrightRed("one-time"), + ) + bar := initializeProgressBar(len(accountNames), "Migrating accounts...") + decryptor := keystorev4.New() + privKeys := make([][]byte, len(accountNames)) + pubKeys := make([][]byte, len(accountNames)) + // Next up, we retrieve every single keystore for each + // account and attempt to unlock. + for i, name := range accountNames { + password, err := dr.wallet.ReadPasswordFromDisk(ctx, name+PasswordFileSuffix) + if err != nil { + return errors.Wrapf(err, "could not read password for account %s", name) + } + encoded, err := dr.wallet.ReadFileAtPath(ctx, name, KeystoreFileName) + if err != nil { + return errors.Wrapf(err, "could not read keystore file for account %s", name) + } + keystoreFile := &v2keymanager.Keystore{} + if err := json.Unmarshal(encoded, keystoreFile); err != nil { + return errors.Wrapf(err, "could not decode keystore file for account %s", name) + } + // We extract the validator signing private key from the keystore + // by utilizing the password. + privKeyBytes, err := decryptor.Decrypt(keystoreFile.Crypto, password) + if err != nil { + return errors.Wrapf(err, "could not decrypt signing key for account %s", name) + } + publicKeyBytes, err := hex.DecodeString(keystoreFile.Pubkey) + if err != nil { + return err + } + privKeys[i] = privKeyBytes + pubKeys[i] = publicKeyBytes + if err := bar.Add(1); err != nil { + return err + } + } + accountsKeystore, err := dr.createAccountsKeystore(ctx, privKeys, pubKeys) + if err != nil { + return err + } + encodedAccounts, err := json.MarshalIndent(accountsKeystore, "", "\t") + if err != nil { + return err + } + fileName := fmt.Sprintf(accountsKeystoreFileNameFormat, roughtime.Now().Unix()) + return dr.wallet.WriteFileAtPath(ctx, AccountsPath, fileName, encodedAccounts) +} diff --git a/validator/keymanager/v2/direct/migrate_test.go b/validator/keymanager/v2/direct/migrate_test.go new file mode 100644 index 0000000000..01b4369ab7 --- /dev/null +++ b/validator/keymanager/v2/direct/migrate_test.go @@ -0,0 +1,103 @@ +package direct + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/prysmaticlabs/prysm/shared/bls" + "github.com/prysmaticlabs/prysm/shared/petnames" + "github.com/prysmaticlabs/prysm/shared/testutil" + "github.com/prysmaticlabs/prysm/shared/testutil/assert" + "github.com/prysmaticlabs/prysm/shared/testutil/require" + mock "github.com/prysmaticlabs/prysm/validator/accounts/v2/testing" + v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" + logTest "github.com/sirupsen/logrus/hooks/test" + keystorev4 "github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4" +) + +func TestDirectKeymanager_MigrateToSingleKeystoreFormat(t *testing.T) { + hook := logTest.NewGlobal() + password := "secretPassw0rd$1999" + wallet := &mock.Wallet{ + Files: make(map[string]map[string][]byte), + AccountPasswords: make(map[string]string), + WalletPassword: password, + } + dr := &Keymanager{ + keysCache: make(map[[48]byte]bls.SecretKey), + wallet: wallet, + accountsStore: &AccountStore{}, + } + ctx := context.Background() + + // Generate several old account keystore and save to the wallet path. + numAccounts := 5 + wallet.Directories = make([]string, numAccounts) + wantedValidatingKeys := make([]bls.SecretKey, numAccounts) + for i := 0; i < numAccounts; i++ { + validatingKey := bls.RandKey() + accountName, keystore := generateOldAccountKeystore(t, dr, validatingKey, password) + wallet.Directories[i] = accountName + wantedValidatingKeys[i] = validatingKey + encodedKeystore, err := json.MarshalIndent(keystore, "", "\t") + require.NoError(t, err) + require.NoError(t, dr.wallet.WritePasswordToDisk(ctx, accountName+".pass", password)) + require.NoError(t, dr.wallet.WriteFileAtPath(ctx, accountName, KeystoreFileName, encodedKeystore)) + } + + // Now, we run the migration strategy. + require.NoError(t, dr.migrateToSingleKeystore(ctx)) + + // We retrieve the new accounts keystore format containing all keys in a single file. + var encodedAccountsFile []byte + for k, v := range wallet.Files[AccountsPath] { + if strings.Contains(k, "keystore") { + encodedAccountsFile = v + } + } + require.NotNil(t, encodedAccountsFile, "could not find keystore file") + accountsKeystore := &v2keymanager.Keystore{} + require.NoError(t, json.Unmarshal(encodedAccountsFile, accountsKeystore)) + + // We extract the accounts from the keystore. + decryptor := keystorev4.New() + encodedAccounts, err := decryptor.Decrypt(accountsKeystore.Crypto, password) + require.NoError(t, err, "Could not decrypt validator accounts") + store := &AccountStore{} + require.NoError(t, json.Unmarshal(encodedAccounts, store)) + + // We expect the migration strategy to have succeeded, with all accounts from earlier + // now being stored in a single keystore file. + require.Equal(t, numAccounts, len(store.PublicKeys)) + require.Equal(t, numAccounts, len(store.PrivateKeys)) + for i := 0; i < numAccounts; i++ { + privKey, err := bls.SecretKeyFromBytes(store.PrivateKeys[i]) + require.NoError(t, err) + assert.DeepEqual(t, privKey.Marshal(), wantedValidatingKeys[i].Marshal()) + } + testutil.AssertLogsContain(t, hook, "Now migrating accounts to a more efficient format") +} + +func generateOldAccountKeystore( + t testing.TB, dr *Keymanager, validatingKey bls.SecretKey, password string, +) (string, *v2keymanager.Keystore) { + accountName := petnames.DeterministicName(validatingKey.PublicKey().Marshal(), "-") + // Generates a new EIP-2335 compliant keystore file + // from a BLS private key and marshals it as JSON. + encryptor := keystorev4.New() + cryptoFields, err := encryptor.Encrypt(validatingKey.Marshal(), password) + require.NoError(t, err) + id, err := uuid.NewRandom() + require.NoError(t, err) + return accountName, &v2keymanager.Keystore{ + Crypto: cryptoFields, + ID: id.String(), + Pubkey: fmt.Sprintf("%x", validatingKey.PublicKey().Marshal()), + Version: encryptor.Version(), + Name: encryptor.Name(), + } +}