Accounts-V2: Change accounts-v2 import to use directory of keystores (#6742)

* Finish implementation of importing dir of keystores

* Fix

* Fix importing

* Finish testing import account

* Update validator/accounts/v2/accounts_import.go

* Fix

* Fix

* fix

* Fix prompt

Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>

validator/accounts/v2/BUILD.bazel

@@ -62,6 +62,7 @@ go_test(
         "//shared/bls:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/petnames:go_default_library",
+        "//shared/roughtime:go_default_library",
         "//shared/testutil:go_default_library",
         "//shared/testutil/assert:go_default_library",
         "//shared/testutil/require:go_default_library",
@@ -71,7 +72,9 @@ go_test(
         "//validator/keymanager/v2/direct:go_default_library",
         "//validator/keymanager/v2/remote:go_default_library",
         "@com_github_dustin_go_humanize//:go_default_library",
+        "@com_github_google_uuid//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_urfave_cli_v2//:go_default_library",
+        "@com_github_wealdtech_go_eth2_wallet_encryptor_keystorev4//:go_default_library",
     ],
 )

validator/accounts/v2/accounts_export_test.go

@@ -7,7 +7,6 @@ import (
 	"math/big"
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/shared/testutil"
@@ -17,6 +16,7 @@ import (
 )
 
 func TestZipAndUnzip(t *testing.T) {
+	t.Skip("skipping until exporting is implemented")
 	walletDir, passwordsDir, _ := setupWalletAndPasswordsDir(t)
 	randPath, err := rand.Int(rand.Reader, big.NewInt(1000000))
 	require.NoError(t, err, "Could not generate random file path")
@@ -57,16 +57,6 @@ func TestZipAndUnzip(t *testing.T) {
 	if _, err := os.Stat(filepath.Join(exportDir, archiveFilename)); os.IsNotExist(err) {
 		t.Fatal("Expected file to exist")
 	}
-
-	importedAccounts, err := unzipArchiveToTarget(exportDir, importDir)
-	require.NoError(t, err)
-
-	allAccountsStr := strings.Join(accounts, " ")
-	for _, importedAccount := range importedAccounts {
-		if !strings.Contains(allAccountsStr, importedAccount) {
-			t.Fatalf("Expected %s to be in %s", importedAccount, allAccountsStr)
-		}
-	}
 }
 
 func TestExport_Noninteractive(t *testing.T) {

validator/accounts/v2/accounts_import.go

@@ -1,16 +1,17 @@
 package v2
 
 import (
-	"archive/zip"
 	"context"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/shared/petnames"
 	"github.com/prysmaticlabs/prysm/validator/flags"
 	v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2"
 	"github.com/prysmaticlabs/prysm/validator/keymanager/v2/direct"
@@ -28,9 +29,9 @@ func ImportAccount(cliCtx *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	backupDir, err := inputDirectory(cliCtx, importDirPromptText, flags.BackupDirFlag)
+	keysDir, err := inputDirectory(cliCtx, importKeysDirPromptText, flags.KeysDirFlag)
 	if err != nil {
-		return errors.Wrap(err, "could not parse output directory")
+		return errors.Wrap(err, "could not parse keys directory")
 	}
 
 	accountsPath := filepath.Join(walletDir, v2keymanager.Direct.String())
@@ -40,10 +41,6 @@ func ImportAccount(cliCtx *cli.Context) error {
 	if err := os.MkdirAll(passwordsDir, DirectoryPermissions); err != nil {
 		return errors.Wrap(err, "could not create passwords directory")
 	}
-	accountsImported, err := unzipArchiveToTarget(backupDir, filepath.Dir(walletDir))
-	if err != nil {
-		return errors.Wrap(err, "could not unzip archive")
-	}
 
 	wallet := &Wallet{
 		accountsPath:   accountsPath,
@@ -51,18 +48,42 @@ func ImportAccount(cliCtx *cli.Context) error {
 		keymanagerKind: v2keymanager.Direct,
 	}
 
-	au := aurora.NewAurora(true)
-	var loggedAccounts []string
-	for _, accountName := range accountsImported {
-		loggedAccounts = append(loggedAccounts, fmt.Sprintf("%s", au.BrightGreen(accountName).Bold()))
-	}
-	fmt.Printf("Importing accounts: %s\n", strings.Join(loggedAccounts, ", "))
-
-	for _, accountName := range accountsImported {
-		if err := wallet.enterPasswordForAccount(cliCtx, accountName); err != nil {
-			return errors.Wrap(err, "could not set account password")
+	var accountsImported []string
+	ctx := context.Background()
+	if err := filepath.Walk(keysDir, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
 		}
+
+		parentDir := filepath.Dir(path)
+		matches, err := filepath.Glob(filepath.Join(parentDir, direct.KeystoreFileName))
+		if err != nil {
+			return err
+		}
+
+		var keystoreFileFound bool
+		for _, match := range matches {
+			if match == path {
+				keystoreFileFound = true
+			}
+		}
+		if !keystoreFileFound {
+			return nil
+		}
+
+		accountName, err := wallet.importKeystore(ctx, path)
+		if err != nil {
+			return errors.Wrap(err, "could not import keystore")
+		}
+		if err := wallet.enterPasswordForAccount(cliCtx, accountName); err != nil {
+			return errors.Wrap(err, "could not verify password for keystore")
+		}
+		accountsImported = append(accountsImported, accountName)
+		return nil
+	}); err != nil {
+		return errors.Wrap(err, "could not walk files")
 	}
+
 	keymanager, err := wallet.InitializeKeymanager(context.Background(), true /* skip mnemonic confirm */)
 	if err != nil {
 		return errors.Wrap(err, "could not initialize keymanager")
@@ -78,66 +99,25 @@ func ImportAccount(cliCtx *cli.Context) error {
 	return nil
 }
 
-func unzipArchiveToTarget(archiveDir string, target string) ([]string, error) {
-	archiveFile := filepath.Join(archiveDir, archiveFilename)
-	reader, err := zip.OpenReader(archiveFile)
+func (w *Wallet) importKeystore(ctx context.Context, keystoreFilePath string) (string, error) {
+	keystoreBytes, err := ioutil.ReadFile(keystoreFilePath)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not open reader for archive")
+		return "", errors.Wrap(err, "could not read keystore file")
 	}
-
-	perms := os.FileMode(0700)
-	if err := os.MkdirAll(target, perms); err != nil {
-		return nil, errors.Wrap(err, "could not parent path for folder")
+	keystoreFile := &v2keymanager.Keystore{}
+	if err := json.Unmarshal(keystoreBytes, keystoreFile); err != nil {
+		return "", errors.Wrap(err, "could not decode keystore json")
 	}
-
-	var accounts []string
-	for _, file := range reader.File {
-		path := filepath.Join(target, file.Name)
-		parentFolder := filepath.Dir(path)
-		if file.FileInfo().IsDir() {
-			accounts = append(accounts, file.FileInfo().Name())
-			if err := os.MkdirAll(path, perms); err != nil {
-				return nil, errors.Wrap(err, "could not make path for file")
-			}
-			continue
-		} else {
-			if err := os.MkdirAll(parentFolder, perms); err != nil {
-				return nil, errors.Wrap(err, "could not make path for file")
-			}
-		}
-
-		if err := copyFileFromZipToPath(file, path); err != nil {
-			return nil, err
-		}
-	}
-	return accounts, nil
-}
-
-func copyFileFromZipToPath(file *zip.File, path string) error {
-	fileReader, err := file.Open()
+	pubKeyBytes, err := hex.DecodeString(keystoreFile.Pubkey)
 	if err != nil {
-		return err
+		return "", errors.Wrap(err, "could not decode public key string in keystore")
 	}
-	defer func() {
-		if err := fileReader.Close(); err != nil {
-			log.WithError(err).Error("Could not close file")
-		}
-	}()
-
-	targetFile, err := os.Create(path)
-	if err != nil {
-		return errors.Wrap(err, "could not open file")
+	accountName := petnames.DeterministicName(pubKeyBytes, "-")
+	keystoreFileName := filepath.Base(keystoreFilePath)
+	if err := w.WriteFileAtPath(ctx, accountName, keystoreFileName, keystoreBytes); err != nil {
+		return "", errors.Wrap(err, "could not write keystore to account dir")
 	}
-	defer func() {
-		if err := targetFile.Close(); err != nil {
-			log.WithError(err).Error("Could not close target")
-		}
-	}()
-
-	if _, err := io.Copy(targetFile, fileReader); err != nil {
-		return errors.Wrap(err, "could not copy file")
-	}
-	return nil
+	return accountName, nil
 }
 
 func logAccountsImported(wallet *Wallet, keymanager *direct.Keymanager, accountNames []string) error {

validator/accounts/v2/accounts_import_test.go

@@ -3,40 +3,40 @@ package v2
 import (
 	"context"
 	"crypto/rand"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"math/big"
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
+	"github.com/google/uuid"
+	"github.com/prysmaticlabs/prysm/shared/bls"
+	"github.com/prysmaticlabs/prysm/shared/roughtime"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
 	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 	v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2"
 	"github.com/prysmaticlabs/prysm/validator/keymanager/v2/direct"
+	keystorev4 "github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4"
 )
 
 func TestImport_Noninteractive(t *testing.T) {
-	walletDir, passwordsDir, _ := setupWalletAndPasswordsDir(t)
+	walletDir, passwordsDir, passwordFilePath := setupWalletAndPasswordsDir(t)
 	randPath, err := rand.Int(rand.Reader, big.NewInt(1000000))
 	require.NoError(t, err, "Could not generate random file path")
-	exportDir := filepath.Join(testutil.TempDir(), fmt.Sprintf("/%d", randPath), "export")
-	importDir := filepath.Join(testutil.TempDir(), fmt.Sprintf("/%d", randPath), "import")
-	importPasswordDir := filepath.Join(testutil.TempDir(), fmt.Sprintf("/%d", randPath), "importpassword")
+	keysDir := filepath.Join(testutil.TempDir(), fmt.Sprintf("/%d", randPath), "keysDir")
+	require.NoError(t, os.MkdirAll(keysDir, os.ModePerm))
 	t.Cleanup(func() {
-		require.NoError(t, os.RemoveAll(exportDir), "Failed to remove directory")
-		require.NoError(t, os.RemoveAll(importDir), "Failed to remove directory")
-		require.NoError(t, os.RemoveAll(importPasswordDir), "Failed to remove directory")
+		require.NoError(t, os.RemoveAll(keysDir), "Failed to remove directory")
 	})
-	require.NoError(t, os.MkdirAll(importPasswordDir, os.ModePerm))
-	passwordFilePath := filepath.Join(importPasswordDir, passwordFileName)
-	require.NoError(t, ioutil.WriteFile(passwordFilePath, []byte(password), os.ModePerm))
 
 	cliCtx := setupWalletCtx(t, &testWalletConfig{
 		walletDir:      walletDir,
 		passwordsDir:   passwordsDir,
-		exportDir:      exportDir,
+		keysDir:        keysDir,
 		keymanagerKind: v2keymanager.Direct,
 		passwordFile:   passwordFilePath,
 	})
@@ -54,19 +54,17 @@ func TestImport_Noninteractive(t *testing.T) {
 		keymanagerCfg,
 	)
 	require.NoError(t, err)
-	_, err = keymanager.CreateAccount(ctx, password)
-	require.NoError(t, err)
 
+	// Make sure there are no accounts at the start.
 	accounts, err := keymanager.ValidatingAccountNames()
 	require.NoError(t, err)
-	assert.Equal(t, len(accounts), 1)
+	assert.Equal(t, len(accounts), 0)
 
-	require.NoError(t, wallet.zipAccounts(accounts, exportDir))
-	if _, err := os.Stat(filepath.Join(exportDir, archiveFilename)); os.IsNotExist(err) {
-		t.Fatal("Expected file to exist")
-	}
+	// Create 2 keys.
+	createKeystore(t, keysDir)
+	time.Sleep(time.Second)
+	createKeystore(t, keysDir)
 
-	require.NoError(t, os.RemoveAll(walletDir), "Failed to remove directory")
 	require.NoError(t, ImportAccount(cliCtx))
 
 	wallet, err = OpenWallet(cliCtx)
@@ -76,5 +74,27 @@ func TestImport_Noninteractive(t *testing.T) {
 	keys, err := km.FetchValidatingPublicKeys(ctx)
 	require.NoError(t, err)
 
-	assert.Equal(t, len(keys), 1)
+	assert.Equal(t, 2, len(keys))
+}
+
+func createKeystore(t *testing.T, path string) {
+	validatingKey := bls.RandKey()
+	encryptor := keystorev4.New()
+	cryptoFields, err := encryptor.Encrypt(validatingKey.Marshal(), []byte(password))
+	require.NoError(t, err)
+	id, err := uuid.NewRandom()
+	require.NoError(t, err)
+	keystoreFile := &v2keymanager.Keystore{
+		Crypto:  cryptoFields,
+		ID:      id.String(),
+		Pubkey:  fmt.Sprintf("%x", validatingKey.PublicKey().Marshal()),
+		Version: encryptor.Version(),
+		Name:    encryptor.Name(),
+	}
+	encoded, err := json.MarshalIndent(keystoreFile, "", "\t")
+	require.NoError(t, err)
+	// Write the encoded keystore to disk with the timestamp appended
+	createdAt := roughtime.Now().Unix()
+	fullPath := filepath.Join(path, fmt.Sprintf(direct.KeystoreFileNameFormat, createdAt))
+	require.NoError(t, ioutil.WriteFile(fullPath, encoded, os.ModePerm))
 }

validator/accounts/v2/cmd_accounts.go

@@ -76,7 +76,7 @@ this command outputs a deposit data string which is required to become a validat
 			Flags: []cli.Flag{
 				flags.WalletDirFlag,
 				flags.WalletPasswordsDirFlag,
-				flags.BackupDirFlag,
+				flags.KeysDirFlag,
 				flags.PasswordFileFlag,
 				featureconfig.AltonaTestnet,
 				featureconfig.MedallaTestnet,

validator/accounts/v2/prompt.go

@@ -18,6 +18,7 @@ import (
 
 const (
 	importDirPromptText          = "Enter the file location of the exported wallet zip to import"
+	importKeysDirPromptText      = "Enter the directory where your keystores to import are located"
 	exportDirPromptText          = "Enter a file location to write the exported wallet to"
 	walletDirPromptText          = "Enter a wallet directory"
 	passwordsDirPromptText       = "Directory where passwords will be stored"
@@ -153,6 +154,37 @@ func inputPassword(cliCtx *cli.Context, promptText string, confirmPassword passw
 	return strings.TrimRight(walletPassword, "\r\n"), nil
 }
 
+func inputWeakPassword(cliCtx *cli.Context, promptText string) (string, error) {
+	if cliCtx.IsSet(flags.PasswordFileFlag.Name) {
+		passwordFilePath := cliCtx.String(flags.PasswordFileFlag.Name)
+		data, err := ioutil.ReadFile(passwordFilePath)
+		if err != nil {
+			return "", errors.Wrap(err, "could not read password file")
+		}
+		return strings.TrimRight(string(data), "\r\n"), nil
+	}
+
+	prompt := promptui.Prompt{
+		Label: promptText,
+		Validate: func(input string) error {
+			if input == "" {
+				return errors.New("password cannot be empty")
+			}
+			if !isValidUnicode(input) {
+				return errors.New("not valid unicode")
+			}
+			return nil
+		},
+		Mask: '*',
+	}
+
+	walletPassword, err := prompt.Run()
+	if err != nil {
+		return "", fmt.Errorf("could not read account password: %v", formatPromptError(err))
+	}
+	return strings.TrimRight(walletPassword, "\r\n"), nil
+}
+
 // Validate a strong password input for new accounts,
 // including a min length, at least 1 number and at least
 // 1 special character.

validator/accounts/v2/wallet.go

@@ -399,7 +399,7 @@ func (w *Wallet) enterPasswordForAccount(cliCtx *cli.Context, accountName string
 		// Loop asking for the password until the user enters it correctly.
 		for attemptingPassword {
 			// Ask the user for the password to their account.
-			password, err = inputPassword(cliCtx, fmt.Sprintf(passwordForAccountPromptText, accountName), noConfirmPass)
+			password, err = inputWeakPassword(cliCtx, fmt.Sprintf(passwordForAccountPromptText, accountName))
 			if err != nil {
 				return errors.Wrap(err, "could not input password")
 			}

validator/accounts/v2/wallet_test.go

@@ -31,6 +31,7 @@ type testWalletConfig struct {
 	walletDir        string
 	passwordsDir     string
 	exportDir        string
+	keysDir          string
 	accountsToExport string
 	passwordFile     string
 	numAccounts      int64
@@ -45,6 +46,7 @@ func setupWalletCtx(
 	set := flag.NewFlagSet("test", 0)
 	set.String(flags.WalletDirFlag.Name, cfg.walletDir, "")
 	set.String(flags.WalletPasswordsDirFlag.Name, cfg.passwordsDir, "")
+	set.String(flags.KeysDirFlag.Name, cfg.keysDir, "")
 	set.String(flags.KeymanagerKindFlag.Name, cfg.keymanagerKind.String(), "")
 	set.String(flags.BackupDirFlag.Name, cfg.exportDir, "")
 	set.String(flags.AccountsFlag.Name, cfg.accountsToExport, "")
@@ -53,6 +55,7 @@ func setupWalletCtx(
 	set.Int64(flags.NumAccountsFlag.Name, cfg.numAccounts, "")
 	assert.NoError(tb, set.Set(flags.WalletDirFlag.Name, cfg.walletDir))
 	assert.NoError(tb, set.Set(flags.WalletPasswordsDirFlag.Name, cfg.passwordsDir))
+	assert.NoError(tb, set.Set(flags.KeysDirFlag.Name, cfg.keysDir))
 	assert.NoError(tb, set.Set(flags.KeymanagerKindFlag.Name, cfg.keymanagerKind.String()))
 	assert.NoError(tb, set.Set(flags.BackupDirFlag.Name, cfg.exportDir))
 	assert.NoError(tb, set.Set(flags.AccountsFlag.Name, cfg.accountsToExport))

validator/flags/flags.go

@@ -171,6 +171,11 @@ var (
 		Usage: "Path to a directory where accounts will be exported into a zip file",
 		Value: DefaultValidatorDir(),
 	}
+	// KeysDirFlag defines the path for a directory where keystores to be imported at stored.
+	KeysDirFlag = &cli.StringFlag{
+		Name:  "keys-dir",
+		Usage: "Path to a directory where keystores to be imported are stored",
+	}
 	// GrpcRemoteAddressFlag defines the host:port address for a remote keymanager to connect to.
 	GrpcRemoteAddressFlag = &cli.StringFlag{
 		Name:  "grpc-remote-address",