diff --git a/beacon-chain/state/stateutil/BUILD.bazel b/beacon-chain/state/stateutil/BUILD.bazel index f6255c0760..b4c29a61fb 100644 --- a/beacon-chain/state/stateutil/BUILD.bazel +++ b/beacon-chain/state/stateutil/BUILD.bazel @@ -63,7 +63,6 @@ go_test( "//shared/testutil/assert:go_default_library", "//shared/testutil/require:go_default_library", "@com_github_google_gofuzz//:go_default_library", - "@com_github_protolambda_zssz//merkle:go_default_library", "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library", "@com_github_prysmaticlabs_go_ssz//:go_default_library", ], diff --git a/beacon-chain/state/stateutil/benchmark_test.go b/beacon-chain/state/stateutil/benchmark_test.go index 90de43a28e..10eb41dae3 100644 --- a/beacon-chain/state/stateutil/benchmark_test.go +++ b/beacon-chain/state/stateutil/benchmark_test.go @@ -3,7 +3,6 @@ package stateutil_test import ( "testing" - "github.com/protolambda/zssz/merkle" ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" "github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil" "github.com/prysmaticlabs/prysm/shared/hashutil" @@ -45,17 +44,11 @@ func BenchmarkBlockHTR(b *testing.B) { }) } -func BenchmarkMerkleize(b *testing.B) { +func BenchmarkMerkleize_Buffered(b *testing.B) { roots := make([][32]byte, 8192) for i := 0; i < 8192; i++ { roots[0] = [32]byte{byte(i)} } - oldMerkleize := func(chunks [][32]byte, count uint64, limit uint64) ([32]byte, error) { - leafIndexer := func(i uint64) []byte { - return chunks[i][:] - } - return merkle.Merkleize(hashutil.CustomSHA256Hasher(), count, limit, leafIndexer), nil - } newMerkleize := func(chunks [][32]byte, count uint64, limit uint64) ([32]byte, error) { leafIndexer := func(i uint64) []byte { @@ -64,24 +57,11 @@ func BenchmarkMerkleize(b *testing.B) { return htrutils.Merkleize(htrutils.NewHasherFunc(hashutil.CustomSHA256Hasher()), count, limit, leafIndexer), nil } - b.Run("Non Buffered Merkleizer", func(b *testing.B) { - b.ResetTimer() - b.ReportAllocs() - b.N = 1000 - for i := 0; i < b.N; i++ { - _, err := oldMerkleize(roots, 8192, 8192) - require.NoError(b, err) - } - }) - - b.Run("Buffered Merkleizer", func(b *testing.B) { - b.ResetTimer() - b.ReportAllocs() - b.N = 1000 - for i := 0; i < b.N; i++ { - _, err := newMerkleize(roots, 8192, 8192) - require.NoError(b, err) - } - }) - + b.ResetTimer() + b.ReportAllocs() + b.N = 1000 + for i := 0; i < b.N; i++ { + _, err := newMerkleize(roots, 8192, 8192) + require.NoError(b, err) + } } diff --git a/deps.bzl b/deps.bzl index 6dbdcbeddf..6e63f95dfe 100644 --- a/deps.bzl +++ b/deps.bzl @@ -697,8 +697,8 @@ def prysm_deps(): go_repository( name = "com_github_herumi_bls_eth_go_binary", importpath = "github.com/herumi/bls-eth-go-binary", - sum = "h1:mu+F5uA3Y68oB6KXZqWlASKMetbNufhQx2stMI+sD+Y=", - version = "v0.0.0-20200522010937-01d282b5380b", + sum = "h1:P8yaFmLwc5ZlUx2sHuawcdQvpv5/0GM+WEGJ07ljN3g=", + version = "v0.0.0-20200706085701-832d8c2c0f7d", ) go_repository( name = "com_github_hpcloud_tail", @@ -1989,8 +1989,8 @@ def prysm_deps(): go_repository( name = "com_github_dgraph_io_ristretto", importpath = "github.com/dgraph-io/ristretto", - sum = "h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po=", - version = "v0.0.2", + sum = "h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI=", + version = "v0.0.3", ) go_repository( name = "com_github_emicklei_dot", @@ -2594,8 +2594,8 @@ def prysm_deps(): go_repository( name = "com_github_protolambda_zssz", importpath = "github.com/protolambda/zssz", - sum = "h1:4jkt8sqwhOVR8B1JebREU/gVX0Ply4GypsV8+RWrDuw=", - version = "v0.1.4", + sum = "h1:7fjJjissZIIaa2QcvmhS/pZISMX21zVITt49sW1ouek=", + version = "v0.1.5", ) go_repository( name = "com_github_prysmaticlabs_ethereumapis", @@ -2661,8 +2661,8 @@ def prysm_deps(): go_repository( name = "com_github_wealdtech_go_eth2_util", importpath = "github.com/wealdtech/go-eth2-util", - sum = "h1:4OPbf2yaEQmqDmOIU6UKBfhKTPNZ7skU4lPhueBLx8o=", - version = "v1.1.5", + sum = "h1:b3fgyvoq/WocW9LkWT7zcO5VCKzKLCc97rPrk/B9oIc=", + version = "v1.5.0", ) go_repository( name = "com_github_wealdtech_go_eth2_wallet", @@ -2823,8 +2823,8 @@ def prysm_deps(): go_repository( name = "org_golang_x_crypto", importpath = "golang.org/x/crypto", - sum = "h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw=", - version = "v0.0.0-20200510223506-06a226fb4e37", + sum = "h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=", + version = "v0.0.0-20200709230013-948cd5f35899", ) go_repository( name = "org_golang_x_exp", @@ -2859,8 +2859,8 @@ def prysm_deps(): go_repository( name = "org_golang_x_sys", importpath = "golang.org/x/sys", - sum = "h1:rITEj+UZHYC927n8GT97eC3zrpzXdb/voyeOuVKS46o=", - version = "v0.0.0-20200523222454-059865788121", + sum = "h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=", + version = "v0.0.0-20200625212154-ddb9806d33ae", ) go_repository( name = "org_golang_x_text", @@ -2882,20 +2882,20 @@ def prysm_deps(): ) go_repository( name = "org_uber_go_automaxprocs", - importpath = "go.uber.org/automaxprocs", - sum = "h1:II28aZoGdaglS5vVNnspf28lnZpXScxtIozx1lAjdb0=", - version = "v1.3.0", build_directives = [ # Do not use this library directly. # Rather, load maxprocs from github.com/prysmaticlabs/shared/maxprocs. "gazelle:go_visibility @prysm//shared/maxprocs:__pkg__", ], + importpath = "go.uber.org/automaxprocs", + sum = "h1:II28aZoGdaglS5vVNnspf28lnZpXScxtIozx1lAjdb0=", + version = "v1.3.0", ) go_repository( name = "com_github_prysmaticlabs_go_ssz", importpath = "github.com/prysmaticlabs/go-ssz", - sum = "h1:V4o7uJqGXAuz6ZpwxhT4cnVjRb/XxpBmTKp/lVVr05k=", - version = "v0.0.0-20200605034351-b6a925e519d0", + sum = "h1:7qd0Af1ozWKBU3c93YW2RH+/09hJns9+ftqWUZyts9c=", + version = "v0.0.0-20200612203617-6d5c9aa213ae", ) go_repository( name = "io_k8s_client_go", @@ -2962,8 +2962,8 @@ def prysm_deps(): "gazelle:resolve go github.com/herumi/bls-eth-go-binary/bls @herumi_bls_eth_go_binary//:go_default_library", ], importpath = "github.com/wealdtech/go-eth2-types/v2", - sum = "h1:2KSUzducArOynCL2prRf4vWU5GjwaPSnSN9oqNgf+dQ=", - version = "v2.3.1", + sum = "h1:L8sl3yoICAbn3134CBLNUt0o5h2voe0Es2KD5O9r8YQ=", + version = "v2.5.0", ) go_repository( name = "io_k8s_sigs_structured_merge_diff", diff --git a/go.mod b/go.mod index f9ea89720c..53dd7373ae 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/confluentinc/confluent-kafka-go v1.4.2 // indirect github.com/d4l3k/messagediff v1.2.1 // indirect github.com/deckarep/golang-set v1.7.1 // indirect - github.com/dgraph-io/ristretto v0.0.2 + github.com/dgraph-io/ristretto v0.0.3 github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0 github.com/edsrzf/mmap-go v1.0.0 // indirect github.com/elastic/gosigar v0.10.5 // indirect @@ -38,7 +38,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/grpc-ecosystem/grpc-gateway v1.14.6 github.com/hashicorp/golang-lru v0.5.4 - github.com/herumi/bls-eth-go-binary v0.0.0-20200522010937-01d282b5380b + github.com/herumi/bls-eth-go-binary v0.0.0-20200706085701-832d8c2c0f7d github.com/ianlancetaylor/cgosymbolizer v0.0.0-20200424224625-be1b05b0b279 github.com/influxdata/influxdb v1.8.0 // indirect github.com/ipfs/go-cid v0.0.6 // indirect @@ -82,10 +82,10 @@ require ( github.com/prestonvanloon/go-recaptcha v0.0.0-20190217191114-0834cef6e8bd github.com/prometheus/client_golang v1.6.0 github.com/prometheus/tsdb v0.10.0 // indirect - github.com/protolambda/zssz v0.1.4 + github.com/protolambda/zssz v0.1.5 github.com/prysmaticlabs/ethereumapis v0.0.0-20200617012222-f52a0eff2886 github.com/prysmaticlabs/go-bitfield v0.0.0-20200618145306-2ae0807bef65 - github.com/prysmaticlabs/go-ssz v0.0.0-20200605034351-b6a925e519d0 + github.com/prysmaticlabs/go-ssz v0.0.0-20200612203617-6d5c9aa213ae github.com/prysmaticlabs/prombbolt v0.0.0-20200324184628-09789ef63796 github.com/rs/cors v1.7.0 github.com/sirupsen/logrus v1.6.0 @@ -94,6 +94,7 @@ require ( github.com/urfave/cli/v2 v2.2.0 github.com/wealdtech/eth2-signer-api v1.3.0 github.com/wealdtech/go-bytesutil v1.1.1 + github.com/wealdtech/go-eth2-util v1.5.0 github.com/wealdtech/go-eth2-wallet v1.9.4 github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4 v1.0.0 github.com/wealdtech/go-eth2-wallet-nd v1.8.0 @@ -103,10 +104,9 @@ require ( go.etcd.io/bbolt v1.3.4 go.opencensus.io v0.22.3 go.uber.org/automaxprocs v1.3.0 - golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 + golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 golang.org/x/exp v0.0.0-20200513190911-00229845015e golang.org/x/net v0.0.0-20200528225125-3c3fba18258b // indirect - golang.org/x/sys v0.0.0-20200523222454-059865788121 // indirect golang.org/x/tools v0.0.0-20200528185414-6be401e3f76e google.golang.org/genproto v0.0.0-20200528191852-705c0b31589b google.golang.org/grpc v1.29.1 diff --git a/go.sum b/go.sum index 0b86b2b7d5..77db167bb4 100644 --- a/go.sum +++ b/go.sum @@ -169,6 +169,8 @@ github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlN github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= +github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -321,6 +323,7 @@ github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190309163659-77426154d546/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -368,8 +371,8 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/hcl v0.0.0-20170914154624-68e816d1c783/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/herumi/bls-eth-go-binary v0.0.0-20200428020417-6dd0e5634b87/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= -github.com/herumi/bls-eth-go-binary v0.0.0-20200522010937-01d282b5380b h1:mu+F5uA3Y68oB6KXZqWlASKMetbNufhQx2stMI+sD+Y= -github.com/herumi/bls-eth-go-binary v0.0.0-20200522010937-01d282b5380b/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= +github.com/herumi/bls-eth-go-binary v0.0.0-20200706085701-832d8c2c0f7d h1:P8yaFmLwc5ZlUx2sHuawcdQvpv5/0GM+WEGJ07ljN3g= +github.com/herumi/bls-eth-go-binary v0.0.0-20200706085701-832d8c2c0f7d/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= @@ -907,6 +910,8 @@ github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSg github.com/protolambda/zssz v0.1.3/go.mod h1:a4iwOX5FE7/JkKA+J/PH0Mjo9oXftN6P8NZyL28gpag= github.com/protolambda/zssz v0.1.4 h1:4jkt8sqwhOVR8B1JebREU/gVX0Ply4GypsV8+RWrDuw= github.com/protolambda/zssz v0.1.4/go.mod h1:a4iwOX5FE7/JkKA+J/PH0Mjo9oXftN6P8NZyL28gpag= +github.com/protolambda/zssz v0.1.5 h1:7fjJjissZIIaa2QcvmhS/pZISMX21zVITt49sW1ouek= +github.com/protolambda/zssz v0.1.5/go.mod h1:a4iwOX5FE7/JkKA+J/PH0Mjo9oXftN6P8NZyL28gpag= github.com/prysmaticlabs/bazel-go-ethereum v0.0.0-20200626171358-a933315235ec h1:9JrPtwqCvV38DXYaHbB855KUIHYMwjJBE88lL8lMu8Q= github.com/prysmaticlabs/bazel-go-ethereum v0.0.0-20200626171358-a933315235ec/go.mod h1:oP8FC5+TbICUyftkTWs+8JryntjIJLJvWvApK3z2AYw= github.com/prysmaticlabs/ethereumapis v0.0.0-20200617012222-f52a0eff2886 h1:0zB+DtS1NdwgYtto4JcvV3OX3m1wmM7ocjLvveNaMgA= @@ -918,8 +923,8 @@ github.com/prysmaticlabs/go-bitfield v0.0.0-20200618145306-2ae0807bef65 h1:hJfAW github.com/prysmaticlabs/go-bitfield v0.0.0-20200618145306-2ae0807bef65/go.mod h1:hCwmef+4qXWjv0jLDbQdWnL0Ol7cS7/lCSS26WR+u6s= github.com/prysmaticlabs/go-ssz v0.0.0-20200101200214-e24db4d9e963 h1:Th5ufPIaL5s/7i3gXHTgiTwfsUhWDP/PwFRiI6qV6v0= github.com/prysmaticlabs/go-ssz v0.0.0-20200101200214-e24db4d9e963/go.mod h1:VecIJZrewdAuhVckySLFt2wAAHRME934bSDurP8ftkc= -github.com/prysmaticlabs/go-ssz v0.0.0-20200605034351-b6a925e519d0 h1:V4o7uJqGXAuz6ZpwxhT4cnVjRb/XxpBmTKp/lVVr05k= -github.com/prysmaticlabs/go-ssz v0.0.0-20200605034351-b6a925e519d0/go.mod h1:VecIJZrewdAuhVckySLFt2wAAHRME934bSDurP8ftkc= +github.com/prysmaticlabs/go-ssz v0.0.0-20200612203617-6d5c9aa213ae h1:7qd0Af1ozWKBU3c93YW2RH+/09hJns9+ftqWUZyts9c= +github.com/prysmaticlabs/go-ssz v0.0.0-20200612203617-6d5c9aa213ae/go.mod h1:VecIJZrewdAuhVckySLFt2wAAHRME934bSDurP8ftkc= github.com/prysmaticlabs/prombbolt v0.0.0-20200324184628-09789ef63796 h1:bVD46NhbqEE6bsIqj42TCS3ELUdumti3WfAw9DXNtkg= github.com/prysmaticlabs/prombbolt v0.0.0-20200324184628-09789ef63796/go.mod h1:5JkKm84FcLZQPNuHwjX8Mtd5emni/PH5CylWCNqnKos= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -1048,8 +1053,12 @@ github.com/wealdtech/go-eth2-types v1.0.0 h1:ggrbQ5HeFcxVm20zxVWr8Sc3uCditaetzWB github.com/wealdtech/go-eth2-types v1.0.0/go.mod h1:fWUgtKQ7hiNVl6263bGeyjlydYuaxkxcUIPIopgz2CM= github.com/wealdtech/go-eth2-types/v2 v2.3.1 h1:2KSUzducArOynCL2prRf4vWU5GjwaPSnSN9oqNgf+dQ= github.com/wealdtech/go-eth2-types/v2 v2.3.1/go.mod h1:FubkGSavaa+rvmHDMTUVoPdFh00wKg0k5QPW6G52mhw= +github.com/wealdtech/go-eth2-types/v2 v2.5.0 h1:L8sl3yoICAbn3134CBLNUt0o5h2voe0Es2KD5O9r8YQ= +github.com/wealdtech/go-eth2-types/v2 v2.5.0/go.mod h1:321w9X26lAnNa/lQJi2A6Lap5IsNORoLwFPoJ1i8QvY= github.com/wealdtech/go-eth2-util v1.1.5 h1:4OPbf2yaEQmqDmOIU6UKBfhKTPNZ7skU4lPhueBLx8o= github.com/wealdtech/go-eth2-util v1.1.5/go.mod h1:wYYmtc9KpQQAaAzWjXSPLgtsJMkoDAmTNN0h6uj3RCA= +github.com/wealdtech/go-eth2-util v1.5.0 h1:b3fgyvoq/WocW9LkWT7zcO5VCKzKLCc97rPrk/B9oIc= +github.com/wealdtech/go-eth2-util v1.5.0/go.mod h1:0PGWeWWc6qjky/aNjdPdguJdZ2HSEHHCA+3cTjvT+Hk= github.com/wealdtech/go-eth2-wallet v1.9.4 h1:9XFM1Y7dsyrgNFFCnE3Gd00PAsrpob70SAQqHSPmsBU= github.com/wealdtech/go-eth2-wallet v1.9.4/go.mod h1:UGd1bAPDEtP+UrFjj3HCbip7jggFGDIQoeGw8/XHMvo= github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4 v1.0.0 h1:IcpS4VpXhYz+TVupB5n6C6IQzaKwG+Rc8nvgCa/da4c= @@ -1156,6 +1165,8 @@ golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200427165652-729f1e841bcc/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1287,8 +1298,8 @@ golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200427175716-29b57079015a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121 h1:rITEj+UZHYC927n8GT97eC3zrpzXdb/voyeOuVKS46o= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/shared/testutil/log.go b/shared/testutil/log.go index 6846aa30ee..47a02d5c92 100644 --- a/shared/testutil/log.go +++ b/shared/testutil/log.go @@ -31,6 +31,15 @@ func assertLogs(t *testing.T, hook *test.Hook, want string, flag bool) { if strings.Contains(msg, want) { match = true } + for _, field := range e.Data { + fieldStr, ok := field.(string) + if !ok { + continue + } + if strings.Contains(fieldStr, want) { + match = true + } + } t.Logf("log: %s", msg) } diff --git a/validator/accounts/v2/BUILD.bazel b/validator/accounts/v2/BUILD.bazel index 48f17dcedd..fa751f2c6b 100644 --- a/validator/accounts/v2/BUILD.bazel +++ b/validator/accounts/v2/BUILD.bazel @@ -25,6 +25,7 @@ go_library( "//shared/params:go_default_library", "//validator/flags:go_default_library", "//validator/keymanager/v2:go_default_library", + "//validator/keymanager/v2/derived:go_default_library", "//validator/keymanager/v2/direct:go_default_library", "//validator/keymanager/v2/remote:go_default_library", "@com_github_dustinkirkland_golang_petname//:go_default_library", diff --git a/validator/accounts/v2/export.go b/validator/accounts/v2/export.go index 8a1a697587..96fb51cd00 100644 --- a/validator/accounts/v2/export.go +++ b/validator/accounts/v2/export.go @@ -154,7 +154,7 @@ func (w *Wallet) zipAccounts(accounts []string, targetPath string) error { if strings.Contains(path, accountName) { // Add all files under the account folder to the archive. isAccount = true - } else if !info.IsDir() && info.Name() == keymanagerConfigFileName { + } else if !info.IsDir() && info.Name() == KeymanagerConfigFileName { // Add the keymanager config file to the archive as well. isAccount = true } diff --git a/validator/accounts/v2/export_test.go b/validator/accounts/v2/export_test.go index ac0eb07e9a..5678ad4c99 100644 --- a/validator/accounts/v2/export_test.go +++ b/validator/accounts/v2/export_test.go @@ -22,7 +22,12 @@ func setupWallet(t *testing.T, testDir string) *Wallet { passwordsDir := filepath.Join(testDir, passwordDirName) ctx := context.Background() - assert.NoError(t, initializeDirectWallet(walletDir, passwordsDir)) + app := cli.App{} + set := flag.NewFlagSet("test", 0) + set.String(flags.WalletPasswordsDirFlag.Name, passwordsDir, "") + assert.NoError(t, set.Set(flags.WalletPasswordsDirFlag.Name, passwordsDir)) + cliCtx := cli.NewContext(&app, set, nil) + assert.NoError(t, createDirectWallet(cliCtx, walletDir)) cfg := &WalletConfig{ WalletDir: walletDir, PasswordsDir: passwordsDir, diff --git a/validator/accounts/v2/iface/BUILD.bazel b/validator/accounts/v2/iface/BUILD.bazel new file mode 100644 index 0000000000..a44fd13c4b --- /dev/null +++ b/validator/accounts/v2/iface/BUILD.bazel @@ -0,0 +1,11 @@ +load("@prysm//tools/go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["wallet.go"], + importpath = "github.com/prysmaticlabs/prysm/validator/accounts/v2/iface", + visibility = [ + "//validator:__pkg__", + "//validator:__subpackages__", + ], +) diff --git a/validator/accounts/v2/iface/wallet.go b/validator/accounts/v2/iface/wallet.go new file mode 100644 index 0000000000..50883fa136 --- /dev/null +++ b/validator/accounts/v2/iface/wallet.go @@ -0,0 +1,25 @@ +package iface + +import ( + "context" + "io" +) + +// Wallet defines a struct which has capabilities and knowledge of how +// to read and write important accounts-related files to the filesystem. +// Useful for keymanagers to have persistent capabilities for accounts on-disk. +type Wallet interface { + // Methods to retrieve wallet and accounts metadata. + AccountNames() ([]string, error) + AccountsDir() string + CanUnlockAccounts() bool + // Read methods for important wallet and accounts-related files. + ReadEncryptedSeedFromDisk(ctx context.Context) (io.ReadCloser, error) + ReadPasswordForAccount(accountName string) (string, error) + ReadFileForAccount(accountName string, fileName string) ([]byte, error) + // Write methods to persist important wallet and accounts-related files to disk. + WriteFileAtPath(ctx context.Context, pathName string, fileName string, data []byte) error + WriteEncryptedSeedToDisk(ctx context.Context, encoded []byte) error + WriteAccountToDisk(ctx context.Context, password string) (string, error) + WriteFileForAccount(ctx context.Context, accountName string, fileName string, data []byte) error +} diff --git a/validator/accounts/v2/list_test.go b/validator/accounts/v2/list_test.go index 9df9fc47a2..f3010596dd 100644 --- a/validator/accounts/v2/list_test.go +++ b/validator/accounts/v2/list_test.go @@ -42,7 +42,7 @@ func TestListAccounts_DirectKeymanager(t *testing.T) { // Generate a directory for the account name and // write its associated password to disk. accountPath := path.Join(wallet.accountsPath, name) - require.NoError(t, os.MkdirAll(accountPath, directoryPermissions)) + require.NoError(t, os.MkdirAll(accountPath, DirectoryPermissions)) require.NoError(t, wallet.writePasswordToFile(name, password)) // Write the deposit data for each account. diff --git a/validator/accounts/v2/new.go b/validator/accounts/v2/new.go index e1417d9034..a15d1267ba 100644 --- a/validator/accounts/v2/new.go +++ b/validator/accounts/v2/new.go @@ -26,8 +26,8 @@ const ( ) var keymanagerKindSelections = map[v2keymanager.Kind]string{ - v2keymanager.Direct: "Direct, On-Disk Wallet (Recommended)", - v2keymanager.Derived: "Derived HD Wallet (Advanced)", + v2keymanager.Derived: "HD Wallet (Recommended)", + v2keymanager.Direct: "Non-HD Wallet (Most Basic)", v2keymanager.Remote: "Remote Signing Wallet (Advanced)", } @@ -48,8 +48,8 @@ func NewAccount(cliCtx *cli.Context) error { } // Only direct keymanagers can create accounts for now. - if keymanagerKind != v2keymanager.Direct { - log.Fatalf("cannot create a new account for a %s keymanager", keymanagerKind) + if keymanagerKind == v2keymanager.Remote { + log.Fatal("Cannot create a new account for a remote keymanager") } // Read the directory for password storage from user input. passwordsDirPath := inputPasswordsDirectory(cliCtx) @@ -63,8 +63,8 @@ func NewAccount(cliCtx *cli.Context) error { log.Fatalf("Could not open wallet: %v", err) } - skipMnemonicConfirm := cliCtx.Bool(flags.SkipMnemonicConfirmFlag.Name) // We initialize a new keymanager depending on the wallet's keymanager kind. + skipMnemonicConfirm := cliCtx.Bool(flags.SkipMnemonicConfirmFlag.Name) keymanager, err := wallet.InitializeKeymanager(ctx, skipMnemonicConfirm) if err != nil { log.Fatalf("Could not initialize keymanager: %v", err) @@ -130,6 +130,53 @@ func inputKeymanagerKind(cliCtx *cli.Context) (v2keymanager.Kind, error) { return v2keymanager.Kind(selection), nil } +func inputNewWalletPassword() (string, error) { + var hasValidPassword bool + var walletPassword string + var err error + for !hasValidPassword { + prompt := promptui.Prompt{ + Label: "New wallet password", + Validate: validatePasswordInput, + Mask: '*', + } + + walletPassword, err = prompt.Run() + if err != nil { + return "", fmt.Errorf("could not read wallet password: %v", formatPromptError(err)) + } + + prompt = promptui.Prompt{ + Label: "Confirm password", + Mask: '*', + } + confirmPassword, err := prompt.Run() + if err != nil { + return "", fmt.Errorf("could not read password confirmation: %v", formatPromptError(err)) + } + if walletPassword != confirmPassword { + log.Error("Passwords do not match") + continue + } + hasValidPassword = true + } + return walletPassword, nil +} + +func inputExistingWalletPassword() (string, error) { + prompt := promptui.Prompt{ + Label: "Wallet password", + Validate: validatePasswordInput, + Mask: '*', + } + + walletPassword, err := prompt.Run() + if err != nil { + return "", fmt.Errorf("could not read wallet password: %v", formatPromptError(err)) + } + return walletPassword, nil +} + func inputNewAccountPassword(cliCtx *cli.Context) (string, error) { if cliCtx.IsSet(flags.PasswordFileFlag.Name) { passwordFilePath := cliCtx.String(flags.PasswordFileFlag.Name) @@ -156,7 +203,7 @@ func inputNewAccountPassword(cliCtx *cli.Context) (string, error) { walletPassword, err = prompt.Run() if err != nil { - return "", fmt.Errorf("could not read wallet password: %v", formatPromptError(err)) + return "", fmt.Errorf("could not read account password: %v", formatPromptError(err)) } prompt = promptui.Prompt{ diff --git a/validator/accounts/v2/testing/mock.go b/validator/accounts/v2/testing/mock.go index e1eba2989c..f42e56476a 100644 --- a/validator/accounts/v2/testing/mock.go +++ b/validator/accounts/v2/testing/mock.go @@ -1,8 +1,11 @@ package mock import ( + "bytes" "context" "errors" + "io" + "io/ioutil" "sync" petname "github.com/dustinkirkland/golang-petname" @@ -10,10 +13,11 @@ import ( // Wallet contains an in-memory, simulated wallet implementation. type Wallet struct { - Files map[string]map[string][]byte - AccountPasswords map[string]string - UnlockAccounts bool - lock sync.RWMutex + Files map[string]map[string][]byte + EncryptedSeedFile []byte + AccountPasswords map[string]string + UnlockAccounts bool + lock sync.RWMutex } // AccountNames -- @@ -85,3 +89,29 @@ func (m *Wallet) ReadFileForAccount(accountName string, fileName string) ([]byte } return nil, errors.New("file not found") } + +// WriteFileAtPath -- +func (m *Wallet) WriteFileAtPath(ctx context.Context, pathName string, fileName string, data []byte) error { + m.lock.Lock() + defer m.lock.Unlock() + if m.Files[pathName] == nil { + m.Files[pathName] = make(map[string][]byte) + } + m.Files[pathName][fileName] = data + return nil +} + +// ReadEncryptedSeedFromDisk -- +func (m *Wallet) ReadEncryptedSeedFromDisk(ctx context.Context) (io.ReadCloser, error) { + m.lock.Lock() + defer m.lock.Unlock() + return ioutil.NopCloser(bytes.NewReader(m.EncryptedSeedFile)), nil +} + +// WriteEncryptedSeedToDisk -- +func (m *Wallet) WriteEncryptedSeedToDisk(ctx context.Context, encoded []byte) error { + m.lock.Lock() + defer m.lock.Unlock() + m.EncryptedSeedFile = encoded + return nil +} diff --git a/validator/accounts/v2/wallet.go b/validator/accounts/v2/wallet.go index e13e9f69ef..c56bfc5d99 100644 --- a/validator/accounts/v2/wallet.go +++ b/validator/accounts/v2/wallet.go @@ -18,6 +18,7 @@ import ( "github.com/prysmaticlabs/prysm/shared/params" "github.com/prysmaticlabs/prysm/validator/flags" v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" + "github.com/prysmaticlabs/prysm/validator/keymanager/v2/derived" "github.com/prysmaticlabs/prysm/validator/keymanager/v2/direct" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" @@ -28,12 +29,19 @@ const ( // WalletDefaultDirName for accounts-v2. WalletDefaultDirName = ".prysm-wallet-v2" // PasswordsDefaultDirName where account passwords are stored. - PasswordsDefaultDirName = ".prysm-wallet-v2-passwords" - keymanagerConfigFileName = "keymanageropts.json" - passwordFileSuffix = ".pass" - numAccountWords = 3 // Number of words in account human-readable names. - accountFilePermissions = os.O_CREATE | os.O_RDWR - directoryPermissions = os.ModePerm + PasswordsDefaultDirName = ".prysm-wallet-v2-passwords" + // KeymanagerConfigFileName for the keymanager used by the wallet: direct, derived, or remote. + KeymanagerConfigFileName = "keymanageropts.json" + // EncryptedSeedFileName for persisting a wallet's seed when using a derived keymanager. + EncryptedSeedFileName = "seed.encrypted.json" + // PasswordFileSuffix for passwords persisted as text to disk. + PasswordFileSuffix = ".pass" + // NumAccountWords for human-readable names in wallets using a direct keymanager. + NumAccountWords = 3 // Number of words in account human-readable names. + // AccountFilePermissions for accounts saved to disk. + AccountFilePermissions = os.O_CREATE | os.O_RDWR + // DirectoryPermissions for directories created under the wallet path. + DirectoryPermissions = os.ModePerm ) var ( @@ -74,11 +82,11 @@ func NewWallet(ctx context.Context, cfg *WalletConfig) (*Wallet, error) { return nil, errors.New("wallet dir and passwords dir cannot be nil") } accountsPath := path.Join(cfg.WalletDir, cfg.KeymanagerKind.String()) - if err := os.MkdirAll(accountsPath, directoryPermissions); err != nil { + if err := os.MkdirAll(accountsPath, DirectoryPermissions); err != nil { return nil, errors.Wrap(err, "could not create wallet directory") } if cfg.PasswordsDir != "" { - if err := os.MkdirAll(cfg.PasswordsDir, directoryPermissions); err != nil { + if err := os.MkdirAll(cfg.PasswordsDir, DirectoryPermissions); err != nil { return nil, errors.Wrap(err, "could not create passwords directory") } } @@ -107,10 +115,10 @@ func OpenWallet(ctx context.Context, cfg *WalletConfig) (*Wallet, error) { // ReadKeymanagerConfigFromDisk opens a keymanager config file // for reading if it exists at the wallet path. func (w *Wallet) ReadKeymanagerConfigFromDisk(ctx context.Context) (io.ReadCloser, error) { - if !fileExists(path.Join(w.accountsPath, keymanagerConfigFileName)) { + if !fileExists(path.Join(w.accountsPath, KeymanagerConfigFileName)) { return nil, fmt.Errorf("no keymanager config file found at path: %s", w.accountsPath) } - configFilePath := path.Join(w.accountsPath, keymanagerConfigFileName) + configFilePath := path.Join(w.accountsPath, KeymanagerConfigFileName) return os.Open(configFilePath) } @@ -180,14 +188,23 @@ func (w *Wallet) InitializeKeymanager( } keymanager, err = direct.NewKeymanager(ctx, w, cfg, skipMnemonicConfirm) if err != nil { - return nil, errors.Wrap(err, "could not initialize keymanager") + return nil, errors.Wrap(err, "could not initialize direct keymanager") } case v2keymanager.Derived: - return nil, errors.New("derived keymanager is unimplemented, work in progress") - case v2keymanager.Remote: - return nil, errors.New("remote keymanager is unimplemented, work in progress") + seedPassword, err := inputExistingWalletPassword() + if err != nil { + return nil, err + } + cfg, err := derived.UnmarshalConfigFile(configFile) + if err != nil { + return nil, errors.Wrap(err, "could not unmarshal keymanager config file") + } + keymanager, err = derived.NewKeymanager(ctx, w, cfg, skipMnemonicConfirm, seedPassword) + if err != nil { + return nil, errors.Wrap(err, "could not initialize derived keymanager") + } default: - return nil, errors.New("keymanager kind must be specified") + return nil, fmt.Errorf("keymanager kind not supported: %s", w.keymanagerKind) } return keymanager, nil } @@ -203,7 +220,7 @@ func (w *Wallet) WriteAccountToDisk(ctx context.Context, password string) (strin // Generate a directory for the new account name and // write its associated password to disk. accountPath := path.Join(w.accountsPath, accountName) - if err := os.MkdirAll(accountPath, directoryPermissions); err != nil { + if err := os.MkdirAll(accountPath, DirectoryPermissions); err != nil { return "", errors.Wrap(err, "could not create account directory") } if err := w.writePasswordToFile(accountName, password); err != nil { @@ -212,6 +229,23 @@ func (w *Wallet) WriteAccountToDisk(ctx context.Context, password string) (strin return accountName, nil } +// WriteFileAtPath within the wallet directory given the desired path, filename, and raw data. +func (w *Wallet) WriteFileAtPath(ctx context.Context, filePath string, fileName string, data []byte) error { + accountPath := path.Join(w.accountsPath, filePath) + if err := os.MkdirAll(accountPath, os.ModePerm); err != nil { + return errors.Wrapf(err, "could not create path: %s", accountPath) + } + fullPath := path.Join(accountPath, fileName) + if err := ioutil.WriteFile(fullPath, data, os.ModePerm); err != nil { + return errors.Wrapf(err, "could not write %s", filePath) + } + log.WithFields(logrus.Fields{ + "path": fullPath, + "fileName": fileName, + }).Debug("Wrote new file at path") + return nil +} + // WriteFileForAccount stores a unique file and its data under an account namespace // in the wallet's directory on-disk. Creates the file if it does not exist // and writes over it otherwise. @@ -238,21 +272,43 @@ func (w *Wallet) WriteFileForAccount(ctx context.Context, accountName string, fi // WriteKeymanagerConfigToDisk takes an encoded keymanager config file // and writes it to the wallet path. func (w *Wallet) WriteKeymanagerConfigToDisk(ctx context.Context, encoded []byte) error { - configFilePath := path.Join(w.accountsPath, keymanagerConfigFileName) + configFilePath := path.Join(w.accountsPath, KeymanagerConfigFileName) // Write the config file to disk. if err := ioutil.WriteFile(configFilePath, encoded, os.ModePerm); err != nil { return errors.Wrapf(err, "could not write %s", configFilePath) } - log.WithField("configFile", configFilePath).Debug("Wrote keymanager config file to disk") + log.WithField("configFilePath", configFilePath).Debug("Wrote keymanager config file to disk") return nil } +// WriteEncryptedSeedToDisk writes the encrypted wallet seed configuration +// within the wallet path. +func (w *Wallet) WriteEncryptedSeedToDisk(ctx context.Context, encoded []byte) error { + seedFilePath := path.Join(w.accountsPath, EncryptedSeedFileName) + // Write the config file to disk. + if err := ioutil.WriteFile(seedFilePath, encoded, os.ModePerm); err != nil { + return errors.Wrapf(err, "could not write %s", seedFilePath) + } + log.WithField("seedFilePath", seedFilePath).Debug("Wrote wallet encrypted seed file to disk") + return nil +} + +// ReadEncryptedSeedFromDisk reads the encrypted wallet seed configuration from +// within the wallet path. +func (w *Wallet) ReadEncryptedSeedFromDisk(ctx context.Context) (io.ReadCloser, error) { + if !fileExists(path.Join(w.accountsPath, EncryptedSeedFileName)) { + return nil, fmt.Errorf("no encrypted seed file found at path: %s", w.accountsPath) + } + configFilePath := path.Join(w.accountsPath, EncryptedSeedFileName) + return os.Open(configFilePath) +} + // ReadPasswordForAccount when given an account name from the wallet's passwords' path. func (w *Wallet) ReadPasswordForAccount(accountName string) (string, error) { if !w.canUnlockAccounts { return "", errors.New("wallet has no permission to read account passwords") } - passwordFilePath := path.Join(w.passwordsDir, accountName+passwordFileSuffix) + passwordFilePath := path.Join(w.passwordsDir, accountName+PasswordFileSuffix) passwordFile, err := os.Open(passwordFilePath) if err != nil { return "", errors.Wrapf(err, "could not read password file from directory: %s", w.passwordsDir) @@ -269,6 +325,29 @@ func (w *Wallet) ReadPasswordForAccount(accountName string) (string, error) { return string(password), nil } +// ReadFileForAccount from the wallet's accounts directory. +func (w *Wallet) ReadFileForAccount(accountName string, fileName string) ([]byte, error) { + accountPath := path.Join(w.accountsPath, accountName) + exists, err := hasDir(accountPath) + if err != nil { + return nil, errors.Wrapf(err, "could not check if account exists in directory: %s", w.accountsPath) + } + if !exists { + return nil, errors.Wrapf(err, "account does not exist in wallet directory: %s", w.accountsPath) + } + filePath := path.Join(accountPath, fileName) + f, err := os.Open(filePath) + if err != nil { + return nil, errors.Wrapf(err, "could not read file for account: %s", filePath) + } + defer func() { + if err := f.Close(); err != nil { + log.Errorf("Could not close file after writing: %s", filePath) + } + }() + return ioutil.ReadAll(f) +} + func (w *Wallet) enterPasswordForAccount(cliCtx *cli.Context, accountName string) error { au := aurora.NewAurora(true) @@ -344,44 +423,21 @@ func (w *Wallet) publicKeyForAccount(accountName string) ([48]byte, error) { return bytesutil.ToBytes48(pubKey), nil } -func (w *Wallet) keystoreForAccount(accountName string) (*direct.Keystore, error) { +func (w *Wallet) keystoreForAccount(accountName string) (*v2keymanager.Keystore, error) { encoded, err := w.ReadFileForAccount(accountName, direct.KeystoreFileName) if err != nil { return nil, errors.Wrap(err, "could not read keystore file") } - keystoreJSON := &direct.Keystore{} + keystoreJSON := &v2keymanager.Keystore{} if err := json.Unmarshal(encoded, &keystoreJSON); err != nil { return nil, errors.Wrap(err, "could not decode json") } return keystoreJSON, nil } -// ReadFileForAccount from the wallet's accounts directory. -func (w *Wallet) ReadFileForAccount(accountName string, fileName string) ([]byte, error) { - accountPath := path.Join(w.accountsPath, accountName) - exists, err := hasDir(accountPath) - if err != nil { - return nil, errors.Wrapf(err, "could not check if account exists in directory: %s", w.accountsPath) - } - if !exists { - return nil, errors.Wrapf(err, "account does not exist in wallet directory: %s", w.accountsPath) - } - filePath := path.Join(accountPath, fileName) - f, err := os.Open(filePath) - if err != nil { - return nil, errors.Wrapf(err, "could not read file for account: %s", filePath) - } - defer func() { - if err := f.Close(); err != nil { - log.Errorf("Could not close file after writing: %s", filePath) - } - }() - return ioutil.ReadAll(f) -} - // Writes the password file for an account namespace in the wallet's passwords directory. func (w *Wallet) writePasswordToFile(accountName string, password string) error { - passwordFilePath := path.Join(w.passwordsDir, accountName+passwordFileSuffix) + passwordFilePath := path.Join(w.passwordsDir, accountName+PasswordFileSuffix) // Removing any file that exists to make sure the existing is overwritten. if _, err := os.Stat(passwordFilePath); os.IsExist(err) { if err := os.Remove(passwordFilePath); err != nil { @@ -412,7 +468,7 @@ func (w *Wallet) generateAccountName() (string, error) { var accountExists bool var accountName string for !accountExists { - accountName = petname.Generate(numAccountWords, "-" /* separator */) + accountName = petname.Generate(NumAccountWords, "-" /* separator */) exists, err := hasDir(path.Join(w.accountsPath, accountName)) if err != nil { return "", errors.Wrapf(err, "could not check if account exists in dir: %s", w.accountsPath) diff --git a/validator/accounts/v2/wallet_create.go b/validator/accounts/v2/wallet_create.go index cc4f7a6eaf..76bbccd0c4 100644 --- a/validator/accounts/v2/wallet_create.go +++ b/validator/accounts/v2/wallet_create.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/prysmaticlabs/prysm/validator/flags" v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" + "github.com/prysmaticlabs/prysm/validator/keymanager/v2/derived" "github.com/prysmaticlabs/prysm/validator/keymanager/v2/direct" "github.com/prysmaticlabs/prysm/validator/keymanager/v2/remote" "github.com/urfave/cli/v2" @@ -43,8 +44,7 @@ func CreateWallet(cliCtx *cli.Context) error { } switch keymanagerKind { case v2keymanager.Direct: - passwordsDirPath := inputPasswordsDirectory(cliCtx) - if err = initializeDirectWallet(walletDir, passwordsDirPath); err != nil { + if err = createDirectWallet(cliCtx, walletDir); err != nil { log.Fatalf("Could not initialize wallet with direct keymanager: %v", err) } log.WithField("wallet-path", walletDir).Infof( @@ -52,9 +52,15 @@ func CreateWallet(cliCtx *cli.Context) error { "Make a new validator account with ./prysm.sh validator accounts-2 new", ) case v2keymanager.Derived: - log.Fatal("Derived keymanager is not yet supported") + if err = createDerivedWallet(cliCtx, walletDir); err != nil { + log.Fatalf("Could not initialize wallet with derived keymanager: %v", err) + } + log.WithField("wallet-path", walletDir).Infof( + "Successfully created HD wallet and saved configuration to disk. " + + "Make a new validator account with ./prysm.sh validator accounts-2 new", + ) case v2keymanager.Remote: - if err = initializeRemoteSignerWallet(cliCtx, walletDir); err != nil { + if err = createRemoteWallet(cliCtx, walletDir); err != nil { log.Fatalf("Could not initialize wallet with remote keymanager: %v", err) } log.WithField("wallet-path", walletDir).Infof( @@ -66,10 +72,11 @@ func CreateWallet(cliCtx *cli.Context) error { return nil } -func initializeDirectWallet(walletDir string, passwordsDir string) error { +func createDirectWallet(cliCtx *cli.Context, walletDir string) error { + passwordsDirPath := inputPasswordsDirectory(cliCtx) walletConfig := &WalletConfig{ WalletDir: walletDir, - PasswordsDir: passwordsDir, + PasswordsDir: passwordsDirPath, KeymanagerKind: v2keymanager.Direct, CanUnlockAccounts: true, } @@ -88,7 +95,46 @@ func initializeDirectWallet(walletDir string, passwordsDir string) error { return nil } -func initializeRemoteSignerWallet(cliCtx *cli.Context, walletDir string) error { +func createDerivedWallet(cliCtx *cli.Context, walletDir string) error { + passwordsDirPath := inputPasswordsDirectory(cliCtx) + walletConfig := &WalletConfig{ + PasswordsDir: passwordsDirPath, + WalletDir: walletDir, + KeymanagerKind: v2keymanager.Derived, + CanUnlockAccounts: true, + } + ctx := context.Background() + walletPassword, err := inputNewWalletPassword() + if err != nil { + return errors.Wrap(err, "could not input new wallet password") + } + skipMnemonicConfirm := cliCtx.Bool(flags.SkipMnemonicConfirmFlag.Name) + seedConfig, err := derived.InitializeWalletSeedFile(ctx, walletPassword, skipMnemonicConfirm) + if err != nil { + return errors.Wrap(err, "could not initialize new wallet seed file") + } + seedConfigFile, err := derived.MarshalEncryptedSeedFile(ctx, seedConfig) + if err != nil { + return errors.Wrap(err, "could not marshal encrypted wallet seed file") + } + wallet, err := NewWallet(ctx, walletConfig) + if err != nil { + return errors.Wrap(err, "could not create new wallet") + } + keymanagerConfig, err := derived.MarshalConfigFile(ctx, derived.DefaultConfig()) + if err != nil { + return errors.Wrap(err, "could not marshal keymanager config file") + } + if err := wallet.WriteKeymanagerConfigToDisk(ctx, keymanagerConfig); err != nil { + return errors.Wrap(err, "could not write keymanager config to disk") + } + if err := wallet.WriteEncryptedSeedToDisk(ctx, seedConfigFile); err != nil { + return errors.Wrap(err, "could not write encrypted wallet seed config to disk") + } + return nil +} + +func createRemoteWallet(cliCtx *cli.Context, walletDir string) error { conf, err := inputRemoteKeymanagerConfig(cliCtx) if err != nil { return errors.Wrap(err, "could not input remote keymanager config") diff --git a/validator/accounts/v2/wallet_test.go b/validator/accounts/v2/wallet_test.go index 9e6d33b7b9..56aa2f228d 100644 --- a/validator/accounts/v2/wallet_test.go +++ b/validator/accounts/v2/wallet_test.go @@ -13,7 +13,6 @@ import ( "github.com/prysmaticlabs/prysm/shared/testutil" "github.com/prysmaticlabs/prysm/shared/testutil/require" v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" - "github.com/prysmaticlabs/prysm/validator/keymanager/v2/direct" mock "github.com/prysmaticlabs/prysm/validator/keymanager/v2/testing" "github.com/sirupsen/logrus" ) @@ -23,8 +22,6 @@ func init() { logrus.SetOutput(ioutil.Discard) } -var _ = direct.Wallet(&Wallet{}) - func setupWalletDir(t testing.TB) (string, string) { randPath, err := rand.Int(rand.Reader, big.NewInt(1000000)) require.NoError(t, err, "Could not generate random file path") @@ -64,7 +61,7 @@ func TestCreateAndReadWallet(t *testing.T) { require.NoError(t, wallet.WriteKeymanagerConfigToDisk(ctx, keymanagerConfig), "Could not write keymanager config file to disk") walletPath := path.Join(walletDir, keymanagerKind.String()) - configFilePath := path.Join(walletPath, keymanagerConfigFileName) + configFilePath := path.Join(walletPath, KeymanagerConfigFileName) require.Equal(t, true, fileExists(configFilePath), "Expected config file to have been created at path: %s", configFilePath) // We should be able to now read the wallet as well. diff --git a/validator/keymanager/v2/BUILD.bazel b/validator/keymanager/v2/BUILD.bazel index 2a44e39ade..b7493ebd4c 100644 --- a/validator/keymanager/v2/BUILD.bazel +++ b/validator/keymanager/v2/BUILD.bazel @@ -19,5 +19,9 @@ go_test( name = "go_default_test", srcs = ["types_test.go"], embed = [":go_default_library"], - deps = ["//validator/keymanager/v2/direct:go_default_library"], + deps = [ + "//validator/keymanager/v2/derived:go_default_library", + "//validator/keymanager/v2/direct:go_default_library", + "//validator/keymanager/v2/remote:go_default_library", + ], ) diff --git a/validator/keymanager/v2/derived/BUILD.bazel b/validator/keymanager/v2/derived/BUILD.bazel new file mode 100644 index 0000000000..17d3d1a4ed --- /dev/null +++ b/validator/keymanager/v2/derived/BUILD.bazel @@ -0,0 +1,50 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_test") +load("@prysm//tools/go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "derived.go", + "mnemonic.go", + ], + importpath = "github.com/prysmaticlabs/prysm/validator/keymanager/v2/derived", + visibility = [ + "//validator:__pkg__", + "//validator:__subpackages__", + ], + deps = [ + "//proto/validator/accounts/v2:go_default_library", + "//shared/bls:go_default_library", + "//shared/rand:go_default_library", + "//shared/roughtime:go_default_library", + "//validator/accounts/v2/iface:go_default_library", + "//validator/keymanager/v2:go_default_library", + "@com_github_google_uuid//:go_default_library", + "@com_github_manifoldco_promptui//:go_default_library", + "@com_github_pkg_errors//:go_default_library", + "@com_github_sirupsen_logrus//:go_default_library", + "@com_github_tyler_smith_go_bip39//:go_default_library", + "@com_github_wealdtech_go_eth2_util//:go_default_library", + "@com_github_wealdtech_go_eth2_wallet_encryptor_keystorev4//:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "derived_test.go", + "mnemonic_test.go", + ], + embed = [":go_default_library"], + deps = [ + "//shared/bls:go_default_library", + "//shared/testutil:go_default_library", + "//shared/testutil/assert:go_default_library", + "//shared/testutil/require:go_default_library", + "//validator/accounts/v2/testing:go_default_library", + "//validator/keymanager/v2:go_default_library", + "@com_github_sirupsen_logrus//hooks/test:go_default_library", + "@com_github_tyler_smith_go_bip39//:go_default_library", + "@com_github_wealdtech_go_eth2_wallet_encryptor_keystorev4//:go_default_library", + ], +) diff --git a/validator/keymanager/v2/derived/derived.go b/validator/keymanager/v2/derived/derived.go new file mode 100644 index 0000000000..6c932ff6eb --- /dev/null +++ b/validator/keymanager/v2/derived/derived.go @@ -0,0 +1,290 @@ +package derived + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "path" + "strconv" + "sync" + + "github.com/google/uuid" + "github.com/pkg/errors" + validatorpb "github.com/prysmaticlabs/prysm/proto/validator/accounts/v2" + "github.com/prysmaticlabs/prysm/shared/bls" + "github.com/prysmaticlabs/prysm/shared/rand" + "github.com/prysmaticlabs/prysm/shared/roughtime" + "github.com/prysmaticlabs/prysm/validator/accounts/v2/iface" + v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" + "github.com/sirupsen/logrus" + util "github.com/wealdtech/go-eth2-util" + keystorev4 "github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4" +) + +var log = logrus.WithField("prefix", "derived-keymanager-v2") + +const ( + // TimestampFileName stores a timestamp for account creation as a + // file for a direct keymanager account. + TimestampFileName = "created_at.txt" + // KeystoreFileName exposes the expected filename for the keystore file for an account. + KeystoreFileName = "keystore.json" + // EIPVersion used by this derived keymanager implementation. + EIPVersion = "EIP-2334" + // WithdrawalKeyDerivationPathTemplate defining the hierarchical path for withdrawal + // keys for Prysm eth2 validators. According to EIP-2334, the format is as follows: + // m / purpose / coin_type / account_index / withdrawal_key + WithdrawalKeyDerivationPathTemplate = "m/12381/3600/%d/0" + // ValidatingKeyDerivationPathTemplate defining the hierarchical path for validating + // keys for Prysm eth2 validators. According to EIP-2334, the format is as follows: + // m / purpose / coin_type / account_index / withdrawal_key / validating_key + ValidatingKeyDerivationPathTemplate = "m/12381/3600/%d/0/0" +) + +// Config for a derived keymanager. +type Config struct { + DerivedPathStructure string + DerivedEIPNumber string +} + +// Keymanager implementation for derived, HD keymanager using EIP-2333 and EIP-2334. +type Keymanager struct { + wallet iface.Wallet + cfg *Config + mnemonicGenerator SeedPhraseFactory + keysCache map[[48]byte]bls.SecretKey + lock sync.RWMutex + seedCfg *SeedConfig + seed []byte +} + +// SeedConfig json file representation as a Go struct. +type SeedConfig struct { + Crypto map[string]interface{} `json:"crypto"` + ID string `json:"uuid"` + NextAccount uint64 `json:"next_account"` + Version uint `json:"version"` + Name string `json:"name"` +} + +// DefaultConfig for a derived keymanager implementation. +func DefaultConfig() *Config { + return &Config{ + DerivedPathStructure: "m / purpose / coin_type / account / withdrawal_key / validating_key", + DerivedEIPNumber: EIPVersion, + } +} + +// NewKeymanager instantiates a new derived keymanager from configuration options. +func NewKeymanager( + ctx context.Context, + wallet iface.Wallet, + cfg *Config, + skipMnemonicConfirm bool, + password string, +) (*Keymanager, error) { + seedConfigFile, err := wallet.ReadEncryptedSeedFromDisk(ctx) + if err != nil { + return nil, errors.Wrap(err, "could not read encrypted seed configuration file from disk") + } + enc, err := ioutil.ReadAll(seedConfigFile) + if err != nil { + return nil, errors.Wrap(err, "could not read seed configuration file contents") + } + defer func() { + if err := seedConfigFile.Close(); err != nil { + log.Errorf("Could not close keymanager config file: %v", err) + } + }() + seedConfig := &SeedConfig{} + if err := json.Unmarshal(enc, seedConfig); err != nil { + return nil, errors.Wrap(err, "could not unmarshal seed configuration") + } + decryptor := keystorev4.New() + seed, err := decryptor.Decrypt(seedConfig.Crypto, []byte(password)) + if err != nil { + return nil, errors.Wrap(err, "could not decrypt seed configuration with password") + } + k := &Keymanager{ + wallet: wallet, + cfg: cfg, + mnemonicGenerator: &EnglishMnemonicGenerator{ + skipMnemonicConfirm: skipMnemonicConfirm, + }, + seedCfg: seedConfig, + seed: seed, + } + return k, nil +} + +// UnmarshalConfigFile attempts to JSON unmarshal a derived keymanager +// configuration file into the *Config{} struct. +func UnmarshalConfigFile(r io.ReadCloser) (*Config, error) { + enc, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + defer func() { + if err := r.Close(); err != nil { + log.Errorf("Could not close keymanager config file: %v", err) + } + }() + cfg := &Config{} + if err := json.Unmarshal(enc, cfg); err != nil { + return nil, err + } + return cfg, nil +} + +// MarshalConfigFile returns a marshaled configuration file for a keymanager. +func MarshalConfigFile(ctx context.Context, cfg *Config) ([]byte, error) { + return json.MarshalIndent(cfg, "", "\t") +} + +// InitializeWalletSeedFile creates a new, encrypted seed using a password input +// and persists its encrypted file metadata to disk under the wallet path. +func InitializeWalletSeedFile(ctx context.Context, password string, skipMnemonicConfirm bool) (*SeedConfig, error) { + walletSeed := make([]byte, 32) + n, err := rand.NewGenerator().Read(walletSeed) + if err != nil { + return nil, errors.Wrap(err, "could not initialize wallet seed") + } + if n != len(walletSeed) { + return nil, errors.New("could not randomly create seed") + } + m := &EnglishMnemonicGenerator{ + skipMnemonicConfirm: skipMnemonicConfirm, + } + phrase, err := m.Generate(walletSeed) + if err != nil { + return nil, errors.Wrap(err, "could not generate wallet seed") + } + if err := m.ConfirmAcknowledgement(phrase); err != nil { + return nil, errors.Wrap(err, "could not confirm mnemonic acknowledgement") + } + encryptor := keystorev4.New() + cryptoFields, err := encryptor.Encrypt(walletSeed, []byte(password)) + if err != nil { + return nil, errors.Wrap(err, "could not encrypt seed phrase into keystore") + } + id, err := uuid.NewRandom() + if err != nil { + return nil, errors.Wrap(err, "could not generate unique UUID") + } + return &SeedConfig{ + Crypto: cryptoFields, + ID: id.String(), + NextAccount: 0, + Version: encryptor.Version(), + Name: encryptor.Name(), + }, nil +} + +// MarshalEncryptedSeedFile json encodes the seed configuration for a derived keymanager. +func MarshalEncryptedSeedFile(ctx context.Context, seedCfg *SeedConfig) ([]byte, error) { + return json.MarshalIndent(seedCfg, "", "\t") +} + +// CreateAccount for a derived keymanager implementation. This utilizes +// the EIP-2335 keystore standard for BLS12-381 keystores. It uses the EIP-2333 and EIP-2334 +// for hierarchical derivation of BLS secret keys and a common derivation path structure for +// persisting accounts to disk. Each account stores the generated keystore.json file. +// The entire derived wallet seed phrase can be recovered from a BIP-39 english mnemonic. +func (dr *Keymanager) CreateAccount(ctx context.Context, password string) (string, error) { + withdrawalKeyPath := fmt.Sprintf(WithdrawalKeyDerivationPathTemplate, dr.seedCfg.NextAccount) + validatingKeyPath := fmt.Sprintf(ValidatingKeyDerivationPathTemplate, dr.seedCfg.NextAccount) + withdrawalKey, err := util.PrivateKeyFromSeedAndPath(dr.seed, withdrawalKeyPath) + if err != nil { + return "", errors.Wrapf(err, "failed to create withdrawal key for account %d", dr.seedCfg.NextAccount) + } + validatingKey, err := util.PrivateKeyFromSeedAndPath(dr.seed, validatingKeyPath) + if err != nil { + return "", errors.Wrapf(err, "failed to create validating key for account %d", dr.seedCfg.NextAccount) + } + + // Create encrypted keystores for both the withdrawal and validating keys. + encodedWithdrawalKeystore, err := dr.generateKeystoreFile( + withdrawalKey.Marshal(), + withdrawalKey.PublicKey().Marshal(), + password, + ) + if err != nil { + return "", errors.Wrap(err, "could not generate keystore file for withdrawal account") + } + encodedValidatingKeystore, err := dr.generateKeystoreFile( + validatingKey.Marshal(), + validatingKey.PublicKey().Marshal(), + password, + ) + if err != nil { + return "", errors.Wrap(err, "could not generate keystore file for validating account") + } + + // Write both keystores to disk at their respective derived paths. + if err := dr.wallet.WriteFileAtPath(ctx, withdrawalKeyPath, KeystoreFileName, encodedWithdrawalKeystore); err != nil { + return "", errors.Wrapf(err, "could not write keystore file for account %d", dr.seedCfg.NextAccount) + } + if err := dr.wallet.WriteFileAtPath(ctx, validatingKeyPath, KeystoreFileName, encodedValidatingKeystore); err != nil { + return "", errors.Wrapf(err, "could not write keystore file for account %d", dr.seedCfg.NextAccount) + } + + // Finally, write the account creation timestamps as a files. + createdAt := roughtime.Now().Unix() + createdAtStr := strconv.FormatInt(createdAt, 10) + if err := dr.wallet.WriteFileAtPath(ctx, withdrawalKeyPath, TimestampFileName, []byte(createdAtStr)); err != nil { + return "", errors.Wrapf(err, "could not write timestamp file for account %d", dr.seedCfg.NextAccount) + } + if err := dr.wallet.WriteFileAtPath(ctx, validatingKeyPath, TimestampFileName, []byte(createdAtStr)); err != nil { + return "", errors.Wrapf(err, "could not write timestamp file for account %d", dr.seedCfg.NextAccount) + } + + newAccountNumber := dr.seedCfg.NextAccount + log.WithFields(logrus.Fields{ + "accountNumber": newAccountNumber, + "withdrawalPublicKey": fmt.Sprintf("%#x", withdrawalKey.PublicKey().Marshal()), + "validatingPublicKey": fmt.Sprintf("%#x", validatingKey.PublicKey().Marshal()), + "withdrawalKeyPath": path.Join(dr.wallet.AccountsDir(), withdrawalKeyPath), + "validatingKeyPath": path.Join(dr.wallet.AccountsDir(), validatingKeyPath), + }).Info("Successfully created new validator account") + dr.seedCfg.NextAccount++ + encodedCfg, err := MarshalEncryptedSeedFile(ctx, dr.seedCfg) + if err != nil { + return "", errors.Wrap(err, "could not marshal encrypted seed file") + } + if err := dr.wallet.WriteEncryptedSeedToDisk(ctx, encodedCfg); err != nil { + return "", errors.Wrap(err, "could not write encrypted seed file to disk") + } + return fmt.Sprintf("%d", newAccountNumber), nil +} + +// FetchValidatingPublicKeys fetches the list of public keys from the direct account keystores. +func (dr *Keymanager) FetchValidatingPublicKeys(ctx context.Context) ([][48]byte, error) { + return nil, errors.New("unimplemented") +} + +// Sign signs a message using a validator key. +func (dr *Keymanager) Sign(ctx context.Context, req *validatorpb.SignRequest) (bls.Signature, error) { + return nil, errors.New("unimplemented") +} + +func (dr *Keymanager) generateKeystoreFile(privateKey []byte, publicKey []byte, password string) ([]byte, error) { + encryptor := keystorev4.New() + cryptoFields, err := encryptor.Encrypt(privateKey, []byte(password)) + if err != nil { + return nil, errors.Wrap(err, "could not encrypt validating key into keystore") + } + id, err := uuid.NewRandom() + if err != nil { + return nil, errors.Wrap(err, "could not generate new, random UUID for keystore") + } + keystoreFile := &v2keymanager.Keystore{ + Crypto: cryptoFields, + ID: id.String(), + Pubkey: fmt.Sprintf("%x", publicKey), + Version: encryptor.Version(), + Name: encryptor.Name(), + } + return json.MarshalIndent(keystoreFile, "", "\t") +} diff --git a/validator/keymanager/v2/derived/derived_test.go b/validator/keymanager/v2/derived/derived_test.go new file mode 100644 index 0000000000..0352015622 --- /dev/null +++ b/validator/keymanager/v2/derived/derived_test.go @@ -0,0 +1,94 @@ +package derived + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "testing" + + "github.com/prysmaticlabs/prysm/shared/bls" + "github.com/prysmaticlabs/prysm/shared/testutil" + "github.com/prysmaticlabs/prysm/shared/testutil/assert" + "github.com/prysmaticlabs/prysm/shared/testutil/require" + mock "github.com/prysmaticlabs/prysm/validator/accounts/v2/testing" + v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" + logTest "github.com/sirupsen/logrus/hooks/test" + keystorev4 "github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4" +) + +func TestDerivedKeymanager_CreateAccount(t *testing.T) { + hook := logTest.NewGlobal() + wallet := &mock.Wallet{ + Files: make(map[string]map[string][]byte), + AccountPasswords: make(map[string]string), + } + seed := make([]byte, 32) + copy(seed, "hello world") + dr := &Keymanager{ + wallet: wallet, + seed: seed, + seedCfg: &SeedConfig{ + NextAccount: 0, + }, + } + ctx := context.Background() + password := "secretPassw0rd$1999" + accountName, err := dr.CreateAccount(ctx, password) + require.NoError(t, err) + assert.Equal(t, "0", accountName) + + // Ensure the keystore file was written to the wallet + // and ensure we can decrypt it using the EIP-2335 standard. + validatingAccount0 := fmt.Sprintf(ValidatingKeyDerivationPathTemplate, 0) + encodedKeystore, ok := wallet.Files[validatingAccount0][KeystoreFileName] + require.Equal(t, ok, true, fmt.Sprintf("Expected to have stored %s in wallet", KeystoreFileName)) + keystoreFile := &v2keymanager.Keystore{} + require.NoError(t, json.Unmarshal(encodedKeystore, keystoreFile)) + + // We extract the validator signing private key from the keystore + // by utilizing the password and initialize a new BLS secret key from + // its raw bytes. + decryptor := keystorev4.New() + rawValidatingKey, err := decryptor.Decrypt(keystoreFile.Crypto, []byte(password)) + require.NoError(t, err, "Could not decrypt validator signing key") + + validatingKey, err := bls.SecretKeyFromBytes(rawValidatingKey) + require.NoError(t, err, "Could not instantiate bls secret key from bytes") + + // Ensure the keystore file was written to the wallet + // and ensure we can decrypt it using the EIP-2335 standard. + withdrawalAccount0 := fmt.Sprintf(WithdrawalKeyDerivationPathTemplate, 0) + encodedKeystore, ok = wallet.Files[withdrawalAccount0][KeystoreFileName] + require.Equal(t, ok, true, fmt.Sprintf("Expected to have stored %s in wallet", KeystoreFileName)) + keystoreFile = &v2keymanager.Keystore{} + require.NoError(t, json.Unmarshal(encodedKeystore, keystoreFile)) + + // We extract the validator signing private key from the keystore + // by utilizing the password and initialize a new BLS secret key from + // its raw bytes. + rawWithdrawalKey, err := decryptor.Decrypt(keystoreFile.Crypto, []byte(password)) + require.NoError(t, err, "Could not decrypt validator withdrawal key") + + withdrawalKey, err := bls.SecretKeyFromBytes(rawWithdrawalKey) + require.NoError(t, err, "Could not instantiate bls secret key from bytes") + + // Assert the new value for next account increased and also + // check the config file was updated on disk with this new value. + assert.Equal(t, uint64(1), dr.seedCfg.NextAccount, "Wrong value for next account") + encryptedSeedFile, err := wallet.ReadEncryptedSeedFromDisk(ctx) + require.NoError(t, err) + enc, err := ioutil.ReadAll(encryptedSeedFile) + require.NoError(t, err) + defer func() { + assert.NoError(t, encryptedSeedFile.Close()) + }() + seedConfig := &SeedConfig{} + require.NoError(t, json.Unmarshal(enc, seedConfig)) + assert.Equal(t, uint64(1), seedConfig.NextAccount, "Wrong value for next account") + + // Ensure the new account information is displayed to stdout. + testutil.AssertLogsContain(t, hook, "Successfully created new validator account") + testutil.AssertLogsContain(t, hook, fmt.Sprintf("%#x", validatingKey.PublicKey().Marshal())) + testutil.AssertLogsContain(t, hook, fmt.Sprintf("%#x", withdrawalKey.PublicKey().Marshal())) +} diff --git a/validator/keymanager/v2/derived/mnemonic.go b/validator/keymanager/v2/derived/mnemonic.go new file mode 100644 index 0000000000..3d9da01a47 --- /dev/null +++ b/validator/keymanager/v2/derived/mnemonic.go @@ -0,0 +1,65 @@ +package derived + +import ( + "fmt" + + "github.com/manifoldco/promptui" + "github.com/tyler-smith/go-bip39" +) + +// SeedPhraseFactory defines a struct which +// can generate new seed phrases in human-readable +// format from a source of entropy in raw bytes. It +// also provides methods for verifying a user has successfully +// acknowledged the mnemonic phrase and written it down offline. +type SeedPhraseFactory interface { + Generate(data []byte) (string, error) + ConfirmAcknowledgement(phrase string) error +} + +// EnglishMnemonicGenerator implements methods for creating +// mnemonic seed phrases in english using a given +// source of entropy such as a private key. +type EnglishMnemonicGenerator struct { + skipMnemonicConfirm bool +} + +// Generate a mnemonic seed phrase in english using a source of +// entropy given as raw bytes. +func (m *EnglishMnemonicGenerator) Generate(data []byte) (string, error) { + return bip39.NewMnemonic(data) +} + +// ConfirmAcknowledgement displays the mnemonic phrase to the user +// and confirms the user has written down the phrase securely offline. +func (m *EnglishMnemonicGenerator) ConfirmAcknowledgement(phrase string) error { + log.Info( + "Write down the sentence below, as it is your only " + + "means of recovering your wallet", + ) + fmt.Printf(` +=================Wallet Seed Recovery Phrase==================== + +%s + +=================================================================== + `, phrase) + if m.skipMnemonicConfirm { + return nil + } + // Confirm the user has written down the mnemonic phrase offline. + prompt := promptui.Prompt{ + Label: "Confirm you have written down the recovery words somewhere safe (offline)", + IsConfirm: true, + } + expected := "y" + var result string + var err error + for result != expected { + result, err = prompt.Run() + if err != nil { + log.Errorf("Could not confirm acknowledgement of prompt, please enter y") + } + } + return nil +} diff --git a/validator/keymanager/v2/derived/mnemonic_test.go b/validator/keymanager/v2/derived/mnemonic_test.go new file mode 100644 index 0000000000..40b35eb011 --- /dev/null +++ b/validator/keymanager/v2/derived/mnemonic_test.go @@ -0,0 +1,25 @@ +package derived + +import ( + "bytes" + "testing" + + "github.com/tyler-smith/go-bip39" +) + +func TestMnemonic_Generate_CanRecover(t *testing.T) { + generator := &EnglishMnemonicGenerator{} + data := make([]byte, 32) + copy(data, []byte("hello-world")) + phrase, err := generator.Generate(data) + if err != nil { + t.Fatal(err) + } + entropy, err := bip39.EntropyFromMnemonic(phrase) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(entropy, data) { + t.Errorf("Expected to recover original data: %v, received %v", data, entropy) + } +} diff --git a/validator/keymanager/v2/direct/BUILD.bazel b/validator/keymanager/v2/direct/BUILD.bazel index f4f70d89cc..cae137872e 100644 --- a/validator/keymanager/v2/direct/BUILD.bazel +++ b/validator/keymanager/v2/direct/BUILD.bazel @@ -21,6 +21,8 @@ go_library( "//shared/depositutil:go_default_library", "//shared/params:go_default_library", "//shared/roughtime:go_default_library", + "//validator/accounts/v2/iface:go_default_library", + "//validator/keymanager/v2:go_default_library", "@com_github_ethereum_go_ethereum//core/types:go_default_library", "@com_github_google_uuid//:go_default_library", "@com_github_manifoldco_promptui//:go_default_library", @@ -47,6 +49,7 @@ go_test( "//shared/depositutil:go_default_library", "//shared/testutil:go_default_library", "//validator/accounts/v2/testing:go_default_library", + "//validator/keymanager/v2:go_default_library", "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library", "@com_github_prysmaticlabs_go_ssz//:go_default_library", "@com_github_sirupsen_logrus//hooks/test:go_default_library", diff --git a/validator/keymanager/v2/direct/direct.go b/validator/keymanager/v2/direct/direct.go index 2819c77a2a..198eaabe86 100644 --- a/validator/keymanager/v2/direct/direct.go +++ b/validator/keymanager/v2/direct/direct.go @@ -22,6 +22,8 @@ import ( "github.com/prysmaticlabs/prysm/shared/depositutil" "github.com/prysmaticlabs/prysm/shared/params" "github.com/prysmaticlabs/prysm/shared/roughtime" + "github.com/prysmaticlabs/prysm/validator/accounts/v2/iface" + v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" "github.com/sirupsen/logrus" keystorev4 "github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4" ) @@ -41,19 +43,6 @@ const ( eipVersion = "EIP-2335" ) -// Wallet defines a struct which has capabilities and knowledge of how -// to read and write important accounts-related files to the filesystem. -// Useful for keymanager to have persistent capabilities for accounts on-disk. -type Wallet interface { - AccountsDir() string - CanUnlockAccounts() bool - AccountNames() ([]string, error) - ReadPasswordForAccount(accountName string) (string, error) - ReadFileForAccount(accountName string, fileName string) ([]byte, error) - WriteAccountToDisk(ctx context.Context, password string) (string, error) - WriteFileForAccount(ctx context.Context, accountName string, fileName string, data []byte) error -} - // Config for a direct keymanager. type Config struct { EIPVersion string `json:"direct_eip_version"` @@ -61,22 +50,13 @@ type Config struct { // Keymanager implementation for direct keystores utilizing EIP-2335. type Keymanager struct { - wallet Wallet + wallet iface.Wallet cfg *Config mnemonicGenerator SeedPhraseFactory keysCache map[[48]byte]bls.SecretKey lock sync.RWMutex } -// Keystore json file representation as a Go struct. -type Keystore struct { - Crypto map[string]interface{} `json:"crypto"` - ID string `json:"uuid"` - Pubkey string `json:"pubkey"` - Version uint `json:"version"` - Name string `json:"name"` -} - // DefaultConfig for a direct keymanager implementation. func DefaultConfig() *Config { return &Config{ @@ -85,7 +65,7 @@ func DefaultConfig() *Config { } // NewKeymanager instantiates a new direct keymanager from configuration options. -func NewKeymanager(ctx context.Context, wallet Wallet, cfg *Config, skipMnemonicConfirm bool) (*Keymanager, error) { +func NewKeymanager(ctx context.Context, wallet iface.Wallet, cfg *Config, skipMnemonicConfirm bool) (*Keymanager, error) { k := &Keymanager{ wallet: wallet, cfg: cfg, @@ -230,7 +210,7 @@ func (dr *Keymanager) FetchValidatingPublicKeys(ctx context.Context) ([][48]byte if err != nil { return nil, errors.Wrapf(err, "could not read keystore file for account %s", name) } - keystoreFile := &Keystore{} + keystoreFile := &v2keymanager.Keystore{} if err := json.Unmarshal(encoded, keystoreFile); err != nil { return nil, errors.Wrapf(err, "could not decode keystore json for account: %s", name) } @@ -273,7 +253,7 @@ func (dr *Keymanager) initializeSecretKeysCache() error { if err != nil { return errors.Wrapf(err, "could not read keystore file for account %s", name) } - keystoreFile := &Keystore{} + keystoreFile := &v2keymanager.Keystore{} if err := json.Unmarshal(encoded, keystoreFile); err != nil { return errors.Wrapf(err, "could not decode keystore json for account: %s", name) } @@ -307,12 +287,13 @@ func (dr *Keymanager) generateKeystoreFile(validatingKey bls.SecretKey, password if err != nil { return nil, err } - keystoreFile := &Keystore{} - keystoreFile.Crypto = cryptoFields - keystoreFile.ID = id.String() - keystoreFile.Pubkey = fmt.Sprintf("%x", validatingKey.PublicKey().Marshal()) - keystoreFile.Version = encryptor.Version() - keystoreFile.Name = encryptor.Name() + keystoreFile := &v2keymanager.Keystore{ + Crypto: cryptoFields, + ID: id.String(), + Pubkey: fmt.Sprintf("%x", validatingKey.PublicKey().Marshal()), + Version: encryptor.Version(), + Name: encryptor.Name(), + } return json.MarshalIndent(keystoreFile, "", "\t") } diff --git a/validator/keymanager/v2/direct/direct_test.go b/validator/keymanager/v2/direct/direct_test.go index 3f81317f68..65024a1126 100644 --- a/validator/keymanager/v2/direct/direct_test.go +++ b/validator/keymanager/v2/direct/direct_test.go @@ -16,6 +16,7 @@ import ( "github.com/prysmaticlabs/prysm/shared/depositutil" "github.com/prysmaticlabs/prysm/shared/testutil" mock "github.com/prysmaticlabs/prysm/validator/accounts/v2/testing" + v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" logTest "github.com/sirupsen/logrus/hooks/test" "github.com/tyler-smith/go-bip39" keystorev4 "github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4" @@ -64,7 +65,7 @@ func TestKeymanager_CreateAccount(t *testing.T) { if !ok { t.Fatalf("Expected to have stored %s in wallet", KeystoreFileName) } - keystoreFile := &Keystore{} + keystoreFile := &v2keymanager.Keystore{} if err := json.Unmarshal(encodedKeystore, keystoreFile); err != nil { t.Fatalf("Could not decode keystore json: %v", err) } diff --git a/validator/keymanager/v2/types.go b/validator/keymanager/v2/types.go index e78b1fdb92..975cb976d1 100644 --- a/validator/keymanager/v2/types.go +++ b/validator/keymanager/v2/types.go @@ -18,6 +18,15 @@ type IKeymanager interface { Sign(context.Context, *validatorpb.SignRequest) (bls.Signature, error) } +// Keystore json file representation as a Go struct. +type Keystore struct { + Crypto map[string]interface{} `json:"crypto"` + ID string `json:"uuid"` + Pubkey string `json:"pubkey"` + Version uint `json:"version"` + Name string `json:"name"` +} + // Kind defines an enum for either direct, derived, or remote-signing // keystores for Prysm wallets. type Kind int diff --git a/validator/keymanager/v2/types_test.go b/validator/keymanager/v2/types_test.go index e2d6420886..849931a312 100644 --- a/validator/keymanager/v2/types_test.go +++ b/validator/keymanager/v2/types_test.go @@ -1,5 +1,14 @@ -package v2 +package v2_test -import "github.com/prysmaticlabs/prysm/validator/keymanager/v2/direct" +import ( + v2keymanager "github.com/prysmaticlabs/prysm/validator/keymanager/v2" + "github.com/prysmaticlabs/prysm/validator/keymanager/v2/derived" + "github.com/prysmaticlabs/prysm/validator/keymanager/v2/direct" + "github.com/prysmaticlabs/prysm/validator/keymanager/v2/remote" +) -var _ = IKeymanager(&direct.Keymanager{}) +var ( + _ = v2keymanager.IKeymanager(&direct.Keymanager{}) + _ = v2keymanager.IKeymanager(&derived.Keymanager{}) + _ = v2keymanager.IKeymanager(&remote.Keymanager{}) +)