diff --git a/INTEROP.md b/INTEROP.md new file mode 100644 index 0000000000..eb2479acec --- /dev/null +++ b/INTEROP.md @@ -0,0 +1,48 @@ +# Prysm Client Interoperability Guide + +This README details how to setup Prysm for interop testing for usage with other Ethereum 2.0 clients. + +**WARNING**: The tool can only generate up to 190 private keys at the moment, as there is a BLS bug that +prevents deterministically generating more keys than those. + +## Installation & Setup + +1. Install [Bazel](https://docs.bazel.build/versions/master/install.html) **(Recommended)** +2. `git clone https://github.com/prysmaticlabs/prysm && cd prysm` +3. `bazel build //...` + +## Starting from Genesis + +Prysm supports a few ways to quickly launch a beacon node from basic configurations: + +- `SSZ Genesis`: Launches a beacon node from a .ssz file containing a SSZ-encoded, genesis beacon state **(Recommended)** +- `Yaml Genesis`: Launches a beacon node from a .yaml file specifying a genesis beacon state - binary data is _base64 encoded_ +- `JSON Genesis`: Launches a beacon node from a .json file specifying a genesis beacon state - binary data is _base64 encoded_ + +## Generating a Genesis State + +To setup the necessary files for these quick starts, Prysm provides a tool to generate `genesis.yaml`, `genesis.ssz`, `genesis.json` from an +a deterministically generated set of validator private keys following the official interop YAML format +[here](https://github.com/ethereum/eth2.0-pm/blob/master/interop/mocked_start). If you already have a genesis state in this format, you can skip this section. + +You can use `bazel run //tools/genesis-state-gen` to create a deterministic genesis state for interop. + +### Usage + +- **--genesis-time** uint: Unix timestamp used as the genesis time in the generated genesis state +- **--mainnet-config** bool: Select whether genesis state should be generated with mainnet or minimal (default) params +- **--num-validators** int: Number of validators to deterministically include in the generated genesis state +- **--output-json** string: Output filename of the JSON marshaling of the generated genesis state +- **--output-ssz** string: Output filename of the SSZ marshaling of the generated genesis state +- **--output-yaml** string: Output filename of the YAML marshaling of the generated genesis state + +The example below creates 10 validator keys, instantiates a genesis state with those 10 validators and with genesis time 1567542540, +and finally writes a YAML encoded output to ~/Desktop/genesis.yaml. This file can be used to kickstart the beacon chain in the next section. + +``` +bazel run //tools/genesis-state-gen -- --output-yaml ~/Desktop/genesis.yaml --num-validators 10 --genesis-time 1567542540 +``` + +## Launching a Beacon Node + Validator Client + +TODO: Add section after incorporating the mock start functionality into the beacon chain and validator bazel binaries. \ No newline at end of file diff --git a/beacon-chain/core/state/BUILD.bazel b/beacon-chain/core/state/BUILD.bazel index 04e0671459..ab31507ebd 100644 --- a/beacon-chain/core/state/BUILD.bazel +++ b/beacon-chain/core/state/BUILD.bazel @@ -7,7 +7,10 @@ go_library( "transition.go", ], importpath = "github.com/prysmaticlabs/prysm/beacon-chain/core/state", - visibility = ["//beacon-chain:__subpackages__"], + visibility = [ + "//beacon-chain:__subpackages__", + "//tools/genesis-state-gen:__pkg__", + ], deps = [ "//beacon-chain/core/blocks:go_default_library", "//beacon-chain/core/epoch:go_default_library", diff --git a/shared/bls/bls.go b/shared/bls/bls.go index e51d5306b9..fc6961cdaa 100644 --- a/shared/bls/bls.go +++ b/shared/bls/bls.go @@ -5,7 +5,6 @@ package bls import ( "encoding/binary" - "fmt" "io" g1 "github.com/phoreproject/bls/g1pubs" @@ -39,9 +38,6 @@ func RandKey(r io.Reader) (*SecretKey, error) { // SecretKeyFromBytes creates a BLS private key from a byte slice. func SecretKeyFromBytes(priv []byte) (*SecretKey, error) { - if len(priv) != 32 { - return nil, fmt.Errorf("expected byte slice of length 32, received: %d", len(priv)) - } k := bytesutil.ToBytes32(priv) val := g1.DeserializeSecretKey(k) if val.GetFRElement() == nil { diff --git a/tools/genesis-state-gen/BUILD.bazel b/tools/genesis-state-gen/BUILD.bazel new file mode 100644 index 0000000000..0897035a1c --- /dev/null +++ b/tools/genesis-state-gen/BUILD.bazel @@ -0,0 +1,37 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "github.com/prysmaticlabs/prysm/tools/genesis-state-gen", + visibility = ["//visibility:private"], + deps = [ + "//beacon-chain/core/state:go_default_library", + "//proto/eth/v1alpha1:go_default_library", + "//shared/bls:go_default_library", + "//shared/hashutil:go_default_library", + "//shared/params:go_default_library", + "//shared/trieutil:go_default_library", + "@com_github_ghodss_yaml//:go_default_library", + "@com_github_pkg_errors//:go_default_library", + "@com_github_prysmaticlabs_go_ssz//:go_default_library", + ], +) + +go_binary( + name = "genesis-state-gen", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["main_test.go"], + embed = [":go_default_library"], + deps = [ + "//beacon-chain/core/state:go_default_library", + "//proto/eth/v1alpha1:go_default_library", + "//shared/params:go_default_library", + "//shared/trieutil:go_default_library", + ], +) diff --git a/tools/genesis-state-gen/main.go b/tools/genesis-state-gen/main.go new file mode 100644 index 0000000000..81c38deef1 --- /dev/null +++ b/tools/genesis-state-gen/main.go @@ -0,0 +1,217 @@ +package main + +import ( + "encoding/binary" + "encoding/json" + "flag" + "io/ioutil" + "log" + "math/big" + + "github.com/ghodss/yaml" + "github.com/pkg/errors" + "github.com/prysmaticlabs/go-ssz" + "github.com/prysmaticlabs/prysm/beacon-chain/core/state" + ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1" + "github.com/prysmaticlabs/prysm/shared/bls" + "github.com/prysmaticlabs/prysm/shared/hashutil" + "github.com/prysmaticlabs/prysm/shared/params" + "github.com/prysmaticlabs/prysm/shared/trieutil" +) + +const ( + blsWithdrawalPrefixByte = byte(0) + blsCurveOrder = "52435875175126190479447740508185965837690552500527637822603658699938581184513" +) + +var ( + domainDeposit = [4]byte{3, 0, 0, 0} + genesisForkVersion = []byte{0, 0, 0, 0} + numValidators = flag.Int("num-validators", 0, "Number of validators to deterministically include in the generated genesis state") + useMainnetConfig = flag.Bool("mainnet-config", false, "Select whether genesis state should be generated with mainnet or minimal (default) params") + genesisTime = flag.Uint64("genesis-time", 0, "Unix timestamp used as the genesis time in the generated genesis state") + sszOutputFile = flag.String("output-ssz", "", "Output filename of the SSZ marshaling of the generated genesis state") + yamlOutputFile = flag.String("output-yaml", "", "Output filename of the YAML marshaling of the generated genesis state") + jsonOutputFile = flag.String("output-json", "", "Output filename of the JSON marshaling of the generated genesis state") + // This is the recommended mock eth1 block hash according to the Eth2 interop guidelines. + // https://github.com/ethereum/eth2.0-pm/blob/a085c9870f3956d6228ed2a40cd37f0c6580ecd7/interop/mocked_start/README.md + mockEth1BlockHash = []byte{66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66} +) + +func main() { + flag.Parse() + // TODO(#3398): Cannot generate more than 190 keys due to BLS errors. + if *numValidators == 0 { + log.Fatal("Expected --num-validators to have been provided, received 0") + } + if *genesisTime == 0 { + log.Print("No --genesis-time specified, defaulting to 0 as the unix timestamp") + } + if *sszOutputFile == "" && *yamlOutputFile == "" && *jsonOutputFile == "" { + log.Fatal("Expected --output-ssz, --output-yaml, or --output-json to have been provided, received nil") + } + if !*useMainnetConfig { + params.OverrideBeaconConfig(params.MinimalSpecConfig()) + } + privKeys, pubKeys, err := deterministicallyGenerateKeys(*numValidators) + if err != nil { + log.Fatalf("Could not deterministically generate keys for %d validators: %v", *numValidators, err) + } + depositDataItems, depositDataRoots, err := depositDataFromKeys(privKeys, pubKeys) + if err != nil { + log.Fatalf("Could not generate deposit data from keys: %v", err) + } + trie, err := trieutil.GenerateTrieFromItems( + depositDataRoots, + int(params.BeaconConfig().DepositContractTreeDepth), + ) + if err != nil { + log.Fatalf("Could not generate Merkle trie for deposit proofs: %v", err) + } + deposits, err := generateDepositsFromData(depositDataItems, trie) + if err != nil { + log.Fatalf("Could not generate deposits from the deposit data provided: %v", err) + } + root := trie.Root() + genesisState, err := state.GenesisBeaconState(deposits, *genesisTime, ðpb.Eth1Data{ + DepositRoot: root[:], + DepositCount: uint64(len(deposits)), + BlockHash: mockEth1BlockHash, + }) + if err != nil { + log.Fatalf("Could not generate genesis beacon state: %v", err) + } + if *sszOutputFile != "" { + encodedState, err := ssz.Marshal(genesisState) + if err != nil { + log.Fatalf("Could not ssz marshal the genesis beacon state: %v", err) + } + if err := ioutil.WriteFile(*sszOutputFile, encodedState, 0644); err != nil { + log.Fatalf("Could not write encoded genesis beacon state to file: %v", err) + } + log.Printf("Done writing to %s", *sszOutputFile) + } + if *yamlOutputFile != "" { + encodedState, err := yaml.Marshal(genesisState) + if err != nil { + log.Fatalf("Could not yaml marshal the genesis beacon state: %v", err) + } + if err := ioutil.WriteFile(*yamlOutputFile, encodedState, 0644); err != nil { + log.Fatalf("Could not write encoded genesis beacon state to file: %v", err) + } + log.Printf("Done writing to %s", *yamlOutputFile) + } + if *jsonOutputFile != "" { + encodedState, err := json.Marshal(genesisState) + if err != nil { + log.Fatalf("Could not json marshal the genesis beacon state: %v", err) + } + if err := ioutil.WriteFile(*jsonOutputFile, encodedState, 0644); err != nil { + log.Fatalf("Could not write encoded genesis beacon state to file: %v", err) + } + log.Printf("Done writing to %s", *jsonOutputFile) + } +} + +// Deterministically creates BLS private keys using a fixed curve order according to +// the algorithm specified in the Eth2.0-Specs interop mock start section found here: +// https://github.com/ethereum/eth2.0-pm/blob/a085c9870f3956d6228ed2a40cd37f0c6580ecd7/interop/mocked_start/README.md +func deterministicallyGenerateKeys(n int) ([]*bls.SecretKey, []*bls.PublicKey, error) { + privKeys := make([]*bls.SecretKey, n) + pubKeys := make([]*bls.PublicKey, n) + for i := 0; i < n; i++ { + enc := make([]byte, 32) + binary.LittleEndian.PutUint32(enc, uint32(i)) + hash := hashutil.Hash(enc) + // Reverse byte order to big endian for use with big ints. + b := reverseByteOrder(hash[:]) + num := new(big.Int) + num = num.SetBytes(b) + order := new(big.Int) + var ok bool + order, ok = order.SetString(blsCurveOrder, 10) + if !ok { + return nil, nil, errors.New("could not set bls curve order as big int") + } + num = num.Mod(num, order) + priv, err := bls.SecretKeyFromBytes(num.Bytes()) + if err != nil { + return nil, nil, errors.Wrap(err, "could not create bls secret key from raw bytes") + } + privKeys[i] = priv + pubKeys[i] = priv.PublicKey() + } + return privKeys, pubKeys, nil +} + +// Generates a list of deposit items by creating proofs for each of them from a sparse Merkle trie. +func generateDepositsFromData(depositDataItems []*ethpb.Deposit_Data, trie *trieutil.MerkleTrie) ([]*ethpb.Deposit, error) { + deposits := make([]*ethpb.Deposit, len(depositDataItems)) + for i, item := range depositDataItems { + proof, err := trie.MerkleProof(i) + if err != nil { + return nil, errors.Wrapf(err, "could not generate proof for deposit %d", i) + } + deposits[i] = ðpb.Deposit{ + Proof: proof, + Data: item, + } + } + return deposits, nil +} + +// Generates a list of deposit data items from a set of BLS validator keys. +func depositDataFromKeys(privKeys []*bls.SecretKey, pubKeys []*bls.PublicKey) ([]*ethpb.Deposit_Data, [][]byte, error) { + dataRoots := make([][]byte, len(privKeys)) + depositDataItems := make([]*ethpb.Deposit_Data, len(privKeys)) + for i := 0; i < len(privKeys); i++ { + data, err := createDepositData(privKeys[i], pubKeys[i]) + if err != nil { + return nil, nil, errors.Wrapf(err, "could not create deposit data for key: %#x", privKeys[i].Marshal()) + } + hash, err := ssz.HashTreeRoot(data) + if err != nil { + return nil, nil, errors.Wrap(err, "could not hash tree root deposit data item") + } + dataRoots[i] = hash[:] + depositDataItems[i] = data + } + return depositDataItems, dataRoots, nil +} + +// Generates a deposit data item from BLS keys and signs the hash tree root of the data. +func createDepositData(privKey *bls.SecretKey, pubKey *bls.PublicKey) (*ethpb.Deposit_Data, error) { + di := ðpb.Deposit_Data{ + PublicKey: pubKey.Marshal(), + WithdrawalCredentials: withdrawalCredentialsHash(pubKey.Marshal()), + Amount: params.BeaconConfig().MaxEffectiveBalance, + } + sr, err := ssz.HashTreeRoot(di) + if err != nil { + return nil, err + } + domain := bls.Domain(domainDeposit[:], genesisForkVersion) + di.Signature = privKey.Sign(sr[:], domain).Marshal() + return di, nil +} + +// withdrawalCredentialsHash forms a 32 byte hash of the withdrawal public +// address. +// +// The specification is as follows: +// withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX_BYTE +// withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:] +// where withdrawal_credentials is of type bytes32. +func withdrawalCredentialsHash(pubKey []byte) []byte { + h := hashutil.HashKeccak256(pubKey) + return append([]byte{blsWithdrawalPrefixByte}, h[0:]...)[:32] +} + +// Switch the endianness of a byte slice by reversing its order. +func reverseByteOrder(input []byte) []byte { + b := input + for i := 0; i < len(b)/2; i++ { + b[i], b[len(b)-i-1] = b[len(b)-i-1], b[i] + } + return b +} diff --git a/tools/genesis-state-gen/main_test.go b/tools/genesis-state-gen/main_test.go new file mode 100644 index 0000000000..010001aec7 --- /dev/null +++ b/tools/genesis-state-gen/main_test.go @@ -0,0 +1,52 @@ +package main + +import ( + "testing" + + "github.com/prysmaticlabs/prysm/beacon-chain/core/state" + ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1" + "github.com/prysmaticlabs/prysm/shared/params" + "github.com/prysmaticlabs/prysm/shared/trieutil" +) + +func TestGenerateGenesisState(t *testing.T) { + numValidators := 64 + privKeys, pubKeys, err := deterministicallyGenerateKeys(numValidators) + if err != nil { + t.Fatal(err) + } + depositDataItems, depositDataRoots, err := depositDataFromKeys(privKeys, pubKeys) + if err != nil { + t.Fatal(err) + } + trie, err := trieutil.GenerateTrieFromItems( + depositDataRoots, + int(params.BeaconConfig().DepositContractTreeDepth), + ) + if err != nil { + t.Fatal(err) + } + deposits, err := generateDepositsFromData(depositDataItems, trie) + if err != nil { + t.Fatal(err) + } + root := trie.Root() + genesisState, err := state.GenesisBeaconState(deposits, 0, ðpb.Eth1Data{ + DepositRoot: root[:], + DepositCount: uint64(len(deposits)), + BlockHash: mockEth1BlockHash, + }) + if err != nil { + t.Fatal(err) + } + want := numValidators + if len(genesisState.Validators) != want { + t.Errorf("Wanted %d validators, received %d", want, len(genesisState.Validators)) + } + if len(genesisState.Validators) != want { + t.Errorf("Wanted %d validators, received %v", want, len(genesisState.Validators)) + } + if genesisState.GenesisTime != 0 { + t.Errorf("Wanted genesis time 0, received %d", genesisState.GenesisTime) + } +}