Genesis State Generator + Interop Docs (#3405)

* add main.go

* interop readme

* proper visibility

* standardize and abstract into simpler funcs

* formatting

* no os pkg

* add test

* no panics anywhere, properly and nicely handle errors

* proper comments

* fix broken test

* readme

* comment

* recommend ssz

* install

* tool now works

* README

* build

* readme

* 64 validators

* rem print

INTEROP.md

@@ -0,0 +1,48 @@
+# Prysm Client Interoperability Guide
+
+This README details how to setup Prysm for interop testing for usage with other Ethereum 2.0 clients.
+
+**WARNING**: The tool can only generate up to 190 private keys at the moment, as there is a BLS bug that
+prevents deterministically generating more keys than those. 
+
+## Installation & Setup
+
+1. Install [Bazel](https://docs.bazel.build/versions/master/install.html) **(Recommended)**
+2. `git clone https://github.com/prysmaticlabs/prysm && cd prysm`
+3. `bazel build //...`
+
+## Starting from Genesis
+
+Prysm supports a few ways to quickly launch a beacon node from basic configurations:
+
+- `SSZ Genesis`: Launches a beacon node  from a .ssz file containing a SSZ-encoded, genesis beacon state  **(Recommended)**
+- `Yaml Genesis`: Launches a beacon node from a .yaml file specifying a genesis beacon state - binary data is _base64 encoded_
+- `JSON Genesis`: Launches a beacon node  from a .json file specifying a genesis beacon state - binary data is _base64 encoded_
+
+## Generating a Genesis State
+
+To setup the necessary files for these quick starts, Prysm provides a tool to generate `genesis.yaml`, `genesis.ssz`, `genesis.json` from an
+a deterministically generated set of validator private keys following the official interop YAML format 
+[here](https://github.com/ethereum/eth2.0-pm/blob/master/interop/mocked_start). If you already have a genesis state in this format, you can skip this section.
+
+You can use `bazel run //tools/genesis-state-gen` to create a deterministic genesis state for interop.
+
+### Usage
+
+- **--genesis-time** uint: Unix timestamp used as the genesis time in the generated genesis state
+- **--mainnet-config** bool: Select whether genesis state should be generated with mainnet or minimal (default) params
+- **--num-validators** int: Number of validators to deterministically include in the generated genesis state
+- **--output-json** string: Output filename of the JSON marshaling of the generated genesis state
+- **--output-ssz** string: Output filename of the SSZ marshaling of the generated genesis state
+- **--output-yaml** string: Output filename of the YAML marshaling of the generated genesis state
+
+The example below creates 10 validator keys, instantiates a genesis state with those 10 validators and with genesis time 1567542540,
+and finally writes a YAML encoded output to ~/Desktop/genesis.yaml. This file can be used to kickstart the beacon chain in the next section.
+
+```
+bazel run //tools/genesis-state-gen -- --output-yaml ~/Desktop/genesis.yaml --num-validators 10 --genesis-time 1567542540
+```
+
+## Launching a Beacon Node + Validator Client
+
+TODO: Add section after incorporating the mock start functionality into the beacon chain and validator bazel binaries.

beacon-chain/core/state/BUILD.bazel

@@ -7,7 +7,10 @@ go_library(
         "transition.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/core/state",
-    visibility = ["//beacon-chain:__subpackages__"],
+    visibility = [
+        "//beacon-chain:__subpackages__",
+        "//tools/genesis-state-gen:__pkg__",
+    ],
     deps = [
         "//beacon-chain/core/blocks:go_default_library",
         "//beacon-chain/core/epoch:go_default_library",

shared/bls/bls.go

@@ -5,7 +5,6 @@ package bls
 
 import (
 	"encoding/binary"
-	"fmt"
 	"io"
 
 	g1 "github.com/phoreproject/bls/g1pubs"
@@ -39,9 +38,6 @@ func RandKey(r io.Reader) (*SecretKey, error) {
 
 // SecretKeyFromBytes creates a BLS private key from a byte slice.
 func SecretKeyFromBytes(priv []byte) (*SecretKey, error) {
-	if len(priv) != 32 {
-		return nil, fmt.Errorf("expected byte slice of length 32, received: %d", len(priv))
-	}
 	k := bytesutil.ToBytes32(priv)
 	val := g1.DeserializeSecretKey(k)
 	if val.GetFRElement() == nil {

tools/genesis-state-gen/BUILD.bazel

@@ -0,0 +1,37 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "github.com/prysmaticlabs/prysm/tools/genesis-state-gen",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//beacon-chain/core/state:go_default_library",
+        "//proto/eth/v1alpha1:go_default_library",
+        "//shared/bls:go_default_library",
+        "//shared/hashutil:go_default_library",
+        "//shared/params:go_default_library",
+        "//shared/trieutil:go_default_library",
+        "@com_github_ghodss_yaml//:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+        "@com_github_prysmaticlabs_go_ssz//:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "genesis-state-gen",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["main_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//beacon-chain/core/state:go_default_library",
+        "//proto/eth/v1alpha1:go_default_library",
+        "//shared/params:go_default_library",
+        "//shared/trieutil:go_default_library",
+    ],
+)

tools/genesis-state-gen/main.go

@@ -0,0 +1,217 @@
+package main
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"flag"
+	"io/ioutil"
+	"log"
+	"math/big"
+
+	"github.com/ghodss/yaml"
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/shared/bls"
+	"github.com/prysmaticlabs/prysm/shared/hashutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/trieutil"
+)
+
+const (
+	blsWithdrawalPrefixByte = byte(0)
+	blsCurveOrder           = "52435875175126190479447740508185965837690552500527637822603658699938581184513"
+)
+
+var (
+	domainDeposit      = [4]byte{3, 0, 0, 0}
+	genesisForkVersion = []byte{0, 0, 0, 0}
+	numValidators      = flag.Int("num-validators", 0, "Number of validators to deterministically include in the generated genesis state")
+	useMainnetConfig   = flag.Bool("mainnet-config", false, "Select whether genesis state should be generated with mainnet or minimal (default) params")
+	genesisTime        = flag.Uint64("genesis-time", 0, "Unix timestamp used as the genesis time in the generated genesis state")
+	sszOutputFile      = flag.String("output-ssz", "", "Output filename of the SSZ marshaling of the generated genesis state")
+	yamlOutputFile     = flag.String("output-yaml", "", "Output filename of the YAML marshaling of the generated genesis state")
+	jsonOutputFile     = flag.String("output-json", "", "Output filename of the JSON marshaling of the generated genesis state")
+	// This is the recommended mock eth1 block hash according to the Eth2 interop guidelines.
+	// https://github.com/ethereum/eth2.0-pm/blob/a085c9870f3956d6228ed2a40cd37f0c6580ecd7/interop/mocked_start/README.md
+	mockEth1BlockHash = []byte{66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66}
+)
+
+func main() {
+	flag.Parse()
+	// TODO(#3398): Cannot generate more than 190 keys due to BLS errors.
+	if *numValidators == 0 {
+		log.Fatal("Expected --num-validators to have been provided, received 0")
+	}
+	if *genesisTime == 0 {
+		log.Print("No --genesis-time specified, defaulting to 0 as the unix timestamp")
+	}
+	if *sszOutputFile == "" && *yamlOutputFile == "" && *jsonOutputFile == "" {
+		log.Fatal("Expected --output-ssz, --output-yaml, or --output-json to have been provided, received nil")
+	}
+	if !*useMainnetConfig {
+		params.OverrideBeaconConfig(params.MinimalSpecConfig())
+	}
+	privKeys, pubKeys, err := deterministicallyGenerateKeys(*numValidators)
+	if err != nil {
+		log.Fatalf("Could not deterministically generate keys for %d validators: %v", *numValidators, err)
+	}
+	depositDataItems, depositDataRoots, err := depositDataFromKeys(privKeys, pubKeys)
+	if err != nil {
+		log.Fatalf("Could not generate deposit data from keys: %v", err)
+	}
+	trie, err := trieutil.GenerateTrieFromItems(
+		depositDataRoots,
+		int(params.BeaconConfig().DepositContractTreeDepth),
+	)
+	if err != nil {
+		log.Fatalf("Could not generate Merkle trie for deposit proofs: %v", err)
+	}
+	deposits, err := generateDepositsFromData(depositDataItems, trie)
+	if err != nil {
+		log.Fatalf("Could not generate deposits from the deposit data provided: %v", err)
+	}
+	root := trie.Root()
+	genesisState, err := state.GenesisBeaconState(deposits, *genesisTime, &ethpb.Eth1Data{
+		DepositRoot:  root[:],
+		DepositCount: uint64(len(deposits)),
+		BlockHash:    mockEth1BlockHash,
+	})
+	if err != nil {
+		log.Fatalf("Could not generate genesis beacon state: %v", err)
+	}
+	if *sszOutputFile != "" {
+		encodedState, err := ssz.Marshal(genesisState)
+		if err != nil {
+			log.Fatalf("Could not ssz marshal the genesis beacon state: %v", err)
+		}
+		if err := ioutil.WriteFile(*sszOutputFile, encodedState, 0644); err != nil {
+			log.Fatalf("Could not write encoded genesis beacon state to file: %v", err)
+		}
+		log.Printf("Done writing to %s", *sszOutputFile)
+	}
+	if *yamlOutputFile != "" {
+		encodedState, err := yaml.Marshal(genesisState)
+		if err != nil {
+			log.Fatalf("Could not yaml marshal the genesis beacon state: %v", err)
+		}
+		if err := ioutil.WriteFile(*yamlOutputFile, encodedState, 0644); err != nil {
+			log.Fatalf("Could not write encoded genesis beacon state to file: %v", err)
+		}
+		log.Printf("Done writing to %s", *yamlOutputFile)
+	}
+	if *jsonOutputFile != "" {
+		encodedState, err := json.Marshal(genesisState)
+		if err != nil {
+			log.Fatalf("Could not json marshal the genesis beacon state: %v", err)
+		}
+		if err := ioutil.WriteFile(*jsonOutputFile, encodedState, 0644); err != nil {
+			log.Fatalf("Could not write encoded genesis beacon state to file: %v", err)
+		}
+		log.Printf("Done writing to %s", *jsonOutputFile)
+	}
+}
+
+// Deterministically creates BLS private keys using a fixed curve order according to
+// the algorithm specified in the Eth2.0-Specs interop mock start section found here:
+// https://github.com/ethereum/eth2.0-pm/blob/a085c9870f3956d6228ed2a40cd37f0c6580ecd7/interop/mocked_start/README.md
+func deterministicallyGenerateKeys(n int) ([]*bls.SecretKey, []*bls.PublicKey, error) {
+	privKeys := make([]*bls.SecretKey, n)
+	pubKeys := make([]*bls.PublicKey, n)
+	for i := 0; i < n; i++ {
+		enc := make([]byte, 32)
+		binary.LittleEndian.PutUint32(enc, uint32(i))
+		hash := hashutil.Hash(enc)
+		// Reverse byte order to big endian for use with big ints.
+		b := reverseByteOrder(hash[:])
+		num := new(big.Int)
+		num = num.SetBytes(b)
+		order := new(big.Int)
+		var ok bool
+		order, ok = order.SetString(blsCurveOrder, 10)
+		if !ok {
+			return nil, nil, errors.New("could not set bls curve order as big int")
+		}
+		num = num.Mod(num, order)
+		priv, err := bls.SecretKeyFromBytes(num.Bytes())
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "could not create bls secret key from raw bytes")
+		}
+		privKeys[i] = priv
+		pubKeys[i] = priv.PublicKey()
+	}
+	return privKeys, pubKeys, nil
+}
+
+// Generates a list of deposit items by creating proofs for each of them from a sparse Merkle trie.
+func generateDepositsFromData(depositDataItems []*ethpb.Deposit_Data, trie *trieutil.MerkleTrie) ([]*ethpb.Deposit, error) {
+	deposits := make([]*ethpb.Deposit, len(depositDataItems))
+	for i, item := range depositDataItems {
+		proof, err := trie.MerkleProof(i)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not generate proof for deposit %d", i)
+		}
+		deposits[i] = &ethpb.Deposit{
+			Proof: proof,
+			Data:  item,
+		}
+	}
+	return deposits, nil
+}
+
+// Generates a list of deposit data items from a set of BLS validator keys.
+func depositDataFromKeys(privKeys []*bls.SecretKey, pubKeys []*bls.PublicKey) ([]*ethpb.Deposit_Data, [][]byte, error) {
+	dataRoots := make([][]byte, len(privKeys))
+	depositDataItems := make([]*ethpb.Deposit_Data, len(privKeys))
+	for i := 0; i < len(privKeys); i++ {
+		data, err := createDepositData(privKeys[i], pubKeys[i])
+		if err != nil {
+			return nil, nil, errors.Wrapf(err, "could not create deposit data for key: %#x", privKeys[i].Marshal())
+		}
+		hash, err := ssz.HashTreeRoot(data)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "could not hash tree root deposit data item")
+		}
+		dataRoots[i] = hash[:]
+		depositDataItems[i] = data
+	}
+	return depositDataItems, dataRoots, nil
+}
+
+// Generates a deposit data item from BLS keys and signs the hash tree root of the data.
+func createDepositData(privKey *bls.SecretKey, pubKey *bls.PublicKey) (*ethpb.Deposit_Data, error) {
+	di := &ethpb.Deposit_Data{
+		PublicKey:             pubKey.Marshal(),
+		WithdrawalCredentials: withdrawalCredentialsHash(pubKey.Marshal()),
+		Amount:                params.BeaconConfig().MaxEffectiveBalance,
+	}
+	sr, err := ssz.HashTreeRoot(di)
+	if err != nil {
+		return nil, err
+	}
+	domain := bls.Domain(domainDeposit[:], genesisForkVersion)
+	di.Signature = privKey.Sign(sr[:], domain).Marshal()
+	return di, nil
+}
+
+// withdrawalCredentialsHash forms a 32 byte hash of the withdrawal public
+// address.
+//
+// The specification is as follows:
+//   withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX_BYTE
+//   withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]
+// where withdrawal_credentials is of type bytes32.
+func withdrawalCredentialsHash(pubKey []byte) []byte {
+	h := hashutil.HashKeccak256(pubKey)
+	return append([]byte{blsWithdrawalPrefixByte}, h[0:]...)[:32]
+}
+
+// Switch the endianness of a byte slice by reversing its order.
+func reverseByteOrder(input []byte) []byte {
+	b := input
+	for i := 0; i < len(b)/2; i++ {
+		b[i], b[len(b)-i-1] = b[len(b)-i-1], b[i]
+	}
+	return b
+}

tools/genesis-state-gen/main_test.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/trieutil"
+)
+
+func TestGenerateGenesisState(t *testing.T) {
+	numValidators := 64
+	privKeys, pubKeys, err := deterministicallyGenerateKeys(numValidators)
+	if err != nil {
+		t.Fatal(err)
+	}
+	depositDataItems, depositDataRoots, err := depositDataFromKeys(privKeys, pubKeys)
+	if err != nil {
+		t.Fatal(err)
+	}
+	trie, err := trieutil.GenerateTrieFromItems(
+		depositDataRoots,
+		int(params.BeaconConfig().DepositContractTreeDepth),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	deposits, err := generateDepositsFromData(depositDataItems, trie)
+	if err != nil {
+		t.Fatal(err)
+	}
+	root := trie.Root()
+	genesisState, err := state.GenesisBeaconState(deposits, 0, &ethpb.Eth1Data{
+		DepositRoot:  root[:],
+		DepositCount: uint64(len(deposits)),
+		BlockHash:    mockEth1BlockHash,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := numValidators
+	if len(genesisState.Validators) != want {
+		t.Errorf("Wanted %d validators, received %d", want, len(genesisState.Validators))
+	}
+	if len(genesisState.Validators) != want {
+		t.Errorf("Wanted %d validators, received %v", want, len(genesisState.Validators))
+	}
+	if genesisState.GenesisTime != 0 {
+		t.Errorf("Wanted genesis time 0, received %d", genesisState.GenesisTime)
+	}
+}