Miscellaneous Packages from Shared Into Proper Folders (#9638)

* slashutil

* builds

* interop

* viz

Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com>

runtime/interop/BUILD.bazel

@@ -0,0 +1,47 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "generate_genesis_state.go",
+        "generate_keys.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/runtime/interop",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//async:go_default_library",
+        "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/core/transition:go_default_library",
+        "//beacon-chain/state/v1:go_default_library",
+        "//container/trie:go_default_library",
+        "//crypto/bls:go_default_library",
+        "//crypto/hash:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//shared/params:go_default_library",
+        "//time:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "generate_genesis_state_test.go",
+        "generate_keys_test.go",
+    ],
+    data = [
+        "keygen_test_vector.yaml",
+    ],
+    deps = [
+        ":go_default_library",
+        "//beacon-chain/core/transition:go_default_library",
+        "//container/trie:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//shared/params:go_default_library",
+        "//shared/testutil/assert:go_default_library",
+        "//shared/testutil/require:go_default_library",
+        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
+        "@com_github_go_yaml_yaml//:go_default_library",
+        "@io_bazel_rules_go//go/tools/bazel:go_default_library",
+    ],
+)

runtime/interop/generate_genesis_state.go

@@ -0,0 +1,192 @@
+// Package interop contains deterministic utilities for generating
+// genesis states and keys.
+package interop
+
+import (
+	"context"
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/async"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	coreState "github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
+	v1 "github.com/prysmaticlabs/prysm/beacon-chain/state/v1"
+	"github.com/prysmaticlabs/prysm/container/trie"
+	"github.com/prysmaticlabs/prysm/crypto/bls"
+	"github.com/prysmaticlabs/prysm/crypto/hash"
+	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/time"
+)
+
+var (
+	// This is the recommended mock eth1 block hash according to the Ethereum consensus interop guidelines.
+	// https://github.com/ethereum/eth2.0-pm/blob/a085c9870f3956d6228ed2a40cd37f0c6580ecd7/interop/mocked_start/README.md
+	mockEth1BlockHash = []byte{66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66}
+)
+
+// GenerateGenesisState deterministically given a genesis time and number of validators.
+// If a genesis time of 0 is supplied it is set to the current time.
+func GenerateGenesisState(ctx context.Context, genesisTime, numValidators uint64) (*ethpb.BeaconState, []*ethpb.Deposit, error) {
+	privKeys, pubKeys, err := DeterministicallyGenerateKeys(0 /*startIndex*/, numValidators)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "could not deterministically generate keys for %d validators", numValidators)
+	}
+	depositDataItems, depositDataRoots, err := DepositDataFromKeys(privKeys, pubKeys)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not generate deposit data from keys")
+	}
+	return GenerateGenesisStateFromDepositData(ctx, genesisTime, depositDataItems, depositDataRoots)
+}
+
+// GenerateGenesisStateFromDepositData creates a genesis state given a list of
+// deposit data items and their corresponding roots.
+func GenerateGenesisStateFromDepositData(
+	ctx context.Context, genesisTime uint64, depositData []*ethpb.Deposit_Data, depositDataRoots [][]byte,
+) (*ethpb.BeaconState, []*ethpb.Deposit, error) {
+	trie, err := trie.GenerateTrieFromItems(depositDataRoots, params.BeaconConfig().DepositContractTreeDepth)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not generate Merkle trie for deposit proofs")
+	}
+	deposits, err := GenerateDepositsFromData(depositData, trie)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not generate deposits from the deposit data provided")
+	}
+	root := trie.HashTreeRoot()
+	if genesisTime == 0 {
+		genesisTime = uint64(time.Now().Unix())
+	}
+	beaconState, err := coreState.GenesisBeaconState(ctx, deposits, genesisTime, &ethpb.Eth1Data{
+		DepositRoot:  root[:],
+		DepositCount: uint64(len(deposits)),
+		BlockHash:    mockEth1BlockHash,
+	})
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not generate genesis state")
+	}
+
+	pbState, err := v1.ProtobufBeaconState(beaconState.CloneInnerState())
+	if err != nil {
+		return nil, nil, err
+	}
+	return pbState, deposits, nil
+}
+
+// GenerateDepositsFromData a list of deposit items by creating proofs for each of them from a sparse Merkle trie.
+func GenerateDepositsFromData(depositDataItems []*ethpb.Deposit_Data, trie *trie.SparseMerkleTrie) ([]*ethpb.Deposit, error) {
+	deposits := make([]*ethpb.Deposit, len(depositDataItems))
+	results, err := async.Scatter(len(depositDataItems), func(offset int, entries int, _ *sync.RWMutex) (interface{}, error) {
+		return generateDepositsFromData(depositDataItems[offset:offset+entries], offset, trie)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate deposits from data")
+	}
+	for _, result := range results {
+		if depositExtent, ok := result.Extent.([]*ethpb.Deposit); ok {
+			copy(deposits[result.Offset:], depositExtent)
+		} else {
+			return nil, errors.New("extent not of expected type")
+		}
+	}
+	return deposits, nil
+}
+
+// generateDepositsFromData a list of deposit items by creating proofs for each of them from a sparse Merkle trie.
+func generateDepositsFromData(depositDataItems []*ethpb.Deposit_Data, offset int, trie *trie.SparseMerkleTrie) ([]*ethpb.Deposit, error) {
+	deposits := make([]*ethpb.Deposit, len(depositDataItems))
+	for i, item := range depositDataItems {
+		proof, err := trie.MerkleProof(i + offset)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not generate proof for deposit %d", i+offset)
+		}
+		deposits[i] = &ethpb.Deposit{
+			Proof: proof,
+			Data:  item,
+		}
+	}
+	return deposits, nil
+}
+
+// DepositDataFromKeys generates a list of deposit data items from a set of BLS validator keys.
+func DepositDataFromKeys(privKeys []bls.SecretKey, pubKeys []bls.PublicKey) ([]*ethpb.Deposit_Data, [][]byte, error) {
+	type depositData struct {
+		items []*ethpb.Deposit_Data
+		roots [][]byte
+	}
+	depositDataItems := make([]*ethpb.Deposit_Data, len(privKeys))
+	depositDataRoots := make([][]byte, len(privKeys))
+	results, err := async.Scatter(len(privKeys), func(offset int, entries int, _ *sync.RWMutex) (interface{}, error) {
+		items, roots, err := depositDataFromKeys(privKeys[offset:offset+entries], pubKeys[offset:offset+entries])
+		return &depositData{items: items, roots: roots}, err
+	})
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to generate deposit data from keys")
+	}
+	for _, result := range results {
+		if depositDataExtent, ok := result.Extent.(*depositData); ok {
+			copy(depositDataItems[result.Offset:], depositDataExtent.items)
+			copy(depositDataRoots[result.Offset:], depositDataExtent.roots)
+		} else {
+			return nil, nil, errors.New("extent not of expected type")
+		}
+	}
+	return depositDataItems, depositDataRoots, nil
+}
+
+func depositDataFromKeys(privKeys []bls.SecretKey, pubKeys []bls.PublicKey) ([]*ethpb.Deposit_Data, [][]byte, error) {
+	dataRoots := make([][]byte, len(privKeys))
+	depositDataItems := make([]*ethpb.Deposit_Data, len(privKeys))
+	for i := 0; i < len(privKeys); i++ {
+		data, err := createDepositData(privKeys[i], pubKeys[i])
+		if err != nil {
+			return nil, nil, errors.Wrapf(err, "could not create deposit data for key: %#x", privKeys[i].Marshal())
+		}
+		hash, err := data.HashTreeRoot()
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "could not hash tree root deposit data item")
+		}
+		dataRoots[i] = hash[:]
+		depositDataItems[i] = data
+	}
+	return depositDataItems, dataRoots, nil
+}
+
+// Generates a deposit data item from BLS keys and signs the hash tree root of the data.
+func createDepositData(privKey bls.SecretKey, pubKey bls.PublicKey) (*ethpb.Deposit_Data, error) {
+	depositMessage := &ethpb.DepositMessage{
+		PublicKey:             pubKey.Marshal(),
+		WithdrawalCredentials: withdrawalCredentialsHash(pubKey.Marshal()),
+		Amount:                params.BeaconConfig().MaxEffectiveBalance,
+	}
+	sr, err := depositMessage.HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	domain, err := helpers.ComputeDomain(params.BeaconConfig().DomainDeposit, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	root, err := (&ethpb.SigningData{ObjectRoot: sr[:], Domain: domain}).HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	di := &ethpb.Deposit_Data{
+		PublicKey:             depositMessage.PublicKey,
+		WithdrawalCredentials: depositMessage.WithdrawalCredentials,
+		Amount:                depositMessage.Amount,
+		Signature:             privKey.Sign(root[:]).Marshal(),
+	}
+	return di, nil
+}
+
+// withdrawalCredentialsHash forms a 32 byte hash of the withdrawal public
+// address.
+//
+// The specification is as follows:
+//   withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX_BYTE
+//   withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]
+// where withdrawal_credentials is of type bytes32.
+func withdrawalCredentialsHash(pubKey []byte) []byte {
+	h := hash.Hash(pubKey)
+	return append([]byte{blsWithdrawalPrefixByte}, h[1:]...)[:32]
+}

runtime/interop/generate_genesis_state_test.go

@@ -0,0 +1,35 @@
+package interop_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
+	"github.com/prysmaticlabs/prysm/container/trie"
+	eth "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/runtime/interop"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+func TestGenerateGenesisState(t *testing.T) {
+	numValidators := uint64(64)
+	privKeys, pubKeys, err := interop.DeterministicallyGenerateKeys(0 /*startIndex*/, numValidators)
+	require.NoError(t, err)
+	depositDataItems, depositDataRoots, err := interop.DepositDataFromKeys(privKeys, pubKeys)
+	require.NoError(t, err)
+	tr, err := trie.GenerateTrieFromItems(depositDataRoots, params.BeaconConfig().DepositContractTreeDepth)
+	require.NoError(t, err)
+	deposits, err := interop.GenerateDepositsFromData(depositDataItems, tr)
+	require.NoError(t, err)
+	root := tr.HashTreeRoot()
+	genesisState, err := transition.GenesisBeaconState(context.Background(), deposits, 0, &eth.Eth1Data{
+		DepositRoot:  root[:],
+		DepositCount: uint64(len(deposits)),
+	})
+	require.NoError(t, err)
+	want := int(numValidators)
+	assert.Equal(t, want, genesisState.NumValidators())
+	assert.Equal(t, uint64(0), genesisState.GenesisTime())
+}

runtime/interop/generate_keys.go

@@ -0,0 +1,87 @@
+package interop
+
+import (
+	"encoding/binary"
+	"math/big"
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/async"
+	"github.com/prysmaticlabs/prysm/crypto/bls"
+	"github.com/prysmaticlabs/prysm/crypto/hash"
+)
+
+const (
+	blsWithdrawalPrefixByte = byte(0)
+)
+
+// DeterministicallyGenerateKeys creates BLS private keys using a fixed curve order according to
+// the algorithm specified in the Ethereum beacon chain specification interop mock start section found here:
+// https://github.com/ethereum/eth2.0-pm/blob/a085c9870f3956d6228ed2a40cd37f0c6580ecd7/interop/mocked_start/README.md
+func DeterministicallyGenerateKeys(startIndex, numKeys uint64) ([]bls.SecretKey, []bls.PublicKey, error) {
+	privKeys := make([]bls.SecretKey, numKeys)
+	pubKeys := make([]bls.PublicKey, numKeys)
+	type keys struct {
+		secrets []bls.SecretKey
+		publics []bls.PublicKey
+	}
+	results, err := async.Scatter(int(numKeys), func(offset int, entries int, _ *sync.RWMutex) (interface{}, error) {
+		secs, pubs, err := deterministicallyGenerateKeys(uint64(offset)+startIndex, uint64(entries))
+		return &keys{secrets: secs, publics: pubs}, err
+	})
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to generate keys")
+	}
+	for _, result := range results {
+		if keysExtent, ok := result.Extent.(*keys); ok {
+			copy(privKeys[result.Offset:], keysExtent.secrets)
+			copy(pubKeys[result.Offset:], keysExtent.publics)
+		} else {
+			return nil, nil, errors.New("extent not of expected type")
+		}
+	}
+	return privKeys, pubKeys, nil
+}
+
+func deterministicallyGenerateKeys(startIndex, numKeys uint64) ([]bls.SecretKey, []bls.PublicKey, error) {
+	privKeys := make([]bls.SecretKey, numKeys)
+	pubKeys := make([]bls.PublicKey, numKeys)
+	for i := startIndex; i < startIndex+numKeys; i++ {
+		enc := make([]byte, 32)
+		binary.LittleEndian.PutUint32(enc, uint32(i))
+		hash := hash.Hash(enc)
+		// Reverse byte order to big endian for use with big ints.
+		b := reverseByteOrder(hash[:])
+		num := new(big.Int)
+		num = num.SetBytes(b)
+		order := new(big.Int)
+		var ok bool
+		order, ok = order.SetString(bls.CurveOrder, 10)
+		if !ok {
+			return nil, nil, errors.New("could not set bls curve order as big int")
+		}
+		num = num.Mod(num, order)
+		numBytes := num.Bytes()
+		// pad key at the start with zero bytes to make it into a 32 byte key
+		if len(numBytes) < 32 {
+			emptyBytes := make([]byte, 32-len(numBytes))
+			numBytes = append(emptyBytes, numBytes...)
+		}
+		priv, err := bls.SecretKeyFromBytes(numBytes)
+		if err != nil {
+			return nil, nil, errors.Wrapf(err, "could not create bls secret key at index %d from raw bytes", i)
+		}
+		privKeys[i-startIndex] = priv
+		pubKeys[i-startIndex] = priv.PublicKey()
+	}
+	return privKeys, pubKeys, nil
+}
+
+// Switch the endianness of a byte slice by reversing its order.
+func reverseByteOrder(input []byte) []byte {
+	b := input
+	for i := 0; i < len(b)/2; i++ {
+		b[i], b[len(b)-i-1] = b[len(b)-i-1], b[i]
+	}
+	return b
+}

runtime/interop/generate_keys_test.go

@@ -0,0 +1,42 @@
+package interop_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/bazelbuild/rules_go/go/tools/bazel"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/go-yaml/yaml"
+	"github.com/prysmaticlabs/prysm/runtime/interop"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+type TestCase struct {
+	Privkey string `yaml:"privkey"`
+}
+
+type KeyTest struct {
+	TestCases []*TestCase `yaml:"test_cases"`
+}
+
+func TestKeyGenerator(t *testing.T) {
+	path, err := bazel.Runfile("keygen_test_vector.yaml")
+	require.NoError(t, err)
+	file, err := ioutil.ReadFile(path)
+	require.NoError(t, err)
+	testCases := &KeyTest{}
+	require.NoError(t, yaml.Unmarshal(file, testCases))
+	priv, _, err := interop.DeterministicallyGenerateKeys(0, 1000)
+	require.NoError(t, err)
+	// cross-check with the first 1000 keys generated from the python spec
+	for i, key := range priv {
+		hexKey := testCases.TestCases[i].Privkey
+		nKey, err := hexutil.Decode("0x" + hexKey)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		assert.DeepEqual(t, key.Marshal(), nKey)
+	}
+}

runtime/tos/BUILD.bazel

@@ -6,9 +6,9 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/runtime/tos",
     visibility = ["//visibility:public"],
     deps = [
+        "//cmd:go_default_library",
         "//io/file:go_default_library",
         "//io/prompt:go_default_library",
-        "//shared/cmd:go_default_library",
         "@com_github_logrusorgru_aurora//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_urfave_cli_v2//:go_default_library",
@@ -21,7 +21,7 @@ go_test(
     srcs = ["tos_test.go"],
     embed = [":go_default_library"],
     deps = [
-        "//shared/cmd:go_default_library",
+        "//cmd:go_default_library",
         "//shared/testutil/require:go_default_library",
         "@com_github_urfave_cli_v2//:go_default_library",
     ],

runtime/tos/tos.go

@@ -6,9 +6,9 @@ import (
 	"strings"
 
 	"github.com/logrusorgru/aurora"
+	"github.com/prysmaticlabs/prysm/cmd"
 	"github.com/prysmaticlabs/prysm/io/file"
 	"github.com/prysmaticlabs/prysm/io/prompt"
-	"github.com/prysmaticlabs/prysm/shared/cmd"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )

runtime/tos/tos_test.go

@@ -7,7 +7,7 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/prysmaticlabs/prysm/shared/cmd"
+	"github.com/prysmaticlabs/prysm/cmd"
 	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 	"github.com/urfave/cli/v2"
 )