diff --git a/testing/endtoend/BUILD.bazel b/testing/endtoend/BUILD.bazel index 0427cbb413..14577a088f 100644 --- a/testing/endtoend/BUILD.bazel +++ b/testing/endtoend/BUILD.bazel @@ -40,6 +40,7 @@ go_test( "//proto/prysm/v1alpha1:go_default_library", "//testing/assert:go_default_library", "//testing/endtoend/components:go_default_library", + "//testing/endtoend/components/eth1:go_default_library", "//testing/endtoend/evaluators:go_default_library", "//testing/endtoend/helpers:go_default_library", "//testing/endtoend/params:go_default_library", @@ -93,6 +94,7 @@ go_test( "//proto/prysm/v1alpha1:go_default_library", "//testing/assert:go_default_library", "//testing/endtoend/components:go_default_library", + "//testing/endtoend/components/eth1:go_default_library", "//testing/endtoend/evaluators:go_default_library", "//testing/endtoend/helpers:go_default_library", "//testing/endtoend/params:go_default_library", diff --git a/testing/endtoend/components/BUILD.bazel b/testing/endtoend/components/BUILD.bazel index 3b099c4aa8..9f044c19d1 100644 --- a/testing/endtoend/components/BUILD.bazel +++ b/testing/endtoend/components/BUILD.bazel @@ -14,7 +14,10 @@ go_library( "validator.go", "web3remotesigner.go", ], - data = ["@lighthouse//:lighthouse_bin"], + data = [ + "//testing/endtoend/static-files/eth1:eth1data", + "@lighthouse//:lighthouse_bin", + ], importpath = "github.com/prysmaticlabs/prysm/testing/endtoend/components", visibility = ["//testing/endtoend:__subpackages__"], deps = [ @@ -29,6 +32,7 @@ go_library( "//encoding/bytesutil:go_default_library", "//io/file:go_default_library", "//runtime/interop:go_default_library", + "//testing/endtoend/components/eth1:go_default_library", "//testing/endtoend/helpers:go_default_library", "//testing/endtoend/params:go_default_library", "//testing/endtoend/types:go_default_library", diff --git a/testing/endtoend/components/beacon_node.go b/testing/endtoend/components/beacon_node.go index ed73de176e..cffb0ce217 100644 --- a/testing/endtoend/components/beacon_node.go +++ b/testing/endtoend/components/beacon_node.go @@ -112,14 +112,16 @@ func (node *BeaconNode) Start(ctx context.Context) error { return err } expectedNumOfPeers := e2e.TestParams.BeaconNodeCount + e2e.TestParams.LighthouseBeaconNodeCount - 1 - + if node.config.TestSync { + expectedNumOfPeers += 1 + } args := []string{ fmt.Sprintf("--%s=%s/eth2-beacon-node-%d", cmdshared.DataDirFlag.Name, e2e.TestParams.TestPath, index), fmt.Sprintf("--%s=%s", cmdshared.LogFileName.Name, stdOutFile.Name()), fmt.Sprintf("--%s=%s", flags.DepositContractFlag.Name, e2e.TestParams.ContractAddress.Hex()), fmt.Sprintf("--%s=%d", flags.RPCPort.Name, e2e.TestParams.Ports.PrysmBeaconNodeRPCPort+index), fmt.Sprintf("--%s=http://127.0.0.1:%d", flags.HTTPWeb3ProviderFlag.Name, e2e.TestParams.Ports.Eth1RPCPort), - fmt.Sprintf("--%s=%d", flags.MinSyncPeers.Name, e2e.TestParams.BeaconNodeCount-1), + fmt.Sprintf("--%s=%d", flags.MinSyncPeers.Name, 1), fmt.Sprintf("--%s=%d", cmdshared.P2PUDPPort.Name, e2e.TestParams.Ports.PrysmBeaconNodeUDPPort+index), fmt.Sprintf("--%s=%d", cmdshared.P2PTCPPort.Name, e2e.TestParams.Ports.PrysmBeaconNodeTCPPort+index), fmt.Sprintf("--%s=%d", cmdshared.P2PMaxPeers.Name, expectedNumOfPeers), diff --git a/testing/endtoend/components/eth1/BUILD.bazel b/testing/endtoend/components/eth1/BUILD.bazel new file mode 100644 index 0000000000..b696f1c07b --- /dev/null +++ b/testing/endtoend/components/eth1/BUILD.bazel @@ -0,0 +1,30 @@ +load("@prysm//tools/go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + testonly = True, + srcs = [ + "helpers.go", + "miner.go", + "node.go", + "node_set.go", + ], + importpath = "github.com/prysmaticlabs/prysm/testing/endtoend/components/eth1", + visibility = ["//testing/endtoend:__subpackages__"], + deps = [ + "//config/params:go_default_library", + "//contracts/deposit/mock:go_default_library", + "//io/file:go_default_library", + "//testing/endtoend/helpers:go_default_library", + "//testing/endtoend/params:go_default_library", + "//testing/endtoend/types:go_default_library", + "@com_github_ethereum_go_ethereum//accounts/abi/bind:go_default_library", + "@com_github_ethereum_go_ethereum//accounts/keystore:go_default_library", + "@com_github_ethereum_go_ethereum//core/types:go_default_library", + "@com_github_ethereum_go_ethereum//ethclient:go_default_library", + "@com_github_ethereum_go_ethereum//rpc:go_default_library", + "@com_github_pkg_errors//:go_default_library", + "@com_github_sirupsen_logrus//:go_default_library", + "@io_bazel_rules_go//go/tools/bazel:go_default_library", + ], +) diff --git a/testing/endtoend/components/eth1/helpers.go b/testing/endtoend/components/eth1/helpers.go new file mode 100644 index 0000000000..d72ebd65bc --- /dev/null +++ b/testing/endtoend/components/eth1/helpers.go @@ -0,0 +1,64 @@ +package eth1 + +import ( + "context" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types" +) + +// NetworkId is the ID of the ETH1 chain. +const NetworkId = 1337 + +// KeystorePassword is the password used to decrypt ETH1 keystores. +const KeystorePassword = "password" + +const MinerPort = 30303 +const minerPasswordFile = "password.txt" +const minerFile = "UTC--2021-12-22T19-14-08.590377700Z--878705ba3f8bc32fcf7f4caa1a35e72af65cf766" +const timeGapPerTX = 100 * time.Millisecond +const staticFilesPath = "/testing/endtoend/static-files/eth1" +const timeGapPerMiningTX = 250 * time.Millisecond + +var _ e2etypes.ComponentRunner = (*NodeSet)(nil) +var _ e2etypes.ComponentRunner = (*Miner)(nil) +var _ e2etypes.ComponentRunner = (*Node)(nil) + +// WaitForBlocks waits for a certain amount of blocks to be mined by the ETH1 chain before returning. +func WaitForBlocks(web3 *ethclient.Client, keystore *keystore.Key, blocksToWait uint64) error { + nonce, err := web3.PendingNonceAt(context.Background(), keystore.Address) + if err != nil { + return err + } + chainID, err := web3.NetworkID(context.Background()) + if err != nil { + return err + } + block, err := web3.BlockByNumber(context.Background(), nil) + if err != nil { + return err + } + finishBlock := block.NumberU64() + blocksToWait + + for block.NumberU64() <= finishBlock { + spamTX := types.NewTransaction(nonce, keystore.Address, big.NewInt(0), 21000, big.NewInt(1e6), []byte{}) + signed, err := types.SignTx(spamTX, types.NewEIP155Signer(chainID), keystore.PrivateKey) + if err != nil { + return err + } + if err = web3.SendTransaction(context.Background(), signed); err != nil { + return err + } + nonce++ + time.Sleep(timeGapPerMiningTX) + block, err = web3.BlockByNumber(context.Background(), nil) + if err != nil { + return err + } + } + return nil +} diff --git a/testing/endtoend/components/eth1/miner.go b/testing/endtoend/components/eth1/miner.go new file mode 100644 index 0000000000..1d4cfc3c49 --- /dev/null +++ b/testing/endtoend/components/eth1/miner.go @@ -0,0 +1,254 @@ +package eth1 + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "math/big" + "os" + "os/exec" + "path" + "strings" + "time" + + "github.com/bazelbuild/rules_go/go/tools/bazel" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + "github.com/pkg/errors" + "github.com/prysmaticlabs/prysm/config/params" + contracts "github.com/prysmaticlabs/prysm/contracts/deposit/mock" + io "github.com/prysmaticlabs/prysm/io/file" + "github.com/prysmaticlabs/prysm/testing/endtoend/helpers" + e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params" + e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types" + log "github.com/sirupsen/logrus" +) + +// Miner represents an ETH1 node which mines blocks. +type Miner struct { + e2etypes.ComponentRunner + started chan struct{} + bootstrapEnr string + enr string + keystorePath string +} + +// NewMiner creates and returns an ETH1 node miner. +func NewMiner() *Miner { + return &Miner{ + started: make(chan struct{}, 1), + } +} + +// KeystorePath returns the path of the keystore file. +func (m *Miner) KeystorePath() string { + return m.keystorePath +} + +// ENR returns the miner's enode. +func (m *Miner) ENR() string { + return m.enr +} + +// SetBootstrapENR sets the bootstrap record. +func (m *Miner) SetBootstrapENR(bootstrapEnr string) { + m.bootstrapEnr = bootstrapEnr +} + +// Start runs a mining ETH1 node. +// The miner is responsible for moving the ETH1 chain forward and for deploying the deposit contract. +func (m *Miner) Start(ctx context.Context) error { + binaryPath, found := bazel.FindBinary("cmd/geth", "geth") + if !found { + return errors.New("go-ethereum binary not found") + } + + eth1Path := path.Join(e2e.TestParams.TestPath, "eth1data/miner/") + // Clear out potentially existing dir to prevent issues. + if _, err := os.Stat(eth1Path); !os.IsNotExist(err) { + if err = os.RemoveAll(eth1Path); err != nil { + return err + } + } + + genesisSrcPath, err := bazel.Runfile(path.Join(staticFilesPath, "genesis.json")) + if err != nil { + return err + } + genesisDstPath := binaryPath[:strings.LastIndex(binaryPath, "/")] + cpCmd := exec.CommandContext(ctx, "cp", genesisSrcPath, genesisDstPath) // #nosec G204 -- Safe + if err = cpCmd.Start(); err != nil { + return err + } + if err = cpCmd.Wait(); err != nil { + return err + } + + initCmd := exec.CommandContext( + ctx, + binaryPath, + "init", + genesisDstPath+"/genesis.json", + fmt.Sprintf("--datadir=%s", eth1Path)) // #nosec G204 -- Safe + initFile, err := helpers.DeleteAndCreateFile(e2e.TestParams.LogPath, "eth1-init_miner.log") + if err != nil { + return err + } + initCmd.Stderr = initFile + if err = initCmd.Start(); err != nil { + return err + } + if err = initCmd.Wait(); err != nil { + return err + } + + args := []string{ + fmt.Sprintf("--datadir=%s", eth1Path), + fmt.Sprintf("--http.port=%d", e2e.TestParams.Ports.Eth1RPCPort), + fmt.Sprintf("--ws.port=%d", e2e.TestParams.Ports.Eth1WSPort), + fmt.Sprintf("--bootnodes=%s", m.bootstrapEnr), + fmt.Sprintf("--port=%d", e2e.TestParams.Ports.Eth1Port), + fmt.Sprintf("--networkid=%d", NetworkId), + "--http", + "--http.addr=127.0.0.1", + "--http.corsdomain=\"*\"", + "--http.vhosts=\"*\"", + "--rpc.allow-unprotected-txs", + "--ws", + "--ws.addr=127.0.0.1", + "--ws.origins=\"*\"", + "--ipcdisable", + "--verbosity=4", + "--mine", + "--unlock=0x878705ba3f8bc32fcf7f4caa1a35e72af65cf766", + "--allow-insecure-unlock", + fmt.Sprintf("--password=%s", eth1Path+"/keystore/"+minerPasswordFile), + } + + keystorePath, err := bazel.Runfile(path.Join(staticFilesPath, minerFile)) + if err != nil { + return err + } + jsonBytes, err := ioutil.ReadFile(keystorePath) // #nosec G304 -- ReadFile is safe + if err != nil { + return err + } + err = io.WriteFile(eth1Path+"/keystore/"+minerFile, jsonBytes) + if err != nil { + return err + } + err = io.WriteFile(eth1Path+"/keystore/"+minerPasswordFile, []byte(KeystorePassword)) + if err != nil { + return err + } + + runCmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 -- Safe + file, err := helpers.DeleteAndCreateFile(e2e.TestParams.LogPath, "eth1_miner.log") + if err != nil { + return err + } + runCmd.Stdout = file + runCmd.Stderr = file + log.Infof("Starting eth1 miner with flags: %s", strings.Join(args[2:], " ")) + + if err = runCmd.Start(); err != nil { + return fmt.Errorf("failed to start eth1 chain: %w", err) + } + + if err = helpers.WaitForTextInFile(file, "Commit new mining work"); err != nil { + return fmt.Errorf("mining log not found, this means the eth1 chain had issues starting: %w", err) + } + if err = helpers.WaitForTextInFile(file, "Started P2P networking"); err != nil { + return fmt.Errorf("P2P log not found, this means the eth1 chain had issues starting: %w", err) + } + + enode, err := enodeFromLogFile(file.Name()) + if err != nil { + return err + } + enode = "enode://" + enode + "@127.0.0.1:" + fmt.Sprintf("%d", e2e.TestParams.Ports.Eth1Port) + m.enr = enode + log.Infof("Communicated enode. Enode is %s", enode) + + // Connect to the started geth dev chain. + client, err := rpc.DialHTTP(fmt.Sprintf("http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1RPCPort)) + if err != nil { + return fmt.Errorf("failed to connect to ipc: %w", err) + } + web3 := ethclient.NewClient(client) + + // Deploy the contract. + store, err := keystore.DecryptKey(jsonBytes, KeystorePassword) + if err != nil { + return err + } + // Advancing the blocks eth1follow distance to prevent issues reading the chain. + if err = WaitForBlocks(web3, store, params.BeaconConfig().Eth1FollowDistance); err != nil { + return fmt.Errorf("unable to advance chain: %w", err) + } + txOpts, err := bind.NewTransactorWithChainID(bytes.NewReader(jsonBytes), KeystorePassword, big.NewInt(NetworkId)) + if err != nil { + return err + } + nonce, err := web3.PendingNonceAt(ctx, store.Address) + if err != nil { + return err + } + txOpts.Nonce = big.NewInt(int64(nonce)) + txOpts.Context = ctx + contractAddr, tx, _, err := contracts.DeployDepositContract(txOpts, web3) + if err != nil { + return fmt.Errorf("failed to deploy deposit contract: %w", err) + } + e2e.TestParams.ContractAddress = contractAddr + + // Wait for contract to mine. + for pending := true; pending; _, pending, err = web3.TransactionByHash(ctx, tx.Hash()) { + if err != nil { + return err + } + time.Sleep(timeGapPerTX) + } + + // Advancing the blocks another eth1follow distance to prevent issues reading the chain. + if err = WaitForBlocks(web3, store, params.BeaconConfig().Eth1FollowDistance); err != nil { + return fmt.Errorf("unable to advance chain: %w", err) + } + + // Save keystore path (used for saving and mining deposits). + m.keystorePath = keystorePath + + // Mark node as ready. + close(m.started) + + return runCmd.Wait() +} + +// Started checks whether ETH1 node is started and ready to be queried. +func (m *Miner) Started() <-chan struct{} { + return m.started +} + +func enodeFromLogFile(name string) (string, error) { + byteContent, err := ioutil.ReadFile(name) // #nosec G304 + if err != nil { + return "", err + } + contents := string(byteContent) + + searchText := "self=enode://" + startIdx := strings.Index(contents, searchText) + if startIdx == -1 { + return "", fmt.Errorf("did not find ENR text in %s", contents) + } + startIdx += len(searchText) + endIdx := strings.Index(contents[startIdx:], "@") + if endIdx == -1 { + return "", fmt.Errorf("did not find ENR text in %s", contents) + } + enode := contents[startIdx : startIdx+endIdx] + return strings.TrimPrefix(enode, "-"), nil +} diff --git a/testing/endtoend/components/eth1/node.go b/testing/endtoend/components/eth1/node.go new file mode 100644 index 0000000000..1f6932eefd --- /dev/null +++ b/testing/endtoend/components/eth1/node.go @@ -0,0 +1,112 @@ +package eth1 + +import ( + "context" + "fmt" + "os" + "os/exec" + "path" + "strconv" + "strings" + + "github.com/bazelbuild/rules_go/go/tools/bazel" + "github.com/pkg/errors" + "github.com/prysmaticlabs/prysm/testing/endtoend/helpers" + e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params" + e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types" + log "github.com/sirupsen/logrus" +) + +// Node represents an ETH1 node. +type Node struct { + e2etypes.ComponentRunner + started chan struct{} + index int + enr string +} + +// NewNode creates and returns ETH1 node. +func NewNode(index int, enr string) *Node { + return &Node{ + started: make(chan struct{}, 1), + index: index, + enr: enr, + } +} + +// Start runs a non-mining ETH1 node. +// To connect to a miner and start working properly, this node should be a part of a NodeSet. +func (node *Node) Start(ctx context.Context) error { + binaryPath, found := bazel.FindBinary("cmd/geth", "geth") + if !found { + return errors.New("go-ethereum binary not found") + } + + eth1Path := path.Join(e2e.TestParams.TestPath, "eth1data/"+strconv.Itoa(node.index)+"/") + // Clear out potentially existing dir to prevent issues. + if _, err := os.Stat(eth1Path); !os.IsNotExist(err) { + if err = os.RemoveAll(eth1Path); err != nil { + return err + } + } + + initCmd := exec.CommandContext( + ctx, + binaryPath, + "init", + binaryPath[:strings.LastIndex(binaryPath, "/")]+"/genesis.json", + fmt.Sprintf("--datadir=%s", eth1Path)) // #nosec G204 -- Safe + initFile, err := helpers.DeleteAndCreateFile(e2e.TestParams.LogPath, "eth1-init_"+strconv.Itoa(node.index)+".log") + if err != nil { + return err + } + initCmd.Stderr = initFile + if err = initCmd.Start(); err != nil { + return err + } + if err = initCmd.Wait(); err != nil { + return err + } + + args := []string{ + fmt.Sprintf("--datadir=%s", eth1Path), + fmt.Sprintf("--http.port=%d", e2e.TestParams.Ports.Eth1RPCPort+node.index), + fmt.Sprintf("--ws.port=%d", e2e.TestParams.Ports.Eth1WSPort+node.index), + fmt.Sprintf("--bootnodes=%s", node.enr), + fmt.Sprintf("--port=%d", e2e.TestParams.Ports.Eth1Port+node.index), + fmt.Sprintf("--networkid=%d", NetworkId), + "--http", + "--http.addr=127.0.0.1", + "--http.corsdomain=\"*\"", + "--http.vhosts=\"*\"", + "--rpc.allow-unprotected-txs", + "--ws", + "--ws.addr=127.0.0.1", + "--ws.origins=\"*\"", + "--ipcdisable", + "--verbosity=4", + } + + runCmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 -- Safe + file, err := helpers.DeleteAndCreateFile(e2e.TestParams.LogPath, "eth1_"+strconv.Itoa(node.index)+".log") + if err != nil { + return err + } + runCmd.Stdout = file + runCmd.Stderr = file + log.Infof("Starting eth1 node %d with flags: %s", node.index, strings.Join(args[2:], " ")) + + if err = runCmd.Start(); err != nil { + return fmt.Errorf("failed to start eth1 chain: %w", err) + } + + // Mark node as ready. + close(node.started) + + return runCmd.Wait() +} + +// Started checks whether ETH1 node is started and ready to be queried. +func (node *Node) Started() <-chan struct{} { + return node.started +} diff --git a/testing/endtoend/components/eth1/node_set.go b/testing/endtoend/components/eth1/node_set.go new file mode 100644 index 0000000000..509a845494 --- /dev/null +++ b/testing/endtoend/components/eth1/node_set.go @@ -0,0 +1,54 @@ +package eth1 + +import ( + "context" + + "github.com/prysmaticlabs/prysm/testing/endtoend/helpers" + e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params" + e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types" +) + +// NodeSet represents a set of Eth1 nodes, none of which is a mining node. +type NodeSet struct { + e2etypes.ComponentRunner + started chan struct{} + enr string +} + +// NewNodeSet creates and returns a set of Eth1 nodes. +func NewNodeSet() *NodeSet { + return &NodeSet{ + started: make(chan struct{}, 1), + } +} + +// SetMinerENR sets the miner's enode, used to connect to the miner through P2P. +func (s *NodeSet) SetMinerENR(enr string) { + s.enr = enr +} + +// Start starts all the beacon nodes in set. +func (s *NodeSet) Start(ctx context.Context) error { + // Create Eth1 nodes. The number of nodes is the same as the number of beacon nodes. + // We want each beacon node to connect to its own Eth1 node. + // We start up one Eth1 node less than the beacon node count because the first + // beacon node will connect to the already existing Eth1 miner. + nodes := make([]e2etypes.ComponentRunner, e2e.TestParams.BeaconNodeCount-1) + for i := 0; i < e2e.TestParams.BeaconNodeCount-1; i++ { + // We start indexing nodes from 1 because the miner has an implicit 0 index. + node := NewNode(i+1, s.enr) + nodes[i] = node + } + + // Wait for all nodes to finish their job (blocking). + // Once nodes are ready passed in handler function will be called. + return helpers.WaitOnNodes(ctx, nodes, func() { + // All nodes started, close channel, so that all services waiting on a set, can proceed. + close(s.started) + }) +} + +// Started checks whether beacon node set is started and all nodes are ready to be queried. +func (s *NodeSet) Started() <-chan struct{} { + return s.started +} diff --git a/testing/endtoend/components/tracing_sink.go b/testing/endtoend/components/tracing_sink.go index ac856a9230..09d2ea8e6a 100644 --- a/testing/endtoend/components/tracing_sink.go +++ b/testing/endtoend/components/tracing_sink.go @@ -10,6 +10,7 @@ import ( "os/signal" "syscall" + "github.com/pkg/errors" "github.com/prysmaticlabs/prysm/testing/endtoend/helpers" e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params" ) @@ -40,6 +41,9 @@ func NewTracingSink(endpoint string) *TracingSink { // Start the tracing sink. func (ts *TracingSink) Start(_ context.Context) error { + if ts.endpoint == "" { + return errors.New("empty endpoint provided") + } go ts.initializeSink() close(ts.started) return nil diff --git a/testing/endtoend/components/validator.go b/testing/endtoend/components/validator.go index 2e05d0c0c2..886219958c 100644 --- a/testing/endtoend/components/validator.go +++ b/testing/endtoend/components/validator.go @@ -25,6 +25,7 @@ import ( contracts "github.com/prysmaticlabs/prysm/contracts/deposit" "github.com/prysmaticlabs/prysm/encoding/bytesutil" "github.com/prysmaticlabs/prysm/runtime/interop" + "github.com/prysmaticlabs/prysm/testing/endtoend/components/eth1" "github.com/prysmaticlabs/prysm/testing/endtoend/helpers" e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params" e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types" @@ -229,11 +230,11 @@ func SendAndMineDeposits(keystorePath string, validatorNum, offset int, partial if err = sendDeposits(web3, keystoreBytes, validatorNum, offset, partial); err != nil { return err } - mineKey, err := keystore.DecryptKey(keystoreBytes, "" /*password*/) + mineKey, err := keystore.DecryptKey(keystoreBytes, eth1.KeystorePassword) if err != nil { return err } - if err = mineBlocks(web3, mineKey, params.BeaconConfig().Eth1FollowDistance); err != nil { + if err = eth1.WaitForBlocks(web3, mineKey, params.BeaconConfig().Eth1FollowDistance); err != nil { return fmt.Errorf("failed to mine blocks %w", err) } return nil @@ -241,7 +242,7 @@ func SendAndMineDeposits(keystorePath string, validatorNum, offset int, partial // sendDeposits uses the passed in web3 and keystore bytes to send the requested deposits. func sendDeposits(web3 *ethclient.Client, keystoreBytes []byte, num, offset int, partial bool) error { - txOps, err := bind.NewTransactorWithChainID(bytes.NewReader(keystoreBytes), "" /*password*/, big.NewInt(1337)) + txOps, err := bind.NewTransactorWithChainID(bytes.NewReader(keystoreBytes), eth1.KeystorePassword, big.NewInt(eth1.NetworkId)) if err != nil { return err } diff --git a/testing/endtoend/endtoend_test.go b/testing/endtoend/endtoend_test.go index 8088ef1570..1efa466623 100644 --- a/testing/endtoend/endtoend_test.go +++ b/testing/endtoend/endtoend_test.go @@ -21,6 +21,7 @@ import ( eth "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1" "github.com/prysmaticlabs/prysm/testing/assert" "github.com/prysmaticlabs/prysm/testing/endtoend/components" + "github.com/prysmaticlabs/prysm/testing/endtoend/components/eth1" ev "github.com/prysmaticlabs/prysm/testing/endtoend/evaluators" "github.com/prysmaticlabs/prysm/testing/endtoend/helpers" e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params" @@ -88,24 +89,6 @@ func (r *testRunner) run() { }) } - // ETH1 node. - eth1Node := components.NewEth1Node() - g.Go(func() error { - if err := eth1Node.Start(ctx); err != nil { - return errors.Wrap(err, "failed to start eth1node") - } - return nil - }) - g.Go(func() error { - if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Node}); err != nil { - return errors.Wrap(err, "sending and mining deposits require ETH1 node to run") - } - if err := components.SendAndMineDeposits(eth1Node.KeystorePath(), minGenesisActiveCount, 0, true /* partial */); err != nil { - return errors.Wrap(err, "failed to send and mine deposits") - } - return nil - }) - // Boot node. bootNode := components.NewBootNode() g.Go(func() error { @@ -114,10 +97,46 @@ func (r *testRunner) run() { } return nil }) + + // ETH1 miner. + eth1Miner := eth1.NewMiner() + g.Go(func() error { + if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{bootNode}); err != nil { + return errors.Wrap(err, "sending and mining deposits require ETH1 nodes to run") + } + eth1Miner.SetBootstrapENR(bootNode.ENR()) + if err := eth1Miner.Start(ctx); err != nil { + return errors.Wrap(err, "failed to start the ETH1 miner") + } + return nil + }) + + // ETH1 non-mining nodes. + eth1Nodes := eth1.NewNodeSet() + g.Go(func() error { + if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Miner}); err != nil { + return errors.Wrap(err, "sending and mining deposits require ETH1 nodes to run") + } + eth1Nodes.SetMinerENR(eth1Miner.ENR()) + if err := eth1Nodes.Start(ctx); err != nil { + return errors.Wrap(err, "failed to start ETH1 nodes") + } + return nil + }) + g.Go(func() error { + if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Nodes}); err != nil { + return errors.Wrap(err, "sending and mining deposits require ETH1 nodes to run") + } + if err := components.SendAndMineDeposits(eth1Miner.KeystorePath(), minGenesisActiveCount, 0, true /* partial */); err != nil { + return errors.Wrap(err, "failed to send and mine deposits") + } + return nil + }) + // Beacon nodes. beaconNodes := components.NewBeaconNodes(config) g.Go(func() error { - if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Node, bootNode}); err != nil { + if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Nodes, bootNode}); err != nil { return errors.Wrap(err, "beacon nodes require ETH1 and boot node to run") } beaconNodes.SetENR(bootNode.ENR()) @@ -142,7 +161,7 @@ func (r *testRunner) run() { if multiClientActive { lighthouseNodes = components.NewLighthouseBeaconNodes(config) g.Go(func() error { - if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Node, bootNode, beaconNodes}); err != nil { + if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Nodes, bootNode, beaconNodes}); err != nil { return errors.Wrap(err, "lighthouse beacon nodes require ETH1 and boot node to run") } lighthouseNodes.SetENR(bootNode.ENR()) @@ -192,7 +211,7 @@ func (r *testRunner) run() { // Wait for all required nodes to start. requiredComponents := []e2etypes.ComponentRunner{ - tracingSink, eth1Node, bootNode, beaconNodes, validatorNodes, + tracingSink, eth1Nodes, bootNode, beaconNodes, validatorNodes, } if multiClientActive { requiredComponents = append(requiredComponents, []e2etypes.ComponentRunner{keyGen, lighthouseNodes, lighthouseValidatorNodes}...) @@ -224,7 +243,7 @@ func (r *testRunner) run() { if config.TestDeposits { log.Info("Running deposit tests") - r.testDeposits(ctx, g, eth1Node, []e2etypes.ComponentRunner{beaconNodes}) + r.testDeposits(ctx, g, eth1Miner.KeystorePath(), []e2etypes.ComponentRunner{beaconNodes}) } // Create GRPC connection to beacon nodes. @@ -247,7 +266,7 @@ func (r *testRunner) run() { if !config.TestSync { return nil } - if err := r.testBeaconChainSync(ctx, g, conns, tickingStartTime, bootNode.ENR()); err != nil { + if err := r.testBeaconChainSync(ctx, g, conns, tickingStartTime, bootNode.ENR(), eth1Miner.ENR()); err != nil { return errors.Wrap(err, "beacon chain sync test failed") } if err := r.testDoppelGangerProtection(ctx); err != nil { @@ -314,7 +333,7 @@ func (r *testRunner) runEvaluators(conns []*grpc.ClientConn, tickingStartTime ti // testDeposits runs tests when config.TestDeposits is enabled. func (r *testRunner) testDeposits(ctx context.Context, g *errgroup.Group, - eth1Node *components.Eth1Node, requiredNodes []e2etypes.ComponentRunner) { + keystorePath string, requiredNodes []e2etypes.ComponentRunner) { minGenesisActiveCount := int(params.BeaconConfig().MinGenesisActiveValidatorCount) depositCheckValidator := components.NewValidatorNode(r.config, int(e2e.DepositCount), e2e.TestParams.BeaconNodeCount, minGenesisActiveCount) @@ -323,7 +342,7 @@ func (r *testRunner) testDeposits(ctx context.Context, g *errgroup.Group, return fmt.Errorf("deposit check validator node requires beacon nodes to run: %w", err) } go func() { - err := components.SendAndMineDeposits(eth1Node.KeystorePath(), int(e2e.DepositCount), minGenesisActiveCount, false /* partial */) + err := components.SendAndMineDeposits(keystorePath, int(e2e.DepositCount), minGenesisActiveCount, false /* partial */) if err != nil { r.t.Fatal(err) } @@ -334,14 +353,18 @@ func (r *testRunner) testDeposits(ctx context.Context, g *errgroup.Group, // testBeaconChainSync creates another beacon node, and tests whether it can sync to head using previous nodes. func (r *testRunner) testBeaconChainSync(ctx context.Context, g *errgroup.Group, - conns []*grpc.ClientConn, tickingStartTime time.Time, enr string) error { + conns []*grpc.ClientConn, tickingStartTime time.Time, bootnodeEnr, minerEnr string) error { t, config := r.t, r.config index := e2e.TestParams.BeaconNodeCount - syncBeaconNode := components.NewBeaconNode(config, index, enr) + ethNode := eth1.NewNode(index, minerEnr) + g.Go(func() error { + return ethNode.Start(ctx) + }) + syncBeaconNode := components.NewBeaconNode(config, index, bootnodeEnr) g.Go(func() error { return syncBeaconNode.Start(ctx) }) - if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{syncBeaconNode}); err != nil { + if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{ethNode, syncBeaconNode}); err != nil { return fmt.Errorf("sync beacon node not ready: %w", err) } syncConn, err := grpc.Dial(fmt.Sprintf("127.0.0.1:%d", e2e.TestParams.Ports.PrysmBeaconNodeRPCPort+index), grpc.WithInsecure()) diff --git a/testing/endtoend/mainnet_e2e_test.go b/testing/endtoend/mainnet_e2e_test.go index ad13cacf8a..bd6286262a 100644 --- a/testing/endtoend/mainnet_e2e_test.go +++ b/testing/endtoend/mainnet_e2e_test.go @@ -36,7 +36,7 @@ func e2eMainnet(t *testing.T, usePrysmSh bool) { // TODO(#9166): remove this block once v2 changes are live. epochsToRun = helpers.AltairE2EForkEpoch - 1 } - tracingPort := 9411 + e2eParams.TestParams.TestShardIndex + tracingPort := e2eParams.TestParams.Ports.JaegerTracingPort tracingEndpoint := fmt.Sprintf("127.0.0.1:%d", tracingPort) evals := []types.Evaluator{ ev.PeersConnect, diff --git a/testing/endtoend/minimal_e2e_test.go b/testing/endtoend/minimal_e2e_test.go index 817616a467..31a53a95c4 100644 --- a/testing/endtoend/minimal_e2e_test.go +++ b/testing/endtoend/minimal_e2e_test.go @@ -59,7 +59,7 @@ func e2eMinimal(t *testing.T, args *testArgs) { // TODO(#9166): remove this block once v2 changes are live. epochsToRun = helpers.AltairE2EForkEpoch - 1 } - tracingPort := 9411 + e2eParams.TestParams.TestShardIndex + tracingPort := e2eParams.TestParams.Ports.JaegerTracingPort tracingEndpoint := fmt.Sprintf("127.0.0.1:%d", tracingPort) evals := []types.Evaluator{ ev.PeersConnect, diff --git a/testing/endtoend/minimal_slashing_e2e_test.go b/testing/endtoend/minimal_slashing_e2e_test.go index bdac53a4d3..7aa1fc9cd6 100644 --- a/testing/endtoend/minimal_slashing_e2e_test.go +++ b/testing/endtoend/minimal_slashing_e2e_test.go @@ -1,6 +1,7 @@ package endtoend import ( + "fmt" "testing" "github.com/prysmaticlabs/prysm/config/params" @@ -14,6 +15,9 @@ func TestEndToEnd_Slasher_MinimalConfig(t *testing.T) { params.UseE2EConfig() require.NoError(t, e2eParams.Init(e2eParams.StandardBeaconCount)) + tracingPort := e2eParams.TestParams.Ports.JaegerTracingPort + tracingEndpoint := fmt.Sprintf("127.0.0.1:%d", tracingPort) + testConfig := &types.E2EConfig{ BeaconFlags: []string{ "--slasher", @@ -30,6 +34,7 @@ func TestEndToEnd_Slasher_MinimalConfig(t *testing.T) { ev.InjectDoubleVoteOnEpoch(2), ev.InjectDoubleBlockOnEpoch(2), }, + TracingSinkEndpoint: tracingEndpoint, } newTestRunner(t, testConfig).run() diff --git a/testing/endtoend/params/params.go b/testing/endtoend/params/params.go index e218c8ad6a..b453e7b582 100644 --- a/testing/endtoend/params/params.go +++ b/testing/endtoend/params/params.go @@ -27,6 +27,7 @@ type params struct { type ports struct { BootNodePort int BootNodeMetricsPort int + Eth1Port int Eth1RPCPort int Eth1WSPort int PrysmBeaconNodeRPCPort int @@ -40,6 +41,7 @@ type ports struct { LighthouseBeaconNodeMetricsPort int ValidatorMetricsPort int ValidatorGatewayPort int + JaegerTracingPort int } // TestParams is the globally accessible var for getting config elements. @@ -73,8 +75,9 @@ const ( BootNodePort = 2150 BootNodeMetricsPort = BootNodePort + portSpan - Eth1RPCPort = 3150 - Eth1WSPort = Eth1RPCPort + portSpan + Eth1Port = 3150 + Eth1RPCPort = Eth1Port + portSpan + Eth1WSPort = Eth1Port + 2*portSpan PrysmBeaconNodeRPCPort = 4150 PrysmBeaconNodeUDPPort = PrysmBeaconNodeRPCPort + portSpan @@ -89,6 +92,8 @@ const ( ValidatorGatewayPort = 6150 ValidatorMetricsPort = ValidatorGatewayPort + portSpan + + JaegerTracingPort = 9150 ) // Init initializes the E2E config, properly handling test sharding. @@ -124,6 +129,10 @@ func Init(beaconNodeCount int) error { if err != nil { return err } + eth1Port, err := port(Eth1Port, testTotalShards, testShardIndex, &existingRegistrations) + if err != nil { + return err + } eth1RPCPort, err := port(Eth1RPCPort, testTotalShards, testShardIndex, &existingRegistrations) if err != nil { return err @@ -164,9 +173,14 @@ func Init(beaconNodeCount int) error { if err != nil { return err } + jaegerTracingPort, err := port(JaegerTracingPort, testTotalShards, testShardIndex, &existingRegistrations) + if err != nil { + return err + } testPorts := &ports{ BootNodePort: bootnodePort, BootNodeMetricsPort: bootnodeMetricsPort, + Eth1Port: eth1Port, Eth1RPCPort: eth1RPCPort, Eth1WSPort: eth1WSPort, PrysmBeaconNodeRPCPort: beaconNodeRPCPort, @@ -177,6 +191,7 @@ func Init(beaconNodeCount int) error { PrysmBeaconNodePprofPort: beaconNodePprofPort, ValidatorMetricsPort: validatorMetricsPort, ValidatorGatewayPort: validatorGatewayPort, + JaegerTracingPort: jaegerTracingPort, } TestParams = ¶ms{ @@ -222,6 +237,10 @@ func InitMultiClient(beaconNodeCount int, lighthouseNodeCount int) error { if err != nil { return err } + eth1Port, err := port(Eth1Port, testTotalShards, testShardIndex, &existingRegistrations) + if err != nil { + return err + } eth1RPCPort, err := port(Eth1RPCPort, testTotalShards, testShardIndex, &existingRegistrations) if err != nil { return err @@ -277,6 +296,7 @@ func InitMultiClient(beaconNodeCount int, lighthouseNodeCount int) error { testPorts := &ports{ BootNodePort: bootnodePort, BootNodeMetricsPort: bootnodeMetricsPort, + Eth1Port: eth1Port, Eth1RPCPort: eth1RPCPort, Eth1WSPort: eth1WSPort, PrysmBeaconNodeRPCPort: prysmBeaconNodeRPCPort, diff --git a/testing/endtoend/static-files/eth1/BUILD.bazel b/testing/endtoend/static-files/eth1/BUILD.bazel new file mode 100644 index 0000000000..68a4d35fd2 --- /dev/null +++ b/testing/endtoend/static-files/eth1/BUILD.bazel @@ -0,0 +1,8 @@ +filegroup( + name = "eth1data", + srcs = [ + "UTC--2021-12-22T19-14-08.590377700Z--878705ba3f8bc32fcf7f4caa1a35e72af65cf766", + "genesis.json", + ], + visibility = ["//testing/endtoend:__subpackages__"], +) diff --git a/testing/endtoend/static-files/eth1/UTC--2021-12-22T19-14-08.590377700Z--878705ba3f8bc32fcf7f4caa1a35e72af65cf766 b/testing/endtoend/static-files/eth1/UTC--2021-12-22T19-14-08.590377700Z--878705ba3f8bc32fcf7f4caa1a35e72af65cf766 new file mode 100644 index 0000000000..a451203cc8 --- /dev/null +++ b/testing/endtoend/static-files/eth1/UTC--2021-12-22T19-14-08.590377700Z--878705ba3f8bc32fcf7f4caa1a35e72af65cf766 @@ -0,0 +1 @@ +{"address":"878705ba3f8bc32fcf7f4caa1a35e72af65cf766","crypto":{"cipher":"aes-128-ctr","ciphertext":"f02daebbf456faf787c5cd61a33ce780857c1ca10b00972aa451f0e9688e4ead","cipherparams":{"iv":"ef1668814155862f0653f15dae845e58"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"55e5ee70d3e882d2f00a073eda252ff01437abf51d7bfa76c06dcc73f7e8f1a3"},"mac":"d8d04625d0769fe286756734f946c78663961b74f0caaff1d768f0d255632f04"},"id":"5fb9083a-a221-412b-b0e0-921e22cc9645","version":3} \ No newline at end of file diff --git a/testing/endtoend/static-files/eth1/genesis.json b/testing/endtoend/static-files/eth1/genesis.json new file mode 100644 index 0000000000..3abbb24503 --- /dev/null +++ b/testing/endtoend/static-files/eth1/genesis.json @@ -0,0 +1,30 @@ +{ + "config": { + "chainId": 1337, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "clique": { + "period": 2, + "epoch": 30000 + } + }, + "alloc": { + "0x878705ba3f8bc32fcf7f4caa1a35e72af65cf766": {"balance": "100000000000000000000000000000"} + }, + "coinbase" : "0x0000000000000000000000000000000000000000", + "difficulty": "1", + "extradata": "0x0000000000000000000000000000000000000000000000000000000000000000878705ba3f8bc32fcf7f4caa1a35e72af65cf7660000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit" : "0xffffff", + "nonce" : "0x0000000000000042", + "mixhash" : "0x0000000000000000000000000000000000000000000000000000000000000000", + "parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000", + "timestamp" : "0x00" +} \ No newline at end of file