Improve E2E to be more consistent with timing, and allow for custom flags (#4620)

* Add committees helper, benchmark, results show 62ms for 8k validators which was previously 4 minutes

* Add regression test with same data

* fix epoch conversion

* lint

* undo and lint

* Begin work on adding mainnet config benchmark

* Try more to get mainnet e2e

* Try to fix delay

* Get past chainstart on e2e

* Try to fix flaky

* Get demo config working

* Remove unneeded changes

* Change how flags are enabled

* Lower shard count

* Temp skip

* Fix e2e

* Fix testing to run until last epoch

* Fix

* Add ending time log and remove att cache flag

* Fix ordering

* Reenable flag

* Change ports from default

* Add no log for if there are no err logs

* Add block evaluator

* Try to improve evaluators

* Progress on attestation evaluator

* Remove attestation evaluator

* Fix e2e

Co-authored-by: Preston Van Loon <preston@prysmaticlabs.com>

endtoend/beacon_node.go

@@ -16,6 +16,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/pkg/errors"
 	ev "github.com/prysmaticlabs/prysm/endtoend/evaluators"
+	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 type beaconNodeInfo struct {
@@ -28,12 +29,12 @@ type beaconNodeInfo struct {
 }
 
 type end2EndConfig struct {
-	minimalConfig  bool
+	beaconFlags    []string
+	validatorFlags []string
 	tmpPath        string
 	epochsToRun    uint64
 	numValidators  uint64
 	numBeaconNodes uint64
-	enableSSZCache bool
 	contractAddr   common.Address
 	evaluators     []ev.Evaluator
 }
@@ -69,31 +70,21 @@ func startNewBeaconNode(t *testing.T, config *end2EndConfig, beaconNodes []*beac
 
 	args := []string{
 		"--no-genesis-delay",
-		"--verbosity=debug",
 		"--force-clear-db",
 		"--no-discovery",
-		"--new-cache",
-		"--enable-shuffled-index-cache",
-		"--enable-skip-slots-cache",
-		"--enable-attestation-cache",
-		"--http-web3provider=http://127.0.0.1:8545",
-		"--web3provider=ws://127.0.0.1:8546",
+		"--http-web3provider=http://127.0.0.1:8745",
+		"--web3provider=ws://127.0.0.1:8746",
 		fmt.Sprintf("--datadir=%s/eth2-beacon-node-%d", tmpPath, index),
 		fmt.Sprintf("--deposit-contract=%s", config.contractAddr.Hex()),
-		fmt.Sprintf("--rpc-port=%d", 4000+index),
-		fmt.Sprintf("--p2p-udp-port=%d", 12000+index),
-		fmt.Sprintf("--p2p-tcp-port=%d", 13000+index),
-		fmt.Sprintf("--monitoring-port=%d", 8080+index),
-		fmt.Sprintf("--grpc-gateway-port=%d", 3200+index),
+		fmt.Sprintf("--rpc-port=%d", 4200+index),
+		fmt.Sprintf("--p2p-udp-port=%d", 12200+index),
+		fmt.Sprintf("--p2p-tcp-port=%d", 13200+index),
+		fmt.Sprintf("--monitoring-port=%d", 8280+index),
+		fmt.Sprintf("--grpc-gateway-port=%d", 3400+index),
 		fmt.Sprintf("--contract-deployment-block=%d", 0),
+		fmt.Sprintf("--rpc-max-page-size=%d", params.BeaconConfig().MinGenesisActiveValidatorCount),
 	}
-
-	if config.minimalConfig {
-		args = append(args, "--minimal-config")
-	}
-	if config.enableSSZCache {
-		args = append(args, "--enable-ssz-cache")
-	}
+	args = append(args, config.beaconFlags...)
 
 	// After the first node is made, have all following nodes connect to all previously made nodes.
 	if index >= 1 {
@@ -102,7 +93,7 @@ func startNewBeaconNode(t *testing.T, config *end2EndConfig, beaconNodes []*beac
 		}
 	}
 
-	t.Logf("Starting beacon chain with flags: %s", strings.Join(args, " "))
+	t.Logf("Starting beacon chain %d with flags: %s", index, strings.Join(args, " "))
 	cmd := exec.Command(binaryPath, args...)
 	cmd.Stdout = stdOutFile
 	cmd.Stderr = stdOutFile
@@ -122,9 +113,9 @@ func startNewBeaconNode(t *testing.T, config *end2EndConfig, beaconNodes []*beac
 	return &beaconNodeInfo{
 		processID:   cmd.Process.Pid,
 		datadir:     fmt.Sprintf("%s/eth2-beacon-node-%d", tmpPath, index),
-		rpcPort:     4000 + uint64(index),
-		monitorPort: 8080 + uint64(index),
-		grpcPort:    3200 + uint64(index),
+		rpcPort:     4200 + uint64(index),
+		monitorPort: 8280 + uint64(index),
+		grpcPort:    3400 + uint64(index),
 		multiAddr:   multiAddr,
 	}
 }

endtoend/demo_e2e_test.go

@@ -14,9 +14,15 @@ func TestEndToEnd_DemoConfig(t *testing.T) {
 	params.UseDemoBeaconConfig()
 
 	demoConfig := &end2EndConfig{
-		minimalConfig:  false,
+		beaconFlags: []string{
+			"--enable-ssz-cache",
+			"--cache-proposer-indices",
+			"--cache-filtered-block-tree",
+			"--enable-skip-slots-cache",
+			"--enable-attestation-cache",
+		},
 		epochsToRun:    5,
-		numBeaconNodes: 4,
+		numBeaconNodes: 2,
 		numValidators:  params.BeaconConfig().MinGenesisActiveValidatorCount,
 		evaluators: []ev.Evaluator{
 			ev.ValidatorsAreActive,

endtoend/endtoend_test.go

@@ -55,8 +55,8 @@ func runEndToEndTest(t *testing.T, config *end2EndConfig) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := waitForTextInFile(beaconLogFile, "Sending genesis time notification"); err != nil {
-		t.Fatalf("failed to find genesis in logs, this means the chain did not start: %v", err)
+	if err := waitForTextInFile(beaconLogFile, "Chain started within the last epoch"); err != nil {
+		t.Fatalf("failed to find chain start in logs, this means the chain did not start: %v", err)
 	}
 
 	// Failing early in case chain doesn't start.
@@ -64,7 +64,7 @@ func runEndToEndTest(t *testing.T, config *end2EndConfig) {
 		return
 	}
 
-	conn, err := grpc.Dial("127.0.0.1:4000", grpc.WithInsecure())
+	conn, err := grpc.Dial("127.0.0.1:4200", grpc.WithInsecure())
 	if err != nil {
 		t.Fatalf("Failed to dial: %v", err)
 	}
@@ -78,14 +78,8 @@ func runEndToEndTest(t *testing.T, config *end2EndConfig) {
 	// Small offset so evaluators perform in the middle of an epoch.
 	epochSeconds := params.BeaconConfig().SecondsPerSlot * params.BeaconConfig().SlotsPerEpoch
 	genesisTime := time.Unix(genesis.GenesisTime.Seconds+int64(epochSeconds/2), 0)
-	currentEpoch := uint64(0)
 	ticker := GetEpochTicker(genesisTime, epochSeconds)
-	for c := range ticker.C() {
-		if c >= config.epochsToRun || t.Failed() {
-			ticker.Done()
-			break
-		}
-
+	for currentEpoch := range ticker.C() {
 		for _, evaluator := range config.evaluators {
 			// Only run if the policy says so.
 			if !evaluator.Policy(currentEpoch) {
@@ -93,15 +87,15 @@ func runEndToEndTest(t *testing.T, config *end2EndConfig) {
 			}
 			t.Run(fmt.Sprintf(evaluator.Name, currentEpoch), func(t *testing.T) {
 				if err := evaluator.Evaluation(beaconClient); err != nil {
-					t.Fatalf("evaluation failed for epoch %d: %v", currentEpoch, err)
+					t.Errorf("evaluation failed for epoch %d: %v", currentEpoch, err)
 				}
 			})
 		}
-		currentEpoch++
-	}
 
-	if currentEpoch < config.epochsToRun {
-		t.Fatalf("Test ended prematurely, only reached epoch %d", currentEpoch)
+		if t.Failed() || currentEpoch >= config.epochsToRun {
+			ticker.Done()
+			break
+		}
 	}
 }
 
@@ -151,22 +145,21 @@ func killProcesses(t *testing.T, pIDs []int) {
 }
 
 func logOutput(t *testing.T, tmpPath string, config *end2EndConfig) {
-	if t.Failed() {
-		// Log out errors from beacon chain nodes.
-		for i := uint64(0); i < config.numBeaconNodes; i++ {
-			beaconLogFile, err := os.Open(path.Join(tmpPath, fmt.Sprintf(beaconNodeLogFileName, i)))
-			if err != nil {
-				t.Fatal(err)
-			}
-			logErrorOutput(t, beaconLogFile, "beacon chain node", i)
-
-			validatorLogFile, err := os.Open(path.Join(tmpPath, fmt.Sprintf(validatorLogFileName, i)))
-			if err != nil {
-				t.Fatal(err)
-			}
-			logErrorOutput(t, validatorLogFile, "validator client", i)
+	// Log out errors from beacon chain nodes.
+	for i := uint64(0); i < config.numBeaconNodes; i++ {
+		beaconLogFile, err := os.Open(path.Join(tmpPath, fmt.Sprintf(beaconNodeLogFileName, i)))
+		if err != nil {
+			t.Fatal(err)
 		}
+		logErrorOutput(t, beaconLogFile, "beacon chain node", i)
+
+		validatorLogFile, err := os.Open(path.Join(tmpPath, fmt.Sprintf(validatorLogFileName, i)))
+		if err != nil {
+			t.Fatal(err)
+		}
+		logErrorOutput(t, validatorLogFile, "validator client", i)
 	}
+	t.Logf("Ending time: %s\n", time.Now().String())
 }
 
 func logErrorOutput(t *testing.T, file *os.File, title string, index uint64) {
@@ -181,6 +174,7 @@ func logErrorOutput(t *testing.T, file *os.File, title string, index uint64) {
 	}
 
 	if len(errorLines) < 1 {
+		t.Logf("No error logs detected for %s %d", title, index)
 		return
 	}
 

endtoend/eth1.go

@@ -45,9 +45,11 @@ func startEth1(t *testing.T, tmpPath string) (common.Address, string, int) {
 		"--rpcaddr=0.0.0.0",
 		"--rpccorsdomain=\"*\"",
 		"--rpcvhosts=\"*\"",
+		"--rpcport=8745",
 		"--ws",
 		"--wsaddr=0.0.0.0",
 		"--wsorigins=\"*\"",
+		"--wsport=8746",
 		"--dev",
 		"--dev.period=0",
 		"--ipcdisable",
@@ -68,7 +70,7 @@ func startEth1(t *testing.T, tmpPath string) (common.Address, string, int) {
 	}
 
 	// Connect to the started geth dev chain.
-	client, err := rpc.DialHTTP("http://127.0.0.1:8545")
+	client, err := rpc.DialHTTP("http://127.0.0.1:8745")
 	if err != nil {
 		t.Fatalf("Failed to connect to ipc: %v", err)
 	}

endtoend/evaluators/validator.go

@@ -44,7 +44,9 @@ func afterNthEpoch(afterEpoch uint64) func(uint64) bool {
 
 func validatorsAreActive(client eth.BeaconChainClient) error {
 	// Balances actually fluctuate but we just want to check initial balance.
-	validatorRequest := &eth.ListValidatorsRequest{}
+	validatorRequest := &eth.ListValidatorsRequest{
+		PageSize: int32(params.BeaconConfig().MinGenesisActiveValidatorCount),
+	}
 	validators, err := client.ListValidators(context.Background(), validatorRequest)
 	if err != nil {
 		return errors.Wrap(err, "failed to get validators")
@@ -56,24 +58,39 @@ func validatorsAreActive(client eth.BeaconChainClient) error {
 		return fmt.Errorf("expected validator count to be %d, recevied %d", expectedCount, receivedCount)
 	}
 
+	effBalanceLowCount := 0
+	activeEpochWrongCount := 0
+	exitEpochWrongCount := 0
+	withdrawEpochWrongCount := 0
 	for _, item := range validators.ValidatorList {
+		if item.Validator.EffectiveBalance < params.BeaconConfig().MaxEffectiveBalance {
+			effBalanceLowCount++
+		}
 		if item.Validator.ActivationEpoch != 0 {
-			return fmt.Errorf("expected genesis validator epoch to be 0, received %d", item.Validator.ActivationEpoch)
+			activeEpochWrongCount++
 		}
 		if item.Validator.ExitEpoch != params.BeaconConfig().FarFutureEpoch {
-			return fmt.Errorf("expected genesis validator exit epoch to be far future, received %d", item.Validator.ExitEpoch)
+			exitEpochWrongCount++
 		}
 		if item.Validator.WithdrawableEpoch != params.BeaconConfig().FarFutureEpoch {
-			return fmt.Errorf("expected genesis validator withdrawable epoch to be far future, received %d", item.Validator.WithdrawableEpoch)
-		}
-		if item.Validator.EffectiveBalance != params.BeaconConfig().MaxEffectiveBalance {
-			return fmt.Errorf(
-				"expected genesis validator effective balance to be %d, received %d",
-				params.BeaconConfig().MaxEffectiveBalance,
-				item.Validator.EffectiveBalance,
-			)
+			withdrawEpochWrongCount++
 		}
 	}
+
+	if effBalanceLowCount > 0 {
+		return fmt.Errorf(
+			"%d validators did not have genesis validator effective balance of %d",
+			effBalanceLowCount,
+			params.BeaconConfig().MaxEffectiveBalance,
+		)
+	} else if activeEpochWrongCount > 0 {
+		return fmt.Errorf("%d validators did not have genesis validator epoch of 0", activeEpochWrongCount)
+	} else if exitEpochWrongCount > 0 {
+		return fmt.Errorf("%d validators did not have genesis validator exit epoch of far future epoch", exitEpochWrongCount)
+	} else if activeEpochWrongCount > 0 {
+		return fmt.Errorf("%d validators did not have genesis validator withdrawable epoch of far future epoch", activeEpochWrongCount)
+	}
+
 	return nil
 }
 

endtoend/minimal_e2e_test.go

@@ -13,10 +13,19 @@ func TestEndToEnd_MinimalConfig(t *testing.T) {
 	params.UseMinimalConfig()
 
 	minimalConfig := &end2EndConfig{
-		minimalConfig:  true,
+		beaconFlags:  []string{
+			"--minimal-config",
+			"--enable-ssz-cache",
+			"--cache-proposer-indices",
+			"--cache-filtered-block-tree",
+			"--enable-skip-slots-cache",
+			"--enable-attestation-cache",
+		},
+		validatorFlags:  []string{
+			"--minimal-config",
+		},
 		epochsToRun:    5,
 		numBeaconNodes: 4,
-		enableSSZCache: true,
 		numValidators:  params.BeaconConfig().MinGenesisActiveValidatorCount,
 		evaluators: []ev.Evaluator{
 			ev.ValidatorsAreActive,

endtoend/validator.go

@@ -58,27 +58,26 @@ func initializeValidators(
 			"--force-clear-db",
 			fmt.Sprintf("--interop-num-validators=%d", validatorsPerNode),
 			fmt.Sprintf("--interop-start-index=%d", validatorsPerNode*n),
-			fmt.Sprintf("--monitoring-port=%d", 9080+n),
+			fmt.Sprintf("--monitoring-port=%d", 9280+n),
 			fmt.Sprintf("--datadir=%s/eth2-val-%d", tmpPath, n),
-			fmt.Sprintf("--beacon-rpc-provider=localhost:%d", 4000+n),
-		}
-		if config.minimalConfig {
-			args = append(args, "--minimal-config")
+			fmt.Sprintf("--beacon-rpc-provider=localhost:%d", 4200+n),
 		}
+		args = append(args, config.validatorFlags...)
+
 		cmd := exec.Command(binaryPath, args...)
 		cmd.Stdout = file
 		cmd.Stderr = file
-		t.Logf("Starting validator client with flags: %s", strings.Join(args, " "))
+		t.Logf("Starting validator client %d with flags: %s", n, strings.Join(args, " "))
 		if err := cmd.Start(); err != nil {
 			t.Fatal(err)
 		}
 		valClients[n] = &validatorClientInfo{
 			processID:   cmd.Process.Pid,
-			monitorPort: 9080 + n,
+			monitorPort: 9280 + n,
 		}
 	}
 
-	client, err := rpc.DialHTTP("http://127.0.0.1:8545")
+	client, err := rpc.DialHTTP("http://127.0.0.1:8745")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -124,8 +123,7 @@ func initializeValidators(
 		t.Fatal(err)
 	}
 
-	// "Safe" amount of blocks to mine to make sure the deposits are seen.
-	if err := mineBlocks(web3, keystore, 20); err != nil {
+	if err := mineBlocks(web3, keystore, params.BeaconConfig().Eth1FollowDistance); err != nil {
 		t.Fatalf("failed to mine blocks %v", err)
 	}