Prepare E2E for v0.12 (#5991)

* Prepare e2e for v0.12

* Fix e2e

* Comments

* Attempt 100% participation

* Fixes

* Fix

* Update readme

Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com>

endtoend/BUILD.bazel

@@ -7,8 +7,7 @@ go_test(
     srcs = [
         "endtoend_test.go",
         "long_minimal_e2e_test.go",
-        "minimal_antiflake_e2e_1_test.go",
-        "minimal_antiflake_e2e_2_test.go",
+        "minimal_e2e_genesis_test.go",
         "minimal_e2e_test.go",
         "minimal_slashing_e2e_test.go",
     ],

endtoend/README.md

@@ -11,8 +11,8 @@ In order to "evaluate" the state of the beacon chain while the E2E is running, t
 Evaluators have 3 parts, the name for it's test name, a `policy` which declares which epoch(s) the evaluator should run, and then the `evaluation` which uses the beacon chain API to determine if the beacon chain passes certain conditions like finality.
 
 ## Current end-to-end tests
-* Minimal Config - 4 beacon nodes, 64 validators, running for 6 epochs
-* ~~Mainnet Config - 2 beacon nodes, 16,384 validators, running for 5 epochs~~ Disabled for now
+* Minimal Config - 2 beacon nodes, 256 validators, running for 8 epochs
+* Minimal Config Slashing Test - 2 beacon nodes, 256 validators, tests attester and proposer slashing
 
 ## Instructions
 If you wish to run all the E2E tests, you can run them through bazel with:
@@ -20,8 +20,3 @@ If you wish to run all the E2E tests, you can run them through bazel with:
 ```
 bazel test //endtoend:go_default_test --test_output=streamed --test_arg=-test.v --nocache_test_results
 ```
-
-To run the anti-flake E2E tests, run:
-```
-bazel test //endtoend:go_default_test --test_output=streamed --test_filter=TestEndToEnd_AntiFlake_MinimalConfig --test_arg=-test.v --nocache_test_results
-```

endtoend/components/beacon_node.go

@@ -54,8 +54,9 @@ func StartNewBeaconNode(t *testing.T, config *types.E2EConfig, index int, enr st
 		fmt.Sprintf("--grpc-gateway-port=%d", e2e.TestParams.BeaconNodeRPCPort+index+40),
 		fmt.Sprintf("--contract-deployment-block=%d", 0),
 		fmt.Sprintf("--rpc-max-page-size=%d", params.BeaconConfig().MinGenesisActiveValidatorCount),
-		"--force-clear-db",
 		fmt.Sprintf("--bootstrap-node=%s", enr),
+		"--force-clear-db",
+		"--e2e-config",
 	}
 	args = append(args, featureconfig.E2EBeaconChainFlags...)
 	args = append(args, config.BeaconFlags...)

endtoend/components/slasher.go

@@ -37,7 +37,7 @@ func StartSlashers(t *testing.T) []int {
 			fmt.Sprintf("--beacon-rpc-provider=localhost:%d", e2e.TestParams.BeaconNodeRPCPort+i),
 			"--force-clear-db",
 			"--span-map-cache",
-			"--verbosity=debug",
+			"--e2e-config",
 		}
 
 		t.Logf("Starting slasher %d with flags: %s", i, strings.Join(args[2:], " "))

endtoend/components/validator.go

@@ -77,6 +77,7 @@ func StartNewValidatorClient(t *testing.T, config *types.E2EConfig, validatorNum
 		fmt.Sprintf("--beacon-rpc-provider=localhost:%d", beaconRPCPort),
 		"--grpc-headers=dummy=value,foo=bar", // Sending random headers shouldn't break anything.
 		"--force-clear-db",
+		"--e2e-config",
 	}
 	args = append(args, featureconfig.E2EValidatorFlags...)
 	args = append(args, config.ValidatorFlags...)

endtoend/endtoend_test.go

@@ -44,6 +44,8 @@ func runEndToEndTest(t *testing.T, config *types.E2EConfig) {
 	defer helpers.LogOutput(t, config)
 	defer helpers.KillProcesses(t, processIDs)
 
+	// Sleep depending on the count of validators, as generating the genesis state could take some time.
+	time.Sleep(time.Duration(params.BeaconConfig().MinGenesisDelay) * time.Second)
 	beaconLogFile, err := os.Open(path.Join(e2e.TestParams.LogPath, fmt.Sprintf(e2e.BeaconNodeLogFileName, 0)))
 	if err != nil {
 		t.Fatal(err)

endtoend/evaluators/slashing.go

@@ -104,7 +104,7 @@ func insertDoubleAttestationIntoPool(conns ...*grpc.ClientConn) error {
 		return errors.Wrap(err, "could not get chain head")
 	}
 
-	_, privKeys, err := testutil.DeterministicDepositsAndKeys(64)
+	_, privKeys, err := testutil.DeterministicDepositsAndKeys(params.BeaconConfig().MinGenesisActiveValidatorCount)
 	if err != nil {
 		return err
 	}
@@ -192,7 +192,7 @@ func proposeDoubleBlock(conns ...*grpc.ClientConn) error {
 		return errors.Wrap(err, "could not get chain head")
 	}
 
-	_, privKeys, err := testutil.DeterministicDepositsAndKeys(64)
+	_, privKeys, err := testutil.DeterministicDepositsAndKeys(params.BeaconConfig().MinGenesisActiveValidatorCount)
 	if err != nil {
 		return err
 	}

endtoend/evaluators/validator.go

@@ -14,6 +14,8 @@ import (
 	"google.golang.org/grpc"
 )
 
+var expectedParticipation = 1 // 100% participation.
+
 // ValidatorsAreActive ensures the expected amount of validators are active.
 var ValidatorsAreActive = types.Evaluator{
 	Name:       "validators_active_epoch_%d",
@@ -126,7 +128,7 @@ func validatorsParticipating(conns ...*grpc.ClientConn) error {
 	}
 
 	partRate := participation.Participation.GlobalParticipationRate
-	expected := float32(1)
+	expected := float32(expectedParticipation)
 	if partRate < expected {
 		return fmt.Errorf(
 			"validator participation was below for epoch %d, expected %f, received: %f",

endtoend/helpers/helpers.go

@@ -19,7 +19,7 @@ import (
 )
 
 const (
-	maxPollingWaitTime  = 60 * time.Second
+	maxPollingWaitTime  = 60 * time.Second // A minute so timing out doesn't take very long.
 	filePollingInterval = 500 * time.Millisecond
 )
 

endtoend/long_minimal_e2e_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestEndToEnd_Long_MinimalConfig(t *testing.T) {
 	testutil.ResetCache()
-	params.UseMinimalConfig()
+	params.UseE2EConfig()
 
 	epochsToRun := 20
 	var err error
@@ -29,8 +29,8 @@ func TestEndToEnd_Long_MinimalConfig(t *testing.T) {
 	}
 
 	minimalConfig := &types.E2EConfig{
-		BeaconFlags:    []string{"--minimal-config", "--custom-genesis-delay=25"},
-		ValidatorFlags: []string{"--minimal-config"},
+		BeaconFlags:    []string{},
+		ValidatorFlags: []string{},
 		EpochsToRun:    uint64(epochsToRun),
 		TestSync:       false,
 		TestDeposits:   true,

endtoend/minimal_antiflake_e2e_2_test.go

@@ -1,36 +0,0 @@
-package endtoend
-
-import (
-	"testing"
-
-	ev "github.com/prysmaticlabs/prysm/endtoend/evaluators"
-	e2eParams "github.com/prysmaticlabs/prysm/endtoend/params"
-	"github.com/prysmaticlabs/prysm/endtoend/types"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
-)
-
-func TestEndToEnd_AntiFlake_MinimalConfig_2(t *testing.T) {
-	testutil.ResetCache()
-	params.UseMinimalConfig()
-
-	minimalConfig := &types.E2EConfig{
-		BeaconFlags:    []string{"--minimal-config", "--custom-genesis-delay=25"},
-		ValidatorFlags: []string{"--minimal-config"},
-		EpochsToRun:    4,
-		TestSync:       false,
-		TestSlasher:    false,
-		Evaluators: []types.Evaluator{
-			ev.PeersConnect,
-			ev.HealthzCheck,
-			ev.ValidatorsAreActive,
-			ev.ValidatorsParticipating,
-			ev.MetricsCheck,
-		},
-	}
-	if err := e2eParams.Init(2); err != nil {
-		t.Fatal(err)
-	}
-
-	runEndToEndTest(t, minimalConfig)
-}

endtoend/minimal_e2e_genesis_test.go

@@ -10,13 +10,13 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
 
-func TestEndToEnd_AntiFlake_MinimalConfig_1(t *testing.T) {
+func TestEndToEnd_Genesis_MinimalConfig(t *testing.T) {
 	testutil.ResetCache()
-	params.UseMinimalConfig()
+	params.UseE2EConfig()
 
 	minimalConfig := &types.E2EConfig{
-		BeaconFlags:    []string{"--minimal-config", "--custom-genesis-delay=25"},
-		ValidatorFlags: []string{"--minimal-config"},
+		BeaconFlags:    []string{},
+		ValidatorFlags: []string{},
 		EpochsToRun:    4,
 		TestSync:       false,
 		TestSlasher:    false,

endtoend/minimal_e2e_test.go

@@ -12,12 +12,12 @@ import (
 
 func TestEndToEnd_MinimalConfig(t *testing.T) {
 	testutil.ResetCache()
-	params.UseMinimalConfig()
+	params.UseE2EConfig()
 
 	minimalConfig := &types.E2EConfig{
-		BeaconFlags:    []string{"--minimal-config", "--custom-genesis-delay=25"},
-		ValidatorFlags: []string{"--minimal-config"},
-		EpochsToRun:    6,
+		BeaconFlags:    []string{},
+		ValidatorFlags: []string{},
+		EpochsToRun:    8,
 		TestSync:       true,
 		TestSlasher:    true,
 		Evaluators: []types.Evaluator{

endtoend/minimal_slashing_e2e_test.go

@@ -12,11 +12,11 @@ import (
 
 func TestEndToEnd_Slashing_MinimalConfig(t *testing.T) {
 	testutil.ResetCache()
-	params.UseMinimalConfig()
+	params.UseE2EConfig()
 
 	minimalConfig := &types.E2EConfig{
-		BeaconFlags:    []string{"--minimal-config", "--custom-genesis-delay=25"},
-		ValidatorFlags: []string{"--minimal-config"},
+		BeaconFlags:    []string{},
+		ValidatorFlags: []string{},
 		EpochsToRun:    3,
 		TestSync:       false,
 		TestSlasher:    true,

shared/featureconfig/config.go

@@ -29,7 +29,11 @@ var log = logrus.WithField("prefix", "flags")
 
 // Flags is a struct to represent which features the client will perform on runtime.
 type Flags struct {
-	MinimalConfig                              bool // MinimalConfig as defined in the spec.
+	// Configuration related flags.
+	MinimalConfig bool // MinimalConfig as defined in the spec.
+	E2EConfig     bool //E2EConfig made specifically for testing, do not use except in E2E.
+
+	// Feature related flags.
 	WriteSSZStateTransitions                   bool // WriteSSZStateTransitions to tmp directory.
 	InitSyncNoVerify                           bool // InitSyncNoVerify when initial syncing w/o verifying block's contents.
 	DisableDynamicCommitteeSubnets             bool // Disables dynamic attestation committee subnets via p2p.
@@ -280,6 +284,10 @@ func configureConfig(ctx *cli.Context, cfg *Flags) *Flags {
 		log.Warn("Using minimal config")
 		cfg.MinimalConfig = true
 		params.UseMinimalConfig()
+	} else if ctx.Bool(e2eConfigFlag.Name) {
+		log.Warn("Using end-to-end testing config")
+		cfg.MinimalConfig = true
+		params.UseE2EConfig()
 	}
 	return cfg
 }

shared/featureconfig/flags.go

@@ -17,6 +17,10 @@ var (
 		Name:  "minimal-config",
 		Usage: "Use minimal config with parameters as defined in the spec.",
 	}
+	e2eConfigFlag = &cli.BoolFlag{
+		Name:  "e2e-config",
+		Usage: "Use the E2E testing config, only for use within end-to-end testing.",
+	}
 	writeSSZStateTransitionsFlag = &cli.BoolFlag{
 		Name:  "interop-write-ssz-state-transitions",
 		Usage: "Write ssz states to disk after attempted state transition",
@@ -415,6 +419,7 @@ var deprecatedFlags = []cli.Flag{
 // ValidatorFlags contains a list of all the feature flags that apply to the validator client.
 var ValidatorFlags = append(deprecatedFlags, []cli.Flag{
 	minimalConfigFlag,
+	e2eConfigFlag,
 	enableProtectAttesterFlag,
 	enableProtectProposerFlag,
 	enableExternalSlasherProtectionFlag,
@@ -424,6 +429,7 @@ var ValidatorFlags = append(deprecatedFlags, []cli.Flag{
 
 // SlasherFlags contains a list of all the feature flags that apply to the slasher client.
 var SlasherFlags = append(deprecatedFlags, []cli.Flag{
+	e2eConfigFlag,
 	enableHistoricalDetectionFlag,
 	disableLookbackFlag,
 }...)
@@ -440,6 +446,7 @@ var BeaconChainFlags = append(deprecatedFlags, []cli.Flag{
 	devModeFlag,
 	customGenesisDelayFlag,
 	minimalConfigFlag,
+	e2eConfigFlag,
 	writeSSZStateTransitionsFlag,
 	disableForkChoiceUnsafeFlag,
 	disableDynamicCommitteeSubnets,
@@ -469,7 +476,6 @@ var BeaconChainFlags = append(deprecatedFlags, []cli.Flag{
 // E2EBeaconChainFlags contains a list of the beacon chain feature flags to be tested in E2E.
 var E2EBeaconChainFlags = []string{
 	"--cache-filtered-block-tree",
-	"--enable-byte-mempool",
 	"--enable-state-gen-sig-verify",
 	"--check-head-state",
 	"--enable-state-field-trie",

shared/params/config.go

@@ -303,11 +303,32 @@ func MinimalSpecConfig() *BeaconChainConfig {
 	return &minimalConfig
 }
 
+// E2ETestConfig retrieves the configurations made specifically for E2E testing.
+// Warning: This config is only for testing, it is not meant for use outside of E2E.
+func E2ETestConfig() *BeaconChainConfig {
+	e2eConfig := MinimalSpecConfig()
+
+	// Misc.
+	e2eConfig.MinGenesisActiveValidatorCount = 256
+	e2eConfig.MinGenesisDelay = 30 // 30 seconds so E2E has enough time to process deposits and get started.
+
+	// Time parameters.
+	e2eConfig.SecondsPerSlot = 8
+	e2eConfig.SecondsPerETH1Block = 2
+	e2eConfig.Eth1FollowDistance = 4
+	return e2eConfig
+}
+
 // UseMinimalConfig for beacon chain services.
 func UseMinimalConfig() {
 	beaconConfig = MinimalSpecConfig()
 }
 
+// UseE2EConfig for beacon chain services.
+func UseE2EConfig() {
+	beaconConfig = E2ETestConfig()
+}
+
 // UseMainnetConfig for beacon chain services.
 func UseMainnetConfig() {
 	beaconConfig = MainnetConfig()