From 9e4ba75e71b000e848d938bb703b1faf1069f56a Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Wed, 15 Jun 2022 16:02:31 +0800 Subject: [PATCH] Batch Scenario Runs Into Single Test (#10878) * batch scenarios * fix * fix * Update testing/endtoend/endtoend_test.go --- testing/endtoend/endtoend_setup_test.go | 5 +- testing/endtoend/endtoend_test.go | 184 +++++++++--------- testing/endtoend/mainnet_scenario_e2e_test.go | 15 +- testing/endtoend/minimal_scenario_e2e_test.go | 28 +-- testing/endtoend/types/types.go | 6 + 5 files changed, 112 insertions(+), 126 deletions(-) diff --git a/testing/endtoend/endtoend_setup_test.go b/testing/endtoend/endtoend_setup_test.go index 2f03bf0bbb..8e78beee21 100644 --- a/testing/endtoend/endtoend_setup_test.go +++ b/testing/endtoend/endtoend_setup_test.go @@ -86,7 +86,7 @@ func e2eMinimal(t *testing.T, cfgo ...types.E2EConfigOpt) *testRunner { return newTestRunner(t, testConfig) } -func e2eMainnet(t *testing.T, usePrysmSh, useMultiClient bool) *testRunner { +func e2eMainnet(t *testing.T, usePrysmSh, useMultiClient bool, cfgo ...types.E2EConfigOpt) *testRunner { params.SetupTestConfigCleanup(t) params.OverrideBeaconConfig(params.E2EMainnetTestConfig()) if useMultiClient { @@ -155,6 +155,9 @@ func e2eMainnet(t *testing.T, usePrysmSh, useMultiClient bool) *testRunner { EvalInterceptor: defaultInterceptor, Seed: int64(seed), } + for _, o := range cfgo { + o(testConfig) + } return newTestRunner(t, testConfig) } diff --git a/testing/endtoend/endtoend_test.go b/testing/endtoend/endtoend_test.go index ad5213a694..916c8b462c 100644 --- a/testing/endtoend/endtoend_test.go +++ b/testing/endtoend/endtoend_test.go @@ -563,24 +563,17 @@ func (r *testRunner) executeProvidedEvaluators(currentEpoch uint64, conns []*grp wg.Wait() } -func (r *testRunner) singleNodeOffline(epoch uint64, _ []*grpc.ClientConn) bool { - switch epoch { - case 9: - require.NoError(r.t, r.comHandler.beaconNodes.PauseAtIndex(0)) - require.NoError(r.t, r.comHandler.validatorNodes.PauseAtIndex(0)) - return true - case 10: - require.NoError(r.t, r.comHandler.beaconNodes.ResumeAtIndex(0)) - require.NoError(r.t, r.comHandler.validatorNodes.ResumeAtIndex(0)) - return true - case 11, 12: - // Allow 2 epochs for the network to finalize again. - return true - } - return false -} - -func (r *testRunner) singleNodeOfflineMulticlient(epoch uint64, _ []*grpc.ClientConn) bool { +// This interceptor will define the multi scenario run for our minimal tests. +// 1) In the first scenario we will be taking a single prysm node and its validator offline. +// Along with that we will also take a single lighthouse node and its validator offline. +// After 1 epoch we will then attempt to bring it online again. +// +// +// 2) Then we will start testing optimistic sync by engaging our engine proxy. +// After the proxy has been sending `SYNCING` responses to the beacon node, we +// will test this with our optimistic sync evaluator to ensure everything works +// as expected. +func (r *testRunner) multiScenarioMulticlient(epoch uint64, conns []*grpc.ClientConn) bool { switch epoch { case 9: require.NoError(r.t, r.comHandler.beaconNodes.PauseAtIndex(0)) @@ -594,82 +587,7 @@ func (r *testRunner) singleNodeOfflineMulticlient(epoch uint64, _ []*grpc.Client require.NoError(r.t, r.comHandler.lighthouseBeaconNodes.ResumeAtIndex(0)) require.NoError(r.t, r.comHandler.lighthouseValidatorNodes.ResumeAtIndex(0)) return true - case 11, 12: - // Allow 2 epochs for the network to finalize again. - return true - } - return false -} - -func (r *testRunner) eeOffline(epoch uint64, _ []*grpc.ClientConn) bool { - switch epoch { - case 9: - require.NoError(r.t, r.comHandler.eth1Miner.Pause()) - return true - case 10: - require.NoError(r.t, r.comHandler.eth1Miner.Resume()) - return true - case 11, 12: - // Allow 2 epochs for the network to finalize again. - return true - } - return false -} - -func (r *testRunner) allValidatorsOffline(epoch uint64, _ []*grpc.ClientConn) bool { - switch epoch { - case 9: - require.NoError(r.t, r.comHandler.validatorNodes.PauseAtIndex(0)) - require.NoError(r.t, r.comHandler.validatorNodes.PauseAtIndex(1)) - return true - case 10: - require.NoError(r.t, r.comHandler.validatorNodes.ResumeAtIndex(0)) - require.NoError(r.t, r.comHandler.validatorNodes.ResumeAtIndex(1)) - return true - case 11, 12: - // Allow 2 epochs for the network to finalize again. - return true - } - return false -} - -func (r *testRunner) optimisticSync(epoch uint64, conns []*grpc.ClientConn) bool { - switch epoch { - case 9: - component, err := r.comHandler.eth1Proxy.ComponentAtIndex(0) - require.NoError(r.t, err) - component.(e2etypes.EngineProxy).AddRequestInterceptor("engine_newPayloadV1", func() interface{} { - return &enginev1.PayloadStatus{ - Status: enginev1.PayloadStatus_SYNCING, - LatestValidHash: make([]byte, 32), - } - }, func() bool { - return true - }) - return true - case 10: - r.executeProvidedEvaluators(epoch, []*grpc.ClientConn{conns[0]}, []e2etypes.Evaluator{ - ev.OptimisticSyncEnabled, - }) - // Disable Interceptor - component, err := r.comHandler.eth1Proxy.ComponentAtIndex(0) - require.NoError(r.t, err) - engineProxy, ok := component.(e2etypes.EngineProxy) - require.Equal(r.t, true, ok) - engineProxy.RemoveRequestInterceptor("engine_newPayloadV1") - engineProxy.ReleaseBackedUpRequests("engine_newPayloadV1") - - return true - case 11, 12: - // Allow 2 epochs for the network to finalize again. - return true - } - return false -} - -func (r *testRunner) optimisticSyncMulticlient(epoch uint64, conns []*grpc.ClientConn) bool { - switch epoch { - case 9: + case 14: // Set it for prysm beacon node. component, err := r.comHandler.eth1Proxy.ComponentAtIndex(0) require.NoError(r.t, err) @@ -693,7 +611,7 @@ func (r *testRunner) optimisticSyncMulticlient(epoch uint64, conns []*grpc.Clien return true }) return true - case 10: + case 15: r.executeProvidedEvaluators(epoch, []*grpc.ClientConn{conns[0]}, []e2etypes.Evaluator{ ev.OptimisticSyncEnabled, }) @@ -713,6 +631,21 @@ func (r *testRunner) optimisticSyncMulticlient(epoch uint64, conns []*grpc.Clien engineProxy.RemoveRequestInterceptor("engine_newPayloadV1") engineProxy.ReleaseBackedUpRequests("engine_newPayloadV1") + return true + case 11, 12, 16, 17: + // Allow 2 epochs for the network to finalize again. + return true + } + return false +} + +func (r *testRunner) eeOffline(epoch uint64, _ []*grpc.ClientConn) bool { + switch epoch { + case 9: + require.NoError(r.t, r.comHandler.eth1Miner.Pause()) + return true + case 10: + require.NoError(r.t, r.comHandler.eth1Miner.Resume()) return true case 11, 12: // Allow 2 epochs for the network to finalize again. @@ -721,6 +654,67 @@ func (r *testRunner) optimisticSyncMulticlient(epoch uint64, conns []*grpc.Clien return false } +// This interceptor will define the multi scenario run for our minimal tests. +// 1) In the first scenario we will be taking a single node and its validator offline. +// After 1 epoch we will then attempt to bring it online again. +// +// 2) In the second scenario we will be taking all validators offline. After 2 +// epochs we will wait for the network to recover. +// +// 3) Then we will start testing optimistic sync by engaging our engine proxy. +// After the proxy has been sending `SYNCING` responses to the beacon node, we +// will test this with our optimistic sync evaluator to ensure everything works +// as expected. +func (r *testRunner) multiScenario(epoch uint64, conns []*grpc.ClientConn) bool { + switch epoch { + case 9: + require.NoError(r.t, r.comHandler.beaconNodes.PauseAtIndex(0)) + require.NoError(r.t, r.comHandler.validatorNodes.PauseAtIndex(0)) + return true + case 10: + require.NoError(r.t, r.comHandler.beaconNodes.ResumeAtIndex(0)) + require.NoError(r.t, r.comHandler.validatorNodes.ResumeAtIndex(0)) + return true + case 14: + require.NoError(r.t, r.comHandler.validatorNodes.PauseAtIndex(0)) + require.NoError(r.t, r.comHandler.validatorNodes.PauseAtIndex(1)) + return true + case 15: + require.NoError(r.t, r.comHandler.validatorNodes.ResumeAtIndex(0)) + require.NoError(r.t, r.comHandler.validatorNodes.ResumeAtIndex(1)) + return true + case 19: + component, err := r.comHandler.eth1Proxy.ComponentAtIndex(0) + require.NoError(r.t, err) + component.(e2etypes.EngineProxy).AddRequestInterceptor("engine_newPayloadV1", func() interface{} { + return &enginev1.PayloadStatus{ + Status: enginev1.PayloadStatus_SYNCING, + LatestValidHash: make([]byte, 32), + } + }, func() bool { + return true + }) + return true + case 20: + r.executeProvidedEvaluators(epoch, []*grpc.ClientConn{conns[0]}, []e2etypes.Evaluator{ + ev.OptimisticSyncEnabled, + }) + // Disable Interceptor + component, err := r.comHandler.eth1Proxy.ComponentAtIndex(0) + require.NoError(r.t, err) + engineProxy, ok := component.(e2etypes.EngineProxy) + require.Equal(r.t, true, ok) + engineProxy.RemoveRequestInterceptor("engine_newPayloadV1") + engineProxy.ReleaseBackedUpRequests("engine_newPayloadV1") + + return true + case 11, 12, 16, 17, 21, 22: + // Allow 2 epochs for the network to finalize again. + return true + } + return false +} + // All Epochs are valid. func defaultInterceptor(_ uint64, _ []*grpc.ClientConn) bool { return false diff --git a/testing/endtoend/mainnet_scenario_e2e_test.go b/testing/endtoend/mainnet_scenario_e2e_test.go index 1214fba379..08c7953163 100644 --- a/testing/endtoend/mainnet_scenario_e2e_test.go +++ b/testing/endtoend/mainnet_scenario_e2e_test.go @@ -2,22 +2,17 @@ package endtoend import ( "testing" + + "github.com/prysmaticlabs/prysm/testing/endtoend/types" ) func TestEndToEnd_MainnetConfig_MultiClient(t *testing.T) { e2eMainnet(t, false /*usePrysmSh*/, true /*useMultiClient*/).run() } -func TestEndToEnd_ScenarioRun_BeaconOffline_Multiclient(t *testing.T) { - runner := e2eMainnet(t, false /*usePrysmSh*/, true /*useMultiClient*/) +func TestEndToEnd_MultiScenarioRun_Multiclient(t *testing.T) { + runner := e2eMainnet(t, false /*usePrysmSh*/, true /*useMultiClient*/, types.WithEpochs(22)) runner.config.Evaluators = scenarioEvalsMulti() - runner.config.EvalInterceptor = runner.singleNodeOffline - runner.scenarioRunner() -} - -func TestEndToEnd_ScenarioRun_OptimisticSync_Multiclient(t *testing.T) { - runner := e2eMainnet(t, false /*usePrysmSh*/, true /*useMultiClient*/) - runner.config.Evaluators = scenarioEvalsMulti() - runner.config.EvalInterceptor = runner.optimisticSyncMulticlient + runner.config.EvalInterceptor = runner.multiScenarioMulticlient runner.scenarioRunner() } diff --git a/testing/endtoend/minimal_scenario_e2e_test.go b/testing/endtoend/minimal_scenario_e2e_test.go index 82c4bbd016..9adf2c0649 100644 --- a/testing/endtoend/minimal_scenario_e2e_test.go +++ b/testing/endtoend/minimal_scenario_e2e_test.go @@ -1,20 +1,16 @@ package endtoend -import "testing" +import ( + "testing" -func TestEndToEnd_ScenarioRun_BeaconOffline(t *testing.T) { - runner := e2eMinimal(t) + "github.com/prysmaticlabs/prysm/testing/endtoend/types" +) + +func TestEndToEnd_MultiScenarioRun(t *testing.T) { + runner := e2eMinimal(t, types.WithEpochs(22)) runner.config.Evaluators = scenarioEvals() - runner.config.EvalInterceptor = runner.singleNodeOffline - runner.scenarioRunner() -} - -func TestEndToEnd_ScenarioRun_AllvalidatorsOffline(t *testing.T) { - runner := e2eMinimal(t) - - runner.config.Evaluators = scenarioEvals() - runner.config.EvalInterceptor = runner.allValidatorsOffline + runner.config.EvalInterceptor = runner.multiScenario runner.scenarioRunner() } @@ -26,11 +22,3 @@ func TestEndToEnd_ScenarioRun_EEOffline(t *testing.T) { runner.config.EvalInterceptor = runner.eeOffline runner.scenarioRunner() } - -func TestEndToEnd_ScenarioRun_OptimisticSync(t *testing.T) { - runner := e2eMinimal(t) - - runner.config.Evaluators = scenarioEvals() - runner.config.EvalInterceptor = runner.optimisticSync - runner.scenarioRunner() -} diff --git a/testing/endtoend/types/types.go b/testing/endtoend/types/types.go index 384740f61b..6a00b0dd22 100644 --- a/testing/endtoend/types/types.go +++ b/testing/endtoend/types/types.go @@ -17,6 +17,12 @@ func WithExtraEpochs(extra uint64) E2EConfigOpt { } } +func WithEpochs(e uint64) E2EConfigOpt { + return func(cfg *E2EConfig) { + cfg.EpochsToRun = e + } +} + func WithRemoteSigner() E2EConfigOpt { return func(cfg *E2EConfig) { cfg.UseWeb3RemoteSigner = true