Add a test for e2e validator to run against prior release (#9042)

* Add a test for e2e validator to run against prior release

* gofmt

Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>

BUILD.bazel

@@ -157,3 +157,9 @@ string_setting(
         "libfuzzer",
     ],
 )
+
+sh_binary(
+    name = "prysm_sh",
+    srcs = ["prysm.sh"],
+    visibility = ["//visibility:public"],
+)

endtoend/BUILD.bazel

@@ -13,6 +13,7 @@ go_test(
     ],
     args = ["-test.v"],
     data = [
+        "//:prysm_sh",
         "//cmd/beacon-chain",
         "//cmd/slasher",
         "//cmd/validator",
@@ -21,10 +22,10 @@ go_test(
     ],
     shard_count = 2,
     tags = [
-        "block-network",
         "e2e",
         "manual",
         "minimal",
+        "requires-network",
     ],
     deps = [
         "//beacon-chain/core/state:go_default_library",

endtoend/components/validator.go

@@ -6,7 +6,9 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/big"
+	"os"
 	"os/exec"
+	"path"
 	"strings"
 
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
@@ -97,7 +99,15 @@ func NewValidatorNode(config *e2etypes.E2EConfig, validatorNum, index, offset in
 
 // Start starts a validator client.
 func (v *ValidatorNode) Start(ctx context.Context) error {
-	binaryPath, found := bazel.FindBinary("cmd/validator", "validator")
+	var pkg, target string
+	if v.config.UsePrysmShValidator {
+		pkg = ""
+		target = "prysm_sh"
+	} else {
+		pkg = "cmd/validator"
+		target = "validator"
+	}
+	binaryPath, found := bazel.FindBinary(pkg, target)
 	if !found {
 		return errors.New("validator binary not found")
 	}
@@ -135,8 +145,34 @@ func (v *ValidatorNode) Start(ctx context.Context) error {
 	args = append(args, featureconfig.E2EValidatorFlags...)
 	args = append(args, config.ValidatorFlags...)
 
+	if v.config.UsePrysmShValidator {
+		args = append([]string{"validator"}, args...)
+		log.Warning("Using latest release validator via prysm.sh")
+	}
+
 	cmd := exec.CommandContext(ctx, binaryPath, args...)
-	log.Infof("Starting validator client %d with flags: %s", index, strings.Join(args[2:], " "))
+
+	// Write stdout and stderr to log files.
+	stdout, err := os.Create(path.Join(e2e.TestParams.LogPath, fmt.Sprintf("validator_%d_stdout.log", index)))
+	if err != nil {
+		return err
+	}
+	stderr, err := os.Create(path.Join(e2e.TestParams.LogPath, fmt.Sprintf("validator_%d_stderr.log", index)))
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := stdout.Close(); err != nil {
+			log.WithError(err).Error("Failed to close stdout file")
+		}
+		if err := stderr.Close(); err != nil {
+			log.WithError(err).Error("Failed to close stderr file")
+		}
+	}()
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+
+	log.Infof("Starting validator client %d with flags: %s %s", index, binaryPath, strings.Join(args, " "))
 	if err = cmd.Start(); err != nil {
 		return err
 	}

endtoend/minimal_e2e_test.go

@@ -14,7 +14,15 @@ import (
 )
 
 func TestEndToEnd_MinimalConfig(t *testing.T) {
+	e2eMinimal(t, false /*usePrysmSh*/)
+}
 
+// Run minimal e2e config with the current release validator against latest beacon node.
+func TestEndToEnd_MinimalConfig_ValidatorAtCurrentRelease(t *testing.T) {
+	e2eMinimal(t, true /*usePrysmSh*/)
+}
+
+func e2eMinimal(t *testing.T, usePrysmSh bool) {
 	params.UseE2EConfig()
 	require.NoError(t, e2eParams.Init(e2eParams.StandardBeaconCount))
 
@@ -31,12 +39,13 @@ func TestEndToEnd_MinimalConfig(t *testing.T) {
 		BeaconFlags: []string{
 			fmt.Sprintf("--slots-per-archive-point=%d", params.BeaconConfig().SlotsPerEpoch*16),
 		},
-		ValidatorFlags: []string{},
+		ValidatorFlags:      []string{},
-		EpochsToRun:    uint64(epochsToRun),
+		EpochsToRun:         uint64(epochsToRun),
-		TestSync:       true,
+		TestSync:            true,
-		TestDeposits:   true,
+		TestDeposits:        true,
-		TestSlasher:    true,
+		TestSlasher:         true,
-		UsePprof:       !longRunning,
+		UsePrysmShValidator: usePrysmSh,
+		UsePprof:            !longRunning,
 		Evaluators: []types.Evaluator{
 			ev.PeersConnect,
 			ev.HealthzCheck,

endtoend/types/types.go

@@ -11,14 +11,15 @@ import (
 
 // E2EConfig defines the struct for all configurations needed for E2E testing.
 type E2EConfig struct {
-	BeaconFlags    []string
+	BeaconFlags         []string
-	ValidatorFlags []string
+	ValidatorFlags      []string
-	EpochsToRun    uint64
+	EpochsToRun         uint64
-	TestSync       bool
+	TestSync            bool
-	TestSlasher    bool
+	TestSlasher         bool
-	TestDeposits   bool
+	TestDeposits        bool
-	UsePprof       bool
+	UsePprof            bool
-	Evaluators     []Evaluator
+	UsePrysmShValidator bool
+	Evaluators          []Evaluator
 }
 
 // Evaluator defines the structure of the evaluators used to