E2E: beacon APIs Part 1 (#11306)

* adding compare beacon block test

* fixing bazel

* fixing evaluator import

* fixing imports

* changing package name

* fixing bazel

* adding logic to check for checking epoch

* fixing linting

* adding check for attester duties

* handle both blockv1 and blockv2

* making middleware objects public instead

* adding test for block attestations

* fixing typo

* adding blockroot test

* adding test for attestations

* fixing type value

* fixing test

* adding in node endpoints

* fixing bazel

* updating web3signer

* printing beacon blocks on request

* fixing struct

* temp log

* forgot string cast

* adding comparison function

* fixing bazel and evaulators, WIP

* fixing bazel

* changing how to minify json

* trying multiclient

* fixing port problem

* reverting evaluator and making test only for mainnet scenario testing

* removing test data

* fixing linting unused functions
git push

* changed to reflect

* adding in ssz comparison

* fixing tests

* fixing conflict

* fixing tests

* making v2 the standard

* adding better error logging

* fixing type

* adding lighthouse settings and fixing some deepsource items

* testing adding delay to evaluator

* testing without peers check

* changing target peers to try to fix lighthouse peer connections

* temp removing other tests

* fix lint issue

* adding peers connect back in

* adding in state version

* fixing bazel

* fixing path error

* testing changes to state

* fix unmarshal

* simplifying beacon api e2e execution

* fixing missed assertian checks

* improve logging and debugging issue

* trying to fix unmarshal

* still breaking more test edits

* removing fork to test unmarshal

* fixing pathing

* resolving error

* fixing beacon_api

* merging in debug api to beacon_api test

* fixing lint and temp commenting out endpoint

* adding in custom comparison function

* fixing custom evaluator

* adding test for block header data

* fixing header evaluation

* add node apis

* fixing linting,adding tests

* fixing bazel and temp removing unused functions

* fixing deepsource and linting issues

* Update testing/endtoend/evaluators/beaconapi_evaluators/beacon_api.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update testing/endtoend/evaluators/beaconapi_evaluators/beacon_api.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* addressing review

* resolving more review comments

* fixing linting

* removing ssz return value as it's large and possibly not needed

* Update testing/endtoend/evaluators/beaconapi_evaluators/beacon_api.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update testing/endtoend/evaluators/beaconapi_evaluators/beacon_api.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update testing/endtoend/evaluators/beaconapi_evaluators/beacon_api.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update testing/endtoend/evaluators/beaconapi_evaluators/beacon_api.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update testing/endtoend/evaluators/beaconapi_evaluators/beacon_api.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update testing/endtoend/evaluators/beaconapi_evaluators/beacon_api.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* fixing more review comments

* Update testing/endtoend/evaluators/beaconapi_evaluators/beacon_api.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* fixing linting and review iteems

* fixing cognit complexity issue

* fixing linting

* fix log printout

* test build kite only with crossclient

* switching out evaluator to depositedvalidatorsareactive

* removed wrong evaluator switching correct one

* removing skip based on review comments

* fixing pathing issue

* test without participation at epoch

* testing without special lighthouse logic in evaluator

* reducing expected participation when multiclient

* fixing imports

* reducing epochs to see if less flaky

* testing with other tests added back in

* reducing epochs ran further

* testing only cross client again

* testing multi run again

* test reverted scenario for tests

* testing with cross client

* removing commented out function

* testing without peers connect

* adding optimization based on suggestions

* removed the wrong peers connect

* accidently commited something I shouldn't have

* fixing lighthouse flag

* Update testing/endtoend/evaluators/beaconapi_evaluators/beacon_api.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update testing/endtoend/evaluators/beaconapi_evaluators/beacon_api.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update testing/endtoend/evaluators/beaconapi_evaluators/beacon_api.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

.gitignore

@@ -38,3 +38,6 @@ metaData
 
 # execution API authentication
 jwt.hex
+
+# manual testing
+tmp

testing/endtoend/BUILD.bazel

@@ -62,6 +62,7 @@ common_deps = [
     "//testing/endtoend/components:go_default_library",
     "//testing/endtoend/components/eth1:go_default_library",
     "//testing/endtoend/evaluators:go_default_library",
+    "//testing/endtoend/evaluators/beaconapi_evaluators:go_default_library",
     "//testing/endtoend/helpers:go_default_library",
     "//testing/endtoend/params:go_default_library",
     "//testing/endtoend/types:go_default_library",

testing/endtoend/components/lighthouse_beacon.go

@@ -182,9 +182,9 @@ func (node *LighthouseBeaconNode) Start(ctx context.Context) error {
 		fmt.Sprintf("--enr-tcp-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeP2PPort+index),
 		fmt.Sprintf("--port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeP2PPort+index),
 		fmt.Sprintf("--http-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeHTTPPort+index),
-		fmt.Sprintf("--target-peers=%d", 10),
+		fmt.Sprintf("--target-peers=%d", e2e.TestParams.LighthouseBeaconNodeCount),
 		fmt.Sprintf("--eth1-endpoints=http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1RPCPort+prysmNodeCount+index),
-		fmt.Sprintf("--execution-endpoints=http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1ProxyPort+prysmNodeCount+index),
+		fmt.Sprintf("--execution-endpoint=http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1ProxyPort+prysmNodeCount+index),
 		fmt.Sprintf("--jwt-secrets=%s", jwtPath),
 		fmt.Sprintf("--boot-nodes=%s", node.enr),
 		fmt.Sprintf("--metrics-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeMetricsPort+index),
@@ -194,6 +194,7 @@ func (node *LighthouseBeaconNode) Start(ctx context.Context) error {
 		"--enable-private-discovery",
 		"--debug-level=debug",
 		"--merge",
+		"--suggested-fee-recipient=0x878705ba3f8bc32fcf7f4caa1a35e72af65cf766",
 	}
 	if node.config.UseFixedPeerIDs {
 		flagVal := strings.Join(node.config.PeerIDs, ",")

testing/endtoend/components/lighthouse_validator.go

@@ -191,6 +191,7 @@ func (v *LighthouseValidatorNode) Start(ctx context.Context) error {
 		fmt.Sprintf("--datadir=%s", kPath),
 		fmt.Sprintf("--testnet-dir=%s", testNetDir),
 		fmt.Sprintf("--beacon-nodes=http://localhost:%d", httpPort+index),
+		"--suggested-fee-recipient=0x878705ba3f8bc32fcf7f4caa1a35e72af65cf766",
 	}
 
 	cmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 -- Safe

testing/endtoend/components/validator.go

@@ -195,7 +195,7 @@ func (v *ValidatorNode) Start(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-
+	fmt.Printf("validator_%d is starting with offset keys %d", index, offset)
 	_, pubs, err := interop.DeterministicallyGenerateKeys(uint64(offset), uint64(validatorNum))
 	if err != nil {
 		return err

testing/endtoend/deps.bzl

@@ -1,6 +1,6 @@
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")  # gazelle:keep
 
-lighthouse_version = "v3.0.0"
+lighthouse_version = "v3.1.2"
 lighthouse_archive_name = "lighthouse-%s-x86_64-unknown-linux-gnu-portable.tar.gz" % lighthouse_version
 
 def e2e_deps():
@@ -14,7 +14,7 @@ def e2e_deps():
 
     http_archive(
         name = "lighthouse",
-        sha256 = "6e0164d8f5074e083b55a161f3e6ecf1038e505334033ceaca37d6c491436d5d",
+        sha256 = "172bb132d5fdc5bd257d5a66e98d0799498f08cb60502f93a5f4437a70d9c5e0",
         build_file = "@prysm//testing/endtoend:lighthouse.BUILD",
         url = ("https://github.com/sigp/lighthouse/releases/download/%s/" + lighthouse_archive_name) % lighthouse_version,
     )

testing/endtoend/endtoend_setup_test.go

@@ -2,6 +2,7 @@ package endtoend
 
 import (
 	"fmt"
+	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/evaluators/beaconapi_evaluators"
 	"os"
 	"strconv"
 	"testing"
@@ -115,16 +116,17 @@ func e2eMainnet(t *testing.T, usePrysmSh, useMultiClient bool, cfgo ...types.E2E
 		ev.PeersConnect,
 		ev.HealthzCheck,
 		ev.MetricsCheck,
-		ev.ValidatorsAreActive,
 		ev.ValidatorsParticipatingAtEpoch(2),
 		ev.FinalizationOccurs(3),
 		ev.ProposeVoluntaryExit,
 		ev.ValidatorsHaveExited,
+		ev.DepositedValidatorsAreActive,
 		ev.ColdStateCheckpoint,
 		ev.AltairForkTransition,
 		ev.BellatrixForkTransition,
 		ev.APIMiddlewareVerifyIntegrity,
 		ev.APIGatewayV1Alpha1VerifyIntegrity,
+		beaconapi_evaluators.BeaconAPIMultiClientVerifyIntegrity,
 		ev.FinishedSyncing,
 		ev.AllNodesHaveSameHead,
 		ev.FeeRecipientIsPresent,

testing/endtoend/evaluators/beaconapi_evaluators/BUILD.bazel

@@ -0,0 +1,31 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    testonly = True,
+    srcs = [
+        "beacon_api.go",
+        "beacon_api_verify.go",
+        "util.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/v3/testing/endtoend/evaluators/beaconapi_evaluators",
+    visibility = ["//testing/endtoend:__subpackages__"],
+    deps = [
+        "//beacon-chain/rpc/apimiddleware:go_default_library",
+        "//config/params:go_default_library",
+        "//consensus-types/primitives:go_default_library",
+        "//proto/eth/service:go_default_library",
+        "//proto/eth/v1:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//testing/endtoend/helpers:go_default_library",
+        "//testing/endtoend/params:go_default_library",
+        "//testing/endtoend/policies:go_default_library",
+        "//testing/endtoend/types:go_default_library",
+        "//time/slots:go_default_library",
+        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+        "@io_bazel_rules_go//proto/wkt:empty_go_proto",
+        "@org_golang_google_grpc//:go_default_library",
+    ],
+)

testing/endtoend/evaluators/beaconapi_evaluators/beacon_api.go

@@ -0,0 +1,430 @@
+package beaconapi_evaluators
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+
+	v1 "github.com/prysmaticlabs/prysm/v3/proto/eth/v1"
+	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/helpers"
+
+	"github.com/prysmaticlabs/prysm/v3/config/params"
+
+	types "github.com/prysmaticlabs/prysm/v3/consensus-types/primitives"
+
+	"github.com/golang/protobuf/ptypes/empty"
+	"github.com/prysmaticlabs/prysm/v3/proto/eth/service"
+	"github.com/prysmaticlabs/prysm/v3/time/slots"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/v3/beacon-chain/rpc/apimiddleware"
+	ethpb "github.com/prysmaticlabs/prysm/v3/proto/prysm/v1alpha1"
+	"google.golang.org/grpc"
+)
+
+type metadata struct {
+	basepath         string
+	params           func(encoding string, currentEpoch types.Epoch) []string
+	prysmResps       map[string]interface{}
+	lighthouseResps  map[string]interface{}
+	customEvaluation func(interface{}, interface{}) error
+}
+
+var beaconPathsAndObjects = map[string]metadata{
+	"/beacon/genesis": {
+		basepath: v1MiddlewarePathTemplate,
+		params: func(_ string, _ types.Epoch) []string {
+			return []string{}
+		},
+		prysmResps: map[string]interface{}{
+			"json": &apimiddleware.GenesisResponseJson{},
+		},
+		lighthouseResps: map[string]interface{}{
+			"json": &apimiddleware.GenesisResponseJson{},
+		},
+	},
+	"/beacon/states/{param1}/root": {
+		basepath: v1MiddlewarePathTemplate,
+		params: func(_ string, _ types.Epoch) []string {
+			return []string{"head"}
+		},
+		prysmResps: map[string]interface{}{
+			"json": &apimiddleware.StateRootResponseJson{},
+		},
+		lighthouseResps: map[string]interface{}{
+			"json": &apimiddleware.StateRootResponseJson{},
+		},
+	},
+	"/beacon/states/{param1}/finality_checkpoints": {
+		basepath: v1MiddlewarePathTemplate,
+		params: func(_ string, _ types.Epoch) []string {
+			return []string{"head"}
+		},
+		prysmResps: map[string]interface{}{
+			"json": &apimiddleware.StateFinalityCheckpointResponseJson{},
+		},
+		lighthouseResps: map[string]interface{}{
+			"json": &apimiddleware.StateFinalityCheckpointResponseJson{},
+		},
+	},
+	"/beacon/blocks/{param1}": {
+		basepath: v2MiddlewarePathTemplate,
+		params: func(t string, e types.Epoch) []string {
+			if t == "ssz" {
+				if e < 4 {
+					return []string{"genesis"}
+				}
+				return []string{"finalized"}
+			}
+			return []string{"head"}
+
+		},
+		prysmResps: map[string]interface{}{
+			"json": &apimiddleware.BlockResponseJson{},
+			"ssz":  []byte{},
+		},
+		lighthouseResps: map[string]interface{}{
+			"json": &apimiddleware.BlockResponseJson{},
+			"ssz":  []byte{},
+		},
+	},
+	"/beacon/states/{param1}/fork": {
+		basepath: v1MiddlewarePathTemplate,
+		params: func(_ string, _ types.Epoch) []string {
+			return []string{"finalized"}
+		},
+		prysmResps: map[string]interface{}{
+			"json": &apimiddleware.StateForkResponseJson{},
+		},
+		lighthouseResps: map[string]interface{}{
+			"json": &apimiddleware.StateForkResponseJson{},
+		},
+	},
+	"/debug/beacon/states/{param1}": {
+		basepath: v2MiddlewarePathTemplate,
+		params: func(_ string, e types.Epoch) []string {
+			return []string{"head"}
+		},
+		prysmResps: map[string]interface{}{
+			"json": &apimiddleware.BeaconStateV2ResponseJson{},
+		},
+		lighthouseResps: map[string]interface{}{
+			"json": &apimiddleware.BeaconStateV2ResponseJson{},
+		},
+	},
+	"/validator/duties/proposer/{param1}": {
+		basepath: v1MiddlewarePathTemplate,
+		params: func(_ string, e types.Epoch) []string {
+			return []string{fmt.Sprintf("%v", e)}
+		},
+		prysmResps: map[string]interface{}{
+			"json": &apimiddleware.ProposerDutiesResponseJson{},
+		},
+		lighthouseResps: map[string]interface{}{
+			"json": &apimiddleware.ProposerDutiesResponseJson{},
+		},
+		customEvaluation: func(prysmResp interface{}, lhouseResp interface{}) error {
+			castedl, ok := lhouseResp.(*apimiddleware.ProposerDutiesResponseJson)
+			if !ok {
+				return errors.New("failed to cast type")
+			}
+			if castedl.Data[0].Slot == "0" {
+				// remove the first item from lighthouse data since lighthouse is returning a value despite no proposer
+				// there is no proposer on slot 0 so prysm don't return anything for slot 0
+				castedl.Data = castedl.Data[1:]
+			}
+			return compareJSONResponseObjects(prysmResp, castedl)
+		},
+	},
+	"/beacon/headers/{param1}": {
+		basepath: v1MiddlewarePathTemplate,
+		params: func(_ string, e types.Epoch) []string {
+			slot := uint64(0)
+			if e > 0 {
+				slot = (uint64(e) * uint64(params.BeaconConfig().SlotsPerEpoch)) - 1
+			}
+			return []string{fmt.Sprintf("%v", slot)}
+		},
+		prysmResps: map[string]interface{}{
+			"json": &apimiddleware.BlockHeaderResponseJson{},
+		},
+		lighthouseResps: map[string]interface{}{
+			"json": &apimiddleware.BlockHeaderResponseJson{},
+		},
+	},
+	"/node/identity": {
+		basepath: v1MiddlewarePathTemplate,
+		params: func(_ string, _ types.Epoch) []string {
+			return []string{}
+		},
+		prysmResps: map[string]interface{}{
+			"json": &apimiddleware.IdentityResponseJson{},
+		},
+		lighthouseResps: map[string]interface{}{
+			"json": &apimiddleware.IdentityResponseJson{},
+		},
+		customEvaluation: func(prysmResp interface{}, lhouseResp interface{}) error {
+			castedp, ok := prysmResp.(*apimiddleware.IdentityResponseJson)
+			if !ok {
+				return errors.New("failed to cast type")
+			}
+			castedl, ok := lhouseResp.(*apimiddleware.IdentityResponseJson)
+			if !ok {
+				return errors.New("failed to cast type")
+			}
+			if castedp.Data == nil {
+				return errors.New("prysm node identity was empty")
+			}
+			if castedl.Data == nil {
+				return errors.New("lighthouse node identity was empty")
+			}
+			return nil
+		},
+	},
+	"/node/peers": {
+		basepath: v1MiddlewarePathTemplate,
+		params: func(_ string, _ types.Epoch) []string {
+			return []string{}
+		},
+		prysmResps: map[string]interface{}{
+			"json": &apimiddleware.PeersResponseJson{},
+		},
+		lighthouseResps: map[string]interface{}{
+			"json": &apimiddleware.PeersResponseJson{},
+		},
+		customEvaluation: func(prysmResp interface{}, lhouseResp interface{}) error {
+			castedp, ok := prysmResp.(*apimiddleware.PeersResponseJson)
+			if !ok {
+				return errors.New("failed to cast type")
+			}
+			castedl, ok := lhouseResp.(*apimiddleware.PeersResponseJson)
+			if !ok {
+				return errors.New("failed to cast type")
+			}
+			if castedp.Data == nil {
+				return errors.New("prysm node identity was empty")
+			}
+			if castedl.Data == nil {
+				return errors.New("lighthouse node identity was empty")
+			}
+			return nil
+		},
+	},
+}
+
+func withCompareBeaconAPIs(beaconNodeIdx int, conn *grpc.ClientConn) error {
+	ctx := context.Background()
+	beaconClient := service.NewBeaconChainClient(conn)
+	genesisData, err := beaconClient.GetGenesis(ctx, &empty.Empty{})
+	if err != nil {
+		return errors.Wrap(err, "error getting genesis data")
+	}
+	currentEpoch := slots.EpochsSinceGenesis(genesisData.Data.GenesisTime.AsTime())
+
+	for path, meta := range beaconPathsAndObjects {
+		for key := range meta.prysmResps {
+			switch key {
+			case "json":
+				jsonparams := meta.params("json", currentEpoch)
+				apipath := pathFromParams(path, jsonparams)
+				fmt.Printf("executing json api path: %s\n", apipath)
+				if err := compareJSONMulticlient(beaconNodeIdx,
+					meta.basepath,
+					apipath,
+					beaconPathsAndObjects[path].prysmResps[key],
+					beaconPathsAndObjects[path].lighthouseResps[key],
+					meta.customEvaluation,
+				); err != nil {
+					return err
+				}
+			case "ssz":
+				sszparams := meta.params("ssz", currentEpoch)
+				if len(sszparams) == 0 {
+					continue
+				}
+				apipath := pathFromParams(path, sszparams)
+				fmt.Printf("executing ssz api path: %s\n", apipath)
+				prysmr, lighthouser, err := compareSSZMulticlient(beaconNodeIdx, meta.basepath, apipath)
+				if err != nil {
+					return err
+				}
+				beaconPathsAndObjects[path].prysmResps[key] = prysmr
+				beaconPathsAndObjects[path].lighthouseResps[key] = lighthouser
+			default:
+				return fmt.Errorf("unknown encoding type %s", key)
+			}
+		}
+	}
+	return orderedEvaluationOnResponses(beaconPathsAndObjects, genesisData)
+}
+
+func orderedEvaluationOnResponses(beaconPathsAndObjects map[string]metadata, genesisData *v1.GenesisResponse) error {
+	forkPathData := beaconPathsAndObjects["/beacon/states/{param1}/fork"]
+	prysmForkData, ok := forkPathData.prysmResps["json"].(*apimiddleware.StateForkResponseJson)
+	if !ok {
+		return errors.New("failed to cast type")
+	}
+	lighthouseForkData, ok := forkPathData.lighthouseResps["json"].(*apimiddleware.StateForkResponseJson)
+	if !ok {
+		return errors.New("failed to cast type")
+	}
+	if prysmForkData.Data.Epoch != lighthouseForkData.Data.Epoch {
+		return fmt.Errorf("prysm epoch %v does not match lighthouse epoch %v",
+			prysmForkData.Data.Epoch,
+			lighthouseForkData.Data.Epoch)
+	}
+
+	finalizedEpoch, err := strconv.ParseUint(prysmForkData.Data.Epoch, 10, 64)
+	if err != nil {
+		return err
+	}
+	blockPathData := beaconPathsAndObjects["/beacon/blocks/{param1}"]
+	sszrspL, ok := blockPathData.prysmResps["ssz"].([]byte)
+	if !ok {
+		return errors.New("failed to cast type")
+	}
+	sszrspP, ok := blockPathData.lighthouseResps["ssz"].([]byte)
+	if !ok {
+		return errors.New("failed to cast type")
+	}
+	if finalizedEpoch < helpers.AltairE2EForkEpoch+2 {
+		blockP := &ethpb.SignedBeaconBlock{}
+		blockL := &ethpb.SignedBeaconBlock{}
+		if err := blockL.UnmarshalSSZ(sszrspL); err != nil {
+			return errors.Wrap(err, "failed to unmarshal lighthouse ssz")
+		}
+		if err := blockP.UnmarshalSSZ(sszrspP); err != nil {
+			return errors.Wrap(err, "failed to unmarshal rysm ssz")
+		}
+		if len(blockP.Signature) == 0 || len(blockL.Signature) == 0 || hexutil.Encode(blockP.Signature) != hexutil.Encode(blockL.Signature) {
+			return errors.New("prysm signature does not match lighthouse signature")
+		}
+	} else if finalizedEpoch >= helpers.AltairE2EForkEpoch+2 && finalizedEpoch < helpers.BellatrixE2EForkEpoch {
+		blockP := &ethpb.SignedBeaconBlockAltair{}
+		blockL := &ethpb.SignedBeaconBlockAltair{}
+		if err := blockL.UnmarshalSSZ(sszrspL); err != nil {
+			return errors.Wrap(err, "lighthouse ssz error")
+		}
+		if err := blockP.UnmarshalSSZ(sszrspP); err != nil {
+			return errors.Wrap(err, "prysm ssz error")
+		}
+
+		if len(blockP.Signature) == 0 || len(blockL.Signature) == 0 || hexutil.Encode(blockP.Signature) != hexutil.Encode(blockL.Signature) {
+			return fmt.Errorf("prysm response %v does not match lighthouse response %v",
+				blockP,
+				blockL)
+		}
+	} else {
+		blockP := &ethpb.SignedBeaconBlockBellatrix{}
+		blockL := &ethpb.SignedBeaconBlockBellatrix{}
+		if err := blockL.UnmarshalSSZ(sszrspL); err != nil {
+			return errors.Wrap(err, "lighthouse ssz error")
+		}
+		if err := blockP.UnmarshalSSZ(sszrspP); err != nil {
+			return errors.Wrap(err, "prysm ssz error")
+		}
+
+		if len(blockP.Signature) == 0 || len(blockL.Signature) == 0 || hexutil.Encode(blockP.Signature) != hexutil.Encode(blockL.Signature) {
+			return fmt.Errorf("prysm response %v does not match lighthouse response %v",
+				blockP,
+				blockL)
+		}
+	}
+	blockheaderData := beaconPathsAndObjects["/beacon/headers/{param1}"]
+	prysmHeader, ok := blockheaderData.prysmResps["json"].(*apimiddleware.BlockHeaderResponseJson)
+	if !ok {
+		return errors.New("failed to cast type")
+	}
+	proposerdutiesData := beaconPathsAndObjects["/validator/duties/proposer/{param1}"]
+	prysmDuties, ok := proposerdutiesData.prysmResps["json"].(*apimiddleware.ProposerDutiesResponseJson)
+	if !ok {
+		return errors.New("failed to cast type")
+	}
+	if prysmHeader.Data.Root != prysmDuties.DependentRoot {
+		fmt.Printf("current slot: %v\n", slots.CurrentSlot(uint64(genesisData.Data.GenesisTime.AsTime().Unix())))
+		return fmt.Errorf("header root %s does not match duties root %s ", prysmHeader.Data.Root, prysmDuties.DependentRoot)
+	}
+
+	return nil
+}
+
+func compareJSONMulticlient(beaconNodeIdx int, base string, path string, respJSONPrysm interface{}, respJSONLighthouse interface{}, customEvaluator func(interface{}, interface{}) error) error {
+	if err := doMiddlewareJSONGetRequest(
+		base,
+		path,
+		beaconNodeIdx,
+		respJSONPrysm,
+	); err != nil {
+		return errors.Wrap(err, "could not perform GET request for Prysm JSON")
+	}
+
+	if err := doMiddlewareJSONGetRequest(
+		base,
+		path,
+		beaconNodeIdx,
+		respJSONLighthouse,
+		"lighthouse",
+	); err != nil {
+		return errors.Wrap(err, "could not perform GET request for Lighthouse JSON")
+	}
+	if customEvaluator != nil {
+		return customEvaluator(respJSONPrysm, respJSONLighthouse)
+	} else {
+		return compareJSONResponseObjects(respJSONPrysm, respJSONLighthouse)
+	}
+}
+
+func compareSSZMulticlient(beaconNodeIdx int, base string, path string) ([]byte, []byte, error) {
+	sszrspL, err := doMiddlewareSSZGetRequest(
+		base,
+		path,
+		beaconNodeIdx,
+		"lighthouse",
+	)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not perform GET request for Lighthouse SSZ")
+	}
+
+	sszrspP, err := doMiddlewareSSZGetRequest(
+		base,
+		path,
+		beaconNodeIdx,
+	)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not perform GET request for Prysm SSZ")
+	}
+	if !bytes.Equal(sszrspL, sszrspP) {
+		return nil, nil, errors.New("prysm ssz response does not match lighthouse ssz response")
+	}
+	return sszrspP, sszrspL, nil
+}
+
+func compareJSONResponseObjects(prysmResp interface{}, lighthouseResp interface{}) error {
+	if !reflect.DeepEqual(prysmResp, lighthouseResp) {
+		p, err := json.Marshal(prysmResp)
+		if err != nil {
+			return errors.Wrap(err, "failed to marshal Prysm response to JSON")
+		}
+		l, err := json.Marshal(lighthouseResp)
+		if err != nil {
+			return errors.Wrap(err, "failed to marshal Lighthouse response to JSON")
+		}
+		return fmt.Errorf("prysm response %s does not match lighthouse response %s",
+			string(p),
+			string(l))
+	}
+	return nil
+}
+
+func pathFromParams(path string, params []string) string {
+	apiPath := path
+	for index := range params {
+		apiPath = strings.Replace(path, fmt.Sprintf("{param%d}", index+1), params[index], 1)
+	}
+	return apiPath
+}

testing/endtoend/evaluators/beaconapi_evaluators/beacon_api_verify.go

@@ -0,0 +1,46 @@
+package beaconapi_evaluators
+
+import (
+	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/policies"
+	e2etypes "github.com/prysmaticlabs/prysm/v3/testing/endtoend/types"
+	"google.golang.org/grpc"
+)
+
+// BeaconAPIMultiClientVerifyIntegrity tests our API Middleware responses to other beacon nodes such as lighthouse.
+var BeaconAPIMultiClientVerifyIntegrity = e2etypes.Evaluator{
+	Name:       "beacon_api_multi-client_verify_integrity_epoch_%d",
+	Policy:     policies.AllEpochs,
+	Evaluation: beaconAPIVerify,
+}
+
+const (
+	v1MiddlewarePathTemplate = "http://localhost:%d/eth/v1"
+	v2MiddlewarePathTemplate = "http://localhost:%d/eth/v2"
+)
+
+type apiComparisonFunc func(beaconNodeIdx int, conn *grpc.ClientConn) error
+
+func beaconAPIVerify(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+	beacon := []apiComparisonFunc{
+		withCompareBeaconAPIs,
+	}
+	for beaconNodeIdx, conn := range conns {
+		if err := runAPIComparisonFunctions(
+			beaconNodeIdx,
+			conn,
+			beacon...,
+		); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func runAPIComparisonFunctions(beaconNodeIdx int, conn *grpc.ClientConn, fs ...apiComparisonFunc) error {
+	for _, f := range fs {
+		if err := f(beaconNodeIdx, conn); err != nil {
+			return err
+		}
+	}
+	return nil
+}

testing/endtoend/evaluators/beaconapi_evaluators/util.go

@@ -0,0 +1,78 @@
+package beaconapi_evaluators
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/params"
+	log "github.com/sirupsen/logrus"
+)
+
+func doMiddlewareJSONGetRequest(template string, requestPath string, beaconNodeIdx int, dst interface{}, bnType ...string) error {
+	var port int
+	if len(bnType) > 0 {
+		switch bnType[0] {
+		case "lighthouse":
+			port = params.TestParams.Ports.LighthouseBeaconNodeHTTPPort
+		default:
+			port = params.TestParams.Ports.PrysmBeaconNodeGatewayPort
+		}
+	} else {
+		port = params.TestParams.Ports.PrysmBeaconNodeGatewayPort
+	}
+
+	basePath := fmt.Sprintf(template, port+beaconNodeIdx)
+	httpResp, err := http.Get(
+		basePath + requestPath,
+	)
+	if err != nil {
+		return err
+	}
+
+	return json.NewDecoder(httpResp.Body).Decode(&dst)
+}
+
+func doMiddlewareSSZGetRequest(template string, requestPath string, beaconNodeIdx int, bnType ...string) ([]byte, error) {
+	client := &http.Client{}
+	var port int
+	if len(bnType) > 0 {
+		switch bnType[0] {
+		case "lighthouse":
+			port = params.TestParams.Ports.LighthouseBeaconNodeHTTPPort
+		default:
+			port = params.TestParams.Ports.PrysmBeaconNodeGatewayPort
+		}
+	} else {
+		port = params.TestParams.Ports.PrysmBeaconNodeGatewayPort
+	}
+
+	basePath := fmt.Sprintf(template, port+beaconNodeIdx)
+
+	req, err := http.NewRequest("GET", basePath+requestPath, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept", "application/octet-stream")
+	rsp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if rsp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("request failed with response code: %d", rsp.StatusCode)
+	}
+	defer closeBody(rsp.Body)
+	body, err := io.ReadAll(rsp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return body, nil
+}
+
+func closeBody(body io.Closer) {
+	if err := body.Close(); err != nil {
+		log.WithError(err).Error("could not close response body")
+	}
+}

testing/endtoend/evaluators/validator.go

@@ -4,14 +4,13 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/prysmaticlabs/prysm/v3/encoding/bytesutil"
-
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/altair"
 	"github.com/prysmaticlabs/prysm/v3/config/params"
 	"github.com/prysmaticlabs/prysm/v3/consensus-types/blocks"
 	"github.com/prysmaticlabs/prysm/v3/consensus-types/interfaces"
 	ethtypes "github.com/prysmaticlabs/prysm/v3/consensus-types/primitives"
+	"github.com/prysmaticlabs/prysm/v3/encoding/bytesutil"
 	ethpbservice "github.com/prysmaticlabs/prysm/v3/proto/eth/service"
 	"github.com/prysmaticlabs/prysm/v3/proto/eth/v2"
 	ethpb "github.com/prysmaticlabs/prysm/v3/proto/prysm/v1alpha1"
@@ -19,7 +18,6 @@ import (
 	e2eparams "github.com/prysmaticlabs/prysm/v3/testing/endtoend/params"
 	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/policies"
 	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/types"
-
 	"github.com/prysmaticlabs/prysm/v3/time/slots"
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/types/known/emptypb"
@@ -27,7 +25,7 @@ import (
 
 var expectedParticipation = 0.99
 
-var expectedMulticlientParticipation = 0.98
+var expectedMulticlientParticipation = 0.95
 
 var expectedSyncParticipation = 0.99
 

testing/endtoend/mainnet_scenario_e2e_test.go

@@ -1,15 +1,18 @@
 package endtoend
 
 import (
-	"testing"
-
 	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/types"
+	"testing"
 )
 
 func TestEndToEnd_MainnetConfig_MultiClient(t *testing.T) {
 	e2eMainnet(t, false /*usePrysmSh*/, true /*useMultiClient*/).run()
 }
 
+func TestEndToEnd_MainnetConfig_Multiclient_CrossClient(t *testing.T) {
+	e2eMainnet(t, false /*usePrysmSh*/, true /*useMultiClient*/, types.WithValidatorCrossClient()).run()
+}
+
 func TestEndToEnd_MultiScenarioRun_Multiclient(t *testing.T) {
 	runner := e2eMainnet(t, false /*usePrysmSh*/, true /*useMultiClient*/, types.WithEpochs(22))
 	runner.config.Evaluators = scenarioEvalsMulti()

testing/endtoend/types/types.go

@@ -29,6 +29,12 @@ func WithCheckpointSync() E2EConfigOpt {
 	}
 }
 
+func WithValidatorCrossClient() E2EConfigOpt {
+	return func(cfg *E2EConfig) {
+		cfg.UseValidatorCrossClient = true
+	}
+}
+
 // E2EConfig defines the struct for all configurations needed for E2E testing.
 type E2EConfig struct {
 	TestCheckpointSync      bool