wip

api/client/event/event_stream.go

@@ -34,6 +34,18 @@ type Event struct {
 	Data      []byte
 }
 
+// PublishEvent enqueues an event without blocking the producer. If the channel is full,
+// the event is dropped since only the most recent heads are relevant.
+func PublishEvent(eventsChannel chan<- *Event, event *Event) {
+	if eventsChannel == nil || event == nil {
+		return
+	}
+	select {
+	case eventsChannel <- event:
+	default:
+	}
+}
+
 // EventStream is responsible for subscribing to the Beacon API events endpoint
 // and dispatching received events to subscribers.
 type EventStream struct {
@@ -67,19 +79,20 @@ func (h *EventStream) Subscribe(eventsChannel chan<- *Event) {
 	fullUrl := h.host + "/eth/v1/events?topics=" + allTopics
 	req, err := http.NewRequestWithContext(h.ctx, http.MethodGet, fullUrl, nil)
 	if err != nil {
-		eventsChannel <- &Event{
+		PublishEvent(eventsChannel, &Event{
 			EventType: EventConnectionError,
 			Data:      []byte(errors.Wrap(err, "failed to create HTTP request").Error()),
-		}
+		})
+		return
 	}
 	req.Header.Set("Accept", api.EventStreamMediaType)
 	req.Header.Set("Connection", api.KeepAlive)
 	resp, err := h.httpClient.Do(req)
 	if err != nil {
-		eventsChannel <- &Event{
+		PublishEvent(eventsChannel, &Event{
 			EventType: EventConnectionError,
 			Data:      []byte(errors.Wrap(err, client.ErrConnectionIssue.Error()).Error()),
-		}
+		})
 		return
 	}
 
@@ -100,7 +113,6 @@ func (h *EventStream) Subscribe(eventsChannel chan<- *Event) {
 		select {
 		case <-h.ctx.Done():
 			log.Info("Context canceled, stopping event stream")
-			close(eventsChannel)
 			return
 		default:
 			line := scanner.Text()
@@ -109,7 +121,7 @@ func (h *EventStream) Subscribe(eventsChannel chan<- *Event) {
 				// Empty line indicates the end of an event
 				if eventType != "" && data != "" {
 					// Process the event when both eventType and data are set
-					eventsChannel <- &Event{EventType: eventType, Data: []byte(data)}
+					PublishEvent(eventsChannel, &Event{EventType: eventType, Data: []byte(data)})
 				}
 
 				// Reset eventType and data for the next event
@@ -130,9 +142,9 @@ func (h *EventStream) Subscribe(eventsChannel chan<- *Event) {
 	}
 
 	if err := scanner.Err(); err != nil {
-		eventsChannel <- &Event{
+		PublishEvent(eventsChannel, &Event{
 			EventType: EventConnectionError,
 			Data:      []byte(errors.Wrap(err, errors.Wrap(client.ErrConnectionIssue, "scanner failed").Error()).Error()),
-		}
+		})
 	}
 }

api/client/event/event_stream_test.go

@@ -53,7 +53,7 @@ func TestEventStream(t *testing.T) {
 	defer server.Close()
 
 	topics := []string{"head"}
-	eventsChannel := make(chan *Event, 1)
+	eventsChannel := make(chan *Event, 4)
 	stream, err := NewEventStream(t.Context(), http.DefaultClient, server.URL, topics)
 	require.NoError(t, err)
 	go stream.Subscribe(eventsChannel)
@@ -80,7 +80,7 @@ func TestEventStream(t *testing.T) {
 
 func TestEventStreamRequestError(t *testing.T) {
 	topics := []string{"head"}
-	eventsChannel := make(chan *Event, 1)
+	eventsChannel := make(chan *Event, 4)
 	ctx := t.Context()
 
 	// use valid url that will result in failed request with nil body

changelog/james-prysm_deflake-evaluator.md

@@ -1,3 +0,0 @@
-### Ignored
-
-- adding some short retries for some end to end evaluators in an attempt to deflake tests.

changelog/jtraglia-add-specrefs-readme.md

@@ -1,3 +0,0 @@
-### Added
-
-- Added README for maintaining specrefs.

changelog/jtraglia-nightly-reftests-with-run-id.md

@@ -1,3 +0,0 @@
-### Added
-
-- The ability to download the nightly reference tests from a specific day.

specrefs/README.md

@@ -1,35 +0,0 @@
-# Specification References
-
-This directory contains specification reference tracking files managed by
-[ethspecify](https://github.com/jtraglia/ethspecify).
-
-## Installation
-
-Install `ethspecify` with the following command:
-
-```bash
-pipx install ethspecify
-```
-
-> [!NOTE]
-> You can run `ethspecify <cmd>` in the `specrefs` directory or
-> `ethspecify <cmd> --path=specrefs` from the project's root directory.
-
-## Maintenance
-
-When adding support for a new specification version, follow these steps:
-
-0. Change directory into the `specrefs` directory.
-1. Update the version in `.ethspecify.yml` configuration.
-2. Run `ethspecify process` to update/populate specrefs.
-3. Run `ethspecify check` to check specrefs.
-4. If there are errors, use the error message as a guide to fix the issue. If
-   there are new specrefs with empty sources, implement/locate each item and
-   update each specref source list. If you choose not to implement an item,
-   add an exception to the appropriate section the the `.ethspecify.yml`
-   configuration.
-5. Repeat steps 3 and 4 until `ethspecify check` passes.
-6. Run `git diff` to view updated specrefs. If an object/function/etc has
-   changed, make the necessary updates to the implementation.
-7. Lastly, in the project's root directory, run `act -j check-specrefs` to
-   ensure everything is correct.

testing/endtoend/components/eth1/transactions.go

@@ -40,7 +40,6 @@ type TransactionGenerator struct {
 	cancel        context.CancelFunc
 	paused        bool
 	useLargeBlobs bool // Use large blob transactions (6 blobs per tx) for BPO testing
-	blobTxCount   int  // Number of blob transactions per slot (0 means default of 5)
 }
 
 func (t *TransactionGenerator) UnderlyingProcess() *os.Process {
@@ -49,8 +48,8 @@ func (t *TransactionGenerator) UnderlyingProcess() *os.Process {
 	return &os.Process{}
 }
 
-func NewTransactionGenerator(keystore string, seed int64, useLargeBlobs bool, blobTxCount int) *TransactionGenerator {
-	return &TransactionGenerator{keystore: keystore, seed: seed, useLargeBlobs: useLargeBlobs, blobTxCount: blobTxCount}
+func NewTransactionGenerator(keystore string, seed int64, useLargeBlobs bool) *TransactionGenerator {
+	return &TransactionGenerator{keystore: keystore, seed: seed, useLargeBlobs: useLargeBlobs}
 }
 
 func (t *TransactionGenerator) Start(ctx context.Context) error {
@@ -115,7 +114,7 @@ func (t *TransactionGenerator) Start(ctx context.Context) error {
 				continue
 			}
 			backend := ethclient.NewClient(client)
-			err = SendTransaction(client, mineKey.PrivateKey, gasPrice, mineKey.Address.String(), txCount, backend, false, t.useLargeBlobs, t.blobTxCount)
+			err = SendTransaction(client, mineKey.PrivateKey, gasPrice, mineKey.Address.String(), txCount, backend, false, t.useLargeBlobs)
 			if err != nil {
 				return err
 			}
@@ -129,7 +128,7 @@ func (s *TransactionGenerator) Started() <-chan struct{} {
 	return s.started
 }
 
-func SendTransaction(client *rpc.Client, key *ecdsa.PrivateKey, gasPrice *big.Int, addr string, txCount uint64, backend *ethclient.Client, al bool, useLargeBlobs bool, blobTxCount int) error {
+func SendTransaction(client *rpc.Client, key *ecdsa.PrivateKey, gasPrice *big.Int, addr string, txCount uint64, backend *ethclient.Client, al bool, useLargeBlobs bool) error {
 	sender := common.HexToAddress(addr)
 	nonce, err := backend.PendingNonceAt(context.Background(), fundedAccount.Address)
 	if err != nil {
@@ -151,19 +150,14 @@ func SendTransaction(client *rpc.Client, key *ecdsa.PrivateKey, gasPrice *big.In
 	clock := startup.NewClock(e2e.TestParams.CLGenesisTime, [32]byte{})
 	isPostFulu := clock.CurrentEpoch() >= params.BeaconConfig().FuluForkEpoch
 
-	// Default to 5 blob transactions per slot if not configured.
-	numBlobTxs := blobTxCount
-	if numBlobTxs <= 0 {
-		numBlobTxs = 5
-	}
-
 	g, _ := errgroup.WithContext(context.Background())
-	txs := make([]*types.Transaction, numBlobTxs)
+	txs := make([]*types.Transaction, 10)
 
 	// Send blob transactions - use different versions pre/post Fulu
 	if isPostFulu {
 		logrus.Info("Sending blob transactions with cell proofs")
-		for index := range uint64(numBlobTxs) {
+		// Reduced from 10 to 5 to reduce load and prevent builder/EL timeouts
+		for index := range uint64(5) {
 
 			g.Go(func() error {
 				tx, err := RandomBlobCellTx(client, fundedAccount.Address, nonce+index, gasPrice, chainid, al, useLargeBlobs)
@@ -182,7 +176,8 @@ func SendTransaction(client *rpc.Client, key *ecdsa.PrivateKey, gasPrice *big.In
 		}
 	} else {
 		logrus.Info("Sending blob transactions with sidecars")
-		for index := range uint64(numBlobTxs) {
+		// Reduced from 10 to 5 to reduce load and prevent builder/EL timeouts
+		for index := range uint64(5) {
 
 			g.Go(func() error {
 				tx, err := RandomBlobTx(client, fundedAccount.Address, nonce+index, gasPrice, chainid, al, useLargeBlobs)

testing/endtoend/endtoend_test.go

@@ -225,9 +225,9 @@ func (r *testRunner) testDepositsAndTx(ctx context.Context, g *errgroup.Group,
 		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{r.depositor}); err != nil {
 			return errors.Wrap(err, "testDepositsAndTx unable to run, depositor did not Start")
 		}
-		go func() {
-			if r.config.TestDeposits {
-				log.Info("Running deposit tests")
+        go func() {
+            if r.config.TestDeposits {
+                log.Info("Running deposit tests")
 				// The validators with an index < minGenesisActiveCount all have deposits already from the chain start.
 				// Skip all of those chain start validators by seeking to minGenesisActiveCount in the validator list
 				// for further deposit testing.
@@ -238,12 +238,12 @@ func (r *testRunner) testDepositsAndTx(ctx context.Context, g *errgroup.Group,
 						r.t.Error(errors.Wrap(err, "depositor.SendAndMine failed"))
 					}
 				}
-			}
-			// Only generate background transactions when relevant for the test.
-			if r.config.TestDeposits || r.config.TestFeature || r.config.UseBuilder {
-				r.testTxGeneration(ctx, g, keystorePath, []e2etypes.ComponentRunner{})
-			}
-		}()
+            }
+            // Only generate background transactions when relevant for the test.
+            if r.config.TestDeposits || r.config.TestFeature || r.config.UseBuilder {
+                r.testTxGeneration(ctx, g, keystorePath, []e2etypes.ComponentRunner{})
+            }
+        }()
 		if r.config.TestDeposits {
 			return depositCheckValidator.Start(ctx)
 		}
@@ -252,7 +252,7 @@ func (r *testRunner) testDepositsAndTx(ctx context.Context, g *errgroup.Group,
 }
 
 func (r *testRunner) testTxGeneration(ctx context.Context, g *errgroup.Group, keystorePath string, requiredNodes []e2etypes.ComponentRunner) {
-	txGenerator := eth1.NewTransactionGenerator(keystorePath, r.config.Seed, r.config.UseLargeBlobs, r.config.BlobTxCount)
+	txGenerator := eth1.NewTransactionGenerator(keystorePath, r.config.Seed, r.config.UseLargeBlobs)
 	r.comHandler.txGen = txGenerator
 	g.Go(func() error {
 		if err := helpers.ComponentsStarted(ctx, requiredNodes); err != nil {

testing/endtoend/evaluators/node.go

@@ -156,9 +156,19 @@ func waitForMidEpoch(conn *grpc.ClientConn) error {
 	}
 }
 
-// getHeadEpochs fetches the head epoch from all beacon nodes concurrently.
-func getHeadEpochs(conns []*grpc.ClientConn) ([]primitives.Epoch, error) {
-	epochs := make([]primitives.Epoch, len(conns))
+func allNodesHaveSameHead(_ *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+	// Wait until we're at least halfway into the epoch to avoid race conditions
+	// at epoch boundaries where nodes may report different epochs.
+	if err := waitForMidEpoch(conns[0]); err != nil {
+		return errors.Wrap(err, "failed waiting for mid-epoch")
+	}
+
+	headEpochs := make([]primitives.Epoch, len(conns))
+	headBlockRoots := make([][]byte, len(conns))
+	justifiedRoots := make([][]byte, len(conns))
+	prevJustifiedRoots := make([][]byte, len(conns))
+	finalizedRoots := make([][]byte, len(conns))
+	chainHeads := make([]*eth.ChainHead, len(conns))
 	g, _ := errgroup.WithContext(context.Background())
 
 	for i, conn := range conns {
@@ -170,145 +180,63 @@ func getHeadEpochs(conns []*grpc.ClientConn) ([]primitives.Epoch, error) {
 			if err != nil {
 				return errors.Wrapf(err, "connection number=%d", conIdx)
 			}
-			epochs[conIdx] = chainHead.HeadEpoch
+			headEpochs[conIdx] = chainHead.HeadEpoch
+			headBlockRoots[conIdx] = chainHead.HeadBlockRoot
+			justifiedRoots[conIdx] = chainHead.JustifiedBlockRoot
+			prevJustifiedRoots[conIdx] = chainHead.PreviousJustifiedBlockRoot
+			finalizedRoots[conIdx] = chainHead.FinalizedBlockRoot
+			chainHeads[conIdx] = chainHead
 			return nil
 		})
 	}
 	if err := g.Wait(); err != nil {
-		return nil, err
+		return err
 	}
-	return epochs, nil
-}
-
-func allNodesHaveSameHead(_ *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
-	// Wait until we're at least halfway into the epoch to avoid race conditions
-	// at epoch boundaries where nodes may report different epochs.
-	if err := waitForMidEpoch(conns[0]); err != nil {
-		return errors.Wrap(err, "failed waiting for mid-epoch")
-	}
-
-	// First, wait for all nodes to reach the same epoch. Sync nodes may be
-	// behind and need time to catch up. We poll every 2 seconds with a
-	// 60 second timeout - this adapts to actual sync progress rather than
-	// using fixed delays.
-	const epochTimeout = 60 * time.Second
-	const epochPollInterval = 2 * time.Second
-	epochDeadline := time.Now().Add(epochTimeout)
-
-	for time.Now().Before(epochDeadline) {
-		epochs, err := getHeadEpochs(conns)
-		if err != nil {
-			return err
-		}
-		allSame := true
-		for i := 1; i < len(epochs); i++ {
-			if epochs[0] != epochs[i] {
-				allSame = false
-				break
-			}
-		}
-		if allSame {
-			break
-		}
-		time.Sleep(epochPollInterval)
-	}
-
-	// Now that epochs match (or timeout reached), do detailed head comparison
-	// with a few retries to handle block propagation delays.
-	const maxRetries = 5
-	const retryDelay = 1 * time.Second
-	var lastErr error
-
-	for attempt := range maxRetries {
-		if attempt > 0 {
-			time.Sleep(retryDelay)
-		}
-
-		headEpochs := make([]primitives.Epoch, len(conns))
-		headBlockRoots := make([][]byte, len(conns))
-		justifiedRoots := make([][]byte, len(conns))
-		prevJustifiedRoots := make([][]byte, len(conns))
-		finalizedRoots := make([][]byte, len(conns))
-		chainHeads := make([]*eth.ChainHead, len(conns))
-		g, _ := errgroup.WithContext(context.Background())
-
-		for i, conn := range conns {
-			conIdx := i
-			currConn := conn
-			g.Go(func() error {
-				beaconClient := eth.NewBeaconChainClient(currConn)
-				chainHead, err := beaconClient.GetChainHead(context.Background(), &emptypb.Empty{})
-				if err != nil {
-					return errors.Wrapf(err, "connection number=%d", conIdx)
-				}
-				headEpochs[conIdx] = chainHead.HeadEpoch
-				headBlockRoots[conIdx] = chainHead.HeadBlockRoot
-				justifiedRoots[conIdx] = chainHead.JustifiedBlockRoot
-				prevJustifiedRoots[conIdx] = chainHead.PreviousJustifiedBlockRoot
-				finalizedRoots[conIdx] = chainHead.FinalizedBlockRoot
-				chainHeads[conIdx] = chainHead
-				return nil
-			})
-		}
-		if err := g.Wait(); err != nil {
-			return err
-		}
-
-		lastErr = nil
-		for i := range conns {
-			if headEpochs[0] != headEpochs[i] {
-				lastErr = fmt.Errorf(
-					"received conflicting head epochs on node %d, expected %d, received %d",
-					i,
-					headEpochs[0],
-					headEpochs[i],
-				)
-				break
-			}
-			if !bytes.Equal(headBlockRoots[0], headBlockRoots[i]) {
-				lastErr = fmt.Errorf(
-					"received conflicting head block roots on node %d, expected %#x, received %#x",
-					i,
-					headBlockRoots[0],
-					headBlockRoots[i],
-				)
-				break
-			}
-			if !bytes.Equal(justifiedRoots[0], justifiedRoots[i]) {
-				lastErr = fmt.Errorf(
-					"received conflicting justified block roots on node %d, expected %#x, received %#x: %s and %s",
-					i,
-					justifiedRoots[0],
-					justifiedRoots[i],
-					chainHeads[0].String(),
-					chainHeads[i].String(),
-				)
-				break
-			}
-			if !bytes.Equal(prevJustifiedRoots[0], prevJustifiedRoots[i]) {
-				lastErr = fmt.Errorf(
-					"received conflicting previous justified block roots on node %d, expected %#x, received %#x",
-					i,
-					prevJustifiedRoots[0],
-					prevJustifiedRoots[i],
-				)
-				break
-			}
-			if !bytes.Equal(finalizedRoots[0], finalizedRoots[i]) {
-				lastErr = fmt.Errorf(
-					"received conflicting finalized epoch roots on node %d, expected %#x, received %#x",
-					i,
-					finalizedRoots[0],
-					finalizedRoots[i],
-				)
-				break
-			}
-		}
-
-		if lastErr == nil {
-			return nil
-		}
-	}
-
-	return lastErr
+
+	for i := range conns {
+		if headEpochs[0] != headEpochs[i] {
+			return fmt.Errorf(
+				"received conflicting head epochs on node %d, expected %d, received %d",
+				i,
+				headEpochs[0],
+				headEpochs[i],
+			)
+		}
+		if !bytes.Equal(headBlockRoots[0], headBlockRoots[i]) {
+			return fmt.Errorf(
+				"received conflicting head block roots on node %d, expected %#x, received %#x",
+				i,
+				headBlockRoots[0],
+				headBlockRoots[i],
+			)
+		}
+		if !bytes.Equal(justifiedRoots[0], justifiedRoots[i]) {
+			return fmt.Errorf(
+				"received conflicting justified block roots on node %d, expected %#x, received %#x: %s and %s",
+				i,
+				justifiedRoots[0],
+				justifiedRoots[i],
+				chainHeads[0].String(),
+				chainHeads[i].String(),
+			)
+		}
+		if !bytes.Equal(prevJustifiedRoots[0], prevJustifiedRoots[i]) {
+			return fmt.Errorf(
+				"received conflicting previous justified block roots on node %d, expected %#x, received %#x",
+				i,
+				prevJustifiedRoots[0],
+				prevJustifiedRoots[i],
+			)
+		}
+		if !bytes.Equal(finalizedRoots[0], finalizedRoots[i]) {
+			return fmt.Errorf(
+				"received conflicting finalized epoch roots on node %d, expected %#x, received %#x",
+				i,
+				finalizedRoots[0],
+				finalizedRoots[i],
+			)
+		}
+	}
+
+	return nil
 }

testing/endtoend/evaluators/validator.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
-	"time"
 
 	"github.com/OffchainLabs/prysm/v7/api/server/structs"
 	"github.com/OffchainLabs/prysm/v7/beacon-chain/core/altair"
@@ -124,25 +123,6 @@ func validatorsAreActive(ec *types.EvaluationContext, conns ...*grpc.ClientConn)
 
 // validatorsParticipating ensures the validators have an acceptable participation rate.
 func validatorsParticipating(_ *types.EvaluationContext, conns ...*grpc.ClientConn) error {
-	// Retry up to 3 times with 2 second delays to handle timing flakes where
-	// attestations haven't been fully processed yet due to block propagation delays.
-	const maxRetries = 3
-	const retryDelay = 2 * time.Second
-	var lastErr error
-
-	for attempt := range maxRetries {
-		if attempt > 0 {
-			time.Sleep(retryDelay)
-		}
-		lastErr = checkValidatorsParticipating(conns)
-		if lastErr == nil {
-			return nil
-		}
-	}
-	return lastErr
-}
-
-func checkValidatorsParticipating(conns []*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
 	validatorRequest := &ethpb.GetValidatorParticipationRequest{}
@@ -254,25 +234,6 @@ func checkValidatorsParticipating(conns []*grpc.ClientConn) error {
 // validatorsSyncParticipation ensures the validators have an acceptable participation rate for
 // sync committee assignments.
 func validatorsSyncParticipation(_ *types.EvaluationContext, conns ...*grpc.ClientConn) error {
-	// Retry up to 3 times with 2 second delays to handle timing flakes where
-	// sync committee messages haven't fully propagated yet.
-	const maxRetries = 3
-	const retryDelay = 2 * time.Second
-	var lastErr error
-
-	for attempt := range maxRetries {
-		if attempt > 0 {
-			time.Sleep(retryDelay)
-		}
-		lastErr = checkSyncParticipation(conns)
-		if lastErr == nil {
-			return nil
-		}
-	}
-	return lastErr
-}
-
-func checkSyncParticipation(conns []*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewNodeClient(conn)
 	altairClient := ethpb.NewBeaconChainClient(conn)
@@ -311,9 +272,9 @@ func checkSyncParticipation(conns []*grpc.ClientConn) error {
 			// Skip fork slot.
 			continue
 		}
-		// Skip early slots at genesis - validators need time to ramp up after chain start
+		// Skip slots 1-2 at genesis - validators need time to ramp up after chain start
 		// due to doppelganger protection. This is a startup timing issue, not a fork transition issue.
-		if b.Block().Slot() < 5 {
+		if b.Block().Slot() < 3 {
 			continue
 		}
 		expectedParticipation := expectedSyncParticipation
@@ -328,11 +289,6 @@ func checkSyncParticipation(conns []*grpc.ClientConn) error {
 		if err != nil {
 			return err
 		}
-		// Skip blocks with zero sync bits - these are typically empty/anomalous blocks
-		// where the proposer didn't receive sync committee contributions in time.
-		if syncAgg.SyncCommitteeBits.Count() == 0 {
-			continue
-		}
 		threshold := uint64(float64(syncAgg.SyncCommitteeBits.Len()) * expectedParticipation)
 		if syncAgg.SyncCommitteeBits.Count() < threshold {
 			return errors.Errorf("In block of slot %d ,the aggregate bitvector with length of %d only got a count of %d", b.Block().Slot(), threshold, syncAgg.SyncCommitteeBits.Count())
@@ -387,11 +343,6 @@ func checkSyncParticipation(conns []*grpc.ClientConn) error {
 		if err != nil {
 			return err
 		}
-		// Skip blocks with zero sync bits - these are typically empty/anomalous blocks
-		// where the proposer didn't receive sync committee contributions in time.
-		if syncAgg.SyncCommitteeBits.Count() == 0 {
-			continue
-		}
 		threshold := uint64(float64(syncAgg.SyncCommitteeBits.Len()) * expectedSyncParticipation)
 		if syncAgg.SyncCommitteeBits.Count() < threshold {
 			return errors.Errorf("In block of slot %d ,the aggregate bitvector with length of %d only got a count of %d", b.Block().Slot(), threshold, syncAgg.SyncCommitteeBits.Count())

testing/endtoend/minimal_builder_e2e_test.go

@@ -9,11 +9,11 @@ import (
 )
 
 func TestEndToEnd_MinimalConfig_WithBuilder(t *testing.T) {
-	r := e2eMinimal(t, types.InitForkCfg(version.Bellatrix, version.Electra, params.E2ETestConfig()), types.WithCheckpointSync(), types.WithBuilder(), types.WithBlobTxCount(2))
+	r := e2eMinimal(t, types.InitForkCfg(version.Bellatrix, version.Electra, params.E2ETestConfig()), types.WithCheckpointSync(), types.WithBuilder())
 	r.run()
 }
 
 func TestEndToEnd_MinimalConfig_WithBuilder_ValidatorRESTApi(t *testing.T) {
-	r := e2eMinimal(t, types.InitForkCfg(version.Bellatrix, version.Electra, params.E2ETestConfig()), types.WithCheckpointSync(), types.WithBuilder(), types.WithValidatorRESTApi(), types.WithBlobTxCount(2))
+	r := e2eMinimal(t, types.InitForkCfg(version.Bellatrix, version.Electra, params.E2ETestConfig()), types.WithCheckpointSync(), types.WithBuilder(), types.WithValidatorRESTApi())
 	r.run()
 }

testing/endtoend/types/types.go

@@ -68,14 +68,6 @@ func WithLargeBlobs() E2EConfigOpt {
 	}
 }
 
-// WithBlobTxCount sets the number of blob transactions sent per slot.
-// Default is 5 when not specified.
-func WithBlobTxCount(n int) E2EConfigOpt {
-	return func(cfg *E2EConfig) {
-		cfg.BlobTxCount = n
-	}
-}
-
 func WithSSZOnly() E2EConfigOpt {
 	return func(cfg *E2EConfig) {
 		if err := os.Setenv(params.EnvNameOverrideAccept, api.OctetStreamMediaType); err != nil {
@@ -116,7 +108,6 @@ type E2EConfig struct {
 	UseBeaconRestApi        bool
 	UseBuilder              bool
 	UseLargeBlobs           bool // Use large blob transactions (6 blobs per tx) for BPO testing
-	BlobTxCount             int  // Number of blob transactions per slot (0 means default of 5)
 	EpochsToRun             uint64
 	ExitEpoch               primitives.Epoch // Custom epoch for voluntary exit submission (0 means use default)
 	Seed                    int64

testing/spectest/README.md

@@ -21,14 +21,10 @@ There are tests for mainnet and minimal config, so for each config we will add a
 
 ## Running nightly spectests
 
-Since [PR 15312](https://github.com/OffchainLabs/prysm/pull/15312), Prysm has support to download "nightly" spectests from github via a starlark rule configuration by environment variable.
-Set `--repo_env=CONSENSUS_SPEC_TESTS_VERSION=nightly` or `--repo_env=CONSENSUS_SPEC_TESTS_VERSION=nightly-<run_id>` when running spectest to download the "nightly" spectests.
-Note: A GITHUB_TOKEN environment variable is required to be set. The github token does not need to be associated with your main account; it can be from a "burner account". And the token does not need to be a fine-grained token; it can be a classic token.
+Since [PR 15312](https://github.com/OffchainLabs/prysm/pull/15312), Prysm has support to download "nightly" spectests from github via a starlark rule configuration by environment variable. 
+Set `--repo_env=CONSENSUS_SPEC_TESTS_VERSION=nightly` when running spectest to download the "nightly" spectests. 
+Note: A GITHUB_TOKEN environment variable is required to be set. The github token must be a [fine grained token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token).
 
 ```
 bazel test //... --test_tag_filters=spectest --repo_env=CONSENSUS_SPEC_TESTS_VERSION=nightly
 ```
-
-```
-bazel test //... --test_tag_filters=spectest --repo_env=CONSENSUS_SPEC_TESTS_VERSION=nightly-21422848633
-```

tools/download_spectests.bzl

@@ -1,6 +1,5 @@
 # bazel build @consensus_spec_tests//:test_data
 # bazel build @consensus_spec_tests//:test_data --repo_env=CONSENSUS_SPEC_TESTS_VERSION=nightly
-# bazel build @consensus_spec_tests//:test_data --repo_env=CONSENSUS_SPEC_TESTS_VERSION=nightly-<run_id>
 
 def _get_redirected_url(repository_ctx, url, headers):
     if not repository_ctx.which("curl"):
@@ -25,7 +24,7 @@ def _impl(repository_ctx):
     version = repository_ctx.getenv("CONSENSUS_SPEC_TESTS_VERSION") or repository_ctx.attr.version
     token = repository_ctx.getenv("GITHUB_TOKEN") or ""
 
-    if version == "nightly" or version.startswith("nightly-"):
+    if version == "nightly":
         print("Downloading nightly tests")
         if not token:
             fail("Error GITHUB_TOKEN is not set")
@@ -35,22 +34,16 @@ def _impl(repository_ctx):
             "Accept": "application/vnd.github+json",
         }
 
-        if version.startswith("nightly-"):
-            run_id = version.split("nightly-", 1)[1]
-            if not run_id:
-                fail("Error invalid run id")
-        else:
-            repository_ctx.download(
-                "https://api.github.com/repos/%s/actions/workflows/%s/runs?branch=%s&status=success&per_page=1"
-                    % (repository_ctx.attr.repo, repository_ctx.attr.workflow, repository_ctx.attr.branch),
-                headers = headers,
-                output = "runs.json"
-            )
+        repository_ctx.download(
+            "https://api.github.com/repos/%s/actions/workflows/%s/runs?branch=%s&status=success&per_page=1"
+                % (repository_ctx.attr.repo, repository_ctx.attr.workflow, repository_ctx.attr.branch),
+            headers = headers,
+            output = "runs.json"
+        )
 
-            run_id = json.decode(repository_ctx.read("runs.json"))["workflow_runs"][0]["id"]
-            repository_ctx.delete("runs.json")
+        run_id = json.decode(repository_ctx.read("runs.json"))["workflow_runs"][0]["id"]
+        repository_ctx.delete("runs.json")
 
-        print("Run id:", run_id)
         repository_ctx.download(
             "https://api.github.com/repos/%s/actions/runs/%s/artifacts"
                 % (repository_ctx.attr.repo, run_id),
@@ -115,8 +108,8 @@ consensus_spec_tests = repository_rule(
         "version": attr.string(mandatory = True),
         "flavors": attr.string_dict(mandatory = True),
         "repo": attr.string(default = "ethereum/consensus-specs"),
-        "workflow": attr.string(default = "nightly-reftests.yml"),
-        "branch": attr.string(default = "master"),
+        "workflow": attr.string(default = "generate_vectors.yml"),
+        "branch": attr.string(default = "dev"),
         "release_url_template": attr.string(default = "https://github.com/ethereum/consensus-specs/releases/download/%s"),
     },
 )

validator/client/grpc-api/grpc_validator_client.go

@@ -285,10 +285,10 @@ func (c *grpcValidatorClient) StartEventStream(ctx context.Context, topics []str
 	ctx, span := trace.StartSpan(ctx, "validator.gRPCClient.StartEventStream")
 	defer span.End()
 	if len(topics) == 0 {
-		eventsChannel <- &eventClient.Event{
+		eventClient.PublishEvent(eventsChannel, &eventClient.Event{
 			EventType: eventClient.EventError,
 			Data:      []byte(errors.New("no topics were added").Error()),
-		}
+		})
 		return
 	}
 	// TODO(13563): ONLY WORKS WITH HEAD TOPIC.
@@ -299,10 +299,10 @@ func (c *grpcValidatorClient) StartEventStream(ctx context.Context, topics []str
 		}
 	}
 	if !containsHead {
-		eventsChannel <- &eventClient.Event{
+		eventClient.PublishEvent(eventsChannel, &eventClient.Event{
 			EventType: eventClient.EventConnectionError,
 			Data:      []byte(errors.Wrap(client.ErrConnectionIssue, "gRPC only supports the head topic, and head topic was not passed").Error()),
-		}
+		})
 	}
 	if containsHead && len(topics) > 1 {
 		log.Warn("gRPC only supports the head topic, other topics will be ignored")
@@ -310,10 +310,10 @@ func (c *grpcValidatorClient) StartEventStream(ctx context.Context, topics []str
 
 	stream, err := c.beaconNodeValidatorClient.StreamSlots(ctx, &ethpb.StreamSlotsRequest{VerifiedOnly: true})
 	if err != nil {
-		eventsChannel <- &eventClient.Event{
+		eventClient.PublishEvent(eventsChannel, &eventClient.Event{
 			EventType: eventClient.EventConnectionError,
 			Data:      []byte(errors.Wrap(client.ErrConnectionIssue, err.Error()).Error()),
-		}
+		})
 		return
 	}
 	c.isEventStreamRunning = true
@@ -327,25 +327,25 @@ func (c *grpcValidatorClient) StartEventStream(ctx context.Context, topics []str
 			if ctx.Err() != nil {
 				c.isEventStreamRunning = false
 				if errors.Is(ctx.Err(), context.Canceled) {
-					eventsChannel <- &eventClient.Event{
+					eventClient.PublishEvent(eventsChannel, &eventClient.Event{
 						EventType: eventClient.EventConnectionError,
 						Data:      []byte(errors.Wrap(client.ErrConnectionIssue, ctx.Err().Error()).Error()),
-					}
+					})
 					return
 				}
-				eventsChannel <- &eventClient.Event{
+				eventClient.PublishEvent(eventsChannel, &eventClient.Event{
 					EventType: eventClient.EventError,
 					Data:      []byte(ctx.Err().Error()),
-				}
+				})
 				return
 			}
 			res, err := stream.Recv()
 			if err != nil {
 				c.isEventStreamRunning = false
-				eventsChannel <- &eventClient.Event{
+				eventClient.PublishEvent(eventsChannel, &eventClient.Event{
 					EventType: eventClient.EventConnectionError,
 					Data:      []byte(errors.Wrap(client.ErrConnectionIssue, err.Error()).Error()),
-				}
+				})
 				return
 			}
 			if res == nil {
@@ -357,15 +357,15 @@ func (c *grpcValidatorClient) StartEventStream(ctx context.Context, topics []str
 				CurrentDutyDependentRoot:  hexutil.Encode(res.CurrentDutyDependentRoot),
 			})
 			if err != nil {
-				eventsChannel <- &eventClient.Event{
+				eventClient.PublishEvent(eventsChannel, &eventClient.Event{
 					EventType: eventClient.EventError,
 					Data:      []byte(errors.Wrap(err, "failed to marshal Head Event").Error()),
-				}
+				})
 			}
-			eventsChannel <- &eventClient.Event{
+			eventClient.PublishEvent(eventsChannel, &eventClient.Event{
 				EventType: eventClient.EventHead,
 				Data:      b,
-			}
+			})
 		}
 	}
 }

validator/client/grpc-api/grpc_validator_client_test.go

@@ -223,7 +223,7 @@ func TestStartEventStream(t *testing.T) {
 
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			eventsChannel := make(chan *eventClient.Event, 1) // Buffer to prevent blocking
+			eventsChannel := make(chan *eventClient.Event, 4) // Buffer to prevent blocking
 			tc.prepare()                                      // Setup mock expectations
 
 			go grpcClient.StartEventStream(ctx, tc.topics, eventsChannel)

validator/client/runner_test.go

@@ -441,7 +441,6 @@ func TestRunnerPushesProposerSettings_ValidContext(t *testing.T) {
 		defer assertValidContext(t, timedCtx, ctx)
 		delay(t)
 	})
-	vcm.EXPECT().EventStreamIsRunning().Return(true).AnyTimes().Do(func() { delay(t) })
 	vcm.EXPECT().SubmitValidatorRegistrations(liveCtx, gomock.Any()).Do(func(ctx context.Context, _ any) {
 		defer assertValidContext(t, timedCtx, ctx) // This is the specific regression test assertion for PR 15369.
 		delay(t)

validator/client/service.go

@@ -66,6 +66,8 @@ type ValidatorService struct {
 	closeClientFunc         func() // validator client stop function is used here
 }
 
+const eventChannelBufferSize = 32
+
 // Config for the validator service.
 type Config struct {
 	Validator               iface.Validator
@@ -234,7 +236,7 @@ func (v *ValidatorService) Start() {
 		distributed:                    v.distributed,
 		disableDutiesPolling:           v.disableDutiesPolling,
 		accountsChangedChannel:         make(chan [][fieldparams.BLSPubkeyLength]byte, 1),
-		eventsChannel:                  make(chan *eventClient.Event, 1),
+		eventsChannel:                  make(chan *eventClient.Event, eventChannelBufferSize),
 	}
 
 	hm := newHealthMonitor(v.ctx, v.cancel, v.maxHealthChecks, v.validator)

validator/client/validator.go

@@ -64,6 +64,11 @@ var (
 	msgNoKeysFetched     = "No validating keys fetched. Waiting for keys..."
 )
 
+const (
+	eventStreamStopped uint32 = iota
+	eventStreamRunning
+)
+
 type validator struct {
 	logValidatorPerformance            bool
 	distributed                        bool
@@ -82,6 +87,7 @@ type validator struct {
 	cachedAttestationData              *ethpb.AttestationData
 	accountsChangedChannel             chan [][fieldparams.BLSPubkeyLength]byte
 	eventsChannel                      chan *eventClient.Event
+	eventStreamState                   atomic.Uint32
 	highestValidSlot                   primitives.Slot
 	submittedAggregates                map[submittedAttKey]*submittedAtt
 	graffitiStruct                     *graffiti.Graffiti
@@ -1211,12 +1217,40 @@ func (v *validator) PushProposerSettings(ctx context.Context, slot primitives.Sl
 }
 
 func (v *validator) StartEventStream(ctx context.Context, topics []string) {
-	if v.EventStreamIsRunning() {
+	if !v.eventStreamState.CompareAndSwap(eventStreamStopped, eventStreamRunning) {
 		log.Debug("EventStream is already running")
 		return
 	}
 	log.WithField("topics", topics).Info("Starting event stream")
-	v.validatorClient.StartEventStream(ctx, topics, v.eventsChannel)
+	go v.runEventStream(ctx, topics)
+}
+
+func (v *validator) runEventStream(ctx context.Context, topics []string) {
+	defer v.eventStreamState.Store(eventStreamStopped)
+	backoff := time.Second
+	const maxBackoff = 30 * time.Second
+
+	for {
+		v.validatorClient.StartEventStream(ctx, topics, v.eventsChannel)
+		if ctx.Err() != nil {
+			return
+		}
+
+		log.WithField("retryIn", backoff).Warn("Event stream ended unexpectedly, attempting to resubscribe")
+		timer := time.NewTimer(backoff)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return
+		case <-timer.C:
+		}
+		if backoff < maxBackoff {
+			backoff *= 2
+			if backoff > maxBackoff {
+				backoff = maxBackoff
+			}
+		}
+	}
 }
 
 func (v *validator) checkDependentRoots(ctx context.Context, head *structs.HeadEvent) error {
@@ -1303,7 +1337,7 @@ func (v *validator) ProcessEvent(ctx context.Context, event *eventClient.Event)
 }
 
 func (v *validator) EventStreamIsRunning() bool {
-	return v.validatorClient.EventStreamIsRunning()
+	return v.eventStreamState.Load() == eventStreamRunning
 }
 
 func (v *validator) Host() string {