Add proper handling for submit blind block 502 error (#15848)

* Add proper handling for builder relay 502 BadGateway errors

* James feedback

* Change wording

api/client/builder/client.go

@@ -726,6 +726,12 @@ func unexpectedStatusErr(response *http.Response, expected int) error {
 			return errors.Wrap(jsonErr, "unable to read response body")
 		}
 		return errors.Wrap(ErrNotOK, errMessage.Message)
+	case http.StatusBadGateway:
+		log.WithError(ErrBadGateway).Debug(msg)
+		if jsonErr := json.Unmarshal(bodyBytes, &errMessage); jsonErr != nil {
+			return errors.Wrap(jsonErr, "unable to read response body")
+		}
+		return errors.Wrap(ErrBadGateway, errMessage.Message)
 	default:
 		log.WithError(ErrNotOK).Debug(msg)
 		return errors.Wrap(ErrNotOK, fmt.Sprintf("unsupported error code: %d", response.StatusCode))

api/client/builder/client_test.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/OffchainLabs/prysm/v6/api"
 	"github.com/OffchainLabs/prysm/v6/api/server/structs"
-	"github.com/OffchainLabs/prysm/v6/testing/util"
 	"github.com/OffchainLabs/prysm/v6/config/params"
 	"github.com/OffchainLabs/prysm/v6/consensus-types/blocks"
 	"github.com/OffchainLabs/prysm/v6/consensus-types/interfaces"
@@ -22,6 +21,7 @@ import (
 	eth "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
 	"github.com/OffchainLabs/prysm/v6/testing/assert"
 	"github.com/OffchainLabs/prysm/v6/testing/require"
+	"github.com/OffchainLabs/prysm/v6/testing/util"
 	"github.com/prysmaticlabs/go-bitfield"
 	log "github.com/sirupsen/logrus"
 )

api/client/builder/errors.go

@@ -21,3 +21,4 @@ var ErrUnsupportedMediaType = errors.Wrap(ErrNotOK, "The media type in \"Content
 
 // ErrNotAcceptable specifically means that a '406 - Not Acceptable' was received from the API.
 var ErrNotAcceptable = errors.Wrap(ErrNotOK, "The accept header value is not acceptable")
+var ErrBadGateway = errors.Wrap(ErrNotOK, "recv 502 BadGateway response from API")

beacon-chain/blockchain/process_block_test.go

@@ -3302,7 +3302,6 @@ func Test_postBlockProcess_EventSending(t *testing.T) {
 	}
 }
 
-
 func setupLightClientTestRequirements(ctx context.Context, t *testing.T, s *Service, v int, options ...util.LightClientOption) (*util.TestLightClient, *postBlockProcessConfig) {
 	var l *util.TestLightClient
 	switch v {

beacon-chain/rpc/prysm/v1alpha1/validator/proposer.go

@@ -316,6 +316,10 @@ func (vs *Server) ProposeBeaconBlock(ctx context.Context, req *ethpb.GenericSign
 		blobSidecars, dataColumnSidecars, err = vs.handleUnblindedBlock(rob, req)
 	}
 	if err != nil {
+		if errors.Is(err, builderapi.ErrBadGateway) && block.IsBlinded() {
+			log.WithError(err).Info("Optimistically proposed block - builder relay temporarily unavailable, block may arrive over P2P")
+			return &ethpb.ProposeResponse{BlockRoot: root[:]}, nil
+		}
 		return nil, status.Errorf(codes.Internal, "%s: %v", "handle block failed", err)
 	}
 

beacon-chain/rpc/prysm/v1alpha1/validator/proposer_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	builderapi "github.com/OffchainLabs/prysm/v6/api/client/builder"
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/blockchain/kzg"
 	mock "github.com/OffchainLabs/prysm/v6/beacon-chain/blockchain/testing"
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/builder"
@@ -3634,4 +3635,52 @@ func TestServer_ProposeBeaconBlock_PostFuluBlindedBlock(t *testing.T) {
 		require.NotNil(t, res)
 		require.NotEmpty(t, res.BlockRoot)
 	})
+
+	t.Run("blinded block - 502 error handling", func(t *testing.T) {
+		params.SetupTestConfigCleanup(t)
+		cfg := params.BeaconConfig().Copy()
+		cfg.FuluForkEpoch = 10
+		params.OverrideBeaconConfig(cfg)
+
+		mockBuilder := &builderTest.MockBuilderService{
+			HasConfigured:         true,
+			Cfg:                   &builderTest.Config{BeaconDB: db},
+			PayloadDeneb:          &enginev1.ExecutionPayloadDeneb{},
+			ErrSubmitBlindedBlock: builderapi.ErrBadGateway,
+		}
+
+		c := &mock.ChainService{State: beaconState, Root: parentRoot[:]}
+		proposerServer := &Server{
+			ChainStartFetcher: &mockExecution.Chain{},
+			Eth1InfoFetcher:   &mockExecution.Chain{},
+			Eth1BlockFetcher:  &mockExecution.Chain{},
+			BlockReceiver:     c,
+			BlobReceiver:      c,
+			HeadFetcher:       c,
+			BlockNotifier:     c.BlockNotifier(),
+			OperationNotifier: c.OperationNotifier(),
+			StateGen:          stategen.New(db, doublylinkedtree.New()),
+			TimeFetcher:       c,
+			SyncChecker:       &mockSync.Sync{IsSyncing: false},
+			BeaconDB:          db,
+			BlockBuilder:      mockBuilder,
+			P2P:               &mockp2p.MockBroadcaster{},
+		}
+
+		blindedBlock := util.NewBlindedBeaconBlockDeneb()
+		blindedBlock.Message.Slot = 160 // This puts us at epoch 5 (160/32 = 5)
+		blindedBlock.Message.ProposerIndex = 0
+		blindedBlock.Message.ParentRoot = parentRoot[:]
+		blindedBlock.Message.StateRoot = make([]byte, 32)
+
+		req := &ethpb.GenericSignedBeaconBlock{
+			Block: &ethpb.GenericSignedBeaconBlock_BlindedDeneb{BlindedDeneb: blindedBlock},
+		}
+
+		// Should handle 502 error gracefully and continue with original blinded block
+		res, err := proposerServer.ProposeBeaconBlock(ctx, req)
+		require.NoError(t, err)
+		require.NotNil(t, res)
+		require.NotEmpty(t, res.BlockRoot)
+	})
 }

changelog/ttsao_handle-relay-502-errors.md

@@ -0,0 +1,3 @@
+### Changed
+
+- Gracefully handle submit blind block returning 502 errors.

validator/client/beacon-api/beacon_api_validator_client_test.go

@@ -202,10 +202,10 @@ func TestBeaconApiValidatorClient_ProposeBeaconBlockError_ThenPass(t *testing.T)
 
 func TestBeaconApiValidatorClient_ProposeBeaconBlockAllTypes(t *testing.T) {
 	tests := []struct {
-		name          string
-		block         *ethpb.GenericSignedBeaconBlock
-		expectedPath  string
-		wantErr       bool
+		name         string
+		block        *ethpb.GenericSignedBeaconBlock
+		expectedPath string
+		wantErr      bool
 		errorMessage string
 	}{
 		{
@@ -374,7 +374,7 @@ func TestBeaconApiValidatorClient_ProposeBeaconBlockHTTPErrors(t *testing.T) {
 				gomock.Any(),
 				gomock.Any(),
 			).Return(nil, nil, tt.sszError).Times(1)
-			
+
 			if tt.expectJSON {
 				// When SSZ fails, it falls back to JSON
 				jsonRestHandler.EXPECT().Post(

validator/client/beacon-api/mock/json_rest_handler_mock.go

@@ -121,7 +121,7 @@ func (m *MockJsonRestHandler) PostSSZ(ctx context.Context, endpoint string, head
 	ret0, _ := ret[0].([]byte)
 	ret1, _ := ret[1].(http.Header)
 	ret2, _ := ret[2].(error)
-	return ret0,ret1,ret2
+	return ret0, ret1, ret2
 }
 
 // Post indicates an expected call of Post.