diff --git a/beacon-chain/node/config.go b/beacon-chain/node/config.go index 503ed8dae5..a7224e244f 100644 --- a/beacon-chain/node/config.go +++ b/beacon-chain/node/config.go @@ -65,6 +65,24 @@ func configureSafeSlotsToImportOptimistically(cliCtx *cli.Context) error { return nil } +func configureBuilderCircuitBreaker(cliCtx *cli.Context) error { + if cliCtx.IsSet(flags.MaxBuilderConsecutiveMissedSlots.Name) { + c := params.BeaconConfig().Copy() + c.MaxBuilderConsecutiveMissedSlots = types.Slot(cliCtx.Int(flags.MaxBuilderConsecutiveMissedSlots.Name)) + if err := params.SetActive(c); err != nil { + return err + } + } + if cliCtx.IsSet(flags.MaxBuilderEpochMissedSlots.Name) { + c := params.BeaconConfig().Copy() + c.MaxBuilderEpochMissedSlots = types.Slot(cliCtx.Int(flags.MaxBuilderEpochMissedSlots.Name)) + if err := params.SetActive(c); err != nil { + return err + } + } + return nil +} + func configureSlotsPerArchivedPoint(cliCtx *cli.Context) error { if cliCtx.IsSet(flags.SlotsPerArchivedPoint.Name) { c := params.BeaconConfig().Copy() diff --git a/beacon-chain/node/node.go b/beacon-chain/node/node.go index b4c6b08ff0..784684e9ba 100644 --- a/beacon-chain/node/node.go +++ b/beacon-chain/node/node.go @@ -136,6 +136,10 @@ func New(cliCtx *cli.Context, opts ...Option) (*BeaconNode, error) { if err := configureSafeSlotsToImportOptimistically(cliCtx); err != nil { return nil, err } + err := configureBuilderCircuitBreaker(cliCtx) + if err != nil { + return nil, err + } if err := configureSlotsPerArchivedPoint(cliCtx); err != nil { return nil, err } diff --git a/beacon-chain/rpc/prysm/v1alpha1/validator/BUILD.bazel b/beacon-chain/rpc/prysm/v1alpha1/validator/BUILD.bazel index c075268fe7..ca8cba1a68 100644 --- a/beacon-chain/rpc/prysm/v1alpha1/validator/BUILD.bazel +++ b/beacon-chain/rpc/prysm/v1alpha1/validator/BUILD.bazel @@ -134,6 +134,7 @@ go_test( "//beacon-chain/core/transition:go_default_library", "//beacon-chain/db/testing:go_default_library", "//beacon-chain/execution/testing:go_default_library", + "//beacon-chain/forkchoice/protoarray:go_default_library", "//beacon-chain/operations/attestations:go_default_library", "//beacon-chain/operations/slashings:go_default_library", "//beacon-chain/operations/synccommittee:go_default_library", @@ -143,6 +144,7 @@ go_test( "//beacon-chain/state/stategen:go_default_library", "//beacon-chain/state/stategen/mock:go_default_library", "//beacon-chain/state/v1:go_default_library", + "//beacon-chain/state/v3:go_default_library", "//beacon-chain/sync/initial-sync/testing:go_default_library", "//config/fieldparams:go_default_library", "//config/params:go_default_library", diff --git a/beacon-chain/rpc/prysm/v1alpha1/validator/proposer_bellatrix.go b/beacon-chain/rpc/prysm/v1alpha1/validator/proposer_bellatrix.go index bfec0264ac..b4a16fc2b3 100644 --- a/beacon-chain/rpc/prysm/v1alpha1/validator/proposer_bellatrix.go +++ b/beacon-chain/rpc/prysm/v1alpha1/validator/proposer_bellatrix.go @@ -295,6 +295,54 @@ func (vs *Server) readyForBuilder(ctx context.Context) (bool, error) { return blocks.IsExecutionBlock(b.Block().Body()) } +// circuitBreakBuilder returns true if the builder is not allowed to be used due to circuit breaker conditions. +func (vs *Server) circuitBreakBuilder(s types.Slot) (bool, error) { + if vs.ForkFetcher == nil || vs.ForkFetcher.ForkChoicer() == nil { + return true, errors.New("no fork choicer configured") + } + + // Circuit breaker is active if the missing consecutive slots greater than `MaxBuilderConsecutiveMissedSlots`. + highestReceivedSlot := vs.ForkFetcher.ForkChoicer().HighestReceivedBlockSlot() + fallbackSlots := params.BeaconConfig().MaxBuilderConsecutiveMissedSlots + diff, err := s.SafeSubSlot(highestReceivedSlot) + if err != nil { + return true, err + } + if diff > fallbackSlots { + log.WithFields(logrus.Fields{ + "currentSlot": s, + "highestReceivedSlot": highestReceivedSlot, + "fallBackSkipSlots": fallbackSlots, + }).Warn("Builder circuit breaker activated due to missing consecutive slot") + return true, nil + } + + // Not much reason to check missed slots epoch rolling window if input slot is less than epoch. + if s < params.BeaconConfig().SlotsPerEpoch { + return false, nil + } + + // Circuit breaker is active if the missing slots per epoch (rolling window) greater than `MaxBuilderEpochMissedSlots`. + receivedCount, err := vs.ForkFetcher.ForkChoicer().ReceivedBlocksLastEpoch() + if err != nil { + return true, err + } + fallbackSlotsLastEpoch := params.BeaconConfig().MaxBuilderEpochMissedSlots + diff, err = params.BeaconConfig().SlotsPerEpoch.SafeSub(receivedCount) + if err != nil { + return true, err + } + if diff > fallbackSlotsLastEpoch { + log.WithFields(logrus.Fields{ + "totalMissed": receivedCount, + "fallBackSkipSlotsLastEpoch": fallbackSlotsLastEpoch, + }).Warn("Builder circuit breaker activated due to missing enough slots last epoch") + return true, nil + } + + return false, nil +} + // Get and build blind block from builder network. Returns a boolean status, built block and error. // If the status is false that means builder the header block is disallowed. // This routine is time limited by `blockBuilderTimeout`. @@ -313,6 +361,15 @@ func (vs *Server) getAndBuildBlindBlock(ctx context.Context, b *ethpb.BeaconBloc if !ready { return false, nil, nil } + + circuitBreak, err := vs.circuitBreakBuilder(b.Slot) + if err != nil { + return false, nil, errors.Wrap(err, "could not determine if builder circuit breaker condition") + } + if circuitBreak { + return false, nil, nil + } + h, err := vs.getPayloadHeaderFromBuilder(ctx, b.Slot, b.ProposerIndex) if err != nil { return false, nil, errors.Wrap(err, "could not get payload header") diff --git a/beacon-chain/rpc/prysm/v1alpha1/validator/proposer_bellatrix_test.go b/beacon-chain/rpc/prysm/v1alpha1/validator/proposer_bellatrix_test.go index 3ef0b53f76..3cd852168c 100644 --- a/beacon-chain/rpc/prysm/v1alpha1/validator/proposer_bellatrix_test.go +++ b/beacon-chain/rpc/prysm/v1alpha1/validator/proposer_bellatrix_test.go @@ -18,11 +18,14 @@ import ( prysmtime "github.com/prysmaticlabs/prysm/beacon-chain/core/time" dbTest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing" mockExecution "github.com/prysmaticlabs/prysm/beacon-chain/execution/testing" + "github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray" "github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations" "github.com/prysmaticlabs/prysm/beacon-chain/operations/slashings" "github.com/prysmaticlabs/prysm/beacon-chain/operations/synccommittee" "github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits" + "github.com/prysmaticlabs/prysm/beacon-chain/state" "github.com/prysmaticlabs/prysm/beacon-chain/state/stategen" + v3 "github.com/prysmaticlabs/prysm/beacon-chain/state/v3" mockSync "github.com/prysmaticlabs/prysm/beacon-chain/sync/initial-sync/testing" fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams" "github.com/prysmaticlabs/prysm/config/params" @@ -31,6 +34,7 @@ import ( types "github.com/prysmaticlabs/prysm/consensus-types/primitives" "github.com/prysmaticlabs/prysm/crypto/bls" "github.com/prysmaticlabs/prysm/encoding/bytesutil" + enginev1 "github.com/prysmaticlabs/prysm/proto/engine/v1" v1 "github.com/prysmaticlabs/prysm/proto/engine/v1" ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1" "github.com/prysmaticlabs/prysm/testing/require" @@ -331,6 +335,7 @@ func TestServer_getAndBuildHeaderBlock(t *testing.T) { vs.FinalizationFetcher = &blockchainTest.ChainService{FinalizedCheckPoint: ðpb.Checkpoint{Root: wbr1[:]}} vs.HeadFetcher = &blockchainTest.ChainService{Block: wb1} vs.BlockBuilder = &builderTest.MockBuilderService{HasConfigured: true, ErrGetHeader: errors.New("could not get payload")} + vs.ForkFetcher = &blockchainTest.ChainService{ForkChoiceStore: protoarray.New()} ready, _, err = vs.getAndBuildBlindBlock(ctx, ðpb.BeaconBlockAltair{}) require.ErrorContains(t, "could not get payload", err) require.Equal(t, false, ready) @@ -406,6 +411,7 @@ func TestServer_getAndBuildHeaderBlock(t *testing.T) { } vs.BlockBuilder = &builderTest.MockBuilderService{HasConfigured: true, Bid: sBid} vs.TimeFetcher = &blockchainTest.ChainService{Genesis: time.Now()} + vs.ForkFetcher = &blockchainTest.ChainService{ForkChoiceStore: protoarray.New()} ready, builtBlk, err := vs.getAndBuildBlindBlock(ctx, altairBlk.Block) require.NoError(t, err) require.Equal(t, true, ready) @@ -644,12 +650,18 @@ func TestServer_GetBellatrixBeaconBlock_BuilderCase(t *testing.T) { Signature: sk.Sign(sr[:]).Marshal(), } proposerServer.BlockBuilder = &builderTest.MockBuilderService{HasConfigured: true, Bid: sBid} - + proposerServer.ForkFetcher = &blockchainTest.ChainService{ForkChoiceStore: protoarray.New()} randaoReveal, err := util.RandaoReveal(beaconState, 0, privKeys) require.NoError(t, err) require.NoError(t, proposerServer.BeaconDB.SaveRegistrationsByValidatorIDs(ctx, []types.ValidatorIndex{40}, []*ethpb.ValidatorRegistrationV1{{FeeRecipient: bytesutil.PadTo([]byte{}, fieldparams.FeeRecipientLength), Pubkey: bytesutil.PadTo([]byte{}, fieldparams.BLSPubkeyLength)}})) + + params.SetupTestConfigCleanup(t) + cfg.MaxBuilderConsecutiveMissedSlots = bellatrixSlot + 1 + cfg.MaxBuilderEpochMissedSlots = 32 + params.OverrideBeaconConfig(cfg) + block, err := proposerServer.getBellatrixBeaconBlock(ctx, ðpb.BlockRequest{ Slot: bellatrixSlot + 1, RandaoReveal: randaoReveal, @@ -721,3 +733,75 @@ func TestServer_validateBuilderSignature(t *testing.T) { sBid.Message.Value = make([]byte, 32) require.ErrorIs(t, s.validateBuilderSignature(sBid), signing.ErrSigFailedToVerify) } + +func TestServer_circuitBreakBuilder(t *testing.T) { + hook := logTest.NewGlobal() + s := &Server{} + _, err := s.circuitBreakBuilder(0) + require.ErrorContains(t, "no fork choicer configured", err) + + s.ForkFetcher = &blockchainTest.ChainService{ForkChoiceStore: protoarray.New()} + s.ForkFetcher.ForkChoicer().SetGenesisTime(uint64(time.Now().Unix())) + b, err := s.circuitBreakBuilder(params.BeaconConfig().MaxBuilderConsecutiveMissedSlots + 1) + require.NoError(t, err) + require.Equal(t, true, b) + require.LogsContain(t, hook, "Builder circuit breaker activated due to missing consecutive slot") + + ojc := ðpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]} + ofc := ðpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]} + ctx := context.Background() + st, blkRoot, err := createState(1, [32]byte{'a'}, [32]byte{}, params.BeaconConfig().ZeroHash, ojc, ofc) + require.NoError(t, err) + require.NoError(t, s.ForkFetcher.ForkChoicer().InsertNode(ctx, st, blkRoot)) + b, err = s.circuitBreakBuilder(params.BeaconConfig().MaxBuilderConsecutiveMissedSlots + 1) + require.NoError(t, err) + require.Equal(t, false, b) + + params.SetupTestConfigCleanup(t) + params.OverrideBeaconConfig(params.MainnetConfig()) + st, blkRoot, err = createState(params.BeaconConfig().SlotsPerEpoch, [32]byte{'b'}, [32]byte{'a'}, params.BeaconConfig().ZeroHash, ojc, ofc) + require.NoError(t, err) + require.NoError(t, s.ForkFetcher.ForkChoicer().InsertNode(ctx, st, blkRoot)) + b, err = s.circuitBreakBuilder(params.BeaconConfig().SlotsPerEpoch + 1) + require.NoError(t, err) + require.Equal(t, true, b) + require.LogsContain(t, hook, "Builder circuit breaker activated due to missing enough slots last epoch") + + want := params.BeaconConfig().SlotsPerEpoch - params.BeaconConfig().MaxBuilderEpochMissedSlots + for i := types.Slot(2); i <= want+2; i++ { + st, blkRoot, err = createState(i, [32]byte{byte(i)}, [32]byte{'a'}, params.BeaconConfig().ZeroHash, ojc, ofc) + require.NoError(t, err) + require.NoError(t, s.ForkFetcher.ForkChoicer().InsertNode(ctx, st, blkRoot)) + } + b, err = s.circuitBreakBuilder(params.BeaconConfig().SlotsPerEpoch + 1) + require.NoError(t, err) + require.Equal(t, false, b) +} + +func createState( + slot types.Slot, + blockRoot [32]byte, + parentRoot [32]byte, + payloadHash [32]byte, + justified *ethpb.Checkpoint, + finalized *ethpb.Checkpoint, +) (state.BeaconState, [32]byte, error) { + + base := ðpb.BeaconStateBellatrix{ + Slot: slot, + RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector), + BlockRoots: make([][]byte, 1), + CurrentJustifiedCheckpoint: justified, + FinalizedCheckpoint: finalized, + LatestExecutionPayloadHeader: &enginev1.ExecutionPayloadHeader{ + BlockHash: payloadHash[:], + }, + LatestBlockHeader: ðpb.BeaconBlockHeader{ + ParentRoot: parentRoot[:], + }, + } + + base.BlockRoots[0] = append(base.BlockRoots[0], blockRoot[:]...) + st, err := v3.InitializeFromProto(base) + return st, blockRoot, err +} diff --git a/cmd/beacon-chain/flags/base.go b/cmd/beacon-chain/flags/base.go index 3a620b570c..c0a95cc7fc 100644 --- a/cmd/beacon-chain/flags/base.go +++ b/cmd/beacon-chain/flags/base.go @@ -16,6 +16,16 @@ var ( Usage: "A MEV builder relay string http endpoint, this wil be used to interact MEV builder network using API defined in: https://ethereum.github.io/builder-specs/#/Builder", Value: "", } + MaxBuilderConsecutiveMissedSlots = &cli.IntFlag{ + Name: "max-builder-consecutive-missed-slots", + Usage: "Number of consecutive skip slot to fallback from using relay/builder to local execution engine for block construction", + Value: 3, + } + MaxBuilderEpochMissedSlots = &cli.IntFlag{ + Name: "max-builder-epoch-missed-slots", + Usage: "Number of total skip slot to fallback from using relay/builder to local execution engine for block construction in last epoch rolling window", + Value: 8, + } // ExecutionEngineEndpoint provides an HTTP access endpoint to connect to an execution client on the execution layer ExecutionEngineEndpoint = &cli.StringFlag{ Name: "execution-endpoint", diff --git a/config/params/config.go b/config/params/config.go index 16ebd19b32..081b0c7d3a 100644 --- a/config/params/config.go +++ b/config/params/config.go @@ -199,6 +199,10 @@ type BeaconChainConfig struct { DefaultFeeRecipient common.Address // DefaultFeeRecipient where the transaction fee goes to. EthBurnAddressHex string // EthBurnAddressHex is the constant eth address written in hex format to burn fees in that network. the default is 0x0 DefaultBuilderGasLimit uint64 // DefaultBuilderGasLimit is the default used to set the gaslimit for the Builder APIs, typically at around 30M wei. + + // Mev-boost circuit breaker + MaxBuilderConsecutiveMissedSlots types.Slot // MaxBuilderConsecutiveMissedSlots defines the number of consecutive skip slot to fallback from using relay/builder to local execution engine for block construction. + MaxBuilderEpochMissedSlots types.Slot // MaxBuilderEpochMissedSlots is defines the number of total skip slot (per epoch rolling windows) to fallback from using relay/builder to local execution engine for block construction. } // InitializeForkSchedule initializes the schedules forks baked into the config. diff --git a/config/params/mainnet_config.go b/config/params/mainnet_config.go index d2451ac98e..e38e98c0ab 100644 --- a/config/params/mainnet_config.go +++ b/config/params/mainnet_config.go @@ -251,6 +251,10 @@ var mainnetBeaconConfig = &BeaconChainConfig{ TerminalTotalDifficulty: "115792089237316195423570985008687907853269984665640564039457584007913129638912", EthBurnAddressHex: "0x0000000000000000000000000000000000000000", DefaultBuilderGasLimit: uint64(30000000), + + // Mevboost circuit breaker + MaxBuilderConsecutiveMissedSlots: 4, + MaxBuilderEpochMissedSlots: 6, } // MainnetTestConfig provides a version of the mainnet config that has a different name