Test, please ignore

* use new lru cache

* update build

---------

Co-authored-by: Nishant Das <nishdas93@gmail.com>
Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com>

README.md

@@ -35,3 +35,4 @@ Want to get involved? Check out our [Contribution Guide](https://docs.prylabs.ne
 ## Legal Disclaimer
 
 [Terms of Use](/TERMS_OF_SERVICE.md)
+

beacon-chain/blockchain/receive_attestation.go

@@ -86,6 +86,11 @@ func (s *Service) spawnProcessAttestationsRoutine() {
 			}
 			log.Warn("Genesis time received, now available to process attestations")
 		}
+		// Wait for node to be synced before running the routine.
+		if err := s.waitForSync(); err != nil {
+			log.WithError(err).Error("Could not wait to sync")
+			return
+		}
 
 		st := slots.NewSlotTicker(s.genesisTime, params.BeaconConfig().SecondsPerSlot)
 		pat := slots.NewSlotTickerWithOffset(s.genesisTime, -reorgLateBlockCountAttestations, params.BeaconConfig().SecondsPerSlot)

beacon-chain/state/stateutil/BUILD.bazel

@@ -34,6 +34,7 @@ go_library(
         "//math:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
     ],
 )
 

beacon-chain/state/stateutil/field_root_validator.go

@@ -3,12 +3,15 @@ package stateutil
 import (
 	"bytes"
 	"encoding/binary"
+	"runtime"
+	"sync"
 
 	"github.com/pkg/errors"
 	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/crypto/hash/htr"
 	"github.com/prysmaticlabs/prysm/v4/encoding/ssz"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -51,6 +54,20 @@ func validatorRegistryRoot(validators []*ethpb.Validator) ([32]byte, error) {
 	return res, nil
 }
 
+func hashValidatorHelper(validators []*ethpb.Validator, roots [][32]byte, j int, groupSize int, wg *sync.WaitGroup) {
+	defer wg.Done()
+	for i := 0; i < groupSize; i++ {
+		fRoots, err := ValidatorFieldRoots(validators[j*groupSize+i])
+		if err != nil {
+			logrus.WithError(err).Error("could not get validator field roots")
+			return
+		}
+		for k, root := range fRoots {
+			roots[(j*groupSize+i)*validatorFieldRoots+k] = root
+		}
+	}
+}
+
 // OptimizedValidatorRoots uses an optimized routine with gohashtree in order to
 // derive a list of validator roots from a list of validator objects.
 func OptimizedValidatorRoots(validators []*ethpb.Validator) ([][32]byte, error) {
@@ -58,14 +75,25 @@ func OptimizedValidatorRoots(validators []*ethpb.Validator) ([][32]byte, error)
 	if len(validators) == 0 {
 		return [][32]byte{}, nil
 	}
-	roots := make([][32]byte, 0, len(validators)*validatorFieldRoots)
-	for i := 0; i < len(validators); i++ {
+	wg := sync.WaitGroup{}
+	n := runtime.GOMAXPROCS(0)
+	rootsSize := len(validators) * validatorFieldRoots
+	groupSize := len(validators) / n
+	roots := make([][32]byte, rootsSize)
+	wg.Add(n - 1)
+	for j := 0; j < n-1; j++ {
+		go hashValidatorHelper(validators, roots, j, groupSize, &wg)
+	}
+	for i := (n - 1) * groupSize; i < len(validators); i++ {
 		fRoots, err := ValidatorFieldRoots(validators[i])
 		if err != nil {
 			return [][32]byte{}, errors.Wrap(err, "could not compute validators merkleization")
 		}
-		roots = append(roots, fRoots...)
+		for k, root := range fRoots {
+			roots[i*validatorFieldRoots+k] = root
+		}
 	}
+	wg.Wait()
 
 	// A validator's tree can represented with a depth of 3. As log2(8) = 3
 	// Using this property we can lay out all the individual fields of a
@@ -73,9 +101,7 @@ func OptimizedValidatorRoots(validators []*ethpb.Validator) ([][32]byte, error)
 	for i := 0; i < validatorTreeDepth; i++ {
 		// Overwrite input lists as we are hashing by level
 		// and only need the highest level to proceed.
-		outputLen := len(roots) / 2
-		htr.VectorizedSha256(roots, roots)
-		roots = roots[:outputLen]
+		roots = htr.VectorizedSha256(roots)
 	}
 	return roots, nil
 }

beacon-chain/state/stateutil/field_root_validator_test.go

@@ -3,11 +3,13 @@ package stateutil
 import (
 	"reflect"
 	"strings"
+	"sync"
 	"testing"
 
 	mathutil "github.com/prysmaticlabs/prysm/v4/math"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/testing/assert"
+	"github.com/prysmaticlabs/prysm/v4/testing/require"
 )
 
 func TestValidatorConstants(t *testing.T) {
@@ -30,3 +32,28 @@ func TestValidatorConstants(t *testing.T) {
 	_, err := ValidatorRegistryRoot([]*ethpb.Validator{v})
 	assert.NoError(t, err)
 }
+
+func TestHashValidatorHelper(t *testing.T) {
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	v := &ethpb.Validator{}
+	valList := make([]*ethpb.Validator, 10*validatorFieldRoots)
+	for i := range valList {
+		valList[i] = v
+	}
+	roots := make([][32]byte, len(valList))
+	hashValidatorHelper(valList, roots, 2, 2, &wg)
+	for i := 0; i < 4*validatorFieldRoots; i++ {
+		require.Equal(t, [32]byte{}, roots[i])
+	}
+	emptyValRoots, err := ValidatorFieldRoots(v)
+	require.NoError(t, err)
+	for i := 4; i < 6; i++ {
+		for j := 0; j < validatorFieldRoots; j++ {
+			require.Equal(t, emptyValRoots[j], roots[i*validatorFieldRoots+j])
+		}
+	}
+	for i := 6 * validatorFieldRoots; i < 10*validatorFieldRoots; i++ {
+		require.Equal(t, [32]byte{}, roots[i])
+	}
+}

beacon-chain/state/stateutil/sync_committee.root.go

@@ -48,8 +48,7 @@ func merkleizePubkey(pubkey []byte) ([32]byte, error) {
 	if err != nil {
 		return [32]byte{}, err
 	}
-	outputChunk := make([][32]byte, 1)
-	htr.VectorizedSha256(chunks, outputChunk)
+	outputChunk := htr.VectorizedSha256(chunks)
 
 	return outputChunk[0], nil
 }

beacon-chain/state/stateutil/trie_helpers.go

@@ -71,9 +71,7 @@ func ReturnTrieLayerVariable(elements [][32]byte, length uint64) [][]*[32]byte {
 		}
 
 		layers[i+1] = make([]*[32]byte, layerLen/2)
-		newElems := make([][32]byte, layerLen/2)
-		htr.VectorizedSha256(elements, newElems)
-		elements = newElems
+		elements = htr.VectorizedSha256(elements)
 		for j := range elements {
 			layers[i+1][j] = &elements[j]
 		}
@@ -295,9 +293,7 @@ func MerkleizeTrieLeaves(layers [][][32]byte, hashLayer [][32]byte) ([][][32]byt
 		if !math.IsPowerOf2(uint64(len(hashLayer))) {
 			return nil, nil, errors.Errorf("hash layer is a non power of 2: %d", len(hashLayer))
 		}
-		newLayer := make([][32]byte, len(hashLayer)/2)
-		htr.VectorizedSha256(hashLayer, newLayer)
-		hashLayer = newLayer
+		hashLayer = htr.VectorizedSha256(hashLayer)
 		layers[i] = hashLayer
 		i++
 	}

beacon-chain/sync/rpc_beacon_blocks_by_range.go

@@ -75,7 +75,7 @@ func (s *Service) beaconBlocksByRangeRPCHandler(ctx context.Context, msg interfa
 		rpcBlocksByRangeResponseLatency.Observe(float64(time.Since(batchStart).Milliseconds()))
 	}
 	if err := batch.error(); err != nil {
-		log.WithError(err).Info("error in BlocksByRange batch")
+		log.WithError(err).Debug("error in BlocksByRange batch")
 		s.writeErrorResponseToStream(responseCodeServerError, p2ptypes.ErrGeneric.Error(), stream)
 		tracing.AnnotateError(span, err)
 		return err

crypto/bls/blst/BUILD.bazel

@@ -17,7 +17,7 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/v4/crypto/bls/blst",
     visibility = ["//visibility:public"],
     deps = [
-        "//cache/lru:go_default_library",
+        "//cache/nonblocking:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//crypto/bls/common:go_default_library",

crypto/bls/blst/init.go

@@ -3,8 +3,11 @@
 package blst
 
 import (
+	"fmt"
 	"runtime"
 
+	"github.com/prysmaticlabs/prysm/v4/cache/nonblocking"
+	"github.com/prysmaticlabs/prysm/v4/crypto/bls/common"
 	blst "github.com/supranational/blst/bindings/go"
 )
 
@@ -15,4 +18,10 @@ func init() {
 		maxProcs = 1
 	}
 	blst.SetMaxProcs(maxProcs)
+	onEvict := func(_ [48]byte, _ common.PublicKey) {}
+	keysCache, err := nonblocking.NewLRU(maxKeys, onEvict)
+	if err != nil {
+		panic(fmt.Sprintf("Could not initiate public keys cache: %v", err))
+	}
+	pubkeyCache = keysCache
 }

crypto/bls/blst/public_key.go

@@ -6,14 +6,14 @@ import (
 	"fmt"
 
 	"github.com/pkg/errors"
-	lruwrpr "github.com/prysmaticlabs/prysm/v4/cache/lru"
+	"github.com/prysmaticlabs/prysm/v4/cache/nonblocking"
 	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/crypto/bls/common"
 )
 
 var maxKeys = 1_000_000
-var pubkeyCache = lruwrpr.New(maxKeys)
+var pubkeyCache *nonblocking.LRU[[48]byte, common.PublicKey]
 
 // PublicKey used in the BLS signature scheme.
 type PublicKey struct {
@@ -27,7 +27,7 @@ func PublicKeyFromBytes(pubKey []byte) (common.PublicKey, error) {
 	}
 	newKey := (*[fieldparams.BLSPubkeyLength]byte)(pubKey)
 	if cv, ok := pubkeyCache.Get(*newKey); ok {
-		return cv.(*PublicKey).Copy(), nil
+		return cv.Copy(), nil
 	}
 	// Subgroup check NOT done when decompressing pubkey.
 	p := new(blstPublicKey).Uncompress(pubKey)

crypto/hash/htr/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@prysm//tools/go:def.bzl", "go_library")
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -7,3 +7,11 @@ go_library(
     visibility = ["//visibility:public"],
     deps = ["@com_github_prysmaticlabs_gohashtree//:go_default_library"],
 )
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["hashtree_test.go"],
+    embed = [":go_default_library"],
+    deps = ["//testing/require:go_default_library"],
+)

crypto/hash/htr/hashtree.go

@@ -1,17 +1,47 @@
 package htr
 
 import (
+	"runtime"
+	"sync"
+
 	"github.com/prysmaticlabs/gohashtree"
 )
 
+const minSliceSizeToParallelize = 5000
+
+func hashParallel(inputList [][32]byte, outputList [][32]byte, wg *sync.WaitGroup) {
+	defer wg.Done()
+	err := gohashtree.Hash(outputList, inputList)
+	if err != nil {
+		panic(err)
+	}
+}
+
 // VectorizedSha256 takes a list of roots and hashes them using CPU
 // specific vector instructions. Depending on host machine's specific
 // hardware configuration, using this routine can lead to a significant
 // performance improvement compared to the default method of hashing
 // lists.
-func VectorizedSha256(inputList [][32]byte, outputList [][32]byte) {
-	err := gohashtree.Hash(outputList, inputList)
+func VectorizedSha256(inputList [][32]byte) [][32]byte {
+	outputList := make([][32]byte, len(inputList)/2)
+	if len(inputList) < minSliceSizeToParallelize {
+		err := gohashtree.Hash(outputList, inputList)
+		if err != nil {
+			panic(err)
+		}
+		return outputList
+	}
+	n := runtime.GOMAXPROCS(0) - 1
+	wg := sync.WaitGroup{}
+	wg.Add(n)
+	groupSize := len(inputList) / (2 * (n + 1))
+	for j := 0; j < n; j++ {
+		go hashParallel(inputList[j*2*groupSize:(j+1)*2*groupSize], outputList[j*groupSize:], &wg)
+	}
+	err := gohashtree.Hash(outputList[n*groupSize:], inputList[n*2*groupSize:])
 	if err != nil {
 		panic(err)
 	}
+	wg.Wait()
+	return outputList
 }

crypto/hash/htr/hashtree_test.go

@@ -0,0 +1,27 @@
+package htr
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/v4/testing/require"
+)
+
+func Test_VectorizedSha256(t *testing.T) {
+	largeSlice := make([][32]byte, 32*minSliceSizeToParallelize)
+	secondLargeSlice := make([][32]byte, 32*minSliceSizeToParallelize)
+	hash1 := make([][32]byte, 16*minSliceSizeToParallelize)
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		tempHash := VectorizedSha256(largeSlice)
+		copy(hash1, tempHash)
+	}()
+	wg.Wait()
+	hash2 := VectorizedSha256(secondLargeSlice)
+	require.Equal(t, len(hash1), len(hash2))
+	for i, r := range hash1 {
+		require.Equal(t, r, hash2[i])
+	}
+}

encoding/ssz/merkleize.go

@@ -213,9 +213,7 @@ func MerkleizeVector(elements [][32]byte, length uint64) [32]byte {
 			zerohash := trie.ZeroHashes[i]
 			elements = append(elements, zerohash)
 		}
-		outputLen := len(elements) / 2
-		htr.VectorizedSha256(elements, elements)
-		elements = elements[:outputLen]
+		elements = htr.VectorizedSha256(elements)
 	}
 	return elements[0]
 }

testing/endtoend/evaluators/builder.go

@@ -67,6 +67,9 @@ func builderActive(_ *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) err
 		if string(execPayload.ExtraData()) != "prysm-builder" {
 			return errors.Errorf("block with slot %d was not built by the builder. It has an extra data of %s", b.Block().Slot(), string(execPayload.ExtraData()))
 		}
+		if execPayload.GasLimit() == 0 {
+			return errors.Errorf("block with slot %d has a gas limit of 0, when it should be in the 30M range", b.Block().Slot())
+		}
 	}
 	if lowestBound == currEpoch {
 		return nil

testing/spectest/mainnet/capella/operations/BUILD.bazel

@@ -7,10 +7,12 @@ go_test(
         "attestation_test.go",
         "attester_slashing_test.go",
         "block_header_test.go",
+        "bls_to_execution_change_test.go",
         "deposit_test.go",
         "proposer_slashing_test.go",
         "sync_committee_test.go",
         "voluntary_exit_test.go",
+        "withdrawals_test.go",
     ],
     data = [
         "@consensus_spec_tests_mainnet//:test_data",

testing/spectest/mainnet/capella/operations/bls_to_execution_change_test.go

@@ -0,0 +1,11 @@
+package operations
+
+import (
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/v4/testing/spectest/shared/capella/operations"
+)
+
+func TestMainnet_Capella_Operations_BLSToExecutionChange(t *testing.T) {
+	operations.RunBLSToExecutionChangeTest(t, "mainnet")
+}

testing/spectest/mainnet/capella/operations/withdrawals_test.go

@@ -0,0 +1,11 @@
+package operations
+
+import (
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/v4/testing/spectest/shared/capella/operations"
+)
+
+func TestMainnet_Capella_Operations_Withdrawals(t *testing.T) {
+	operations.RunWithdrawalsTest(t, "mainnet")
+}

validator/node/node.go

@@ -559,11 +559,13 @@ func proposerSettings(cliCtx *cli.Context, db iface.ValidatorDB) (*validatorServ
 	}
 
 	if builderConfigFromFlag != nil {
-		config := builderConfigFromFlag
+		config := builderConfigFromFlag.Clone()
 		if config.GasLimit == validator.Uint64(params.BeaconConfig().DefaultBuilderGasLimit) && vpSettings.DefaultConfig.BuilderConfig != nil {
 			config.GasLimit = vpSettings.DefaultConfig.BuilderConfig.GasLimit
 		}
 		vpSettings.DefaultConfig.BuilderConfig = config
+	} else if vpSettings.DefaultConfig.BuilderConfig != nil {
+		vpSettings.DefaultConfig.BuilderConfig.GasLimit = reviewGasLimit(vpSettings.DefaultConfig.BuilderConfig.GasLimit)
 	}
 
 	if psExists {
@@ -573,7 +575,7 @@ func proposerSettings(cliCtx *cli.Context, db iface.ValidatorDB) (*validatorServ
 		}
 	}
 
-	if fileConfig.ProposerConfig != nil {
+	if fileConfig.ProposerConfig != nil && len(fileConfig.ProposerConfig) != 0 {
 		vpSettings.ProposeConfig = make(map[[fieldparams.BLSPubkeyLength]byte]*validatorServiceConfig.ProposerOption)
 		for key, option := range fileConfig.ProposerConfig {
 			decodedKey, err := hexutil.Decode(key)
@@ -583,29 +585,24 @@ func proposerSettings(cliCtx *cli.Context, db iface.ValidatorDB) (*validatorServ
 			if len(decodedKey) != fieldparams.BLSPubkeyLength {
 				return nil, fmt.Errorf("%v  is not a bls public key", key)
 			}
-			if option == nil {
-				return nil, fmt.Errorf("fee recipient is required for proposer %s", key)
-			}
-			if !common.IsHexAddress(option.FeeRecipient) {
-				return nil, errors.New("fee recipient is not a valid eth1 address")
-			}
-			if err := warnNonChecksummedAddress(option.FeeRecipient); err != nil {
+			if err := verifyOption(key, option); err != nil {
 				return nil, err
 			}
+			currentBuilderConfig := validatorServiceConfig.ToBuilderConfig(option.Builder)
 			if builderConfigFromFlag != nil {
-				config := builderConfigFromFlag.ToPayload()
-				if config.GasLimit == validator.Uint64(params.BeaconConfig().DefaultBuilderGasLimit) && option.Builder != nil {
-					config.GasLimit = option.Builder.GasLimit
+				config := builderConfigFromFlag.Clone()
+				if config.GasLimit == validator.Uint64(params.BeaconConfig().DefaultBuilderGasLimit) && currentBuilderConfig != nil {
+					config.GasLimit = currentBuilderConfig.GasLimit
 				}
-				option.Builder = config
-			} else if option.Builder != nil {
-				option.Builder.GasLimit = reviewGasLimit(option.Builder.GasLimit)
+				currentBuilderConfig = config
+			} else if currentBuilderConfig != nil {
+				currentBuilderConfig.GasLimit = reviewGasLimit(currentBuilderConfig.GasLimit)
 			}
 			o := &validatorServiceConfig.ProposerOption{
 				FeeRecipientConfig: &validatorServiceConfig.FeeRecipientConfig{
 					FeeRecipient: common.HexToAddress(option.FeeRecipient),
 				},
-				BuilderConfig: validatorServiceConfig.ToBuilderConfig(option.Builder),
+				BuilderConfig: currentBuilderConfig,
 			}
 			pubkeyB := bytesutil.ToBytes48(decodedKey)
 			vpSettings.ProposeConfig[pubkeyB] = o
@@ -626,6 +623,19 @@ func proposerSettings(cliCtx *cli.Context, db iface.ValidatorDB) (*validatorServ
 	return vpSettings, nil
 }
 
+func verifyOption(key string, option *validatorpb.ProposerOptionPayload) error {
+	if option == nil {
+		return fmt.Errorf("fee recipient is required for proposer %s", key)
+	}
+	if !common.IsHexAddress(option.FeeRecipient) {
+		return errors.New("fee recipient is not a valid eth1 address")
+	}
+	if err := warnNonChecksummedAddress(option.FeeRecipient); err != nil {
+		return err
+	}
+	return nil
+}
+
 func handleNoProposerSettingsFlagsProvided(cliCtx *cli.Context,
 	db iface.ValidatorDB,
 	builderConfigFromFlag *validatorServiceConfig.BuilderConfig) (*validatorServiceConfig.ProposerSettings, error) {

validator/node/node_test.go

@@ -231,6 +231,29 @@ func TestProposerSettings(t *testing.T) {
 		withdb                       func(db iface.ValidatorDB) error
 		validatorRegistrationEnabled bool
 	}{
+		{
+			name: "Happy Path default only proposer settings file with builder settings,",
+			args: args{
+				proposerSettingsFlagValues: &proposerSettingsFlag{
+					dir:        "./testdata/default-only-proposer-config.json",
+					url:        "",
+					defaultfee: "",
+				},
+			},
+			want: func() *validatorserviceconfig.ProposerSettings {
+				return &validatorserviceconfig.ProposerSettings{
+					DefaultConfig: &validatorserviceconfig.ProposerOption{
+						FeeRecipientConfig: &validatorserviceconfig.FeeRecipientConfig{
+							FeeRecipient: common.HexToAddress("0xae967917c465db8578ca9024c205720b1a3651A9"),
+						},
+						BuilderConfig: &validatorserviceconfig.BuilderConfig{
+							Enabled:  true,
+							GasLimit: validator.Uint64(params.BeaconConfig().DefaultBuilderGasLimit),
+						},
+					},
+				}
+			},
+		},
 		{
 			name: "Happy Path Config file File, bad checksum",
 			args: args{

validator/node/testdata/default-only-proposer-config.json

@@ -0,0 +1,7 @@
+{
+  "proposer_config": {},
+  "default_config": {
+    "fee_recipient": "0xAe967917c465db8578ca9024c205720b1a3651A9",
+    "builder": {"enabled": true}
+  }
+}