rm deprecated blob type and db methods

* add it

* pingo

* gaz

* remove pingo

* fix for old forks

.bazelrc

@@ -27,6 +27,7 @@ build:minimal --@io_bazel_rules_go//go/config:tags=minimal
 # Release flags
 build:release --compilation_mode=opt
 build:release --stamp
+build:release --define pgo_enabled=1
 
 # Build binary with cgo symbolizer for debugging / profiling.
 build:cgo_symbolizer --copt=-g

.bazelversion

@@ -1 +1 @@
-6.3.2
+6.4.0

.buildkite-bazelrc

@@ -9,8 +9,8 @@
 #build:remote-cache --strategy=Genrule=standalone
 
 # Prysm specific remote-cache properties.
-#build:remote-cache --disk_cache=
-build:remote-cache --remote_download_toplevel
+build:remote-cache --remote_download_minimal
+build:remote-cache --experimental_remote_build_event_upload=minimal
 build:remote-cache --remote_cache=grpc://bazel-remote-cache:9092
 # Does not work with rules_oci. See https://github.com/bazel-contrib/rules_oci/issues/292
 #build:remote-cache --experimental_remote_downloader=grpc://bazel-remote-cache:9092
@@ -29,7 +29,10 @@ build --experimental_use_hermetic_linux_sandbox
 # Import workspace options.
 import %workspace%/.bazelrc
 
-startup --host_jvm_args=-Xmx4g --host_jvm_args=-Xms2g
+# Enable blake3 once it is supported in remote cache. See: https://github.com/buchgr/bazel-remote/issues/710
+# startup --digest_function=blake3
+
+startup --host_jvm_args=-Xmx8g --host_jvm_args=-Xms4g
 build --experimental_strict_action_env
 build --sandbox_tmpfs_path=/tmp
 build --verbose_failures
@@ -39,6 +42,7 @@ build --curses=no --color=no
 build --keep_going
 build --test_output=errors
 build --flaky_test_attempts=5
+build --build_runfile_links=false # Only build runfile symlink forest when required by local action, test, or run command.
 # Disabled race detection due to unstable test results under constrained environment build kite
 # build --features=race
 

.github/actions/gomodtidy/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:alpine
+FROM golang:1.21-alpine
 
 COPY entrypoint.sh /entrypoint.sh
 

.github/workflows/fuzz.yml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21.5'
       - id: list
         uses: shogo82148/actions-go-fuzz/list@v0
         with:
@@ -36,7 +36,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21.5'
       - uses: shogo82148/actions-go-fuzz/run@v0
         with:
           packages: ${{ matrix.package }}

.github/workflows/go.yml

@@ -5,6 +5,8 @@ on:
     branches: [ master ]
   pull_request:
     branches: [ '*' ]
+  merge_group:
+    types: [checks_requested]
 
 jobs:
   formatting:
@@ -26,15 +28,15 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - name: Set up Go 1.20
+      - name: Set up Go 1.21
         uses: actions/setup-go@v3
         with:
-          go-version: '1.20'
+          go-version: '1.21.5'
       - name: Run Gosec Security Scanner
         run: | # https://github.com/securego/gosec/issues/469
           export PATH=$PATH:$(go env GOPATH)/bin
           go install github.com/securego/gosec/v2/cmd/gosec@v2.15.0
-          gosec -exclude=G307 -exclude-dir=crypto/bls/herumi ./...
+          gosec -exclude-generated -exclude=G307 -exclude-dir=crypto/bls/herumi ./...
 
   lint:
     name: Lint
@@ -43,16 +45,16 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Set up Go 1.20
+      - name: Set up Go 1.21
         uses: actions/setup-go@v3
         with:
-          go-version: '1.20'
+          go-version: '1.21.5'
           id: go
 
       - name: Golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.52.2
+          version: v1.55.2
           args: --config=.golangci.yml --out-${NO_FUTURE}format colored-line-number
 
   build:
@@ -62,7 +64,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v2
         with:
-          go-version: '1.20'
+          go-version: '1.21.5'
         id: go
 
       - name: Check out code into the Go module directory

.golangci.yml

@@ -6,21 +6,83 @@ run:
     - proto
     - tools/analyzers
   timeout: 10m
-  go: '1.19'
+  go: '1.21.5'
 
 linters:
-  disable-all: true
-  enable:
-    - gofmt
-    - goimports
-    - unused
-    - errcheck
-    - gosimple
-    - gocognit
-    - dupword
-    - nilerr
-    - whitespace
-    - misspell
+  enable-all: true
+  disable:
+    # Deprecated linters:
+    - deadcode
+    - exhaustivestruct
+    - golint
+    - govet
+    - ifshort
+    - interfacer
+    - maligned
+    - nosnakecase
+    - scopelint
+    - structcheck
+    - varcheck
+
+    # Disabled for now:
+    - asasalint
+    - bodyclose
+    - containedctx
+    - contextcheck
+    - cyclop
+    - depguard
+    - dogsled
+    - dupl
+    - durationcheck
+    - errorlint
+    - exhaustive
+    - exhaustruct
+    - forbidigo
+    - forcetypeassert
+    - funlen
+    - gci
+    - gochecknoglobals
+    - gochecknoinits
+    - goconst
+    - gocritic
+    - gocyclo
+    - godot
+    - godox
+    - goerr113
+    - gofumpt
+    - gomnd
+    - gomoddirectives
+    - inamedparam
+    - interfacebloat
+    - ireturn
+    - lll
+    - maintidx
+    - makezero
+    - musttag
+    - nakedret
+    - nestif
+    - nilnil
+    - nlreturn
+    - noctx
+    - nolintlint
+    - nonamedreturns
+    - nosprintfhostport
+    - perfsprint
+    - prealloc
+    - predeclared
+    - promlinter
+    - protogetter
+    - revive
+    - staticcheck
+    - stylecheck
+    - tagalign
+    - tagliatelle
+    - thelper
+    - unparam
+    - varnamelen
+    - wastedassign
+    - wrapcheck
+    - wsl
 
 linters-settings:
   gocognit:

BUILD.bazel

@@ -4,6 +4,7 @@ load("@com_github_atlassian_bazel_tools//goimports:def.bzl", "goimports")
 load("@io_kubernetes_build//defs:run_in_workspace.bzl", "workspace_binary")
 load("@io_bazel_rules_go//go:def.bzl", "nogo")
 load("@bazel_skylib//rules:common_settings.bzl", "string_setting")
+load("@prysm//tools/nogo_config:def.bzl", "nogo_config_exclude")
 
 prefix = "github.com/prysmaticlabs/prysm"
 
@@ -82,9 +83,115 @@ workspace_binary(
     cmd = "@com_github_golang_lint//golint",
 )
 
+STATICCHECK_ANALYZERS = [
+    # Enabled static checks. See https://staticcheck.dev/docs/checks/
+    # Please. keep this list sorted. Don't be a bad person by inserting stuff randomly.
+    "sa1000",
+    "sa1001",
+    "sa1002",
+    "sa1003",
+    "sa1004",
+    "sa1005",
+    "sa1006",
+    "sa1007",
+    "sa1008",
+    "sa1010",
+    "sa1011",
+    "sa1012",
+    "sa1013",
+    "sa1014",
+    "sa1015",
+    "sa1016",
+    "sa1017",
+    "sa1018",
+    # "sa1019", # TODO: Fix all uses of deprecated things.
+    "sa1020",
+    "sa1021",
+    "sa1023",
+    "sa1024",
+    "sa1025",
+    "sa1026",
+    "sa1027",
+    "sa1028",
+    "sa1029",
+    "sa1030",
+    "sa2000",
+    "sa2001",
+    "sa2002",
+    "sa2003",
+    "sa3000",
+    "sa3001",
+    "sa4000",
+    "sa4001",
+    "sa4003",
+    "sa4004",
+    "sa4005",
+    "sa4006",
+    "sa4008",
+    "sa4009",
+    "sa4010",
+    "sa4011",
+    "sa4012",
+    "sa4013",
+    "sa4014",
+    "sa4015",
+    "sa4016",
+    "sa4017",
+    "sa4018",
+    "sa4019",
+    "sa4020",
+    "sa4021",
+    "sa4022",
+    "sa4023",
+    "sa4024",
+    "sa4025",
+    "sa4026",
+    "sa4027",
+    "sa4028",
+    "sa4029",
+    "sa4030",
+    "sa4031",
+    "sa4032",
+    "sa5000",
+    "sa5001",
+    "sa5002",
+    "sa5003",
+    "sa5004",
+    "sa5005",
+    "sa5007",
+    "sa5008",
+    "sa5009",
+    "sa5010",
+    "sa5011",
+    "sa5012",
+    "sa6000",
+    "sa6001",
+    "sa6002",
+    "sa6003",
+    "sa6005",
+    "sa6006",
+    "sa9001",
+    "sa9002",
+    #"sa9003", # Doesn't build. See https://github.com/dominikh/go-tools/pull/1483
+    "sa9004",
+    "sa9005",
+    "sa9006",
+    "sa9007",
+    "sa9008",
+]
+
+nogo_config_exclude(
+    name = "nogo_config_with_excludes",
+    checks = [sa.upper() for sa in STATICCHECK_ANALYZERS],
+    exclude_files = [
+        "external/.*",
+    ],
+    input = "nogo_config.json",
+)
+
 nogo(
     name = "nogo",
-    config = "nogo_config.json",
+    config = ":nogo_config_with_excludes",
     visibility = ["//visibility:public"],
     deps = [
         "@org_golang_x_tools//go/analysis/passes/unsafeptr:go_default_library",
@@ -136,7 +243,7 @@ nogo(
             "@org_golang_x_tools//go/analysis/passes/composite:go_default_library",
             "@org_golang_x_tools//go/analysis/passes/lostcancel:go_default_library",
         ],
-    }),
+    }) + ["@co_honnef_go_tools//staticcheck/%s:go_default_library" % c for c in STATICCHECK_ANALYZERS],
 )
 
 config_setting(
@@ -144,6 +251,11 @@ config_setting(
     values = {"define": "coverage_enabled=1"},
 )
 
+config_setting(
+    name = "pgo_enabled",
+    values = {"define": "pgo_enabled=1"},
+)
+
 common_files = {
     "//:LICENSE.md": "LICENSE.md",
     "//:README.md": "README.md",

README.md

@@ -7,7 +7,7 @@
 [![Discord](https://user-images.githubusercontent.com/7288322/34471967-1df7808a-efbb-11e7-9088-ed0b04151291.png)](https://discord.gg/prysmaticlabs)
 [![GitPOAP Badge](https://public-api.gitpoap.io/v1/repo/prysmaticlabs/prysm/badge)](https://www.gitpoap.io/gh/prysmaticlabs/prysm)
 
-This is the core repository for Prysm, a [Golang](https://golang.org/) implementation of the [Ethereum Consensus](https://ethereum.org/en/eth2/) specification, developed by [Prysmatic Labs](https://prysmaticlabs.com). See the [Changelog](https://github.com/prysmaticlabs/prysm/releases) for details of the latest releases and upcoming breaking changes.
+This is the core repository for Prysm, a [Golang](https://golang.org/) implementation of the [Ethereum Consensus](https://ethereum.org/en/developers/docs/consensus-mechanisms/#proof-of-stake) [specification](https://github.com/ethereum/consensus-specs), developed by [Offchain Labs](https://www.offchainlabs.com). See the [Changelog](https://github.com/prysmaticlabs/prysm/releases) for details of the latest releases and upcoming breaking changes.
 
 ### Getting Started
 

TERMS_OF_SERVICE.md

@@ -1,45 +1,53 @@
-## Terms of Use
+# Terms of Use
+Effective as of November 2, 2023
 
-Effective as of Oct 14, 2020
+By downloading, accessing or using the Prysm implementation (“Prysm”), you (referenced herein as “you” or the “user”) certify that you have read and agreed to the terms and conditions below (the “Terms”) which form a binding contract between you and Offchain Labs, Inc. (as successor in interest to Prysmatic Labs LLC) (referenced herein as “Offchain Labs”, “we” or “us”). If you do not agree to the Terms, do not download or use Prysm.  Additionally, the Terms of Use available at https://arbitrum.io/tos (or any successor site, the “OCL Terms of Use”) are hereby incorporated by reference into these Terms.  In the event of any conflict between provisions set forth herein and those set forth in the OCL Terms of Use, the provisions set forth herein shall control.
 
-By downloading, accessing or using the Prysm implementation (“Prysm”), you (referenced herein as “you” or the “user”) certify that you have read and agreed to the terms and conditions below (the “Terms”) which form a binding contract between you and Prysmatic Labs (referenced herein as “we” or “us”). If you do not agree to the Terms, do not download or use Prysm.
+## About Prysm
 
-### About Prysm
-Prysm is a client implementation for Ethereum consensus protocol for a proof-of-stake blockchain. To participate in the network, a user must send ETH from the Eth1.0 chain into a validator deposit contract, which will queue in the user as a validator in the system. Validators participate in proposing and voting on blocks in the protocol, and the network applies rewards/penalties based on their behavior. A detailed set of installation and usage instructions as well as breakdowns of each individual component are available in the official documentation portal, however, we do not warrant the accuracy, completeness or usefulness of this documentation. Any reliance you place on such information is strictly at your own risk.
+Prysm is a client implementation for the Ethereum blockchain’s consensus protocol. To participate in the network, a user must send ETH from the Ethereum mainnet blockchain to a validator deposit smart contract on Ethereum mainnet. Validators participate in proposing and voting on blocks in the protocol, and the network applies rewards/penalties based on their behavior. A detailed set of installation and usage instructions as well as breakdowns of each individual component are available in the official documentation portal, however, we do not warrant the accuracy, completeness or usefulness of this documentation. Any reliance you place on such information is strictly at your own risk.
 
-### Licensing Terms
-Prysm is a fully open-source software program licensed pursuant to the GNU General Public License v3.0.
+## Licensing Terms
+Prysm is an open-source software program licensed pursuant to the GNU General Public License v3.0.
+The Offchain Labs name, the term “Prysm” and all related names, logos, product and service names, designs and slogans are trademarks of Offchain Labs or its affiliates and/or licensors. You must not use such marks without our prior written permission.
+PLEASE READ THESE TERMS CAREFULLY, AS THE OCL TERMS OF USE INCORPORATED BY REFERENCE HEREIN CONTAIN AN AGREEMENT TO ARBITRATE AND OTHER IMPORTANT INFORMATION REGARDING YOUR LEGAL RIGHTS, REMEDIES, AND OBLIGATIONS. THE AGREEMENT TO ARBITRATE REQUIRES (WITH LIMITED EXCEPTION) THAT YOU SUBMIT CLAIMS YOU HAVE AGAINST US TO BINDING AND FINAL ARBITRATION, AND FURTHER (1) YOU WILL ONLY BE PERMITTED TO PURSUE CLAIMS AGAINST OFFCHAIN LABS ON AN INDIVIDUAL BASIS, NOT AS A PLAINTIFF OR CLASS MEMBER IN ANY CLASS OR REPRESENTATIVE ACTION OR PROCEEDING, (2) YOU WILL ONLY BE PERMITTED TO SEEK RELIEF (INCLUDING MONETARY, INJUNCTIVE, AND DECLARATORY RELIEF) ON AN INDIVIDUAL BASIS, AND (3) YOU MAY NOT BE ABLE TO HAVE ANY CLAIMS YOU HAVE AGAINST US RESOLVED BY A JURY OR IN A COURT OF LAW.
 
-The Prysmatic Labs name, the term “Prysm” and all related names, logos, product and service names, designs and slogans are trademarks of Prysmatic Labs or its affiliates and/or licensors. You must not use such marks without our prior written permission.
+## Risks of Operating Prysm
 
-### Risks of Operating Prysm
-The use of Prysm and acting as a validator on the Ethereum network can lead to loss of money. Ethereum is still an experimental system and ETH remains a risky investment. You alone are responsible for your actions on Prysm including the security of your ETH and meeting any applicable minimum system requirements.
+The use of Prysm and acting as a validator on the Ethereum network can lead to loss of money, tokens and value. Ethereum is still an experimental system and ETH remains a risky investment. You alone are responsible for your actions on Prysm, including the security of your ETH and meeting any applicable minimum system requirements.
 
 Use of Prysm and the ability to receive rewards or penalties may be affected at any time by mistakes made by the user or other users, software problems such as bugs, errors, incorrectly constructed transactions, unsafe cryptographic libraries or malware affecting the network, technical failures in the hardware of a user, security problems experienced by a user and/or actions or inactions of third parties and/or events experienced by third parties, among other risks. We cannot and do not guarantee that any user of Prysm will make money, that the Prysm network will operate in accordance with the documentation or that transactions will be effective or secure.
 
-We make no claims that Prysm is appropriate or permitted for use in any specific jurisdiction. Access to Prysm may not be legal by certain persons or in certain jurisdictions or countries. If you access Prysm, you do so on your own initiative and are responsible for compliance with local laws.
+YOU ACKNOWLEDGE THAT WE ARE NOT RESPONSIBLE FOR ANY RISKS ASSOCIATED WITH YOUR USE OF PRYSM, AND CANNOT BE HELD LIABLE FOR ANY RESULTING LOSSES THAT YOU EXPERIENCE WHILE ACCESSING OR USING PRYSM.
 
-Some Internet plans will charge an additional amount for any excess upload bandwidth used that isn’t included in the plan and may terminate your connection without warning because of overuse. We advise that you check whether your Internet connection is subjected to such limitations and monitor your bandwidth use so that you can stop Prysm before you reach your upload limit.
+BY ACCESSING AND USING PRYSM, YOU REPRESENT AND WARRANT THAT YOU UNDERSTAND THE INHERENT RISKS ASSOCIATED WITH USING CRYPTOGRAPHIC AND BLOCKCHAIN-BASED SYSTEMS, AND THAT YOU HAVE A WORKING KNOWLEDGE OF THE USAGE AND INTRICACIES OF DIGITAL ASSETS, SUCH AS THOSE FOLLOWING THE ETHEREUM TOKEN STANDARD (ERC-20). YOU FURTHER UNDERSTAND THAT THE MARKETS FOR DIGITAL ASSETS ARE HIGHLY VOLATILE DUE TO VARIOUS FACTORS, INCLUDING ADOPTION, SPECULATION, TECHNOLOGY, SECURITY, AND REGULATION. YOU ACKNOWLEDGE AND ACCEPT THAT THE COST AND SPEED OF TRANSACTING WITH CRYPTOGRAPHIC AND BLOCKCHAIN-BASED SYSTEMS SUCH AS ETHEREUM ARE VARIABLE AND MAY INCREASE DRAMATICALLY AT ANY TIME. YOU UNDERSTAND THAT ANYONE CAN CREATE A TOKEN, INCLUDING FAKE VERSIONS OF EXISTING TOKENS AND TOKENS THAT FALSELY CLAIM TO REPRESENT PROJECTS, AND ACKNOWLEDGE AND ACCEPT THE RISK THAT YOU MAY MISTAKENLY INTERACT WITH THOSE OR OTHER TOKENS. YOU FURTHER ACKNOWLEDGE THAT WE ARE NOT RESPONSIBLE FOR ANY OF THE VARIABLES OR RISKS DESCRIBED IN THESE TERMS. YOU UNDERSTAND AND AGREE TO ASSUME FULL RESPONSIBILITY FOR ALL OF THE RISKS OF ACCESSING AND USING PRYSM. YOU ARE SOLELY RESPONSIBLE FOR YOUR WALLETS, FOR SAFEGUARDING THE ASSOCIATED PRIVATE KEY AND FOR ANY ACTIVITY THAT OCCURS USING YOUR WALLET. WITHOUT LIMITING THE FOREGOING, YOU ALSO UNDERSTAND THAT THERE MAY BE TAX AND REGULATORY RISKS RELATED TO USING PRYSM. IT IS YOUR SOLE RESPONSIBILITY TO DETERMINE WHETHER, AND TO WHAT EXTENT, ANY TAXES APPLY TO ANY TRANSACTIONS YOU CONDUCT IN CONNECTION WITH YOUR USE OF PRYSM, AND TO WITHHOLD, COLLECT, REPORT AND REMIT THE CORRECT AMOUNTS OF TAXES TO THE APPROPRIATE TAX AUTHORITIES. DIGITAL ASSETS, BLOCKCHAIN TECHNOLOGY, AND ANY RELATED SOFTWARE AND SERVICES ARE ALSO SUBJECT TO LEGAL AND REGULATORY UNCERTAINTY IN THE UNITED STATES AND OTHER JURISDICTIONS. YOU UNDERSTAND THAT LEGISLATIVE AND REGULATORY CHANGES OR ACTIONS MAY ADVERSELY AFFECT THE USAGE, TRANSFERABILITY, TRANSACTABILITY AND ACCESSIBILITY RELATED TO PRYSM.
 
-### Warranty Disclaimer
-PRYSM IS PROVIDED ON AN “AS-IS” BASIS AND MAY INCLUDE ERRORS, OMISSIONS, OR OTHER INACCURACIES. PRYSMATIC LABS AND ITS CONTRIBUTORS MAKE NO REPRESENTATIONS OR WARRANTIES ABOUT PRYSM FOR ANY PURPOSE, AND HEREBY EXPRESSLY DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT OR ANY OTHER IMPLIED WARRANTY UNDER THE UNIFORM COMPUTER INFORMATION TRANSACTIONS ACT AS ENACTED BY ANY STATE. WE ALSO MAKE NO REPRESENTATIONS OR WARRANTIES THAT PRYSM WILL OPERATE ERROR-FREE, UNINTERRUPTED, OR IN A MANNER THAT WILL MEET YOUR REQUIREMENTS AND/OR NEEDS. THEREFORE, YOU ASSUME THE ENTIRE RISK REGARDING THE QUALITY AND/OR PERFORMANCE OF PRYSM AND ANY TRANSACTIONS ENTERED INTO THEREON.
+We make no claims that Prysm is appropriate or permitted for use in any specific jurisdiction. Access to Prysm may not be legal by certain persons or in certain jurisdictions or countries. If you access Prysm, you do so on your own initiative and are responsible for compliance with all Applicable Law (as defined below), including, without limitation, for the avoidance of doubt, local laws.
 
-### Limitation of Liability
-In no event will Prysmatic Labs or any of its contributors be liable, whether in contract, warranty, tort (including negligence, whether active, passive or imputed), product liability, strict liability or other theory, breach of statutory duty or otherwise arising out of, or in connection with, your use of Prysm, for any direct, indirect, incidental, special or consequential damages (including any loss of profits or data, business interruption or other pecuniary loss, or damage, loss or other compromise of data, in each case whether direct, indirect, incidental, special or consequential) arising out of use Prysm, even if we or other users have been advised of the possibility of such damages. The foregoing limitations and disclaimers shall apply to the maximum extent permitted by applicable law, even if any remedy fails of its essential purpose. You acknowledge and agree that the limitations of liability afforded us hereunder constitute a material and actual inducement and condition to entering into these Terms, and are reasonable, fair and equitable in scope to protect our legitimate interests in light of the fact that we are not receiving consideration from you for providing Prysm.
+Some Internet plans will charge additional amounts for bandwidth or any excess upload bandwidth used that isn’t included in the plan and may terminate your connection without warning because of overuse. We advise that you check whether your Internet connection is subjected to any such limitations and monitor your bandwidth use and upload volumes.
 
-### Indemnification
-To the maximum extent permitted by law, you will defend, indemnify and hold Prysmatic Labs and its contributors harmless from and against any and all claims, actions, suits, investigations, or proceedings by any third party (including any party or purported party to or beneficiary or purported beneficiary of any transaction on Prysm), as well as any and all losses, liabilities,
-damages, costs, and expenses (including reasonable attorneys’ fees) arising out of, accruing from, or in any way related to (i) your breach of the terms of this Agreement, (ii) any transaction, or the failure to occur of any transaction on Prysm, and (iii) your negligence, fraud, or willful misconduct.
+## Warranty Disclaimer
 
-### Compliance with Laws and Tax Obligations
-Your use of Prysm is subject to all applicable laws of any governmental authority, including, without limitation, federal, state and foreign securities laws, tax laws, tariff and trade laws, ordinances, judgments, decrees, injunctions, writs and orders or like actions of any governmental authority and rules, regulations, orders, interpretations, licenses, and permits of any federal,
-regional, state, county, municipal or other governmental authority and you agree to comply with all such laws in your use of Prysm. The users of Prysm are solely responsible to determinate what, if any, taxes apply to their ETH transactions. The owners of, or contributors to, Prysm are not responsible for determining the taxes that apply to ETH transactions.
+PRYSM IS PROVIDED ON AN “AS-IS” BASIS AND MAY INCLUDE ERRORS, OMISSIONS, OR OTHER INACCURACIES. WITHOUT LIMITING ANYTHING SET FORTH ELSEWHERE IN THESE TERMS, OFFCHAIN LABS AND ITS CONTRIBUTORS MAKE NO REPRESENTATIONS OR WARRANTIES ABOUT PRYSM FOR ANY PURPOSE, AND HEREBY EXPRESSLY DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT OR ANY OTHER IMPLIED WARRANTY UNDER THE UNIFORM COMPUTER INFORMATION TRANSACTIONS ACT AS ENACTED BY ANY STATE OR OTHER GOVERNMENTAL AUTHORITY. WE ALSO MAKE NO REPRESENTATIONS OR WARRANTIES THAT PRYSM WILL OPERATE ERROR-FREE, UNINTERRUPTED, OR IN A MANNER THAT WILL MEET YOUR REQUIREMENTS AND/OR NEEDS. THEREFORE, YOU ASSUME THE ENTIRE RISK REGARDING THE QUALITY AND/OR PERFORMANCE OF PRYSM AND ANY TRANSACTIONS ENTERED INTO THEREON.
 
-### Miscellaneous
-These Terms will be construed and enforced in accordance with the laws of the state of Illinois as applied to agreements entered into and completely performed in Illinois. You agree to the personal jurisdiction by and venue in Illinois and waive any objection to such jurisdiction or venue.
+## Limitation of Liability
 
-We reserve the right to revise these Terms, and your rights and obligations are at all times subject to the then-current Terms provided on Prysm. Your continued use of Prysm constitutes acceptance of such revised Terms.
+IN NO EVENT WILL OFFCHAIN LABS OR ANY OF ITS AFFILIATES OR ITS OR ANY SUCH AFFILIATE’S DIRECTORS, OFFICERS, EMPLOYEES, AGENTS, OR REPRESENTATIVES OR ANY CONTRIBUTORS (COLLECTIVELY, THE “OCL PARTIES”) BE LIABLE, WHETHER IN CONTRACT, WARRANTY, TORT (INCLUDING NEGLIGENCE, WHETHER ACTIVE, PASSIVE OR IMPUTED), PRODUCT LIABILITY, STRICT LIABILITY OR OTHER THEORY, BREACH OF STATUTORY DUTY OR OTHERWISE ARISING OUT OF, OR IN CONNECTION WITH, YOUR USE OF PRYSM, FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL OR CONSEQUENTIAL DAMAGES (INCLUDING ANY LOSS OF PROFITS OR DATA, BUSINESS INTERRUPTION OR OTHER PECUNIARY LOSS, OR DAMAGE, LOSS OR OTHER COMPROMISE OF DATA, IN EACH CASE WHETHER DIRECT, INDIRECT, INCIDENTAL, SPECIAL OR CONSEQUENTIAL) ARISING OUT OF USE PRYSM, EVEN IF WE OR OTHER USERS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. The foregoing limitations and disclaimers shall apply to the maximum extent permitted by Applicable Law, even if any remedy fails of its essential purpose. You acknowledge and agree that the limitations of liability afforded us hereunder constitute a material and actual inducement and condition to entering into these Terms, and are reasonable, fair and equitable in scope to protect our legitimate interests in light of the fact that we are not receiving consideration from you for providing Prysm.
 
-These Terms constitute the entire agreement between you and Prysmatic Labs regarding use of Prysm and will supersede all prior agreements whether, written or oral. No usage of trade or other regular practice or method of dealing between the parties will be used to modify, interpret, supplement, or alter the terms of these Terms.
+## Indemnification
+
+To the maximum extent permitted by Applicable Law, you will defend, indemnify and hold each OCL Party harmless from and against any and all claims, actions, suits, investigations, or proceedings by any third party (including any party or purported party to or beneficiary or purported beneficiary of any transaction or other activity on Prysm), as well as any and all losses, liabilities, damages, costs, and expenses (including reasonable attorneys’ fees and costs) arising out of, accruing from, or in any way related to (i) your breach of the terms of this Agreement, (ii) any transaction, or the failure to occur of any transaction on Prysm, and (iii) your negligence, fraud, or willful misconduct.
+
+## Compliance with Laws
+
+Your use of Prysm is subject to all applicable laws of any governmental authority, including, without limitation, federal, state and foreign securities laws, tax laws, tariff and trade laws, ordinances, judgments, decrees, injunctions, writs and orders or like actions of any governmental authority and rules, regulations, orders, interpretations, licenses, and permits of any federal, regional, state, county, municipal or other governmental authority (collectively, “Applicable Law”) and you agree to comply with all such Applicable Law in your use of Prysm. The users of Prysm are solely responsible to determinate what, if any, taxes apply to their ETH transactions. The owners of, or contributors to, Prysm are not responsible for determining the taxes that apply to ETH transactions.
+
+## Miscellaneous
+
+These Terms will be governed by the laws of the State of Delaware without regard to its conflict of law provisions. With respect to any disputes or claims not subject to arbitration, as set forth in the OCL Terms of Use, you and Offchain Labs submit to the personal and exclusive jurisdiction of the state and federal courts located within New York, New York and waive any objection to such jurisdiction and venue. The failure of Offchain Labs to exercise or enforce any right or provision of these Terms will not constitute a waiver of such right or provision.
+We reserve the right to revise these Terms, and your rights and obligations are at all times subject to the then-current Terms provided on Prysm. Your  use of Prysm following any such revision to these Terms constitutes acceptance of such revised Terms.
+
+These Terms constitute the entire agreement between you and Offchain Labs regarding use of Prysm and will supersede all prior agreements whether, written or oral. No usage of trade or other regular practice or method of dealing between the parties will be used to modify, interpret, supplement, or alter the terms of these Terms.
+
+If any portion of these Terms is held invalid or unenforceable, such invalidity or enforceability will not affect the other provisions of these Terms, which will remain in full force and effect, and the invalid or unenforceable portion will be given effect to the greatest extent possible. The failure of a party to require performance of any provision will not affect that party’s right to require performance at any time thereafter, nor will a waiver of any breach or default of these Terms or any provision of these Terms constitute a waiver of any subsequent breach or default or a waiver of the provision itself.
 
-If any portion of these Terms is held invalid or unenforceable, such invalidity or enforceability will not affect the other provisions of these Terms, which will remain in full force and effect, and the invalid or unenforceable portion will be given effect to the greatest extent possible. The failure of a party to require performance of any provision will not affect that party’s right to require performance at any time thereafter, nor will a waiver of any breach or default of these Terms or any provision of these Terms constitute a waiver of any subsequent breach or default or a waiver of the provision itself.

WORKSPACE

@@ -27,7 +27,23 @@ http_archive(
 
 load("@hermetic_cc_toolchain//toolchain:defs.bzl", zig_toolchains = "toolchains")
 
-zig_toolchains()
+# Temporarily use a nightly build until 0.12.0 is released.
+# See: https://github.com/prysmaticlabs/prysm/issues/13130
+zig_toolchains(
+    host_platform_sha256 = {
+        "linux-aarch64": "45afb8e32adde825165f4f293fcea9ecea503f7f9ec0e9bf4435afe70e67fb70",
+        "linux-x86_64": "f136c6a8a0f6adcb057d73615fbcd6f88281b3593f7008d5f7ed514ff925c02e",
+        "macos-aarch64": "05d995853c05243151deff47b60bdc2674f1e794a939eaeca0f42312da031cee",
+        "macos-x86_64": "721754ba5a50f31e8a1f0e1a74cace26f8246576878ac4a8591b0ee7b6db1fc1",
+        "windows-x86_64": "93f5248b2ea8c5ee8175e15b1384e133edc1cd49870b3ea259062a2e04164343",
+    },
+    url_formats = [
+        "https://ziglang.org/builds/zig-{host_platform}-{version}.{_ext}",
+        "https://mirror.bazel.build/ziglang.org/builds/zig-{host_platform}-{version}.{_ext}",
+        "https://prysmaticlabs.com/mirror/ziglang.org/builds/zig-{host_platform}-{version}.{_ext}",
+    ],
+    version = "0.12.0-dev.1349+fa022d1ec",
+)
 
 # Register zig sdk toolchains with support for Ubuntu 20.04 (Focal Fossa) which has an EOL date of April, 2025.
 # For ubuntu glibc support, see https://launchpad.net/ubuntu/+source/glibc
@@ -65,10 +81,10 @@ bazel_skylib_workspace()
 
 http_archive(
     name = "bazel_gazelle",
-    sha256 = "29d5dafc2a5582995488c6735115d1d366fcd6a0fc2e2a153f02988706349825",
+    sha256 = "d3fa66a39028e97d76f9e2db8f1b0c11c099e8e01bf363a923074784e451f809",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.31.0/bazel-gazelle-v0.31.0.tar.gz",
-        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.31.0/bazel-gazelle-v0.31.0.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.33.0/bazel-gazelle-v0.33.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.33.0/bazel-gazelle-v0.33.0.tar.gz",
     ],
 )
 
@@ -80,11 +96,22 @@ http_archive(
 )
 
 http_archive(
-    name = "io_bazel_rules_docker",
-    sha256 = "b1e80761a8a8243d03ebca8845e9cc1ba6c82ce7c5179ce2b295cd36f7e394bf",
-    urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.25.0/rules_docker-v0.25.0.tar.gz"],
+    name = "rules_distroless",
+    sha256 = "e64f06e452cd153aeab81f752ccf4642955b3af319e64f7bc7a7c9252f76b10e",
+    strip_prefix = "rules_distroless-f5e678217b57ce3ad2f1c0204bd4e9d416255773",
+    url = "https://github.com/GoogleContainerTools/rules_distroless/archive/f5e678217b57ce3ad2f1c0204bd4e9d416255773.tar.gz",
 )
 
+load("@rules_distroless//distroless:dependencies.bzl", "rules_distroless_dependencies")
+
+rules_distroless_dependencies()
+
+load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "aspect_bazel_lib_register_toolchains")
+
+aspect_bazel_lib_dependencies()
+
+aspect_bazel_lib_register_toolchains()
+
 http_archive(
     name = "rules_oci",
     sha256 = "c71c25ed333a4909d2dd77e0b16c39e9912525a98c7fa85144282be8d04ef54c",
@@ -110,10 +137,10 @@ http_archive(
         # Expose internals of go_test for custom build transitions.
         "//third_party:io_bazel_rules_go_test.patch",
     ],
-    sha256 = "bfc5ce70b9d1634ae54f4e7b495657a18a04e0d596785f672d35d5f505ab491a",
+    sha256 = "d6ab6b57e48c09523e93050f13698f708428cfd5e619252e369d377af6597707",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.40.0/rules_go-v0.40.0.zip",
-        "https://github.com/bazelbuild/rules_go/releases/download/v0.40.0/rules_go-v0.40.0.zip",
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.43.0/rules_go-v0.43.0.zip",
+        "https://github.com/bazelbuild/rules_go/releases/download/v0.43.0/rules_go-v0.43.0.zip",
     ],
 )
 
@@ -132,67 +159,16 @@ git_repository(
     # gazelle args: -go_prefix github.com/gogo/protobuf -proto legacy
 )
 
-load(
-    "@io_bazel_rules_docker//repositories:repositories.bzl",
-    container_repositories = "repositories",
-)
-
-container_repositories()
-
-load(
-    "@io_bazel_rules_docker//container:container.bzl",
-    "container_pull",
-)
-
-# Pulled gcr.io/distroless/cc-debian11:latest on 2022-02-23
-container_pull(
-    name = "cc_image_base_amd64",
-    digest = "sha256:2a0daf90a7deb78465bfca3ef2eee6e91ce0a5706059f05d79d799a51d339523",
-    registry = "gcr.io",
-    repository = "distroless/cc-debian11",
-)
-
-# Pulled gcr.io/distroless/cc-debian11:debug on 2022-02-23
-container_pull(
-    name = "cc_debug_image_base_amd64",
-    digest = "sha256:7bd596f5f200588f13a69c268eea6ce428b222b67cd7428d6a7fef95e75c052a",
-    registry = "gcr.io",
-    repository = "distroless/cc-debian11",
-)
-
-# Pulled from gcr.io/distroless/base-debian11:latest on 2022-02-23
-container_pull(
-    name = "go_image_base_amd64",
-    digest = "sha256:34e682800774ecbd0954b1663d90238505f1ba5543692dbc75feef7dd4839e90",
-    registry = "gcr.io",
-    repository = "distroless/base-debian11",
-)
-
-# Pulled from gcr.io/distroless/base-debian11:debug on 2022-02-23
-container_pull(
-    name = "go_debug_image_base_amd64",
-    digest = "sha256:0f503c6bfd207793bc416f20a35bf6b75d769a903c48f180ad73f60f7b60d7bd",
-    registry = "gcr.io",
-    repository = "distroless/base-debian11",
-)
-
-container_pull(
-    name = "alpine_cc_linux_amd64",
-    digest = "sha256:752aa0c9a88461ffc50c5267bb7497ef03a303e38b2c8f7f2ded9bebe5f1f00e",
-    registry = "index.docker.io",
-    repository = "pinglamb/alpine-glibc",
-)
-
 load("@rules_oci//oci:pull.bzl", "oci_pull")
 
 # A multi-arch base image
 oci_pull(
     name = "linux_debian11_multiarch_base",  # Debian bullseye
-    digest = "sha256:9b8e0854865dcaf49470b4ec305df45957020fbcf17b71eeb50ffd3bc5bf885d",  # 2023-05-17
+    digest = "sha256:b82f113425c5b5c714151aaacd8039bc141821cdcd3c65202d42bdf9c43ae60b",  # 2023-12-12
     image = "gcr.io/distroless/cc-debian11",
     platforms = [
         "linux/amd64",
-        "linux/arm64",
+        "linux/arm64/v8",
     ],
     reproducible = True,
 )
@@ -206,7 +182,7 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe
 go_rules_dependencies()
 
 go_register_toolchains(
-    go_version = "1.20.9",
+    go_version = "1.21.5",
     nogo = "@//:nogo",
 )
 
@@ -228,8 +204,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "91434d5fd5e1c6eb7b0174fed2afe25e09bddf00e1e4c431db931b2cee4e7773",
-    url = "https://github.com/eth-clients/slashing-protection-interchange-tests/archive/b8413ca42dc92308019d0d4db52c87e9e125c4e9.tar.gz",
+    sha256 = "516d551cfb3e50e4ac2f42db0992f4ceb573a7cb1616d727a725c8161485329f",
+    url = "https://github.com/eth-clients/slashing-protection-interchange-tests/archive/refs/tags/v5.3.0.tar.gz",
 )
 
 http_archive(
@@ -247,9 +223,7 @@ filegroup(
     url = "https://github.com/ethereum/EIPs/archive/5480440fe51742ed23342b68cf106cefd427e39d.tar.gz",
 )
 
-consensus_spec_test_version = "v1.4.0-beta.2-hotfix"
-
-consensus_spec_version = "v1.4.0-beta.2"
+consensus_spec_version = "v1.4.0-beta.5"
 
 bls_test_version = "v0.1.1"
 
@@ -265,8 +239,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "99770a001189f66204a4ef79161c8002bcbbcbd8236f1c6479bd5b83a3c68d42",
-    url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/general.tar.gz" % consensus_spec_test_version,
+    sha256 = "9017ffff84d64a7c4c9e6ff9f421f9479f71d3b463b738f54e02158dbb4f50f0",
+    url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/general.tar.gz" % consensus_spec_version,
 )
 
 http_archive(
@@ -281,8 +255,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "56763f6492ee137108271007d62feef60d8e3f1698e53dee4bc4b07e55f7326b",
-    url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/minimal.tar.gz" % consensus_spec_test_version,
+    sha256 = "f08711682553fe7c9362f1400ed8c56b2fa9576df08581fcad4c508ba8ad4788",
+    url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/minimal.tar.gz" % consensus_spec_version,
 )
 
 http_archive(
@@ -297,8 +271,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "bc1cac1a991cdc7426efea14385dcf215df85ed3f0572b824ad6a1d7ca0c89ad",
-    url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/mainnet.tar.gz" % consensus_spec_test_version,
+    sha256 = "7ea3189e3879f2ac62467cbf2945c00b6c94d30cdefb2d645c630b1018c50e10",
+    url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/mainnet.tar.gz" % consensus_spec_version,
 )
 
 http_archive(
@@ -312,7 +286,7 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "c5898001aaab2a5bb38a39ff9d17a52f1f9befcc26e63752cbf556040f0c884e",
+    sha256 = "4119992a2efc79e5cb2bdc07ed08c0b1fa32332cbd0d88e6467f34938df97026",
     strip_prefix = "consensus-specs-" + consensus_spec_version[1:],
     url = "https://github.com/ethereum/consensus-specs/archive/refs/tags/%s.tar.gz" % consensus_spec_version,
 )
@@ -348,6 +322,22 @@ filegroup(
     url = "https://github.com/eth-clients/eth2-networks/archive/7b4897888cebef23801540236f73123e21774954.tar.gz",
 )
 
+http_archive(
+    name = "goerli_testnet",
+    build_file_content = """
+filegroup(
+    name = "configs",
+    srcs = [
+        "prater/config.yaml",
+    ],
+    visibility = ["//visibility:public"],
+)
+    """,
+    sha256 = "43fc0f55ddff7b511713e2de07aa22846a67432df997296fb4fc09cd8ed1dcdb",
+    strip_prefix = "goerli-6522ac6684693740cd4ddcc2a0662e03702aa4a1",
+    url = "https://github.com/eth-clients/goerli/archive/6522ac6684693740cd4ddcc2a0662e03702aa4a1.tar.gz",
+)
+
 http_archive(
     name = "holesky_testnet",
     build_file_content = """
@@ -374,6 +364,22 @@ http_archive(
 )
 
 # External dependencies
+http_archive(
+    name = "googleapis",
+    sha256 = "9d1a930e767c93c825398b8f8692eca3fe353b9aaadedfbcf1fca2282c85df88",
+    strip_prefix = "googleapis-64926d52febbf298cb82a8f472ade4a3969ba922",
+    urls = [
+        "https://github.com/googleapis/googleapis/archive/64926d52febbf298cb82a8f472ade4a3969ba922.zip",
+    ],
+)
+
+load("@googleapis//:repository_rules.bzl", "switched_rules_by_language")
+
+switched_rules_by_language(
+    name = "com_google_googleapis_imports",
+    go = True,
+)
+
 load("//:deps.bzl", "prysm_deps")
 
 # gazelle:repository_macro deps.bzl%prysm_deps
@@ -387,24 +393,6 @@ load("@prysm//testing/endtoend:deps.bzl", "e2e_deps")
 
 e2e_deps()
 
-load(
-    "@io_bazel_rules_docker//go:image.bzl",
-    _go_image_repos = "repositories",
-)
-
-# Golang images
-# This is using gcr.io/distroless/base
-_go_image_repos()
-
-# CC images
-# This is using gcr.io/distroless/base
-load(
-    "@io_bazel_rules_docker//cc:image.bzl",
-    _cc_image_repos = "repositories",
-)
-
-_cc_image_repos()
-
 load("@com_github_atlassian_bazel_tools//gometalinter:deps.bzl", "gometalinter_dependencies")
 
 gometalinter_dependencies()

api/BUILD.bazel

@@ -2,7 +2,10 @@ load("@prysm//tools/go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
-    srcs = ["headers.go"],
+    srcs = [
+        "constants.go",
+        "headers.go",
+    ],
     importpath = "github.com/prysmaticlabs/prysm/v4/api",
     visibility = ["//visibility:public"],
 )

api/client/beacon/BUILD.bazel

@@ -11,9 +11,12 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//api/client:go_default_library",
+        "//api/server:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/rpc/apimiddleware:go_default_library",
+        "//beacon-chain/rpc/eth/beacon:go_default_library",
+        "//beacon-chain/rpc/eth/config:go_default_library",
         "//beacon-chain/rpc/eth/shared:go_default_library",
+        "//beacon-chain/rpc/prysm/beacon:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//consensus-types/interfaces:go_default_library",
         "//consensus-types/primitives:go_default_library",
@@ -21,7 +24,6 @@ go_library(
         "//encoding/ssz/detect:go_default_library",
         "//io/file:go_default_library",
         "//network/forks:go_default_library",
-        "//proto/eth/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//runtime/version:go_default_library",
         "//time/slots:go_default_library",

api/client/beacon/client.go

@@ -13,16 +13,17 @@ import (
 	"strconv"
 	"text/template"
 
-	"github.com/prysmaticlabs/prysm/v4/api/client"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
-	"github.com/prysmaticlabs/prysm/v4/network/forks"
-	v1 "github.com/prysmaticlabs/prysm/v4/proto/eth/v1"
-
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/apimiddleware"
+	"github.com/prysmaticlabs/prysm/v4/api/client"
+	"github.com/prysmaticlabs/prysm/v4/api/server"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/beacon"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/config"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
+	apibeacon "github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/prysm/beacon"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
+	"github.com/prysmaticlabs/prysm/v4/network/forks"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	log "github.com/sirupsen/logrus"
 )
@@ -31,7 +32,7 @@ const (
 	getSignedBlockPath       = "/eth/v2/beacon/blocks"
 	getBlockRootPath         = "/eth/v1/beacon/blocks/{{.Id}}/root"
 	getForkForStatePath      = "/eth/v1/beacon/states/{{.Id}}/fork"
-	getWeakSubjectivityPath  = "/eth/v1/beacon/weak_subjectivity"
+	getWeakSubjectivityPath  = "/prysm/v1/beacon/weak_subjectivity"
 	getForkSchedulePath      = "/eth/v1/config/fork_schedule"
 	getConfigSpecPath        = "/eth/v1/config/spec"
 	getStatePath             = "/eth/v2/debug/beacon/states"
@@ -178,12 +179,12 @@ func (c *Client) GetForkSchedule(ctx context.Context) (forks.OrderedSchedule, er
 }
 
 // GetConfigSpec retrieve the current configs of the network used by the beacon node.
-func (c *Client) GetConfigSpec(ctx context.Context) (*v1.SpecResponse, error) {
+func (c *Client) GetConfigSpec(ctx context.Context) (*config.GetSpecResponse, error) {
 	body, err := c.Get(ctx, getConfigSpecPath)
 	if err != nil {
 		return nil, errors.Wrap(err, "error requesting configSpecPath")
 	}
-	fsr := &v1.SpecResponse{}
+	fsr := &config.GetSpecResponse{}
 	err = json.Unmarshal(body, fsr)
 	if err != nil {
 		return nil, err
@@ -258,16 +259,16 @@ func (c *Client) GetWeakSubjectivity(ctx context.Context) (*WeakSubjectivityData
 	if err != nil {
 		return nil, err
 	}
-	v := &apimiddleware.WeakSubjectivityResponse{}
+	v := &apibeacon.GetWeakSubjectivityResponse{}
 	err = json.Unmarshal(body, v)
 	if err != nil {
 		return nil, err
 	}
-	epoch, err := strconv.ParseUint(v.Data.Checkpoint.Epoch, 10, 64)
+	epoch, err := strconv.ParseUint(v.Data.WsCheckpoint.Epoch, 10, 64)
 	if err != nil {
 		return nil, err
 	}
-	blockRoot, err := hexutil.Decode(v.Data.Checkpoint.Root)
+	blockRoot, err := hexutil.Decode(v.Data.WsCheckpoint.Root)
 	if err != nil {
 		return nil, err
 	}
@@ -284,7 +285,7 @@ func (c *Client) GetWeakSubjectivity(ctx context.Context) (*WeakSubjectivityData
 
 // SubmitChangeBLStoExecution calls a beacon API endpoint to set the withdrawal addresses based on the given signed messages.
 // If the API responds with something other than OK there will be failure messages associated to the corresponding request message.
-func (c *Client) SubmitChangeBLStoExecution(ctx context.Context, request []*apimiddleware.SignedBLSToExecutionChangeJson) error {
+func (c *Client) SubmitChangeBLStoExecution(ctx context.Context, request []*shared.SignedBLSToExecutionChange) error {
 	u := c.BaseURL().ResolveReference(&url.URL{Path: changeBLStoExecutionPath})
 	body, err := json.Marshal(request)
 	if err != nil {
@@ -305,7 +306,7 @@ func (c *Client) SubmitChangeBLStoExecution(ctx context.Context, request []*apim
 	if resp.StatusCode != http.StatusOK {
 		decoder := json.NewDecoder(resp.Body)
 		decoder.DisallowUnknownFields()
-		errorJson := &apimiddleware.IndexedVerificationFailureErrorJson{}
+		errorJson := &server.IndexedVerificationFailureError{}
 		if err := decoder.Decode(errorJson); err != nil {
 			return errors.Wrapf(err, "failed to decode error JSON for %s", resp.Request.URL)
 		}
@@ -323,12 +324,12 @@ func (c *Client) SubmitChangeBLStoExecution(ctx context.Context, request []*apim
 
 // GetBLStoExecutionChanges gets all the set withdrawal messages in the node's operation pool.
 // Returns a struct representation of json response.
-func (c *Client) GetBLStoExecutionChanges(ctx context.Context) (*apimiddleware.BLSToExecutionChangesPoolResponseJson, error) {
+func (c *Client) GetBLStoExecutionChanges(ctx context.Context) (*beacon.BLSToExecutionChangesPoolResponse, error) {
 	body, err := c.Get(ctx, changeBLStoExecutionPath)
 	if err != nil {
 		return nil, err
 	}
-	poolResponse := &apimiddleware.BLSToExecutionChangesPoolResponseJson{}
+	poolResponse := &beacon.BLSToExecutionChangesPoolResponse{}
 	err = json.Unmarshal(body, poolResponse)
 	if err != nil {
 		return nil, err

api/client/builder/BUILD.bazel

@@ -20,8 +20,6 @@ go_library(
         "//encoding/bytesutil:go_default_library",
         "//math:go_default_library",
         "//monitoring/tracing:go_default_library",
-        "//network:go_default_library",
-        "//network/authorization:go_default_library",
         "//proto/engine/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//runtime/version:go_default_library",

api/client/builder/bid.go

@@ -6,7 +6,6 @@ import (
 	consensus_types "github.com/prysmaticlabs/prysm/v4/consensus-types"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
-	enginev1 "github.com/prysmaticlabs/prysm/v4/proto/engine/v1"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/runtime/version"
 )
@@ -22,7 +21,7 @@ type SignedBid interface {
 // Bid is an interface describing the method set of a builder bid.
 type Bid interface {
 	Header() (interfaces.ExecutionData, error)
-	BlindedBlobsBundle() (*enginev1.BlindedBlobsBundle, error)
+	BlobKzgCommitments() ([][]byte, error)
 	Value() []byte
 	Pubkey() []byte
 	Version() int
@@ -115,9 +114,9 @@ func (b builderBid) Header() (interfaces.ExecutionData, error) {
 	return blocks.WrappedExecutionPayloadHeader(b.p.Header)
 }
 
-// BlindedBlobsBundle --
-func (b builderBid) BlindedBlobsBundle() (*enginev1.BlindedBlobsBundle, error) {
-	return nil, errors.New("blinded blobs bundle not available before Deneb")
+// BlobKzgCommitments --
+func (b builderBid) BlobKzgCommitments() ([][]byte, error) {
+	return [][]byte{}, errors.New("blob kzg commitments not available before Deneb")
 }
 
 // Version --
@@ -169,9 +168,9 @@ func (b builderBidCapella) Header() (interfaces.ExecutionData, error) {
 	return blocks.WrappedExecutionPayloadHeaderCapella(b.p.Header, blocks.PayloadValueToGwei(b.p.Value))
 }
 
-// BlindedBlobsBundle --
-func (b builderBidCapella) BlindedBlobsBundle() (*enginev1.BlindedBlobsBundle, error) {
-	return nil, errors.New("blinded blobs bundle not available before Deneb")
+// BlobKzgCommitments --
+func (b builderBidCapella) BlobKzgCommitments() ([][]byte, error) {
+	return [][]byte{}, errors.New("blob kzg commitments not available before Deneb")
 }
 
 // Version --
@@ -253,9 +252,9 @@ func (b builderBidDeneb) Header() (interfaces.ExecutionData, error) {
 	return blocks.WrappedExecutionPayloadHeaderDeneb(b.p.Header, blocks.PayloadValueToGwei(b.p.Value))
 }
 
-// BlindedBlobsBundle --
-func (b builderBidDeneb) BlindedBlobsBundle() (*enginev1.BlindedBlobsBundle, error) {
-	return b.p.BlindedBlobsBundle, nil
+// BlobKzgCommitments --
+func (b builderBidDeneb) BlobKzgCommitments() ([][]byte, error) {
+	return b.p.BlobKzgCommitments, nil
 }
 
 type signedBuilderBidDeneb struct {

api/client/builder/client.go

@@ -19,8 +19,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	"github.com/prysmaticlabs/prysm/v4/monitoring/tracing"
-	"github.com/prysmaticlabs/prysm/v4/network"
-	"github.com/prysmaticlabs/prysm/v4/network/authorization"
 	v1 "github.com/prysmaticlabs/prysm/v4/proto/engine/v1"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/runtime/version"
@@ -89,7 +87,7 @@ type BuilderClient interface {
 	NodeURL() string
 	GetHeader(ctx context.Context, slot primitives.Slot, parentHash [32]byte, pubkey [48]byte) (SignedBid, error)
 	RegisterValidator(ctx context.Context, svr []*ethpb.SignedValidatorRegistrationV1) error
-	SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlySignedBeaconBlock, blobs []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error)
+	SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error)
 	Status(ctx context.Context) error
 }
 
@@ -104,8 +102,7 @@ type Client struct {
 // `host` is the base host + port used to construct request urls. This value can be
 // a URL string, or NewClient will assume an http endpoint if just `host:port` is used.
 func NewClient(host string, opts ...ClientOpt) (*Client, error) {
-	endpoint := covertEndPoint(host)
-	u, err := urlForHost(endpoint.Url)
+	u, err := urlForHost(host)
 	if err != nil {
 		return nil, err
 	}
@@ -121,8 +118,7 @@ func NewClient(host string, opts ...ClientOpt) (*Client, error) {
 
 func urlForHost(h string) (*url.URL, error) {
 	// try to parse as url (being permissive)
-	u, err := url.Parse(h)
-	if err == nil && u.Host != "" {
+	if u, err := url.Parse(h); err == nil && u.Host != "" {
 		return u, nil
 	}
 	// try to parse as host:port
@@ -140,7 +136,7 @@ func (c *Client) NodeURL() string {
 
 type reqOption func(*http.Request)
 
-// do is a generic, opinionated request function to reduce boilerplate amongst the methods in this package api/client/builder/types.go.
+// do is a generic, opinionated request function to reduce boilerplate amongst the methods in this package api/client/builder.
 func (c *Client) do(ctx context.Context, method string, path string, body io.Reader, opts ...reqOption) (res []byte, err error) {
 	ctx, span := trace.StartSpan(ctx, "builder.client.do")
 	defer func() {
@@ -272,11 +268,7 @@ func (c *Client) RegisterValidator(ctx context.Context, svr []*ethpb.SignedValid
 	}
 	vs := make([]*shared.SignedValidatorRegistration, len(svr))
 	for i := 0; i < len(svr); i++ {
-		svrJson, err := shared.SignedValidatorRegistrationFromConsensus(svr[i])
-		if err != nil {
-			return errors.Wrap(err, fmt.Sprintf("failed to encode to SignedValidatorRegistration at index %d", i))
-		}
-		vs[i] = svrJson
+		vs[i] = shared.SignedValidatorRegistrationFromConsensus(svr[i])
 	}
 	body, err := json.Marshal(vs)
 	if err != nil {
@@ -291,7 +283,7 @@ func (c *Client) RegisterValidator(ctx context.Context, svr []*ethpb.SignedValid
 
 // SubmitBlindedBlock calls the builder API endpoint that binds the validator to the builder and submits the block.
 // The response is the full execution payload used to create the blinded block.
-func (c *Client) SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlySignedBeaconBlock, blobs []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
+func (c *Client) SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
 	if !sb.IsBlinded() {
 		return nil, nil, errNotBlinded
 	}
@@ -375,9 +367,9 @@ func (c *Client) SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlyS
 		if err != nil {
 			return nil, nil, errors.Wrapf(err, "could not get protobuf block")
 		}
-		b, err := shared.SignedBlindedBeaconBlockContentsDenebFromConsensus(&ethpb.SignedBlindedBeaconBlockAndBlobsDeneb{SignedBlindedBlock: psb, SignedBlindedBlobSidecars: blobs})
+		b, err := shared.SignedBlindedBeaconBlockDenebFromConsensus(&ethpb.SignedBlindedBeaconBlockDeneb{Message: psb.Message, Signature: bytesutil.SafeCopyBytes(psb.Signature)})
 		if err != nil {
-			return nil, nil, errors.Wrapf(err, "could not convert SignedBlindedBeaconBlockContentsDeneb to json marshalable type")
+			return nil, nil, errors.Wrapf(err, "could not convert SignedBlindedBeaconBlockDeneb to json marshalable type")
 		}
 		body, err := json.Marshal(b)
 		if err != nil {
@@ -431,38 +423,29 @@ func non200Err(response *http.Response) error {
 	}
 	msg := fmt.Sprintf("code=%d, url=%s, body=%s", response.StatusCode, response.Request.URL, body)
 	switch response.StatusCode {
-	case 204:
+	case http.StatusNoContent:
 		log.WithError(ErrNoContent).Debug(msg)
 		return ErrNoContent
-	case 400:
-		if jsonErr := json.Unmarshal(bodyBytes, &errMessage); jsonErr != nil {
-			return errors.Wrap(jsonErr, "unable to read response body")
-		}
+	case http.StatusBadRequest:
 		log.WithError(ErrBadRequest).Debug(msg)
+		if jsonErr := json.Unmarshal(bodyBytes, &errMessage); jsonErr != nil {
+			return errors.Wrap(jsonErr, "unable to read response body")
+		}
 		return errors.Wrap(ErrBadRequest, errMessage.Message)
-	case 404:
-		if jsonErr := json.Unmarshal(bodyBytes, &errMessage); jsonErr != nil {
-			return errors.Wrap(jsonErr, "unable to read response body")
-		}
+	case http.StatusNotFound:
 		log.WithError(ErrNotFound).Debug(msg)
-		return errors.Wrap(ErrNotFound, errMessage.Message)
-	case 500:
 		if jsonErr := json.Unmarshal(bodyBytes, &errMessage); jsonErr != nil {
 			return errors.Wrap(jsonErr, "unable to read response body")
 		}
+		return errors.Wrap(ErrNotFound, errMessage.Message)
+	case http.StatusInternalServerError:
 		log.WithError(ErrNotOK).Debug(msg)
+		if jsonErr := json.Unmarshal(bodyBytes, &errMessage); jsonErr != nil {
+			return errors.Wrap(jsonErr, "unable to read response body")
+		}
 		return errors.Wrap(ErrNotOK, errMessage.Message)
 	default:
 		log.WithError(ErrNotOK).Debug(msg)
 		return errors.Wrap(ErrNotOK, fmt.Sprintf("unsupported error code: %d", response.StatusCode))
 	}
 }
-
-func covertEndPoint(ep string) network.Endpoint {
-	return network.Endpoint{
-		Url: ep,
-		Auth: network.AuthorizationData{ // Auth is not used for builder.
-			Method: authorization.None,
-			Value:  "",
-		}}
-}

api/client/builder/client_test.go

@@ -12,10 +12,8 @@ import (
 	"strconv"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/prysmaticlabs/go-bitfield"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
-	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	types "github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
@@ -270,28 +268,20 @@ func TestClient_GetHeader(t *testing.T) {
 		bidValue := bytesutil.ReverseByteOrder(bid.Value())
 		require.DeepEqual(t, bidValue, value.Bytes())
 		require.DeepEqual(t, big.NewInt(0).SetBytes(bidValue), value.Int)
-		bundle, err := bid.BlindedBlobsBundle()
+		kcgCommitments, err := bid.BlobKzgCommitments()
 		require.NoError(t, err)
-		require.Equal(t, len(bundle.BlobRoots) <= fieldparams.MaxBlobsPerBlock && len(bundle.BlobRoots) > 0, true)
-		for i := range bundle.BlobRoots {
-			require.Equal(t, len(bundle.BlobRoots[i]) == fieldparams.RootLength, true)
-		}
-		require.Equal(t, len(bundle.KzgCommitments) > 0, true)
-		for i := range bundle.KzgCommitments {
-			require.Equal(t, len(bundle.KzgCommitments[i]) == 48, true)
-		}
-		require.Equal(t, len(bundle.Proofs) > 0, true)
-		for i := range bundle.Proofs {
-			require.Equal(t, len(bundle.Proofs[i]) == 48, true)
+		require.Equal(t, len(kcgCommitments) > 0, true)
+		for i := range kcgCommitments {
+			require.Equal(t, len(kcgCommitments[i]) == 48, true)
 		}
 	})
-	t.Run("deneb, no bundle", func(t *testing.T) {
+	t.Run("deneb, too many kzg commitments", func(t *testing.T) {
 		hc := &http.Client{
 			Transport: roundtrip(func(r *http.Request) (*http.Response, error) {
 				require.Equal(t, expectedPath, r.URL.Path)
 				return &http.Response{
 					StatusCode: http.StatusOK,
-					Body:       io.NopCloser(bytes.NewBufferString(testExampleHeaderResponseDenebNoBundle)),
+					Body:       io.NopCloser(bytes.NewBufferString(testExampleHeaderResponseDenebTooManyBlobs)),
 					Request:    r.Clone(ctx),
 				}, nil
 			}),
@@ -300,27 +290,9 @@ func TestClient_GetHeader(t *testing.T) {
 			hc:      hc,
 			baseURL: &url.URL{Host: "localhost:3500", Scheme: "http"},
 		}
-		h, err := c.GetHeader(ctx, slot, bytesutil.ToBytes32(parentHash), bytesutil.ToBytes48(pubkey))
-		require.NoError(t, err)
-		expectedWithdrawalsRoot := ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2")
-		bid, err := h.Message()
-		require.NoError(t, err)
-		bidHeader, err := bid.Header()
-		require.NoError(t, err)
-		withdrawalsRoot, err := bidHeader.WithdrawalsRoot()
-		require.NoError(t, err)
-		require.Equal(t, true, bytes.Equal(expectedWithdrawalsRoot, withdrawalsRoot))
-		value, err := stringToUint256("652312848583266388373324160190187140051835877600158453279131187530910662656")
-		require.NoError(t, err)
-		require.Equal(t, fmt.Sprintf("%#x", value.SSZBytes()), fmt.Sprintf("%#x", bid.Value()))
-		bidValue := bytesutil.ReverseByteOrder(bid.Value())
-		require.DeepEqual(t, bidValue, value.Bytes())
-		require.DeepEqual(t, big.NewInt(0).SetBytes(bidValue), value.Int)
-		bundle, err := bid.BlindedBlobsBundle()
-		require.NoError(t, err)
-		require.Equal(t, (*v1.BlindedBlobsBundle)(nil), bundle)
+		_, err := c.GetHeader(ctx, slot, bytesutil.ToBytes32(parentHash), bytesutil.ToBytes48(pubkey))
+		require.ErrorContains(t, "could not extract proto message from header: too many blob commitments: 7", err)
 	})
-
 	t.Run("unsupported version", func(t *testing.T) {
 		hc := &http.Client{
 			Transport: roundtrip(func(r *http.Request) (*http.Response, error) {
@@ -362,7 +334,7 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		}
 		sbbb, err := blocks.NewSignedBeaconBlock(testSignedBlindedBeaconBlockBellatrix(t))
 		require.NoError(t, err)
-		ep, _, err := c.SubmitBlindedBlock(ctx, sbbb, nil)
+		ep, _, err := c.SubmitBlindedBlock(ctx, sbbb)
 		require.NoError(t, err)
 		require.Equal(t, true, bytes.Equal(ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"), ep.ParentHash()))
 		bfpg, err := stringToUint256("452312848583266388373324160190187140051835877600158453279131187530910662656")
@@ -388,7 +360,7 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		}
 		sbb, err := blocks.NewSignedBeaconBlock(testSignedBlindedBeaconBlockCapella(t))
 		require.NoError(t, err)
-		ep, _, err := c.SubmitBlindedBlock(ctx, sbb, nil)
+		ep, _, err := c.SubmitBlindedBlock(ctx, sbb)
 		require.NoError(t, err)
 		withdrawals, err := ep.Withdrawals()
 		require.NoError(t, err)
@@ -399,18 +371,17 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		assert.Equal(t, uint64(1), withdrawals[0].Amount)
 	})
 	t.Run("deneb", func(t *testing.T) {
-
-		test := testSignedBlindedBeaconBlockAndBlobsDeneb(t)
+		test := testSignedBlindedBeaconBlockDeneb(t)
 		hc := &http.Client{
 			Transport: roundtrip(func(r *http.Request) (*http.Response, error) {
 				require.Equal(t, postBlindedBeaconBlockPath, r.URL.Path)
 				require.Equal(t, "deneb", r.Header.Get("Eth-Consensus-Version"))
-				var req shared.SignedBlindedBeaconBlockContentsDeneb
+				var req shared.SignedBlindedBeaconBlockDeneb
 				err := json.NewDecoder(r.Body).Decode(&req)
 				require.NoError(t, err)
-				block, err := req.SignedBlindedBlock.ToConsensus()
+				block, err := req.ToConsensus()
 				require.NoError(t, err)
-				require.DeepEqual(t, block, test.SignedBlindedBlock)
+				require.DeepEqual(t, block, test)
 				return &http.Response{
 					StatusCode: http.StatusOK,
 					Body:       io.NopCloser(bytes.NewBufferString(testExampleExecutionPayloadDeneb)),
@@ -423,10 +394,10 @@ func TestSubmitBlindedBlock(t *testing.T) {
 			baseURL: &url.URL{Host: "localhost:3500", Scheme: "http"},
 		}
 
-		sbb, err := blocks.NewSignedBeaconBlock(test.SignedBlindedBlock)
+		sbb, err := blocks.NewSignedBeaconBlock(test)
 		require.NoError(t, err)
 
-		ep, blobBundle, err := c.SubmitBlindedBlock(ctx, sbb, test.SignedBlindedBlobSidecars)
+		ep, blobBundle, err := c.SubmitBlindedBlock(ctx, sbb)
 		require.NoError(t, err)
 		withdrawals, err := ep.Withdrawals()
 		require.NoError(t, err)
@@ -436,9 +407,6 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		assert.DeepEqual(t, ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943"), withdrawals[0].Address)
 		assert.Equal(t, uint64(1), withdrawals[0].Amount)
 		require.NotNil(t, blobBundle)
-		require.Equal(t, hexutil.Encode(blobBundle.Blobs[0]), hexutil.Encode(make([]byte, fieldparams.BlobLength)))
-		require.Equal(t, hexutil.Encode(blobBundle.KzgCommitments[0]), "0x8dab030c51e16e84be9caab84ee3d0b8bbec1db4a0e4de76439da8424d9b957370a10a78851f97e4b54d2ce1ab0d686f")
-		require.Equal(t, hexutil.Encode(blobBundle.Proofs[0]), "0xb4021b0de10f743893d4f71e1bf830c019e832958efd6795baf2f83b8699a9eccc5dc99015d8d4d8ec370d0cc333c06a")
 	})
 	t.Run("mismatched versions, expected bellatrix got capella", func(t *testing.T) {
 		hc := &http.Client{
@@ -457,13 +425,13 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		}
 		sbbb, err := blocks.NewSignedBeaconBlock(testSignedBlindedBeaconBlockBellatrix(t))
 		require.NoError(t, err)
-		_, _, err = c.SubmitBlindedBlock(ctx, sbbb, nil)
+		_, _, err = c.SubmitBlindedBlock(ctx, sbbb)
 		require.ErrorContains(t, "not a bellatrix payload", err)
 	})
 	t.Run("not blinded", func(t *testing.T) {
-		sbb, err := blocks.NewSignedBeaconBlock(&eth.SignedBeaconBlockBellatrix{Block: &eth.BeaconBlockBellatrix{Body: &eth.BeaconBlockBodyBellatrix{}}})
+		sbb, err := blocks.NewSignedBeaconBlock(&eth.SignedBeaconBlockBellatrix{Block: &eth.BeaconBlockBellatrix{Body: &eth.BeaconBlockBodyBellatrix{ExecutionPayload: &v1.ExecutionPayload{}}}})
 		require.NoError(t, err)
-		_, _, err = (&Client{}).SubmitBlindedBlock(ctx, sbb, nil)
+		_, _, err = (&Client{}).SubmitBlindedBlock(ctx, sbb)
 		require.ErrorIs(t, err, errNotBlinded)
 	})
 }
@@ -753,91 +721,70 @@ func testSignedBlindedBeaconBlockCapella(t *testing.T) *eth.SignedBlindedBeaconB
 	}
 }
 
-func testSignedBlindedBeaconBlockAndBlobsDeneb(t *testing.T) *eth.SignedBlindedBeaconBlockAndBlobsDeneb {
-	basebytes, err := shared.Uint256ToSSZBytes("14074904626401341155369551180448584754667373453244490859944217516317499064576")
+func testSignedBlindedBeaconBlockDeneb(t *testing.T) *eth.SignedBlindedBeaconBlockDeneb {
+	basebytes, err := bytesutil.Uint256ToSSZBytes("14074904626401341155369551180448584754667373453244490859944217516317499064576")
 	if err != nil {
 		log.Error(err)
 	}
-	return &eth.SignedBlindedBeaconBlockAndBlobsDeneb{
-		SignedBlindedBlock: &eth.SignedBlindedBeaconBlockDeneb{
-			Message: &eth.BlindedBeaconBlockDeneb{
-				Slot:          1,
-				ProposerIndex: 1,
-				ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-				StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-				Body: &eth.BlindedBeaconBlockBodyDeneb{
-					RandaoReveal: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
-					Eth1Data: &eth.Eth1Data{
-						DepositRoot:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						DepositCount: 1,
-						BlockHash:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-					},
-					Graffiti: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-					ProposerSlashings: []*eth.ProposerSlashing{
-						{
-							Header_1: &eth.SignedBeaconBlockHeader{
-								Header: &eth.BeaconBlockHeader{
-									Slot:          1,
-									ProposerIndex: 1,
-									ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									BodyRoot:      ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-								},
-								Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+	return &eth.SignedBlindedBeaconBlockDeneb{
+		Message: &eth.BlindedBeaconBlockDeneb{
+			Slot:          1,
+			ProposerIndex: 1,
+			ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+			StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+			Body: &eth.BlindedBeaconBlockBodyDeneb{
+				RandaoReveal: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+				Eth1Data: &eth.Eth1Data{
+					DepositRoot:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					DepositCount: 1,
+					BlockHash:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+				},
+				Graffiti: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+				ProposerSlashings: []*eth.ProposerSlashing{
+					{
+						Header_1: &eth.SignedBeaconBlockHeader{
+							Header: &eth.BeaconBlockHeader{
+								Slot:          1,
+								ProposerIndex: 1,
+								ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+								StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+								BodyRoot:      ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
 							},
-							Header_2: &eth.SignedBeaconBlockHeader{
-								Header: &eth.BeaconBlockHeader{
-									Slot:          1,
-									ProposerIndex: 1,
-									ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									BodyRoot:      ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-								},
-								Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+							Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+						},
+						Header_2: &eth.SignedBeaconBlockHeader{
+							Header: &eth.BeaconBlockHeader{
+								Slot:          1,
+								ProposerIndex: 1,
+								ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+								StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+								BodyRoot:      ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
 							},
+							Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 						},
 					},
-					AttesterSlashings: []*eth.AttesterSlashing{
-						{
-							Attestation_1: &eth.IndexedAttestation{
-								AttestingIndices: []uint64{1},
-								Data: &eth.AttestationData{
-									Slot:            1,
-									CommitteeIndex:  1,
-									BeaconBlockRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									Source: &eth.Checkpoint{
-										Epoch: 1,
-										Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									},
-									Target: &eth.Checkpoint{
-										Epoch: 1,
-										Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									},
+				},
+				AttesterSlashings: []*eth.AttesterSlashing{
+					{
+						Attestation_1: &eth.IndexedAttestation{
+							AttestingIndices: []uint64{1},
+							Data: &eth.AttestationData{
+								Slot:            1,
+								CommitteeIndex:  1,
+								BeaconBlockRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+								Source: &eth.Checkpoint{
+									Epoch: 1,
+									Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
 								},
-								Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
-							},
-							Attestation_2: &eth.IndexedAttestation{
-								AttestingIndices: []uint64{1},
-								Data: &eth.AttestationData{
-									Slot:            1,
-									CommitteeIndex:  1,
-									BeaconBlockRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									Source: &eth.Checkpoint{
-										Epoch: 1,
-										Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									},
-									Target: &eth.Checkpoint{
-										Epoch: 1,
-										Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									},
+								Target: &eth.Checkpoint{
+									Epoch: 1,
+									Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
 								},
-								Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 							},
+							Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 						},
-					},
-					Attestations: []*eth.Attestation{
-						{
-							AggregationBits: bitfield.Bitlist{0x01},
+						Attestation_2: &eth.IndexedAttestation{
+							AttestingIndices: []uint64{1},
 							Data: &eth.AttestationData{
 								Slot:            1,
 								CommitteeIndex:  1,
@@ -854,68 +801,72 @@ func testSignedBlindedBeaconBlockAndBlobsDeneb(t *testing.T) *eth.SignedBlindedB
 							Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 						},
 					},
-					Deposits: []*eth.Deposit{
-						{
-							Proof: [][]byte{ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2")},
-							Data: &eth.Deposit_Data{
-								PublicKey:             ezDecode(t, "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a"),
-								WithdrawalCredentials: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-								Amount:                1,
-								Signature:             ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+				},
+				Attestations: []*eth.Attestation{
+					{
+						AggregationBits: bitfield.Bitlist{0x01},
+						Data: &eth.AttestationData{
+							Slot:            1,
+							CommitteeIndex:  1,
+							BeaconBlockRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+							Source: &eth.Checkpoint{
+								Epoch: 1,
+								Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+							},
+							Target: &eth.Checkpoint{
+								Epoch: 1,
+								Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
 							},
 						},
+						Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 					},
-					VoluntaryExits: []*eth.SignedVoluntaryExit{
-						{
-							Exit: &eth.VoluntaryExit{
-								Epoch:          1,
-								ValidatorIndex: 1,
-							},
-							Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+				},
+				Deposits: []*eth.Deposit{
+					{
+						Proof: [][]byte{ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2")},
+						Data: &eth.Deposit_Data{
+							PublicKey:             ezDecode(t, "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a"),
+							WithdrawalCredentials: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+							Amount:                1,
+							Signature:             ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 						},
 					},
-					SyncAggregate: &eth.SyncAggregate{
-						SyncCommitteeSignature: make([]byte, 96),
-						SyncCommitteeBits:      bitfield.Bitvector512(ezDecode(t, "0x6451e9f951ebf05edc01de67e593484b672877054f055903ff0df1a1a945cf30ca26bb4d4b154f94a1bc776bcf5d0efb3603e1f9b8ee2499ccdcfe2a18cef458")),
-					},
-					ExecutionPayloadHeader: &v1.ExecutionPayloadHeaderDeneb{
-						ParentHash:       ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						FeeRecipient:     ezDecode(t, "0xabcf8e0d4e9587369b2301d0790347320302cc09"),
-						StateRoot:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						ReceiptsRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						LogsBloom:        ezDecode(t, "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"),
-						PrevRandao:       ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						BlockNumber:      1,
-						GasLimit:         1,
-						GasUsed:          1,
-						Timestamp:        1,
-						ExtraData:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						BaseFeePerGas:    basebytes,
-						BlockHash:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						TransactionsRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						WithdrawalsRoot:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						BlobGasUsed:      1,
-						ExcessBlobGas:    2,
+				},
+				VoluntaryExits: []*eth.SignedVoluntaryExit{
+					{
+						Exit: &eth.VoluntaryExit{
+							Epoch:          1,
+							ValidatorIndex: 1,
+						},
+						Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 					},
 				},
-			},
-			Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
-		},
-		SignedBlindedBlobSidecars: []*eth.SignedBlindedBlobSidecar{
-			{
-				Message: &eth.BlindedBlobSidecar{
-					BlockRoot:       ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-					Index:           0,
-					Slot:            1,
-					BlockParentRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-					ProposerIndex:   1,
-					BlobRoot:        ezDecode(t, "0x24564723180fcb3d994104538d351c8dcbde12d541676bb736cf678018ca4739"),
-					KzgCommitment:   ezDecode(t, "0x8dab030c51e16e84be9caab84ee3d0b8bbec1db4a0e4de76439da8424d9b957370a10a78851f97e4b54d2ce1ab0d686f"),
-					KzgProof:        ezDecode(t, "0xb4021b0de10f743893d4f71e1bf830c019e832958efd6795baf2f83b8699a9eccc5dc99015d8d4d8ec370d0cc333c06a"),
+				SyncAggregate: &eth.SyncAggregate{
+					SyncCommitteeSignature: make([]byte, 96),
+					SyncCommitteeBits:      ezDecode(t, "0x6451e9f951ebf05edc01de67e593484b672877054f055903ff0df1a1a945cf30ca26bb4d4b154f94a1bc776bcf5d0efb3603e1f9b8ee2499ccdcfe2a18cef458"),
+				},
+				ExecutionPayloadHeader: &v1.ExecutionPayloadHeaderDeneb{
+					ParentHash:       ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					FeeRecipient:     ezDecode(t, "0xabcf8e0d4e9587369b2301d0790347320302cc09"),
+					StateRoot:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					ReceiptsRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					LogsBloom:        ezDecode(t, "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"),
+					PrevRandao:       ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					BlockNumber:      1,
+					GasLimit:         1,
+					GasUsed:          1,
+					Timestamp:        1,
+					ExtraData:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					BaseFeePerGas:    basebytes,
+					BlockHash:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					TransactionsRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					WithdrawalsRoot:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					BlobGasUsed:      1,
+					ExcessBlobGas:    2,
 				},
-				Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 			},
 		},
+		Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 	}
 }
 

api/client/builder/testing/mock.go

@@ -41,7 +41,7 @@ func (m MockClient) RegisterValidator(_ context.Context, svr []*ethpb.SignedVali
 }
 
 // SubmitBlindedBlock --
-func (MockClient) SubmitBlindedBlock(_ context.Context, _ interfaces.ReadOnlySignedBeaconBlock, _ []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
+func (MockClient) SubmitBlindedBlock(_ context.Context, _ interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
 	return nil, nil, nil
 }
 

api/client/builder/types.go

@@ -117,7 +117,7 @@ type VersionResponse struct {
 	Version string `json:"version"`
 }
 
-// ExecHeaderResponse is a JSON representation of  the builder API header response for Bellatrix.
+// ExecHeaderResponse is a JSON representation of the builder API header response for Bellatrix.
 type ExecHeaderResponse struct {
 	Version string `json:"version"`
 	Data    struct {
@@ -357,6 +357,45 @@ func FromProtoCapella(payload *v1.ExecutionPayloadCapella) (ExecutionPayloadCape
 	}, nil
 }
 
+func FromProtoDeneb(payload *v1.ExecutionPayloadDeneb) (ExecutionPayloadDeneb, error) {
+	bFee, err := sszBytesToUint256(payload.BaseFeePerGas)
+	if err != nil {
+		return ExecutionPayloadDeneb{}, err
+	}
+	txs := make([]hexutil.Bytes, len(payload.Transactions))
+	for i := range payload.Transactions {
+		txs[i] = bytesutil.SafeCopyBytes(payload.Transactions[i])
+	}
+	withdrawals := make([]Withdrawal, len(payload.Withdrawals))
+	for i, w := range payload.Withdrawals {
+		withdrawals[i] = Withdrawal{
+			Index:          Uint256{Int: big.NewInt(0).SetUint64(w.Index)},
+			ValidatorIndex: Uint256{Int: big.NewInt(0).SetUint64(uint64(w.ValidatorIndex))},
+			Address:        bytesutil.SafeCopyBytes(w.Address),
+			Amount:         Uint256{Int: big.NewInt(0).SetUint64(w.Amount)},
+		}
+	}
+	return ExecutionPayloadDeneb{
+		ParentHash:    bytesutil.SafeCopyBytes(payload.ParentHash),
+		FeeRecipient:  bytesutil.SafeCopyBytes(payload.FeeRecipient),
+		StateRoot:     bytesutil.SafeCopyBytes(payload.StateRoot),
+		ReceiptsRoot:  bytesutil.SafeCopyBytes(payload.ReceiptsRoot),
+		LogsBloom:     bytesutil.SafeCopyBytes(payload.LogsBloom),
+		PrevRandao:    bytesutil.SafeCopyBytes(payload.PrevRandao),
+		BlockNumber:   Uint64String(payload.BlockNumber),
+		GasLimit:      Uint64String(payload.GasLimit),
+		GasUsed:       Uint64String(payload.GasUsed),
+		Timestamp:     Uint64String(payload.Timestamp),
+		ExtraData:     bytesutil.SafeCopyBytes(payload.ExtraData),
+		BaseFeePerGas: bFee,
+		BlockHash:     bytesutil.SafeCopyBytes(payload.BlockHash),
+		Transactions:  txs,
+		Withdrawals:   withdrawals,
+		BlobGasUsed:   Uint64String(payload.BlobGasUsed),
+		ExcessBlobGas: Uint64String(payload.ExcessBlobGas),
+	}, nil
+}
+
 // ExecHeaderResponseCapella is the response of builder API /eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} for Capella.
 type ExecHeaderResponseCapella struct {
 	Data struct {
@@ -869,16 +908,19 @@ func (bb *BuilderBidDeneb) ToProto() (*eth.BuilderBidDeneb, error) {
 	if err != nil {
 		return nil, err
 	}
-	var bundle *v1.BlindedBlobsBundle
-	if bb.BlindedBlobsBundle != nil {
-		bundle, err = bb.BlindedBlobsBundle.ToProto()
-		if err != nil {
-			return nil, err
+	if len(bb.BlobKzgCommitments) > fieldparams.MaxBlobsPerBlock {
+		return nil, fmt.Errorf("too many blob commitments: %d", len(bb.BlobKzgCommitments))
+	}
+	kzgCommitments := make([][]byte, len(bb.BlobKzgCommitments))
+	for i, commit := range bb.BlobKzgCommitments {
+		if len(commit) != fieldparams.BLSPubkeyLength {
+			return nil, fmt.Errorf("commitment length %d is not %d", len(commit), fieldparams.BLSPubkeyLength)
 		}
+		kzgCommitments[i] = bytesutil.SafeCopyBytes(commit)
 	}
 	return &eth.BuilderBidDeneb{
 		Header:             header,
-		BlindedBlobsBundle: bundle,
+		BlobKzgCommitments: kzgCommitments,
 		Value:              bytesutil.SafeCopyBytes(bb.Value.SSZBytes()),
 		Pubkey:             bytesutil.SafeCopyBytes(bb.Pubkey),
 	}, nil
@@ -887,42 +929,11 @@ func (bb *BuilderBidDeneb) ToProto() (*eth.BuilderBidDeneb, error) {
 // BuilderBidDeneb is a field of ExecHeaderResponseDeneb.
 type BuilderBidDeneb struct {
 	Header             *ExecutionPayloadHeaderDeneb `json:"header"`
-	BlindedBlobsBundle *BlindedBlobsBundle          `json:"blinded_blobs_bundle"`
+	BlobKzgCommitments []hexutil.Bytes              `json:"blob_kzg_commitments"`
 	Value              Uint256                      `json:"value"`
 	Pubkey             hexutil.Bytes                `json:"pubkey"`
 }
 
-// BlindedBlobsBundle is a field of BuilderBidDeneb and represents the blinded blobs of the associated header.
-type BlindedBlobsBundle struct {
-	KzgCommitments []hexutil.Bytes `json:"commitments"`
-	Proofs         []hexutil.Bytes `json:"proofs"`
-	BlobRoots      []hexutil.Bytes `json:"blob_roots"`
-}
-
-// ToProto creates a BlindedBlobsBundle Proto from BlindedBlobsBundle.
-func (r *BlindedBlobsBundle) ToProto() (*v1.BlindedBlobsBundle, error) {
-	kzg := make([][]byte, len(r.KzgCommitments))
-	for i := range kzg {
-		kzg[i] = bytesutil.SafeCopyBytes(r.KzgCommitments[i])
-	}
-
-	proofs := make([][]byte, len(r.Proofs))
-	for i := range proofs {
-		proofs[i] = bytesutil.SafeCopyBytes(r.Proofs[i])
-	}
-
-	blobRoots := make([][]byte, len(r.BlobRoots))
-	for i := range blobRoots {
-		blobRoots[i] = bytesutil.SafeCopyBytes(r.BlobRoots[i])
-	}
-
-	return &v1.BlindedBlobsBundle{
-		KzgCommitments: kzg,
-		Proofs:         proofs,
-		BlobRoots:      blobRoots,
-	}, nil
-}
-
 // ExecutionPayloadHeaderDeneb a field part of the BuilderBidDeneb.
 type ExecutionPayloadHeaderDeneb struct {
 	ParentHash       hexutil.Bytes `json:"parent_hash"`
@@ -1052,6 +1063,16 @@ type BlobsBundle struct {
 
 // ToProto returns a BlobsBundle Proto.
 func (b BlobsBundle) ToProto() (*v1.BlobsBundle, error) {
+	if len(b.Blobs) > fieldparams.MaxBlobCommitmentsPerBlock {
+		return nil, fmt.Errorf("blobs length %d is more than max %d", len(b.Blobs), fieldparams.MaxBlobCommitmentsPerBlock)
+	}
+	if len(b.Commitments) != len(b.Blobs) {
+		return nil, fmt.Errorf("commitments length %d does not equal blobs length %d", len(b.Commitments), len(b.Blobs))
+	}
+	if len(b.Proofs) != len(b.Blobs) {
+		return nil, fmt.Errorf("proofs length %d does not equal blobs length %d", len(b.Proofs), len(b.Blobs))
+	}
+
 	commitments := make([][]byte, len(b.Commitments))
 	for i := range b.Commitments {
 		if len(b.Commitments[i]) != fieldparams.BLSPubkeyLength {
@@ -1066,9 +1087,6 @@ func (b BlobsBundle) ToProto() (*v1.BlobsBundle, error) {
 		}
 		proofs[i] = bytesutil.SafeCopyBytes(b.Proofs[i])
 	}
-	if len(b.Blobs) > fieldparams.MaxBlobsPerBlock {
-		return nil, fmt.Errorf("blobs length %d is more than max %d", len(b.Blobs), fieldparams.MaxBlobsPerBlock)
-	}
 	blobs := make([][]byte, len(b.Blobs))
 	for i := range b.Blobs {
 		if len(b.Blobs[i]) != fieldparams.BlobLength {
@@ -1083,6 +1101,28 @@ func (b BlobsBundle) ToProto() (*v1.BlobsBundle, error) {
 	}, nil
 }
 
+// FromBundleProto converts the proto bundle type to the builder
+// type.
+func FromBundleProto(bundle *v1.BlobsBundle) *BlobsBundle {
+	commitments := make([]hexutil.Bytes, len(bundle.KzgCommitments))
+	for i := range bundle.KzgCommitments {
+		commitments[i] = bytesutil.SafeCopyBytes(bundle.KzgCommitments[i])
+	}
+	proofs := make([]hexutil.Bytes, len(bundle.Proofs))
+	for i := range bundle.Proofs {
+		proofs[i] = bytesutil.SafeCopyBytes(bundle.Proofs[i])
+	}
+	blobs := make([]hexutil.Bytes, len(bundle.Blobs))
+	for i := range bundle.Blobs {
+		blobs[i] = bytesutil.SafeCopyBytes(bundle.Blobs[i])
+	}
+	return &BlobsBundle{
+		Commitments: commitments,
+		Proofs:      proofs,
+		Blobs:       blobs,
+	}
+}
+
 // ToProto returns ExecutionPayloadDeneb Proto and BlobsBundle Proto separately.
 func (r *ExecPayloadResponseDeneb) ToProto() (*v1.ExecutionPayloadDeneb, *v1.BlobsBundle, error) {
 	if r.Data == nil {

api/client/builder/types_test.go

@@ -38,8 +38,7 @@ func TestSignedValidatorRegistration_MarshalJSON(t *testing.T) {
 		},
 		Signature: make([]byte, 96),
 	}
-	a, err := shared.SignedValidatorRegistrationFromConsensus(svr)
-	require.NoError(t, err)
+	a := shared.SignedValidatorRegistrationFromConsensus(svr)
 	je, err := json.Marshal(a)
 	require.NoError(t, err)
 	// decode with a struct w/ plain strings so we can check the string encoding of the hex fields
@@ -142,17 +141,9 @@ var testExampleHeaderResponseDeneb = `{
 		"blob_gas_used": "1",	
 		"excess_blob_gas": "2"	
       },
-      "blinded_blobs_bundle": {
-        "commitments": [
+      "blob_kzg_commitments": [
           "0x8dab030c51e16e84be9caab84ee3d0b8bbec1db4a0e4de76439da8424d9b957370a10a78851f97e4b54d2ce1ab0d686f"
-        ],
-        "proofs": [
-          "0xb4021b0de10f743893d4f71e1bf830c019e832958efd6795baf2f83b8699a9eccc5dc99015d8d4d8ec370d0cc333c06a"
-        ],
-        "blob_roots": [
-          "0x24564723180fcb3d994104538d351c8dcbde12d541676bb736cf678018ca4739"
-        ]
-      },
+	  ],
       "value": "652312848583266388373324160190187140051835877600158453279131187530910662656",
       "pubkey": "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a"
     },
@@ -218,6 +209,39 @@ var testExampleHeaderResponseUnknownVersion = `{
   }
 }`
 
+var testExampleHeaderResponseDenebTooManyBlobs = `{
+  "version": "deneb",
+  "data": {
+    "message": {
+      "header": {
+        "parent_hash": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+        "fee_recipient": "0xabcf8e0d4e9587369b2301d0790347320302cc09",
+        "state_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+        "receipts_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+        "logs_bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+        "prev_randao": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+        "block_number": "1",
+        "gas_limit": "1",
+        "gas_used": "1",
+        "timestamp": "1",
+        "extra_data": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+        "base_fee_per_gas": "452312848583266388373324160190187140051835877600158453279131187530910662656",
+		"block_hash": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+        "transactions_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+        "withdrawals_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"blob_gas_used": "1",	
+		"excess_blob_gas": "2"	
+      },
+      "blob_kzg_commitments": [
+          "","","","","","",""
+	  ],
+      "value": "652312848583266388373324160190187140051835877600158453279131187530910662656",
+      "pubkey": "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a"
+    },
+    "signature": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+  }
+}`
+
 func TestExecutionHeaderResponseUnmarshal(t *testing.T) {
 	hr := &ExecHeaderResponse{}
 	require.NoError(t, json.Unmarshal([]byte(testExampleHeaderResponse), hr))
@@ -637,6 +661,151 @@ var testExampleExecutionPayloadDeneb = fmt.Sprintf(`{
   }
 }`, hexutil.Encode(make([]byte, fieldparams.BlobLength)))
 
+var testExampleExecutionPayloadDenebTooManyBlobs = fmt.Sprintf(`{
+  "version": "deneb",
+  "data": {
+	"execution_payload":{
+		"parent_hash": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"fee_recipient": "0xabcf8e0d4e9587369b2301d0790347320302cc09",
+		"state_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"receipts_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"logs_bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+		"prev_randao": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"block_number": "1",
+		"gas_limit": "1",
+		"gas_used": "1",
+		"timestamp": "1",
+		"extra_data": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"base_fee_per_gas": "452312848583266388373324160190187140051835877600158453279131187530910662656",
+		"block_hash": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"transactions": [
+		  "0x02f878831469668303f51d843b9ac9f9843b9aca0082520894c93269b73096998db66be0441e836d873535cb9c8894a19041886f000080c001a031cc29234036afbf9a1fb9476b463367cb1f957ac0b919b69bbc798436e604aaa018c4e9c3914eb27aadd0b91e10b18655739fcf8c1fc398763a9f1beecb8ddc86"
+		],
+		"withdrawals": [
+		  {
+			"index": "1",
+			"validator_index": "1",
+			"address": "0xcf8e0d4e9587369b2301d0790347320302cc0943",
+			"amount": "1"
+		  }
+		],
+		"blob_gas_used": "2",
+ 		"excess_blob_gas": "3"
+	  },
+	"blobs_bundle": {
+      "commitments": [
+        "0x8dab030c51e16e84be9caab84ee3d0b8bbec1db4a0e4de76439da8424d9b957370a10a78851f97e4b54d2ce1ab0d686f"
+      ],
+      "proofs": [
+        "0xb4021b0de10f743893d4f71e1bf830c019e832958efd6795baf2f83b8699a9eccc5dc99015d8d4d8ec370d0cc333c06a"
+      ],
+      "blobs": %s
+    }
+  }
+}`, beyondMaxEmptyBlobs())
+
+func beyondMaxEmptyBlobs() string {
+	moreThanMax := fieldparams.MaxBlobCommitmentsPerBlock + 2
+	blobs := make([]string, moreThanMax)
+	b, err := json.Marshal(blobs)
+	if err != nil {
+		panic(err)
+	}
+	return string(b)
+}
+
+var testExampleExecutionPayloadDenebDifferentCommitmentCount = fmt.Sprintf(`{
+  "version": "deneb",
+  "data": {
+	"execution_payload":{
+		"parent_hash": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"fee_recipient": "0xabcf8e0d4e9587369b2301d0790347320302cc09",
+		"state_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"receipts_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"logs_bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+		"prev_randao": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"block_number": "1",
+		"gas_limit": "1",
+		"gas_used": "1",
+		"timestamp": "1",
+		"extra_data": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"base_fee_per_gas": "452312848583266388373324160190187140051835877600158453279131187530910662656",
+		"block_hash": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"transactions": [
+		  "0x02f878831469668303f51d843b9ac9f9843b9aca0082520894c93269b73096998db66be0441e836d873535cb9c8894a19041886f000080c001a031cc29234036afbf9a1fb9476b463367cb1f957ac0b919b69bbc798436e604aaa018c4e9c3914eb27aadd0b91e10b18655739fcf8c1fc398763a9f1beecb8ddc86"
+		],
+		"withdrawals": [
+		  {
+			"index": "1",
+			"validator_index": "1",
+			"address": "0xcf8e0d4e9587369b2301d0790347320302cc0943",
+			"amount": "1"
+		  }
+		],
+		"blob_gas_used": "2",
+ 		"excess_blob_gas": "3"
+	  },
+	"blobs_bundle": {
+      "commitments": [
+        "0x8dab030c51e16e84be9caab84ee3d0b8bbec1db4a0e4de76439da8424d9b957370a10a78851f97e4b54d2ce1ab0d686f",
+        "0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
+      ],
+      "proofs": [
+        "0xb4021b0de10f743893d4f71e1bf830c019e832958efd6795baf2f83b8699a9eccc5dc99015d8d4d8ec370d0cc333c06a"
+      ],
+      "blobs": [
+        "%s"
+      ]
+    }
+  }
+}`, hexutil.Encode(make([]byte, fieldparams.BlobLength)))
+
+var testExampleExecutionPayloadDenebDifferentProofCount = fmt.Sprintf(`{
+  "version": "deneb",
+  "data": {
+	"execution_payload":{
+		"parent_hash": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"fee_recipient": "0xabcf8e0d4e9587369b2301d0790347320302cc09",
+		"state_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"receipts_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"logs_bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+		"prev_randao": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"block_number": "1",
+		"gas_limit": "1",
+		"gas_used": "1",
+		"timestamp": "1",
+		"extra_data": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"base_fee_per_gas": "452312848583266388373324160190187140051835877600158453279131187530910662656",
+		"block_hash": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"transactions": [
+		  "0x02f878831469668303f51d843b9ac9f9843b9aca0082520894c93269b73096998db66be0441e836d873535cb9c8894a19041886f000080c001a031cc29234036afbf9a1fb9476b463367cb1f957ac0b919b69bbc798436e604aaa018c4e9c3914eb27aadd0b91e10b18655739fcf8c1fc398763a9f1beecb8ddc86"
+		],
+		"withdrawals": [
+		  {
+			"index": "1",
+			"validator_index": "1",
+			"address": "0xcf8e0d4e9587369b2301d0790347320302cc0943",
+			"amount": "1"
+		  }
+		],
+		"blob_gas_used": "2",
+ 		"excess_blob_gas": "3"
+	  },
+	"blobs_bundle": {
+      "commitments": [
+        "0x8dab030c51e16e84be9caab84ee3d0b8bbec1db4a0e4de76439da8424d9b957370a10a78851f97e4b54d2ce1ab0d686f"
+      ],
+      "proofs": [
+        "0xb4021b0de10f743893d4f71e1bf830c019e832958efd6795baf2f83b8699a9eccc5dc99015d8d4d8ec370d0cc333c06a",
+        "0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
+      ],
+      "blobs": [
+        "%s"
+      ]
+    }
+  }
+}`, hexutil.Encode(make([]byte, fieldparams.BlobLength)))
+
 func TestExecutionPayloadResponseUnmarshal(t *testing.T) {
 	epr := &ExecPayloadResponse{}
 	require.NoError(t, json.Unmarshal([]byte(testExampleExecutionPayload), epr))
@@ -1013,7 +1182,6 @@ func TestExecutionPayloadResponseCapellaToProto(t *testing.T) {
 		},
 	}
 	require.DeepEqual(t, expected, p)
-
 }
 
 func TestExecutionPayloadResponseDenebToProto(t *testing.T) {
@@ -1092,7 +1260,27 @@ func TestExecutionPayloadResponseDenebToProto(t *testing.T) {
 	}
 
 	require.DeepEqual(t, blobsBundle, expectedBlobs)
+}
 
+func TestExecutionPayloadResponseDenebToProtoInvalidBlobCount(t *testing.T) {
+	hr := &ExecPayloadResponseDeneb{}
+	require.NoError(t, json.Unmarshal([]byte(testExampleExecutionPayloadDenebTooManyBlobs), hr))
+	_, _, err := hr.ToProto()
+	require.ErrorContains(t, fmt.Sprintf("blobs length %d is more than max %d", fieldparams.MaxBlobCommitmentsPerBlock+2, fieldparams.MaxBlobCommitmentsPerBlock), err)
+}
+
+func TestExecutionPayloadResponseDenebToProtoDifferentCommitmentCount(t *testing.T) {
+	hr := &ExecPayloadResponseDeneb{}
+	require.NoError(t, json.Unmarshal([]byte(testExampleExecutionPayloadDenebDifferentCommitmentCount), hr))
+	_, _, err := hr.ToProto()
+	require.ErrorContains(t, "commitments length 2 does not equal blobs length 1", err)
+}
+
+func TestExecutionPayloadResponseDenebToProtoDifferentProofCount(t *testing.T) {
+	hr := &ExecPayloadResponseDeneb{}
+	require.NoError(t, json.Unmarshal([]byte(testExampleExecutionPayloadDenebDifferentProofCount), hr))
+	_, _, err := hr.ToProto()
+	require.ErrorContains(t, "proofs length 2 does not equal blobs length 1", err)
 }
 
 func pbEth1Data() *eth.Eth1Data {

api/client/errors.go

@@ -32,7 +32,7 @@ func Non200Err(response *http.Response) error {
 	}
 	msg := fmt.Sprintf("code=%d, url=%s, body=%s", response.StatusCode, response.Request.URL, body)
 	switch response.StatusCode {
-	case 404:
+	case http.StatusNotFound:
 		return errors.Wrap(ErrNotFound, msg)
 	default:
 		return errors.Wrap(ErrNotOK, msg)

api/client/validator/BUILD.bazel

@@ -7,7 +7,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//api/client:go_default_library",
-        "//validator/rpc/apimiddleware:go_default_library",
+        "//validator/rpc:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
     ],
 )

api/client/validator/client.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v4/api/client"
-	"github.com/prysmaticlabs/prysm/v4/validator/rpc/apimiddleware"
+	"github.com/prysmaticlabs/prysm/v4/validator/rpc"
 )
 
 const (
@@ -41,17 +41,17 @@ func (c *Client) GetValidatorPubKeys(ctx context.Context) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	if len(jsonlocal.Keystores) == 0 && len(jsonremote.Keystores) == 0 {
+	if len(jsonlocal.Data) == 0 && len(jsonremote.Data) == 0 {
 		return nil, errors.New("there are no local keys or remote keys on the validator")
 	}
 
 	hexKeys := make(map[string]bool)
 
-	for index := range jsonlocal.Keystores {
-		hexKeys[jsonlocal.Keystores[index].ValidatingPubkey] = true
+	for index := range jsonlocal.Data {
+		hexKeys[jsonlocal.Data[index].ValidatingPubkey] = true
 	}
-	for index := range jsonremote.Keystores {
-		hexKeys[jsonremote.Keystores[index].Pubkey] = true
+	for index := range jsonremote.Data {
+		hexKeys[jsonremote.Data[index].Pubkey] = true
 	}
 	keys := make([]string, 0)
 	for k := range hexKeys {
@@ -61,12 +61,12 @@ func (c *Client) GetValidatorPubKeys(ctx context.Context) ([]string, error) {
 }
 
 // GetLocalValidatorKeys calls the keymanager APIs for local validator keys
-func (c *Client) GetLocalValidatorKeys(ctx context.Context) (*apimiddleware.ListKeystoresResponseJson, error) {
+func (c *Client) GetLocalValidatorKeys(ctx context.Context) (*rpc.ListKeystoresResponse, error) {
 	localBytes, err := c.Get(ctx, localKeysPath, client.WithAuthorizationToken(c.Token()))
 	if err != nil {
 		return nil, err
 	}
-	jsonlocal := &apimiddleware.ListKeystoresResponseJson{}
+	jsonlocal := &rpc.ListKeystoresResponse{}
 	if err := json.Unmarshal(localBytes, jsonlocal); err != nil {
 		return nil, errors.Wrap(err, "failed to parse local keystore list")
 	}
@@ -74,14 +74,14 @@ func (c *Client) GetLocalValidatorKeys(ctx context.Context) (*apimiddleware.List
 }
 
 // GetRemoteValidatorKeys calls the keymanager APIs for web3signer validator keys
-func (c *Client) GetRemoteValidatorKeys(ctx context.Context) (*apimiddleware.ListRemoteKeysResponseJson, error) {
+func (c *Client) GetRemoteValidatorKeys(ctx context.Context) (*rpc.ListRemoteKeysResponse, error) {
 	remoteBytes, err := c.Get(ctx, remoteKeysPath, client.WithAuthorizationToken(c.Token()))
 	if err != nil {
 		if !strings.Contains(err.Error(), "Prysm Wallet is not of type Web3Signer") {
 			return nil, err
 		}
 	}
-	jsonremote := &apimiddleware.ListRemoteKeysResponseJson{}
+	jsonremote := &rpc.ListRemoteKeysResponse{}
 	if len(remoteBytes) != 0 {
 		if err := json.Unmarshal(remoteBytes, jsonremote); err != nil {
 			return nil, errors.Wrap(err, "failed to parse remote keystore list")
@@ -107,13 +107,13 @@ func (c *Client) GetFeeRecipientAddresses(ctx context.Context, validators []stri
 }
 
 // GetFeeRecipientAddress takes a public key and calls the keymanager API to return its fee recipient.
-func (c *Client) GetFeeRecipientAddress(ctx context.Context, pubkey string) (*apimiddleware.GetFeeRecipientByPubkeyResponseJson, error) {
+func (c *Client) GetFeeRecipientAddress(ctx context.Context, pubkey string) (*rpc.GetFeeRecipientByPubkeyResponse, error) {
 	path := strings.Replace(feeRecipientPath, "{pubkey}", pubkey, 1)
 	b, err := c.Get(ctx, path, client.WithAuthorizationToken(c.Token()))
 	if err != nil {
 		return nil, err
 	}
-	feejson := &apimiddleware.GetFeeRecipientByPubkeyResponseJson{}
+	feejson := &rpc.GetFeeRecipientByPubkeyResponse{}
 	if err := json.Unmarshal(b, feejson); err != nil {
 		return nil, errors.Wrap(err, "failed to parse fee recipient")
 	}

api/constants.go

@@ -0,0 +1,3 @@
+package api
+
+const WebUrlPrefix = "/v2/validator/"

api/gateway/BUILD.bazel

@@ -14,16 +14,16 @@ go_library(
         "//validator:__subpackages__",
     ],
     deps = [
-        "//api/gateway/apimiddleware:go_default_library",
+        "//api/server:go_default_library",
         "//runtime:go_default_library",
         "@com_github_gorilla_mux//:go_default_library",
         "@com_github_grpc_ecosystem_grpc_gateway_v2//runtime:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
-        "@com_github_rs_cors//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@org_golang_google_grpc//:go_default_library",
         "@org_golang_google_grpc//connectivity:go_default_library",
         "@org_golang_google_grpc//credentials:go_default_library",
+        "@org_golang_google_grpc//credentials/insecure:go_default_library",
         "@org_golang_google_protobuf//proto:go_default_library",
     ],
 )
@@ -33,7 +33,6 @@ go_test(
     srcs = ["gateway_test.go"],
     embed = [":go_default_library"],
     deps = [
-        "//api/gateway/apimiddleware:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/require:go_default_library",

api/gateway/apimiddleware/BUILD.bazel

@@ -1,43 +0,0 @@
-load("@prysm//tools/go:def.bzl", "go_library", "go_test")
-
-go_library(
-    name = "go_default_library",
-    srcs = [
-        "api_middleware.go",
-        "log.go",
-        "param_handling.go",
-        "process_field.go",
-        "process_request.go",
-        "structs.go",
-    ],
-    importpath = "github.com/prysmaticlabs/prysm/v4/api/gateway/apimiddleware",
-    visibility = ["//visibility:public"],
-    deps = [
-        "//api:go_default_library",
-        "//api/grpc:go_default_library",
-        "//encoding/bytesutil:go_default_library",
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
-        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
-        "@com_github_gorilla_mux//:go_default_library",
-        "@com_github_pkg_errors//:go_default_library",
-        "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_wealdtech_go_bytesutil//:go_default_library",
-    ],
-)
-
-go_test(
-    name = "go_default_test",
-    srcs = [
-        "param_handling_test.go",
-        "process_request_test.go",
-    ],
-    embed = [":go_default_library"],
-    deps = [
-        "//api:go_default_library",
-        "//api/grpc:go_default_library",
-        "//testing/assert:go_default_library",
-        "//testing/require:go_default_library",
-        "@com_github_gorilla_mux//:go_default_library",
-        "@com_github_sirupsen_logrus//hooks/test:go_default_library",
-    ],
-)

api/gateway/apimiddleware/api_middleware.go

@@ -1,265 +0,0 @@
-package apimiddleware
-
-import (
-	"net/http"
-	"reflect"
-	"time"
-
-	"github.com/gorilla/mux"
-)
-
-// ApiProxyMiddleware is a proxy between an Ethereum consensus API HTTP client and grpc-gateway.
-// The purpose of the proxy is to handle HTTP requests and gRPC responses in such a way that:
-//   - Ethereum consensus API requests can be handled by grpc-gateway correctly
-//   - gRPC responses can be returned as spec-compliant Ethereum consensus API responses
-type ApiProxyMiddleware struct {
-	GatewayAddress  string
-	EndpointCreator EndpointFactory
-	Timeout         time.Duration
-	router          *mux.Router
-}
-
-// EndpointFactory is responsible for creating new instances of Endpoint values.
-type EndpointFactory interface {
-	Create(path string) (*Endpoint, error)
-	Paths() []string
-	IsNil() bool
-}
-
-// Endpoint is a representation of an API HTTP endpoint that should be proxied by the middleware.
-type Endpoint struct {
-	Path               string          // The path of the HTTP endpoint.
-	GetResponse        interface{}     // The struct corresponding to the JSON structure used in a GET response.
-	PostRequest        interface{}     // The struct corresponding to the JSON structure used in a POST request.
-	PostResponse       interface{}     // The struct corresponding to the JSON structure used in a POST response.
-	DeleteRequest      interface{}     // The struct corresponding to the JSON structure used in a DELETE request.
-	DeleteResponse     interface{}     // The struct corresponding to the JSON structure used in a DELETE response.
-	RequestURLLiterals []string        // Names of URL parameters that should not be base64-encoded.
-	RequestQueryParams []QueryParam    // Query parameters of the request.
-	Err                ErrorJson       // The struct corresponding to the error that should be returned in case of a request failure.
-	Hooks              HookCollection  // A collection of functions that can be invoked at various stages of the request/response cycle.
-	CustomHandlers     []CustomHandler // Functions that will be executed instead of the default request/response behaviour.
-}
-
-// RunDefault expresses whether the default processing logic should be carried out after running a pre hook.
-type RunDefault bool
-
-// DefaultEndpoint returns an Endpoint with default configuration, e.g. DefaultErrorJson for error handling.
-func DefaultEndpoint() Endpoint {
-	return Endpoint{
-		Err: &DefaultErrorJson{},
-	}
-}
-
-// QueryParam represents a single query parameter's metadata.
-type QueryParam struct {
-	Name string
-	Hex  bool
-	Enum bool
-}
-
-// CustomHandler is a function that can be invoked at the very beginning of the request,
-// essentially replacing the whole default request/response logic with custom logic for a specific endpoint.
-type CustomHandler = func(m *ApiProxyMiddleware, endpoint Endpoint, w http.ResponseWriter, req *http.Request) (handled bool)
-
-// HookCollection contains hooks that can be used to amend the default request/response cycle with custom logic for a specific endpoint.
-type HookCollection struct {
-	OnPreDeserializeRequestBodyIntoContainer      func(endpoint *Endpoint, w http.ResponseWriter, req *http.Request) (RunDefault, ErrorJson)
-	OnPostDeserializeRequestBodyIntoContainer     func(endpoint *Endpoint, w http.ResponseWriter, req *http.Request) ErrorJson
-	OnPreDeserializeGrpcResponseBodyIntoContainer func([]byte, interface{}) (RunDefault, ErrorJson)
-	OnPreSerializeMiddlewareResponseIntoJson      func(interface{}) (RunDefault, []byte, ErrorJson)
-}
-
-// fieldProcessor applies the processing function f to a value when the tag is present on the field.
-type fieldProcessor struct {
-	tag string
-	f   func(value reflect.Value) error
-}
-
-// Run starts the proxy, registering all proxy endpoints.
-func (m *ApiProxyMiddleware) Run(gatewayRouter *mux.Router) {
-	for _, path := range m.EndpointCreator.Paths() {
-		gatewayRouter.HandleFunc(path, m.WithMiddleware(path))
-	}
-	m.router = gatewayRouter
-}
-
-// ServeHTTP for the proxy middleware.
-func (m *ApiProxyMiddleware) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	m.router.ServeHTTP(w, req)
-}
-
-// WithMiddleware wraps the given endpoint handler with the middleware logic.
-func (m *ApiProxyMiddleware) WithMiddleware(path string) http.HandlerFunc {
-	return func(w http.ResponseWriter, req *http.Request) {
-		endpoint, err := m.EndpointCreator.Create(path)
-		if err != nil {
-			log.WithError(err).Errorf("Could not create endpoint for path: %s", path)
-			return
-		}
-
-		for _, handler := range endpoint.CustomHandlers {
-			if handler(m, *endpoint, w, req) {
-				return
-			}
-		}
-
-		if req.Method == "POST" {
-			if errJson := handlePostRequestForEndpoint(endpoint, w, req); errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-		}
-
-		if req.Method == "DELETE" && req.Body != http.NoBody {
-			if errJson := handleDeleteRequestForEndpoint(endpoint, req); errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-		}
-
-		if errJson := m.PrepareRequestForProxying(*endpoint, req); errJson != nil {
-			WriteError(w, errJson, nil)
-			return
-		}
-		grpcResp, errJson := m.ProxyRequest(req)
-		if errJson != nil {
-			WriteError(w, errJson, nil)
-			return
-		}
-		grpcRespBody, errJson := ReadGrpcResponseBody(grpcResp.Body)
-		if errJson != nil {
-			WriteError(w, errJson, nil)
-			return
-		}
-
-		var respJson []byte
-		if !GrpcResponseIsEmpty(grpcRespBody) {
-			respHasError, errJson := HandleGrpcResponseError(endpoint.Err, grpcResp, grpcRespBody, w)
-			if errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-			if respHasError {
-				return
-			}
-
-			var resp interface{}
-			if req.Method == "GET" {
-				resp = endpoint.GetResponse
-			} else if req.Method == "DELETE" {
-				resp = endpoint.DeleteResponse
-			} else {
-				resp = endpoint.PostResponse
-			}
-			if errJson := deserializeGrpcResponseBodyIntoContainerWrapped(endpoint, grpcRespBody, resp); errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-			if errJson := ProcessMiddlewareResponseFields(resp); errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-
-			respJson, errJson = serializeMiddlewareResponseIntoJsonWrapped(endpoint, respJson, resp)
-			if errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-		}
-
-		if errJson := WriteMiddlewareResponseHeadersAndBody(grpcResp, respJson, w); errJson != nil {
-			WriteError(w, errJson, nil)
-			return
-		}
-		if errJson := Cleanup(grpcResp.Body); errJson != nil {
-			WriteError(w, errJson, nil)
-			return
-		}
-	}
-}
-
-func handlePostRequestForEndpoint(endpoint *Endpoint, w http.ResponseWriter, req *http.Request) ErrorJson {
-	if errJson := deserializeRequestBodyIntoContainerWrapped(endpoint, req, w); errJson != nil {
-		return errJson
-	}
-	if errJson := ProcessRequestContainerFields(endpoint.PostRequest); errJson != nil {
-		return errJson
-	}
-	return SetRequestBodyToRequestContainer(endpoint.PostRequest, req)
-}
-
-func handleDeleteRequestForEndpoint(endpoint *Endpoint, req *http.Request) ErrorJson {
-	if errJson := DeserializeRequestBodyIntoContainer(req.Body, endpoint.DeleteRequest); errJson != nil {
-		return errJson
-	}
-	if errJson := ProcessRequestContainerFields(endpoint.DeleteRequest); errJson != nil {
-		return errJson
-	}
-	return SetRequestBodyToRequestContainer(endpoint.DeleteRequest, req)
-}
-
-func deserializeRequestBodyIntoContainerWrapped(endpoint *Endpoint, req *http.Request, w http.ResponseWriter) ErrorJson {
-	runDefault := true
-	if endpoint.Hooks.OnPreDeserializeRequestBodyIntoContainer != nil {
-		run, errJson := endpoint.Hooks.OnPreDeserializeRequestBodyIntoContainer(endpoint, w, req)
-		if errJson != nil {
-			return errJson
-		}
-		if !run {
-			runDefault = false
-		}
-	}
-	if runDefault {
-		if errJson := DeserializeRequestBodyIntoContainer(req.Body, endpoint.PostRequest); errJson != nil {
-			return errJson
-		}
-	}
-	if endpoint.Hooks.OnPostDeserializeRequestBodyIntoContainer != nil {
-		if errJson := endpoint.Hooks.OnPostDeserializeRequestBodyIntoContainer(endpoint, w, req); errJson != nil {
-			return errJson
-		}
-	}
-	return nil
-}
-
-func deserializeGrpcResponseBodyIntoContainerWrapped(endpoint *Endpoint, grpcResponseBody []byte, resp interface{}) ErrorJson {
-	runDefault := true
-	if endpoint.Hooks.OnPreDeserializeGrpcResponseBodyIntoContainer != nil {
-		run, errJson := endpoint.Hooks.OnPreDeserializeGrpcResponseBodyIntoContainer(grpcResponseBody, resp)
-		if errJson != nil {
-			return errJson
-		}
-		if !run {
-			runDefault = false
-		}
-	}
-	if runDefault {
-		if errJson := DeserializeGrpcResponseBodyIntoContainer(grpcResponseBody, resp); errJson != nil {
-			return errJson
-		}
-	}
-	return nil
-}
-
-func serializeMiddlewareResponseIntoJsonWrapped(endpoint *Endpoint, respJson []byte, resp interface{}) ([]byte, ErrorJson) {
-	runDefault := true
-	var errJson ErrorJson
-	if endpoint.Hooks.OnPreSerializeMiddlewareResponseIntoJson != nil {
-		var run RunDefault
-		run, respJson, errJson = endpoint.Hooks.OnPreSerializeMiddlewareResponseIntoJson(resp)
-		if errJson != nil {
-			return nil, errJson
-		}
-		if !run {
-			runDefault = false
-		}
-	}
-	if runDefault {
-		respJson, errJson = SerializeMiddlewareResponseIntoJson(resp)
-		if errJson != nil {
-			return nil, errJson
-		}
-	}
-	return respJson, nil
-}

api/gateway/apimiddleware/log.go

@@ -1,5 +0,0 @@
-package apimiddleware
-
-import "github.com/sirupsen/logrus"
-
-var log = logrus.WithField("prefix", "apimiddleware")

api/gateway/apimiddleware/param_handling.go

@@ -1,103 +0,0 @@
-package apimiddleware
-
-import (
-	"encoding/base64"
-	"net/http"
-	"net/url"
-	"strings"
-
-	"github.com/gorilla/mux"
-	butil "github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
-	"github.com/wealdtech/go-bytesutil"
-)
-
-// HandleURLParameters processes URL parameters, allowing parameterized URLs to be safely and correctly proxied to grpc-gateway.
-func HandleURLParameters(url string, req *http.Request, literals []string) ErrorJson {
-	segments := strings.Split(url, "/")
-
-segmentsLoop:
-	for i, s := range segments {
-		// We only care about segments which are parameterized.
-		if isRequestParam(s) {
-			// Don't do anything with parameters which should be forwarded literally to gRPC.
-			for _, l := range literals {
-				if s == "{"+l+"}" {
-					continue segmentsLoop
-				}
-			}
-
-			routeVar := mux.Vars(req)[s[1:len(s)-1]]
-			bRouteVar := []byte(routeVar)
-			if butil.IsHex(bRouteVar) {
-				var err error
-				bRouteVar, err = bytesutil.FromHexString(string(bRouteVar))
-				if err != nil {
-					return InternalServerErrorWithMessage(err, "could not process URL parameter")
-				}
-			}
-			// Converting hex to base64 may result in a value which malforms the URL.
-			// We use URLEncoding to safely escape such values.
-			base64RouteVar := base64.URLEncoding.EncodeToString(bRouteVar)
-
-			// Merge segments back into the full URL.
-			splitPath := strings.Split(req.URL.Path, "/")
-			splitPath[i] = base64RouteVar
-			req.URL.Path = strings.Join(splitPath, "/")
-		}
-	}
-	return nil
-}
-
-// HandleQueryParameters processes query parameters, allowing them to be safely and correctly proxied to grpc-gateway.
-func HandleQueryParameters(req *http.Request, params []QueryParam) ErrorJson {
-	queryParams := req.URL.Query()
-
-	normalizeQueryValues(queryParams)
-
-	for key, vals := range queryParams {
-		for _, p := range params {
-			if key == p.Name {
-				if p.Hex {
-					queryParams.Del(key)
-					for _, v := range vals {
-						b := []byte(v)
-						if butil.IsHex(b) {
-							var err error
-							b, err = bytesutil.FromHexString(v)
-							if err != nil {
-								return InternalServerErrorWithMessage(err, "could not process query parameter")
-							}
-						}
-						queryParams.Add(key, base64.URLEncoding.EncodeToString(b))
-					}
-				}
-				if p.Enum {
-					queryParams.Del(key)
-					for _, v := range vals {
-						// gRPC expects uppercase enum values.
-						queryParams.Add(key, strings.ToUpper(v))
-					}
-				}
-			}
-		}
-	}
-	req.URL.RawQuery = queryParams.Encode()
-	return nil
-}
-
-// isRequestParam verifies whether the passed string is a request parameter.
-// Request parameters are enclosed in { and }.
-func isRequestParam(s string) bool {
-	return len(s) > 2 && s[0] == '{' && s[len(s)-1] == '}'
-}
-
-func normalizeQueryValues(queryParams url.Values) {
-	// Replace comma-separated values with individual values.
-	for key, vals := range queryParams {
-		splitVals := make([]string, 0)
-		for _, v := range vals {
-			splitVals = append(splitVals, strings.Split(v, ",")...)
-		}
-		queryParams[key] = splitVals
-	}
-}

api/gateway/apimiddleware/param_handling_test.go

@@ -1,124 +0,0 @@
-package apimiddleware
-
-import (
-	"bytes"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/gorilla/mux"
-	"github.com/prysmaticlabs/prysm/v4/testing/assert"
-	"github.com/prysmaticlabs/prysm/v4/testing/require"
-)
-
-func TestHandleURLParameters(t *testing.T) {
-	var body bytes.Buffer
-
-	t.Run("no_params", func(t *testing.T) {
-		request := httptest.NewRequest("GET", "http://foo.example/bar", &body)
-
-		errJson := HandleURLParameters("/not_param", request, []string{})
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "/bar", request.URL.Path)
-	})
-
-	t.Run("with_params", func(t *testing.T) {
-		muxVars := make(map[string]string)
-		muxVars["bar_param"] = "bar"
-		muxVars["quux_param"] = "quux"
-		request := httptest.NewRequest("GET", "http://foo.example/bar/baz/quux", &body)
-		request = mux.SetURLVars(request, muxVars)
-
-		errJson := HandleURLParameters("/{bar_param}/not_param/{quux_param}", request, []string{})
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "/YmFy/baz/cXV1eA==", request.URL.Path)
-	})
-
-	t.Run("with_literal", func(t *testing.T) {
-		muxVars := make(map[string]string)
-		muxVars["bar_param"] = "bar"
-		request := httptest.NewRequest("GET", "http://foo.example/bar/baz", &body)
-		request = mux.SetURLVars(request, muxVars)
-
-		errJson := HandleURLParameters("/{bar_param}/not_param/", request, []string{"bar_param"})
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "/bar/baz", request.URL.Path)
-	})
-
-	t.Run("with_hex", func(t *testing.T) {
-		muxVars := make(map[string]string)
-		muxVars["hex_param"] = "0x626172"
-		request := httptest.NewRequest("GET", "http://foo.example/0x626172/baz", &body)
-		request = mux.SetURLVars(request, muxVars)
-
-		errJson := HandleURLParameters("/{hex_param}/not_param/", request, []string{})
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "/YmFy/baz", request.URL.Path)
-	})
-}
-
-func TestHandleQueryParameters(t *testing.T) {
-	var body bytes.Buffer
-
-	t.Run("regular_params", func(t *testing.T) {
-		request := httptest.NewRequest("GET", "http://foo.example?bar=bar&baz=baz", &body)
-
-		errJson := HandleQueryParameters(request, []QueryParam{{Name: "bar"}, {Name: "baz"}})
-		require.Equal(t, true, errJson == nil)
-		query := request.URL.Query()
-		v, ok := query["bar"]
-		require.Equal(t, true, ok, "query param not found")
-		require.Equal(t, 1, len(v), "wrong number of query param values")
-		assert.Equal(t, "bar", v[0])
-		v, ok = query["baz"]
-		require.Equal(t, true, ok, "query param not found")
-		require.Equal(t, 1, len(v), "wrong number of query param values")
-		assert.Equal(t, "baz", v[0])
-	})
-
-	t.Run("hex_and_enum_params", func(t *testing.T) {
-		request := httptest.NewRequest("GET", "http://foo.example?hex=0x626172&baz=baz", &body)
-
-		errJson := HandleQueryParameters(request, []QueryParam{{Name: "hex", Hex: true}, {Name: "baz", Enum: true}})
-		require.Equal(t, true, errJson == nil)
-		query := request.URL.Query()
-		v, ok := query["hex"]
-		require.Equal(t, true, ok, "query param not found")
-		require.Equal(t, 1, len(v), "wrong number of query param values")
-		assert.Equal(t, "YmFy", v[0])
-		v, ok = query["baz"]
-		require.Equal(t, true, ok, "query param not found")
-		require.Equal(t, 1, len(v), "wrong number of query param values")
-		assert.Equal(t, "BAZ", v[0])
-	})
-}
-
-func TestIsRequestParam(t *testing.T) {
-	tests := []struct {
-		s string
-		b bool
-	}{
-		{"", false},
-		{"{", false},
-		{"}", false},
-		{"{}", false},
-		{"{x}", true},
-		{"{very_long_parameter_name_with_underscores}", true},
-	}
-	for _, tt := range tests {
-		b := isRequestParam(tt.s)
-		assert.Equal(t, tt.b, b)
-	}
-}
-
-func TestNormalizeQueryValues(t *testing.T) {
-	input := make(map[string][]string)
-	input["key"] = []string{"value1", "value2,value3,value4", "value5"}
-
-	normalizeQueryValues(input)
-	require.Equal(t, 5, len(input["key"]))
-	assert.Equal(t, "value1", input["key"][0])
-	assert.Equal(t, "value2", input["key"][1])
-	assert.Equal(t, "value3", input["key"][2])
-	assert.Equal(t, "value4", input["key"][3])
-	assert.Equal(t, "value5", input["key"][4])
-}

api/gateway/apimiddleware/process_field.go

@@ -1,179 +0,0 @@
-package apimiddleware
-
-import (
-	"encoding/base64"
-	"fmt"
-	"math/big"
-	"reflect"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/pkg/errors"
-	"github.com/wealdtech/go-bytesutil"
-)
-
-// processField calls each processor function on any field that has the matching tag set.
-// It is a recursive function.
-func processField(s interface{}, processors []fieldProcessor) error {
-	kind := reflect.TypeOf(s).Kind()
-	if kind != reflect.Ptr && kind != reflect.Slice && kind != reflect.Array {
-		return fmt.Errorf("processing fields of kind '%v' is unsupported", kind)
-	}
-
-	t := reflect.TypeOf(s).Elem()
-	v := reflect.Indirect(reflect.ValueOf(s))
-
-	for i := 0; i < t.NumField(); i++ {
-		switch v.Field(i).Kind() {
-		case reflect.Slice:
-			sliceElem := t.Field(i).Type.Elem()
-			kind := sliceElem.Kind()
-			// Recursively process slices to struct pointers.
-			switch {
-			case kind == reflect.Ptr && sliceElem.Elem().Kind() == reflect.Struct:
-				for j := 0; j < v.Field(i).Len(); j++ {
-					if err := processField(v.Field(i).Index(j).Interface(), processors); err != nil {
-						return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
-					}
-				}
-			// Process each string in string slices.
-			case kind == reflect.String:
-				for _, proc := range processors {
-					_, hasTag := t.Field(i).Tag.Lookup(proc.tag)
-					if !hasTag {
-						continue
-					}
-					for j := 0; j < v.Field(i).Len(); j++ {
-						if err := proc.f(v.Field(i).Index(j)); err != nil {
-							return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
-						}
-					}
-				}
-			}
-		// Recursively process struct pointers.
-		case reflect.Ptr:
-			if v.Field(i).Elem().Kind() == reflect.Struct {
-				if err := processField(v.Field(i).Interface(), processors); err != nil {
-					return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
-				}
-			}
-		default:
-			field := t.Field(i)
-			for _, proc := range processors {
-				if _, hasTag := field.Tag.Lookup(proc.tag); hasTag {
-					if err := proc.f(v.Field(i)); err != nil {
-						return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
-					}
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func hexToBase64Processor(v reflect.Value) error {
-	if v.String() == "0x" {
-		v.SetString("")
-		return nil
-	}
-	b, err := bytesutil.FromHexString(v.String())
-	if err != nil {
-		return err
-	}
-	v.SetString(base64.StdEncoding.EncodeToString(b))
-	return nil
-}
-
-func base64ToHexProcessor(v reflect.Value) error {
-	if v.String() == "" {
-		// Empty hex values are represented as "0x".
-		v.SetString("0x")
-		return nil
-	}
-	b, err := base64.StdEncoding.DecodeString(v.String())
-	if err != nil {
-		return err
-	}
-	v.SetString(hexutil.Encode(b))
-	return nil
-}
-
-func base64ToChecksumAddressProcessor(v reflect.Value) error {
-	if v.String() == "" {
-		// Empty hex values are represented as "0x".
-		v.SetString("0x")
-		return nil
-	}
-	b, err := base64.StdEncoding.DecodeString(v.String())
-	if err != nil {
-		return err
-	}
-	v.SetString(common.BytesToAddress(b).Hex())
-	return nil
-}
-
-func base64ToUint256Processor(v reflect.Value) error {
-	if v.String() == "" {
-		return nil
-	}
-	littleEndian, err := base64.StdEncoding.DecodeString(v.String())
-	if err != nil {
-		return err
-	}
-	if len(littleEndian) != 32 {
-		return errors.New("invalid length for Uint256")
-	}
-
-	// Integers are stored as little-endian, but
-	// big.Int expects big-endian. So we need to reverse
-	// the byte order before decoding.
-	var bigEndian [32]byte
-	for i := 0; i < len(littleEndian); i++ {
-		bigEndian[i] = littleEndian[len(littleEndian)-1-i]
-	}
-	var uint256 big.Int
-	uint256.SetBytes(bigEndian[:])
-	v.SetString(uint256.String())
-	return nil
-}
-
-func uint256ToBase64Processor(v reflect.Value) error {
-	if v.String() == "" {
-		return nil
-	}
-	uint256, ok := new(big.Int).SetString(v.String(), 10)
-	if !ok {
-		return fmt.Errorf("could not parse Uint256")
-	}
-	bigEndian := uint256.Bytes()
-	if len(bigEndian) > 32 {
-		return fmt.Errorf("number too big for Uint256")
-	}
-
-	// Integers are stored as little-endian, but
-	// big.Int gives big-endian. So we need to reverse
-	// the byte order before encoding.
-	var littleEndian [32]byte
-	for i := 0; i < len(bigEndian); i++ {
-		littleEndian[i] = bigEndian[len(bigEndian)-1-i]
-	}
-	v.SetString(base64.StdEncoding.EncodeToString(littleEndian[:]))
-	return nil
-}
-
-func enumToLowercaseProcessor(v reflect.Value) error {
-	v.SetString(strings.ToLower(v.String()))
-	return nil
-}
-
-func timeToUnixProcessor(v reflect.Value) error {
-	t, err := time.Parse(time.RFC3339, v.String())
-	if err != nil {
-		return err
-	}
-	v.SetString(strconv.FormatUint(uint64(t.Unix()), 10))
-	return nil
-}

api/gateway/apimiddleware/process_request.go

@@ -1,283 +0,0 @@
-package apimiddleware
-
-import (
-	"bytes"
-	"encoding/json"
-	"io"
-	"net"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/v4/api"
-	"github.com/prysmaticlabs/prysm/v4/api/grpc"
-)
-
-// DeserializeRequestBodyIntoContainer deserializes the request's body into an endpoint-specific struct.
-func DeserializeRequestBodyIntoContainer(body io.Reader, requestContainer interface{}) ErrorJson {
-	decoder := json.NewDecoder(body)
-	decoder.DisallowUnknownFields()
-	if err := decoder.Decode(&requestContainer); err != nil {
-		if strings.Contains(err.Error(), "json: unknown field") {
-			e := errors.Wrap(err, "could not decode request body")
-			return &DefaultErrorJson{
-				Message: e.Error(),
-				Code:    http.StatusBadRequest,
-			}
-		}
-		return InternalServerErrorWithMessage(err, "could not decode request body")
-	}
-	return nil
-}
-
-// ProcessRequestContainerFields processes fields of an endpoint-specific container according to field tags.
-func ProcessRequestContainerFields(requestContainer interface{}) ErrorJson {
-	if err := processField(requestContainer, []fieldProcessor{
-		{
-			tag: "hex",
-			f:   hexToBase64Processor,
-		},
-		{
-			tag: "uint256",
-			f:   uint256ToBase64Processor,
-		},
-	}); err != nil {
-		return InternalServerErrorWithMessage(err, "could not process request data")
-	}
-	return nil
-}
-
-// SetRequestBodyToRequestContainer makes the endpoint-specific container the new body of the request.
-func SetRequestBodyToRequestContainer(requestContainer interface{}, req *http.Request) ErrorJson {
-	// Serialize the struct, which now includes a base64-encoded value, into JSON.
-	j, err := json.Marshal(requestContainer)
-	if err != nil {
-		return InternalServerErrorWithMessage(err, "could not marshal request")
-	}
-	// Set the body to the new JSON.
-	req.Body = io.NopCloser(bytes.NewReader(j))
-	req.Header.Set("Content-Length", strconv.Itoa(len(j)))
-	req.ContentLength = int64(len(j))
-	return nil
-}
-
-// PrepareRequestForProxying applies additional logic to the request so that it can be correctly proxied to grpc-gateway.
-func (m *ApiProxyMiddleware) PrepareRequestForProxying(endpoint Endpoint, req *http.Request) ErrorJson {
-	req.URL.Scheme = "http"
-	req.URL.Host = m.GatewayAddress
-	req.RequestURI = ""
-	if errJson := HandleURLParameters(endpoint.Path, req, endpoint.RequestURLLiterals); errJson != nil {
-		return errJson
-	}
-	if errJson := HandleQueryParameters(req, endpoint.RequestQueryParams); errJson != nil {
-		return errJson
-	}
-	// We have to add the prefix after handling parameters because adding the prefix changes URL segment indexing.
-	req.URL.Path = "/internal" + req.URL.Path
-	return nil
-}
-
-// ProxyRequest proxies the request to grpc-gateway.
-func (m *ApiProxyMiddleware) ProxyRequest(req *http.Request) (*http.Response, ErrorJson) {
-	// We do not use http.DefaultClient because it does not have any timeout.
-	netClient := &http.Client{Timeout: m.Timeout}
-	grpcResp, err := netClient.Do(req)
-	if err != nil {
-		if err, ok := err.(net.Error); ok && err.Timeout() {
-			return nil, TimeoutError()
-		}
-		return nil, InternalServerErrorWithMessage(err, "could not proxy request")
-	}
-	if grpcResp == nil {
-		return nil, &DefaultErrorJson{Message: "nil response from gRPC-gateway", Code: http.StatusInternalServerError}
-	}
-	return grpcResp, nil
-}
-
-// ReadGrpcResponseBody reads the body from the grpc-gateway's response.
-func ReadGrpcResponseBody(r io.Reader) ([]byte, ErrorJson) {
-	body, err := io.ReadAll(r)
-	if err != nil {
-		return nil, InternalServerErrorWithMessage(err, "could not read response body")
-	}
-	return body, nil
-}
-
-// HandleGrpcResponseError acts on an error that resulted from a grpc-gateway's response.
-// Whether there was an error is indicated by the bool return value. In case of an error,
-// there is no need to write to the response because it's taken care of by the function.
-func HandleGrpcResponseError(errJson ErrorJson, resp *http.Response, respBody []byte, w http.ResponseWriter) (bool, ErrorJson) {
-	responseHasError := false
-	if err := json.Unmarshal(respBody, errJson); err != nil {
-		return false, InternalServerErrorWithMessage(err, "could not unmarshal error")
-	}
-	if errJson.Msg() != "" {
-		responseHasError = true
-		// Something went wrong, but the request completed, meaning we can write headers and the error message.
-		for h, vs := range resp.Header {
-			for _, v := range vs {
-				if strings.HasSuffix(h, api.VersionHeader) {
-					w.Header().Set(api.VersionHeader, v)
-				} else {
-					w.Header().Set(h, v)
-				}
-			}
-		}
-		// Handle gRPC timeout.
-		if resp.StatusCode == http.StatusGatewayTimeout {
-			WriteError(w, TimeoutError(), resp.Header)
-		} else {
-			// Set code to HTTP code because unmarshalled body contained gRPC code.
-			errJson.SetCode(resp.StatusCode)
-			WriteError(w, errJson, resp.Header)
-		}
-	}
-	return responseHasError, nil
-}
-
-// GrpcResponseIsEmpty determines whether the grpc-gateway's response body contains no data.
-func GrpcResponseIsEmpty(grpcResponseBody []byte) bool {
-	return len(grpcResponseBody) == 0 || string(grpcResponseBody) == "{}"
-}
-
-// DeserializeGrpcResponseBodyIntoContainer deserializes the grpc-gateway's response body into an endpoint-specific struct.
-func DeserializeGrpcResponseBodyIntoContainer(body []byte, responseContainer interface{}) ErrorJson {
-	if err := json.Unmarshal(body, &responseContainer); err != nil {
-		return InternalServerErrorWithMessage(err, "could not unmarshal response")
-	}
-	return nil
-}
-
-// ProcessMiddlewareResponseFields processes fields of an endpoint-specific container according to field tags.
-func ProcessMiddlewareResponseFields(responseContainer interface{}) ErrorJson {
-	if err := processField(responseContainer, []fieldProcessor{
-		{
-			tag: "hex",
-			f:   base64ToHexProcessor,
-		},
-		{
-			tag: "address",
-			f:   base64ToChecksumAddressProcessor,
-		},
-		{
-			tag: "enum",
-			f:   enumToLowercaseProcessor,
-		},
-		{
-			tag: "time",
-			f:   timeToUnixProcessor,
-		},
-		{
-			tag: "uint256",
-			f:   base64ToUint256Processor,
-		},
-	}); err != nil {
-		return InternalServerErrorWithMessage(err, "could not process response data")
-	}
-	return nil
-}
-
-// SerializeMiddlewareResponseIntoJson serializes the endpoint-specific response struct into a JSON representation.
-func SerializeMiddlewareResponseIntoJson(responseContainer interface{}) (jsonResponse []byte, errJson ErrorJson) {
-	j, err := json.Marshal(responseContainer)
-	if err != nil {
-		return nil, InternalServerErrorWithMessage(err, "could not marshal response")
-	}
-	return j, nil
-}
-
-// WriteMiddlewareResponseHeadersAndBody populates headers and the body of the final response.
-func WriteMiddlewareResponseHeadersAndBody(grpcResp *http.Response, responseJson []byte, w http.ResponseWriter) ErrorJson {
-	var statusCodeHeader string
-	for h, vs := range grpcResp.Header {
-		// We don't want to expose any gRPC metadata in the HTTP response, so we skip forwarding metadata headers.
-		if strings.HasPrefix(h, grpc.MetadataPrefix) {
-			if h == grpc.WithPrefix(grpc.HttpCodeMetadataKey) {
-				statusCodeHeader = vs[0]
-			} else if strings.HasSuffix(h, api.VersionHeader) {
-				w.Header().Set(api.VersionHeader, vs[0])
-			}
-		} else {
-			for _, v := range vs {
-				w.Header().Set(h, v)
-			}
-		}
-	}
-	if !GrpcResponseIsEmpty(responseJson) {
-		w.Header().Set("Content-Length", strconv.Itoa(len(responseJson)))
-		if statusCodeHeader != "" {
-			code, err := strconv.Atoi(statusCodeHeader)
-			if err != nil {
-				return InternalServerErrorWithMessage(err, "could not parse status code")
-			}
-			w.WriteHeader(code)
-		} else {
-			w.WriteHeader(grpcResp.StatusCode)
-		}
-		if _, err := io.Copy(w, io.NopCloser(bytes.NewReader(responseJson))); err != nil {
-			return InternalServerErrorWithMessage(err, "could not write response message")
-		}
-	} else {
-		w.Header().Set("Content-Length", "0")
-		w.WriteHeader(grpcResp.StatusCode)
-	}
-	return nil
-}
-
-// WriteError writes the error by manipulating headers and the body of the final response.
-func WriteError(w http.ResponseWriter, errJson ErrorJson, responseHeader http.Header) {
-	// Include custom error in the error JSON.
-	hasCustomError := false
-	if responseHeader != nil {
-		customError, ok := responseHeader[grpc.WithPrefix(grpc.CustomErrorMetadataKey)]
-		if ok {
-			hasCustomError = true
-			// Assume header has only one value and read the 0 index.
-			if err := json.Unmarshal([]byte(customError[0]), errJson); err != nil {
-				log.WithError(err).Error("Could not unmarshal custom error message")
-				return
-			}
-		}
-	}
-
-	var j []byte
-	if hasCustomError {
-		var err error
-		j, err = json.Marshal(errJson)
-		if err != nil {
-			log.WithError(err).Error("Could not marshal error message")
-			return
-		}
-	} else {
-		var err error
-		// We marshal the response body into a DefaultErrorJson if the custom error is not present.
-		// This is because the ErrorJson argument is the endpoint's error definition, which may contain custom fields.
-		// In such a scenario marhaling the endpoint's error would populate the resulting JSON
-		// with these fields even if they are not present in the gRPC header.
-		d := &DefaultErrorJson{
-			Message: errJson.Msg(),
-			Code:    errJson.StatusCode(),
-		}
-		j, err = json.Marshal(d)
-		if err != nil {
-			log.WithError(err).Error("Could not marshal error message")
-			return
-		}
-	}
-
-	w.Header().Set("Content-Length", strconv.Itoa(len(j)))
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(errJson.StatusCode())
-	if _, err := io.Copy(w, io.NopCloser(bytes.NewReader(j))); err != nil {
-		log.WithError(err).Error("Could not write error message")
-	}
-}
-
-// Cleanup performs final cleanup on the initial response from grpc-gateway.
-func Cleanup(grpcResponseBody io.ReadCloser) ErrorJson {
-	if err := grpcResponseBody.Close(); err != nil {
-		return InternalServerErrorWithMessage(err, "could not close response body")
-	}
-	return nil
-}

api/gateway/apimiddleware/process_request_test.go

@@ -1,435 +0,0 @@
-package apimiddleware
-
-import (
-	"bytes"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/prysmaticlabs/prysm/v4/api"
-	"github.com/prysmaticlabs/prysm/v4/api/grpc"
-	"github.com/prysmaticlabs/prysm/v4/testing/assert"
-	"github.com/prysmaticlabs/prysm/v4/testing/require"
-	"github.com/sirupsen/logrus/hooks/test"
-)
-
-type testRequestContainer struct {
-	TestString         string
-	TestHexString      string `hex:"true"`
-	TestEmptyHexString string `hex:"true"`
-	TestUint256String  string `uint256:"true"`
-}
-
-func defaultRequestContainer() *testRequestContainer {
-	return &testRequestContainer{
-		TestString:         "test string",
-		TestHexString:      "0x666F6F", // hex encoding of "foo"
-		TestEmptyHexString: "0x",
-		TestUint256String:  "4196",
-	}
-}
-
-type testResponseContainer struct {
-	TestString       string
-	TestHex          string `hex:"true"`
-	TestEmptyHex     string `hex:"true"`
-	TestAddress      string `address:"true"`
-	TestEmptyAddress string `address:"true"`
-	TestUint256      string `uint256:"true"`
-	TestEnum         string `enum:"true"`
-	TestTime         string `time:"true"`
-}
-
-func defaultResponseContainer() *testResponseContainer {
-	return &testResponseContainer{
-		TestString:       "test string",
-		TestHex:          "Zm9v", // base64 encoding of "foo"
-		TestEmptyHex:     "",
-		TestAddress:      "Zm9v",
-		TestEmptyAddress: "",
-		TestEnum:         "Test Enum",
-		TestTime:         "2006-01-02T15:04:05Z",
-
-		// base64 encoding of 4196 in little-endian
-		TestUint256: "ZBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
-	}
-}
-
-type testErrorJson struct {
-	Message     string
-	Code        int
-	CustomField string
-}
-
-// StatusCode returns the error's underlying error code.
-func (e *testErrorJson) StatusCode() int {
-	return e.Code
-}
-
-// Msg returns the error's underlying message.
-func (e *testErrorJson) Msg() string {
-	return e.Message
-}
-
-// SetCode sets the error's underlying error code.
-func (e *testErrorJson) SetCode(code int) {
-	e.Code = code
-}
-
-// SetMsg sets the error's underlying message.
-func (e *testErrorJson) SetMsg(msg string) {
-	e.Message = msg
-}
-
-func TestDeserializeRequestBodyIntoContainer(t *testing.T) {
-	t.Run("ok", func(t *testing.T) {
-		var bodyJson bytes.Buffer
-		err := json.NewEncoder(&bodyJson).Encode(defaultRequestContainer())
-		require.NoError(t, err)
-
-		container := &testRequestContainer{}
-		errJson := DeserializeRequestBodyIntoContainer(&bodyJson, container)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "test string", container.TestString)
-	})
-
-	t.Run("error", func(t *testing.T) {
-		var bodyJson bytes.Buffer
-		bodyJson.Write([]byte("foo"))
-		errJson := DeserializeRequestBodyIntoContainer(&bodyJson, &testRequestContainer{})
-		require.NotNil(t, errJson)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not decode request body"))
-		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
-	})
-
-	t.Run("unknown field", func(t *testing.T) {
-		var bodyJson bytes.Buffer
-		bodyJson.Write([]byte("{\"foo\":\"foo\"}"))
-		errJson := DeserializeRequestBodyIntoContainer(&bodyJson, &testRequestContainer{})
-		require.NotNil(t, errJson)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not decode request body"))
-		assert.Equal(t, http.StatusBadRequest, errJson.StatusCode())
-	})
-}
-
-func TestProcessRequestContainerFields(t *testing.T) {
-	t.Run("ok", func(t *testing.T) {
-		container := defaultRequestContainer()
-
-		errJson := ProcessRequestContainerFields(container)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "Zm9v", container.TestHexString)
-		assert.Equal(t, "", container.TestEmptyHexString)
-		assert.Equal(t, "ZBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", container.TestUint256String)
-	})
-
-	t.Run("error", func(t *testing.T) {
-		errJson := ProcessRequestContainerFields("foo")
-		require.NotNil(t, errJson)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not process request data"))
-		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
-	})
-}
-
-func TestSetRequestBodyToRequestContainer(t *testing.T) {
-	var body bytes.Buffer
-	request := httptest.NewRequest("GET", "http://foo.example", &body)
-
-	errJson := SetRequestBodyToRequestContainer(defaultRequestContainer(), request)
-	require.Equal(t, true, errJson == nil)
-	container := &testRequestContainer{}
-	require.NoError(t, json.NewDecoder(request.Body).Decode(container))
-	assert.Equal(t, "test string", container.TestString)
-	contentLengthHeader, ok := request.Header["Content-Length"]
-	require.Equal(t, true, ok)
-	require.Equal(t, 1, len(contentLengthHeader), "wrong number of header values")
-	assert.Equal(t, "108", contentLengthHeader[0])
-	assert.Equal(t, int64(108), request.ContentLength)
-}
-
-func TestPrepareRequestForProxying(t *testing.T) {
-	middleware := &ApiProxyMiddleware{
-		GatewayAddress: "http://gateway.example",
-	}
-	// We will set some params to make the request more interesting.
-	endpoint := Endpoint{
-		Path:               "/{url_param}",
-		RequestURLLiterals: []string{"url_param"},
-		RequestQueryParams: []QueryParam{{Name: "query_param"}},
-	}
-	var body bytes.Buffer
-	request := httptest.NewRequest("GET", "http://foo.example?query_param=bar", &body)
-
-	errJson := middleware.PrepareRequestForProxying(endpoint, request)
-	require.Equal(t, true, errJson == nil)
-	assert.Equal(t, "http", request.URL.Scheme)
-	assert.Equal(t, middleware.GatewayAddress, request.URL.Host)
-	assert.Equal(t, "", request.RequestURI)
-}
-
-func TestReadGrpcResponseBody(t *testing.T) {
-	var b bytes.Buffer
-	b.Write([]byte("foo"))
-
-	body, jsonErr := ReadGrpcResponseBody(&b)
-	require.Equal(t, true, jsonErr == nil)
-	assert.Equal(t, "foo", string(body))
-}
-
-func TestHandleGrpcResponseError(t *testing.T) {
-	response := &http.Response{
-		StatusCode: 400,
-		Header: http.Header{
-			"Foo": []string{"foo"},
-			"Bar": []string{"bar"},
-		},
-	}
-	writer := httptest.NewRecorder()
-	errJson := &testErrorJson{
-		Message: "foo",
-		Code:    400,
-	}
-	b, err := json.Marshal(errJson)
-	require.NoError(t, err)
-
-	hasError, e := HandleGrpcResponseError(errJson, response, b, writer)
-	require.Equal(t, true, e == nil)
-	assert.Equal(t, true, hasError)
-	v, ok := writer.Header()["Foo"]
-	require.Equal(t, true, ok, "header not found")
-	require.Equal(t, 1, len(v), "wrong number of header values")
-	assert.Equal(t, "foo", v[0])
-	v, ok = writer.Header()["Bar"]
-	require.Equal(t, true, ok, "header not found")
-	require.Equal(t, 1, len(v), "wrong number of header values")
-	assert.Equal(t, "bar", v[0])
-	assert.Equal(t, 400, errJson.StatusCode())
-}
-
-func TestGrpcResponseIsEmpty(t *testing.T) {
-	t.Run("nil", func(t *testing.T) {
-		assert.Equal(t, true, GrpcResponseIsEmpty(nil))
-	})
-	t.Run("empty_slice", func(t *testing.T) {
-		assert.Equal(t, true, GrpcResponseIsEmpty(make([]byte, 0)))
-	})
-	t.Run("empty_brackets", func(t *testing.T) {
-		assert.Equal(t, true, GrpcResponseIsEmpty([]byte("{}")))
-	})
-	t.Run("non_empty", func(t *testing.T) {
-		assert.Equal(t, false, GrpcResponseIsEmpty([]byte("{\"foo\":\"bar\"})")))
-	})
-}
-
-func TestDeserializeGrpcResponseBodyIntoContainer(t *testing.T) {
-	t.Run("ok", func(t *testing.T) {
-		body, err := json.Marshal(defaultRequestContainer())
-		require.NoError(t, err)
-
-		container := &testRequestContainer{}
-		errJson := DeserializeGrpcResponseBodyIntoContainer(body, container)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "test string", container.TestString)
-	})
-
-	t.Run("error", func(t *testing.T) {
-		var bodyJson bytes.Buffer
-		bodyJson.Write([]byte("foo"))
-		errJson := DeserializeGrpcResponseBodyIntoContainer(bodyJson.Bytes(), &testRequestContainer{})
-		require.NotNil(t, errJson)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not unmarshal response"))
-		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
-	})
-}
-
-func TestProcessMiddlewareResponseFields(t *testing.T) {
-	t.Run("Ok", func(t *testing.T) {
-		container := defaultResponseContainer()
-
-		errJson := ProcessMiddlewareResponseFields(container)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "0x666f6f", container.TestHex)
-		assert.Equal(t, "0x", container.TestEmptyHex)
-		assert.Equal(t, "0x0000000000000000000000000000000000666F6f", container.TestAddress)
-		assert.Equal(t, "0x", container.TestEmptyAddress)
-		assert.Equal(t, "4196", container.TestUint256)
-		assert.Equal(t, "test enum", container.TestEnum)
-		assert.Equal(t, "1136214245", container.TestTime)
-	})
-
-	t.Run("error", func(t *testing.T) {
-		errJson := ProcessMiddlewareResponseFields("foo")
-		require.NotNil(t, errJson)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not process response data"))
-		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
-	})
-}
-
-func TestSerializeMiddlewareResponseIntoJson(t *testing.T) {
-	container := defaultResponseContainer()
-	j, errJson := SerializeMiddlewareResponseIntoJson(container)
-	assert.Equal(t, true, errJson == nil)
-	cToDeserialize := &testResponseContainer{}
-	require.NoError(t, json.Unmarshal(j, cToDeserialize))
-	assert.Equal(t, "test string", cToDeserialize.TestString)
-}
-
-func TestWriteMiddlewareResponseHeadersAndBody(t *testing.T) {
-	t.Run("GET", func(t *testing.T) {
-		response := &http.Response{
-			Header: http.Header{
-				"Foo": []string{"foo"},
-				grpc.WithPrefix(grpc.HttpCodeMetadataKey): []string{"204"},
-				grpc.WithPrefix(api.VersionHeader):        []string{"capella"},
-			},
-		}
-		container := defaultResponseContainer()
-		responseJson, err := json.Marshal(container)
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-		writer.Body = &bytes.Buffer{}
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, true, errJson == nil)
-		v, ok := writer.Header()["Foo"]
-		require.Equal(t, true, ok, "header not found")
-		require.Equal(t, 1, len(v), "wrong number of header values")
-		assert.Equal(t, "foo", v[0])
-		v, ok = writer.Header()["Content-Length"]
-		require.Equal(t, true, ok, "header not found")
-		require.Equal(t, 1, len(v), "wrong number of header values")
-		assert.Equal(t, "224", v[0])
-		v, ok = writer.Header()["Eth-Consensus-Version"]
-		require.Equal(t, true, ok, "header not found")
-		assert.Equal(t, "capella", v[0])
-		assert.Equal(t, 204, writer.Code)
-		assert.DeepEqual(t, responseJson, writer.Body.Bytes())
-	})
-
-	t.Run("GET_no_grpc_status_code_header", func(t *testing.T) {
-		response := &http.Response{
-			Header:     http.Header{},
-			StatusCode: 204,
-		}
-		container := defaultResponseContainer()
-		responseJson, err := json.Marshal(container)
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, 204, writer.Code)
-	})
-
-	t.Run("GET_invalid_status_code", func(t *testing.T) {
-		response := &http.Response{
-			Header: http.Header{"Grpc-Metadata-Eth-Consensus-Version": []string{"capella"}},
-		}
-
-		// Set invalid status code.
-		response.Header[grpc.WithPrefix(grpc.HttpCodeMetadataKey)] = []string{"invalid"}
-		response.Header[grpc.WithPrefix(api.VersionHeader)] = []string{"capella"}
-
-		container := defaultResponseContainer()
-		responseJson, err := json.Marshal(container)
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, false, errJson == nil)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not parse status code"))
-		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
-	})
-
-	t.Run("POST", func(t *testing.T) {
-		response := &http.Response{
-			Header:     http.Header{},
-			StatusCode: 204,
-		}
-		container := defaultResponseContainer()
-		responseJson, err := json.Marshal(container)
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, 204, writer.Code)
-	})
-
-	t.Run("POST_with_response_body", func(t *testing.T) {
-		response := &http.Response{
-			Header:     http.Header{},
-			StatusCode: 204,
-		}
-		container := defaultResponseContainer()
-		responseJson, err := json.Marshal(container)
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-		writer.Body = &bytes.Buffer{}
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, 204, writer.Code)
-		assert.DeepEqual(t, responseJson, writer.Body.Bytes())
-	})
-
-	t.Run("POST_with_empty_json_body", func(t *testing.T) {
-		response := &http.Response{
-			Header:     http.Header{},
-			StatusCode: 204,
-		}
-		responseJson, err := json.Marshal(struct{}{})
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-		writer.Body = &bytes.Buffer{}
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, 204, writer.Code)
-		assert.DeepEqual(t, []byte(nil), writer.Body.Bytes())
-		assert.Equal(t, "0", writer.Header()["Content-Length"][0])
-	})
-}
-
-func TestWriteError(t *testing.T) {
-	t.Run("ok", func(t *testing.T) {
-		responseHeader := http.Header{
-			grpc.WithPrefix(grpc.CustomErrorMetadataKey): []string{"{\"CustomField\":\"bar\"}"},
-		}
-		errJson := &testErrorJson{
-			Message: "foo",
-			Code:    500,
-		}
-		writer := httptest.NewRecorder()
-		writer.Body = &bytes.Buffer{}
-
-		WriteError(writer, errJson, responseHeader)
-		v, ok := writer.Header()["Content-Length"]
-		require.Equal(t, true, ok, "header not found")
-		require.Equal(t, 1, len(v), "wrong number of header values")
-		assert.Equal(t, "48", v[0])
-		v, ok = writer.Header()["Content-Type"]
-		require.Equal(t, true, ok, "header not found")
-		require.Equal(t, 1, len(v), "wrong number of header values")
-		assert.Equal(t, "application/json", v[0])
-		assert.Equal(t, 500, writer.Code)
-		eDeserialize := &testErrorJson{}
-		require.NoError(t, json.Unmarshal(writer.Body.Bytes(), eDeserialize))
-		assert.Equal(t, "foo", eDeserialize.Message)
-		assert.Equal(t, 500, eDeserialize.Code)
-		assert.Equal(t, "bar", eDeserialize.CustomField)
-	})
-
-	t.Run("invalid_custom_error_header", func(t *testing.T) {
-		logHook := test.NewGlobal()
-
-		responseHeader := http.Header{
-			grpc.WithPrefix(grpc.CustomErrorMetadataKey): []string{"invalid"},
-		}
-
-		WriteError(httptest.NewRecorder(), &testErrorJson{}, responseHeader)
-		assert.LogsContain(t, logHook, "Could not unmarshal custom error message")
-	})
-}

api/gateway/apimiddleware/structs.go

@@ -1,69 +0,0 @@
-package apimiddleware
-
-import (
-	"net/http"
-
-	"github.com/pkg/errors"
-)
-
-// ---------------
-// Error handling.
-// ---------------
-
-// ErrorJson describes common functionality of all JSON error representations.
-type ErrorJson interface {
-	StatusCode() int
-	SetCode(code int)
-	Msg() string
-	SetMsg(msg string)
-}
-
-// DefaultErrorJson is a JSON representation of a simple error value, containing only a message and an error code.
-type DefaultErrorJson struct {
-	Message string `json:"message"`
-	Code    int    `json:"code"`
-}
-
-// InternalServerErrorWithMessage returns a DefaultErrorJson with 500 code and a custom message.
-func InternalServerErrorWithMessage(err error, message string) *DefaultErrorJson {
-	e := errors.Wrapf(err, message)
-	return &DefaultErrorJson{
-		Message: e.Error(),
-		Code:    http.StatusInternalServerError,
-	}
-}
-
-// InternalServerError returns a DefaultErrorJson with 500 code.
-func InternalServerError(err error) *DefaultErrorJson {
-	return &DefaultErrorJson{
-		Message: err.Error(),
-		Code:    http.StatusInternalServerError,
-	}
-}
-
-func TimeoutError() *DefaultErrorJson {
-	return &DefaultErrorJson{
-		Message: "Request timeout",
-		Code:    http.StatusRequestTimeout,
-	}
-}
-
-// StatusCode returns the error's underlying error code.
-func (e *DefaultErrorJson) StatusCode() int {
-	return e.Code
-}
-
-// Msg returns the error's underlying message.
-func (e *DefaultErrorJson) Msg() string {
-	return e.Message
-}
-
-// SetCode sets the error's underlying error code.
-func (e *DefaultErrorJson) SetCode(code int) {
-	e.Code = code
-}
-
-// SetMsg sets the error's underlying message.
-func (e *DefaultErrorJson) SetMsg(msg string) {
-	e.Message = msg
-}

api/gateway/gateway.go

@@ -6,19 +6,17 @@ import (
 	"fmt"
 	"net"
 	"net/http"
-	"path"
-	"strings"
 	"time"
 
 	"github.com/gorilla/mux"
 	gwruntime "github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/v4/api/gateway/apimiddleware"
+	"github.com/prysmaticlabs/prysm/v4/api/server"
 	"github.com/prysmaticlabs/prysm/v4/runtime"
-	"github.com/rs/cors"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
 )
 
 var _ runtime.Service = (*Gateway)(nil)
@@ -35,7 +33,6 @@ type PbHandlerRegistration func(context.Context, *gwruntime.ServeMux, *grpc.Clie
 
 // MuxHandler is a function that implements the mux handler functionality.
 type MuxHandler func(
-	apiMiddlewareHandler *apimiddleware.ApiProxyMiddleware,
 	h http.HandlerFunc,
 	w http.ResponseWriter,
 	req *http.Request,
@@ -43,16 +40,15 @@ type MuxHandler func(
 
 // Config parameters for setting up the gateway service.
 type config struct {
-	maxCallRecvMsgSize           uint64
-	remoteCert                   string
-	gatewayAddr                  string
-	remoteAddr                   string
-	allowedOrigins               []string
-	apiMiddlewareEndpointFactory apimiddleware.EndpointFactory
-	muxHandler                   MuxHandler
-	pbHandlers                   []*PbMux
-	router                       *mux.Router
-	timeout                      time.Duration
+	maxCallRecvMsgSize uint64
+	remoteCert         string
+	gatewayAddr        string
+	remoteAddr         string
+	allowedOrigins     []string
+	muxHandler         MuxHandler
+	pbHandlers         []*PbMux
+	router             *mux.Router
+	timeout            time.Duration
 }
 
 // Gateway is the gRPC gateway to serve HTTP JSON traffic as a proxy and forward it to the gRPC server.
@@ -61,7 +57,6 @@ type Gateway struct {
 	conn         *grpc.ClientConn
 	server       *http.Server
 	cancel       context.CancelFunc
-	proxy        *apimiddleware.ApiProxyMiddleware
 	ctx          context.Context
 	startFailure error
 }
@@ -109,15 +104,11 @@ func (g *Gateway) Start() {
 		}
 	}
 
-	corsMux := g.corsMiddleware(g.cfg.router)
-
-	if g.cfg.apiMiddlewareEndpointFactory != nil && !g.cfg.apiMiddlewareEndpointFactory.IsNil() {
-		g.registerApiMiddleware()
-	}
+	corsMux := server.CorsHandler(g.cfg.allowedOrigins).Middleware(g.cfg.router)
 
 	if g.cfg.muxHandler != nil {
 		g.cfg.router.PathPrefix("/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			g.cfg.muxHandler(g.proxy, corsMux.ServeHTTP, w, r)
+			g.cfg.muxHandler(corsMux.ServeHTTP, w, r)
 		})
 	}
 
@@ -167,35 +158,6 @@ func (g *Gateway) Stop() error {
 	return nil
 }
 
-func (g *Gateway) corsMiddleware(h http.Handler) http.Handler {
-	c := cors.New(cors.Options{
-		AllowedOrigins:   g.cfg.allowedOrigins,
-		AllowedMethods:   []string{http.MethodPost, http.MethodGet, http.MethodDelete, http.MethodOptions},
-		AllowCredentials: true,
-		MaxAge:           600,
-		AllowedHeaders:   []string{"*"},
-	})
-	return c.Handler(h)
-}
-
-const swaggerDir = "proto/prysm/v1alpha1/"
-
-// SwaggerServer returns swagger specification files located under "/swagger/"
-func SwaggerServer() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		if !strings.HasSuffix(r.URL.Path, ".swagger.json") {
-			log.Debugf("Not found: %s", r.URL.Path)
-			http.NotFound(w, r)
-			return
-		}
-
-		log.Debugf("Serving %s\n", r.URL.Path)
-		p := strings.TrimPrefix(r.URL.Path, "/swagger/")
-		p = path.Join(swaggerDir, p)
-		http.ServeFile(w, r, p)
-	}
-}
-
 // dial the gRPC server.
 func (g *Gateway) dial(ctx context.Context, network, addr string) (*grpc.ClientConn, error) {
 	switch network {
@@ -211,19 +173,21 @@ func (g *Gateway) dial(ctx context.Context, network, addr string) (*grpc.ClientC
 // dialTCP creates a client connection via TCP.
 // "addr" must be a valid TCP address with a port number.
 func (g *Gateway) dialTCP(ctx context.Context, addr string) (*grpc.ClientConn, error) {
-	security := grpc.WithInsecure()
+	var security grpc.DialOption
 	if len(g.cfg.remoteCert) > 0 {
 		creds, err := credentials.NewClientTLSFromFile(g.cfg.remoteCert, "")
 		if err != nil {
 			return nil, err
 		}
 		security = grpc.WithTransportCredentials(creds)
+	} else {
+		// Use insecure credentials when there's no remote cert provided.
+		security = grpc.WithTransportCredentials(insecure.NewCredentials())
 	}
 	opts := []grpc.DialOption{
 		security,
 		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(g.cfg.maxCallRecvMsgSize))),
 	}
-
 	return grpc.DialContext(ctx, addr, opts...)
 }
 
@@ -240,19 +204,9 @@ func (g *Gateway) dialUnix(ctx context.Context, addr string) (*grpc.ClientConn,
 		return d(addr, 0)
 	}
 	opts := []grpc.DialOption{
-		grpc.WithInsecure(),
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
 		grpc.WithContextDialer(f),
 		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(g.cfg.maxCallRecvMsgSize))),
 	}
 	return grpc.DialContext(ctx, addr, opts...)
 }
-
-func (g *Gateway) registerApiMiddleware() {
-	g.proxy = &apimiddleware.ApiProxyMiddleware{
-		GatewayAddress:  g.cfg.gatewayAddr,
-		EndpointCreator: g.cfg.apiMiddlewareEndpointFactory,
-		Timeout:         g.cfg.timeout,
-	}
-	log.Info("Starting API middleware")
-	g.proxy.Run(g.cfg.router)
-}

api/gateway/gateway_test.go

@@ -10,7 +10,6 @@ import (
 	"testing"
 
 	"github.com/gorilla/mux"
-	"github.com/prysmaticlabs/prysm/v4/api/gateway/apimiddleware"
 	"github.com/prysmaticlabs/prysm/v4/cmd/beacon-chain/flags"
 	"github.com/prysmaticlabs/prysm/v4/testing/assert"
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
@@ -18,36 +17,18 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
-type mockEndpointFactory struct {
-}
-
-func (*mockEndpointFactory) Paths() []string {
-	return []string{}
-}
-
-func (*mockEndpointFactory) Create(_ string) (*apimiddleware.Endpoint, error) {
-	return nil, nil
-}
-
-func (*mockEndpointFactory) IsNil() bool {
-	return false
-}
-
 func TestGateway_Customized(t *testing.T) {
 	r := mux.NewRouter()
 	cert := "cert"
 	origins := []string{"origin"}
 	size := uint64(100)
-	endpointFactory := &mockEndpointFactory{}
 
 	opts := []Option{
 		WithRouter(r),
 		WithRemoteCert(cert),
 		WithAllowedOrigins(origins),
 		WithMaxCallRecvMsgSize(size),
-		WithApiMiddleware(endpointFactory),
 		WithMuxHandler(func(
-			_ *apimiddleware.ApiProxyMiddleware,
 			_ http.HandlerFunc,
 			_ http.ResponseWriter,
 			_ *http.Request,
@@ -63,7 +44,6 @@ func TestGateway_Customized(t *testing.T) {
 	require.Equal(t, 1, len(g.cfg.allowedOrigins))
 	assert.Equal(t, origins[0], g.cfg.allowedOrigins[0])
 	assert.Equal(t, size, g.cfg.maxCallRecvMsgSize)
-	assert.Equal(t, endpointFactory, g.cfg.apiMiddlewareEndpointFactory)
 }
 
 func TestGateway_StartStop(t *testing.T) {
@@ -83,7 +63,6 @@ func TestGateway_StartStop(t *testing.T) {
 		WithGatewayAddr(gatewayAddress),
 		WithRemoteAddr(selfAddress),
 		WithMuxHandler(func(
-			_ *apimiddleware.ApiProxyMiddleware,
 			_ http.HandlerFunc,
 			_ http.ResponseWriter,
 			_ *http.Request,

api/gateway/options.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/gorilla/mux"
 	gwruntime "github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
-	"github.com/prysmaticlabs/prysm/v4/api/gateway/apimiddleware"
 )
 
 type Option func(g *Gateway) error
@@ -70,14 +69,6 @@ func WithMaxCallRecvMsgSize(size uint64) Option {
 	}
 }
 
-// WithApiMiddleware allows adding an API middleware proxy to the gateway.
-func WithApiMiddleware(endpointFactory apimiddleware.EndpointFactory) Option {
-	return func(g *Gateway) error {
-		g.cfg.apiMiddlewareEndpointFactory = endpointFactory
-		return nil
-	}
-}
-
 // WithTimeout allows changing the timeout value for API calls.
 func WithTimeout(seconds uint64) Option {
 	return func(g *Gateway) error {

api/headers.go

@@ -4,6 +4,7 @@ const (
 	VersionHeader                 = "Eth-Consensus-Version"
 	ExecutionPayloadBlindedHeader = "Eth-Execution-Payload-Blinded"
 	ExecutionPayloadValueHeader   = "Eth-Execution-Payload-Value"
+	ConsensusBlockValueHeader     = "Eth-Consensus-Block-Value"
 	JsonMediaType                 = "application/json"
 	OctetStreamMediaType          = "application/octet-stream"
 )

api/server/BUILD.bazel

@@ -0,0 +1,30 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "error.go",
+        "middleware.go",
+        "util.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/v4/api/server",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_gorilla_mux//:go_default_library",
+        "@com_github_rs_cors//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "error_test.go",
+        "middleware_test.go",
+        "util_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = [
+        "//testing/assert:go_default_library",
+        "//testing/require:go_default_library",
+    ],
+)

api/server/error.go

@@ -0,0 +1,45 @@
+package server
+
+import (
+	"fmt"
+	"strings"
+)
+
+// DecodeError represents an error resulting from trying to decode an HTTP request.
+// It tracks the full field name for which decoding failed.
+type DecodeError struct {
+	path []string
+	err  error
+}
+
+// NewDecodeError wraps an error (either the initial decoding error or another DecodeError).
+// The current field that failed decoding must be passed in.
+func NewDecodeError(err error, field string) *DecodeError {
+	de, ok := err.(*DecodeError)
+	if ok {
+		return &DecodeError{path: append([]string{field}, de.path...), err: de.err}
+	}
+	return &DecodeError{path: []string{field}, err: err}
+}
+
+// Error returns the formatted error message which contains the full field name and the actual decoding error.
+func (e *DecodeError) Error() string {
+	return fmt.Sprintf("could not decode %s: %s", strings.Join(e.path, "."), e.err.Error())
+}
+
+// IndexedVerificationFailureError wraps a collection of verification failures.
+type IndexedVerificationFailureError struct {
+	Message  string                        `json:"message"`
+	Code     int                           `json:"code"`
+	Failures []*IndexedVerificationFailure `json:"failures"`
+}
+
+func (e *IndexedVerificationFailureError) StatusCode() int {
+	return e.Code
+}
+
+// IndexedVerificationFailure represents an issue when verifying a single indexed object e.g. an item in an array.
+type IndexedVerificationFailure struct {
+	Index   int    `json:"index"`
+	Message string `json:"message"`
+}

api/server/error_test.go

@@ -0,0 +1,16 @@
+package server
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/v4/testing/assert"
+)
+
+func TestDecodeError(t *testing.T) {
+	e := errors.New("not a number")
+	de := NewDecodeError(e, "Z")
+	de = NewDecodeError(de, "Y")
+	de = NewDecodeError(de, "X")
+	assert.Equal(t, "could not decode X.Y.Z: not a number", de.Error())
+}

api/server/middleware.go

@@ -0,0 +1,32 @@
+package server
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/rs/cors"
+)
+
+// NormalizeQueryValuesHandler normalizes an input query of "key=value1,value2,value3" to "key=value1&key=value2&key=value3"
+func NormalizeQueryValuesHandler(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		query := r.URL.Query()
+		NormalizeQueryValues(query)
+		r.URL.RawQuery = query.Encode()
+
+		next.ServeHTTP(w, r)
+	})
+}
+
+// CorsHandler sets the cors settings on api endpoints
+func CorsHandler(allowOrigins []string) mux.MiddlewareFunc {
+	c := cors.New(cors.Options{
+		AllowedOrigins:   allowOrigins,
+		AllowedMethods:   []string{http.MethodPost, http.MethodGet, http.MethodDelete, http.MethodOptions},
+		AllowCredentials: true,
+		MaxAge:           600,
+		AllowedHeaders:   []string{"*"},
+	})
+
+	return c.Handler
+}

api/server/middleware_test.go

@@ -0,0 +1,54 @@
+package server
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/v4/testing/require"
+)
+
+func TestNormalizeQueryValuesHandler(t *testing.T) {
+	nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, err := w.Write([]byte("next handler"))
+		require.NoError(t, err)
+	})
+
+	handler := NormalizeQueryValuesHandler(nextHandler)
+
+	tests := []struct {
+		name          string
+		inputQuery    string
+		expectedQuery string
+	}{
+		{
+			name:          "3 values",
+			inputQuery:    "key=value1,value2,value3",
+			expectedQuery: "key=value1&key=value2&key=value3", // replace with expected normalized value
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			req, err := http.NewRequest("GET", "/test?"+test.inputQuery, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			rr := httptest.NewRecorder()
+			handler.ServeHTTP(rr, req)
+
+			if rr.Code != http.StatusOK {
+				t.Errorf("handler returned wrong status code: got %v want %v", rr.Code, http.StatusOK)
+			}
+
+			if req.URL.RawQuery != test.expectedQuery {
+				t.Errorf("query not normalized: got %v want %v", req.URL.RawQuery, test.expectedQuery)
+			}
+
+			if rr.Body.String() != "next handler" {
+				t.Errorf("next handler was not executed")
+			}
+		})
+	}
+}

api/server/util.go

@@ -1,4 +1,4 @@
-package helpers
+package server
 
 import (
 	"net/url"

api/server/util_test.go

@@ -1,4 +1,4 @@
-package helpers
+package server
 
 import (
 	"testing"

beacon-chain/BUILD.bazel

@@ -1,19 +0,0 @@
-load("//tools:target_migration.bzl", "moved_targets")
-
-moved_targets(
-    [
-        ":push_images_debug",
-        ":push_images_alpine",
-        ":push_images",
-        ":image_bundle_debug",
-        ":image_debug",
-        ":image_bundle_alpine",
-        ":image_bundle",
-        ":image_with_creation_time",
-        ":image_alpine",
-        ":image",
-        ":go_default_test",
-        ":beacon-chain",
-    ],
-    "//cmd/beacon-chain",
-)

beacon-chain/blockchain/BUILD.bazel

@@ -26,6 +26,7 @@ go_library(
         "receive_blob.go",
         "receive_block.go",
         "service.go",
+        "tracked_proposer.go",
         "weak_subjectivity_checks.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain",
@@ -49,9 +50,10 @@ go_library(
         "//beacon-chain/core/signing:go_default_library",
         "//beacon-chain/core/time:go_default_library",
         "//beacon-chain/core/transition:go_default_library",
+        "//beacon-chain/das:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/db/filesystem:go_default_library",
         "//beacon-chain/db/filters:go_default_library",
-        "//beacon-chain/db/kv:go_default_library",
         "//beacon-chain/execution:go_default_library",
         "//beacon-chain/forkchoice:go_default_library",
         "//beacon-chain/forkchoice/doubly-linked-tree:go_default_library",
@@ -69,6 +71,7 @@ go_library(
         "//config/params:go_default_library",
         "//consensus-types:go_default_library",
         "//consensus-types/blocks:go_default_library",
+        "//consensus-types/forkchoice:go_default_library",
         "//consensus-types/interfaces:go_default_library",
         "//consensus-types/payload-attribute:go_default_library",
         "//consensus-types/primitives:go_default_library",
@@ -97,26 +100,22 @@ go_library(
     ],
 )
 
-test_suite(
-    name = "go_default_test",
-    tests = [
-        ":go_raceoff_test",
-        ":go_raceon_test",
-    ],
-)
-
 go_test(
-    name = "go_raceoff_test",
+    name = "go_default_test",
     size = "medium",
     srcs = [
         "blockchain_test.go",
+        "chain_info_norace_test.go",
         "chain_info_test.go",
         "checktags_test.go",
+        "error_test.go",
         "execution_engine_test.go",
         "forkchoice_update_execution_test.go",
         "head_sync_committee_info_test.go",
         "head_test.go",
+        "init_sync_process_block_test.go",
         "init_test.go",
+        "lightclient_test.go",
         "log_test.go",
         "metrics_test.go",
         "mock_test.go",
@@ -125,98 +124,67 @@ go_test(
         "process_block_test.go",
         "receive_attestation_test.go",
         "receive_block_test.go",
+        "service_norace_test.go",
         "service_test.go",
         "setup_test.go",
         "weak_subjectivity_checks_test.go",
     ],
     embed = [":go_default_library"],
     gotags = ["develop"],
+    tags = ["CI_race_detection"],
     deps = [
         "//async/event:go_default_library",
         "//beacon-chain/blockchain/testing:go_default_library",
+        "//beacon-chain/cache:go_default_library",
         "//beacon-chain/cache/depositcache:go_default_library",
         "//beacon-chain/core/blocks:go_default_library",
+        "//beacon-chain/core/feed/state:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/core/signing:go_default_library",
         "//beacon-chain/core/transition:go_default_library",
+        "//beacon-chain/das:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/db/filesystem:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/execution:go_default_library",
         "//beacon-chain/execution/testing:go_default_library",
+        "//beacon-chain/forkchoice:go_default_library",
+        "//beacon-chain/forkchoice/doubly-linked-tree:go_default_library",
         "//beacon-chain/forkchoice/types:go_default_library",
+        "//beacon-chain/operations/attestations:go_default_library",
+        "//beacon-chain/operations/blstoexec:go_default_library",
+        "//beacon-chain/operations/slashings:go_default_library",
+        "//beacon-chain/operations/voluntaryexits:go_default_library",
         "//beacon-chain/p2p:go_default_library",
+        "//beacon-chain/startup:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/state-native:go_default_library",
-        "//beacon-chain/state/stateutil:go_default_library",
+        "//beacon-chain/state/stategen:go_default_library",
+        "//config/features:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//consensus-types/blocks:go_default_library",
-        "//consensus-types/blocks/testing:go_default_library",
+        "//consensus-types/interfaces:go_default_library",
+        "//consensus-types/primitives:go_default_library",
         "//container/trie:go_default_library",
+        "//crypto/bls:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//proto/engine/v1:go_default_library",
+        "//proto/eth/v1:go_default_library",
+        "//proto/eth/v2:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
+        "//runtime/version:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/require:go_default_library",
         "//testing/util:go_default_library",
-        "@com_github_ethereum_go_ethereum//:go_default_library",
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
-        "@com_github_ethereum_go_ethereum//core/types:go_default_library",
-        "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_sirupsen_logrus//hooks/test:go_default_library",
-        "@in_gopkg_d4l3k_messagediff_v1//:go_default_library",
-        "@org_golang_google_protobuf//proto:go_default_library",
-        "@org_golang_x_net//context:go_default_library",
-    ],
-)
-
-go_test(
-    name = "go_raceon_test",
-    srcs = [
-        "chain_info_norace_test.go",
-        "checktags_test.go",
-        "init_test.go",
-        "mock_test.go",
-        "receive_block_test.go",
-        "service_norace_test.go",
-        "setup_test.go",
-    ],
-    embed = [":go_default_library"],
-    gc_goopts = [
-        # Go 1.14 enables checkptr by default when building with -race or -msan. There is a pointer
-        # issue in boltdb, so must disable checkptr at compile time. This flag can be removed once
-        # the project is migrated to etcd's version of boltdb and the issue has been fixed.
-        # See: https://github.com/etcd-io/bbolt/issues/187.
-        "-d=checkptr=0",
-    ],
-    gotags = ["develop"],
-    race = "on",
-    tags = ["race_on"],
-    deps = [
-        "//async/event:go_default_library",
-        "//beacon-chain/blockchain/testing:go_default_library",
-        "//beacon-chain/cache/depositcache:go_default_library",
-        "//beacon-chain/core/blocks:go_default_library",
-        "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/core/transition:go_default_library",
-        "//beacon-chain/db:go_default_library",
-        "//beacon-chain/db/testing:go_default_library",
-        "//beacon-chain/execution:go_default_library",
-        "//beacon-chain/execution/testing:go_default_library",
-        "//beacon-chain/forkchoice/types:go_default_library",
-        "//beacon-chain/p2p:go_default_library",
-        "//config/params:go_default_library",
-        "//consensus-types/blocks:go_default_library",
-        "//consensus-types/blocks/testing:go_default_library",
-        "//container/trie:go_default_library",
-        "//encoding/bytesutil:go_default_library",
-        "//proto/prysm/v1alpha1:go_default_library",
-        "//testing/assert:go_default_library",
-        "//testing/require:go_default_library",
-        "//testing/util:go_default_library",
-        "@com_github_ethereum_go_ethereum//:go_default_library",
+        "//time:go_default_library",
+        "//time/slots:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
+        "@com_github_holiman_uint256//:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
         "@org_golang_google_protobuf//proto:go_default_library",
-        "@org_golang_x_net//context:go_default_library",
     ],
 )

beacon-chain/blockchain/chain_info.go

@@ -12,10 +12,10 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
 	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/forkchoice"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
-	ethpbv1 "github.com/prysmaticlabs/prysm/v4/proto/eth/v1"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/time/slots"
 	"go.opencensus.io/trace"
@@ -45,7 +45,7 @@ type ForkchoiceFetcher interface {
 	HighestReceivedBlockSlot() primitives.Slot
 	ReceivedBlocksLastEpoch() (uint64, error)
 	InsertNode(context.Context, state.BeaconState, [32]byte) error
-	ForkChoiceDump(context.Context) (*ethpbv1.ForkChoiceDump, error)
+	ForkChoiceDump(context.Context) (*forkchoice.Dump, error)
 	NewSlot(context.Context, primitives.Slot) error
 	ProposerBoost() [32]byte
 }
@@ -75,6 +75,7 @@ type HeadFetcher interface {
 	HeadPublicKeyToValidatorIndex(pubKey [fieldparams.BLSPubkeyLength]byte) (primitives.ValidatorIndex, bool)
 	HeadValidatorIndexToPublicKey(ctx context.Context, index primitives.ValidatorIndex) ([fieldparams.BLSPubkeyLength]byte, error)
 	ChainHeads() ([][32]byte, []primitives.Slot)
+	TargetRootForEpoch([32]byte, primitives.Epoch) ([32]byte, error)
 	HeadSyncCommitteeFetcher
 	HeadDomainFetcher
 }
@@ -464,6 +465,13 @@ func (s *Service) IsOptimisticForRoot(ctx context.Context, root [32]byte) (bool,
 	return !isCanonical, nil
 }
 
+// TargetRootForEpoch wraps the corresponding method in forkchoice
+func (s *Service) TargetRootForEpoch(root [32]byte, epoch primitives.Epoch) ([32]byte, error) {
+	s.cfg.ForkChoiceStore.RLock()
+	defer s.cfg.ForkChoiceStore.RUnlock()
+	return s.cfg.ForkChoiceStore.TargetRootForEpoch(root, epoch)
+}
+
 // Ancestor returns the block root of an ancestry block from the input block root.
 //
 // Spec pseudocode definition:
@@ -530,3 +538,10 @@ func (s *Service) recoverStateSummary(ctx context.Context, blockRoot [32]byte) (
 func (s *Service) BlockBeingSynced(root [32]byte) bool {
 	return s.blockBeingSynced.isSyncing(root)
 }
+
+// RecentBlockSlot returns block slot form fork choice store
+func (s *Service) RecentBlockSlot(root [32]byte) (primitives.Slot, error) {
+	s.cfg.ForkChoiceStore.RLock()
+	defer s.cfg.ForkChoiceStore.RUnlock()
+	return s.cfg.ForkChoiceStore.Slot(root)
+}

beacon-chain/blockchain/chain_info_forkchoice.go

@@ -4,8 +4,8 @@ import (
 	"context"
 
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/forkchoice"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
-	ethpbv1 "github.com/prysmaticlabs/prysm/v4/proto/eth/v1"
 )
 
 // CachedHeadRoot returns the corresponding value from Forkchoice
@@ -22,6 +22,13 @@ func (s *Service) GetProposerHead() [32]byte {
 	return s.cfg.ForkChoiceStore.GetProposerHead()
 }
 
+// ShouldOverrideFCU returns the corresponding value from forkchoice
+func (s *Service) ShouldOverrideFCU() bool {
+	s.cfg.ForkChoiceStore.RLock()
+	defer s.cfg.ForkChoiceStore.RUnlock()
+	return s.cfg.ForkChoiceStore.ShouldOverrideFCU()
+}
+
 // SetForkChoiceGenesisTime sets the genesis time in Forkchoice
 func (s *Service) SetForkChoiceGenesisTime(timestamp uint64) {
 	s.cfg.ForkChoiceStore.Lock()
@@ -51,7 +58,7 @@ func (s *Service) InsertNode(ctx context.Context, st state.BeaconState, root [32
 }
 
 // ForkChoiceDump returns the corresponding value from forkchoice
-func (s *Service) ForkChoiceDump(ctx context.Context) (*ethpbv1.ForkChoiceDump, error) {
+func (s *Service) ForkChoiceDump(ctx context.Context) (*forkchoice.Dump, error) {
 	s.cfg.ForkChoiceStore.RLock()
 	defer s.cfg.ForkChoiceStore.RUnlock()
 	return s.cfg.ForkChoiceStore.ForkChoiceDump(ctx)

beacon-chain/blockchain/error.go

@@ -30,6 +30,8 @@ var (
 	ErrNotCheckpoint = errors.New("not a checkpoint in forkchoice")
 )
 
+var errMaxBlobsExceeded = errors.New("Expected commitments in block exceeds MAX_BLOBS_PER_BLOCK")
+
 // An invalid block is the block that fails state transition based on the core protocol rules.
 // The beacon node shall not be accepting nor building blocks that branch off from an invalid block.
 // Some examples of invalid blocks are:

beacon-chain/blockchain/execution_engine.go

@@ -7,11 +7,11 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/kv"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/execution"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/v4/config/features"
@@ -32,21 +32,18 @@ const blobCommitmentVersionKZG uint8 = 0x01
 
 var defaultLatestValidHash = bytesutil.PadTo([]byte{0xff}, 32)
 
-// notifyForkchoiceUpdateArg is the argument for the forkchoice update notification `notifyForkchoiceUpdate`.
-type notifyForkchoiceUpdateArg struct {
-	headState state.BeaconState
-	headRoot  [32]byte
-	headBlock interfaces.ReadOnlyBeaconBlock
-}
-
 // notifyForkchoiceUpdate signals execution engine the fork choice updates. Execution engine should:
 // 1. Re-organizes the execution payload chain and corresponding state to make head_block_hash the head.
 // 2. Applies finality to the execution state: it irreversibly persists the chain of all execution payloads and corresponding state, up to and including finalized_block_hash.
-func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *notifyForkchoiceUpdateArg) (*enginev1.PayloadIDBytes, error) {
+func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *fcuConfig) (*enginev1.PayloadIDBytes, error) {
 	ctx, span := trace.StartSpan(ctx, "blockChain.notifyForkchoiceUpdate")
 	defer span.End()
 
-	headBlk := arg.headBlock
+	if arg.headBlock.IsNil() {
+		log.Error("Head block is nil")
+		return nil, nil
+	}
+	headBlk := arg.headBlock.Block()
 	if headBlk == nil || headBlk.IsNil() || headBlk.Body().IsNil() {
 		log.Error("Head block is nil")
 		return nil, nil
@@ -72,11 +69,10 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *notifyForkcho
 		SafeBlockHash:      justifiedHash[:],
 		FinalizedBlockHash: finalizedHash[:],
 	}
-
-	nextSlot := s.CurrentSlot() + 1 // Cache payload ID for next slot proposer.
-	hasAttr, attr, proposerId := s.getPayloadAttribute(ctx, arg.headState, nextSlot, arg.headRoot[:])
-
-	payloadID, lastValidHash, err := s.cfg.ExecutionEngineCaller.ForkchoiceUpdated(ctx, fcs, attr)
+	if arg.attributes == nil {
+		arg.attributes = payloadattribute.EmptyWithVersion(headBlk.Version())
+	}
+	payloadID, lastValidHash, err := s.cfg.ExecutionEngineCaller.ForkchoiceUpdated(ctx, fcs, arg.attributes)
 	if err != nil {
 		switch err {
 		case execution.ErrAcceptedSyncingPayloadStatus:
@@ -122,10 +118,11 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *notifyForkcho
 				log.WithError(err).Error("Could not get head state")
 				return nil, nil
 			}
-			pid, err := s.notifyForkchoiceUpdate(ctx, &notifyForkchoiceUpdateArg{
-				headState: st,
-				headRoot:  r,
-				headBlock: b.Block(),
+			pid, err := s.notifyForkchoiceUpdate(ctx, &fcuConfig{
+				headState:  st,
+				headRoot:   r,
+				headBlock:  b,
+				attributes: arg.attributes,
 			})
 			if err != nil {
 				return nil, err // Returning err because it's recursive here.
@@ -153,11 +150,18 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *notifyForkcho
 		log.WithError(err).Error("Could not set head root to valid")
 		return nil, nil
 	}
-	// If the forkchoice update call has an attribute, update the proposer payload ID cache.
+	// If the forkchoice update call has an attribute, update the payload ID cache.
+	hasAttr := arg.attributes != nil && !arg.attributes.IsEmpty()
+	nextSlot := s.CurrentSlot() + 1
 	if hasAttr && payloadID != nil {
 		var pId [8]byte
 		copy(pId[:], payloadID[:])
-		s.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(nextSlot, proposerId, pId, arg.headRoot)
+		log.WithFields(logrus.Fields{
+			"blockRoot": fmt.Sprintf("%#x", bytesutil.Trunc(arg.headRoot[:])),
+			"headSlot":  headBlk.Slot(),
+			"payloadID": fmt.Sprintf("%#x", bytesutil.Trunc(payloadID[:])),
+		}).Info("Forkchoice updated with payload attributes for proposal")
+		s.cfg.PayloadIDCache.Set(nextSlot, arg.headRoot, pId)
 	} else if hasAttr && payloadID == nil && !features.Get().PrepareAllPayloads {
 		log.WithFields(logrus.Fields{
 			"blockHash": fmt.Sprintf("%#x", headPayload.BlockHash()),
@@ -272,56 +276,50 @@ func (s *Service) pruneInvalidBlock(ctx context.Context, root, parentRoot, lvh [
 
 // getPayloadAttributes returns the payload attributes for the given state and slot.
 // The attribute is required to initiate a payload build process in the context of an `engine_forkchoiceUpdated` call.
-func (s *Service) getPayloadAttribute(ctx context.Context, st state.BeaconState, slot primitives.Slot, headRoot []byte) (bool, payloadattribute.Attributer, primitives.ValidatorIndex) {
+func (s *Service) getPayloadAttribute(ctx context.Context, st state.BeaconState, slot primitives.Slot, headRoot []byte) payloadattribute.Attributer {
 	emptyAttri := payloadattribute.EmptyWithVersion(st.Version())
-	// Root is `[32]byte{}` since we are retrieving proposer ID of a given slot. During insertion at assignment the root was not known.
-	proposerID, _, ok := s.cfg.ProposerSlotIndexCache.GetProposerPayloadIDs(slot, [32]byte{} /* root */)
-	if !ok && !features.Get().PrepareAllPayloads { // There's no need to build attribute if there is no proposer for slot.
-		return false, emptyAttri, 0
-	}
 
-	// Get previous randao.
+	// If it is an epoch boundary then process slots to get the right
+	// shuffling before checking if the proposer is tracked. Otherwise
+	// perform this check before. This is cheap as the NSC has already been updated.
+	var val cache.TrackedValidator
+	var ok bool
+	e := slots.ToEpoch(slot)
+	stateEpoch := slots.ToEpoch(st.Slot())
+	if e == stateEpoch {
+		val, ok = s.trackedProposer(st, slot)
+		if !ok {
+			return emptyAttri
+		}
+	}
 	st = st.Copy()
 	if slot > st.Slot() {
 		var err error
 		st, err = transition.ProcessSlotsUsingNextSlotCache(ctx, st, headRoot, slot)
 		if err != nil {
 			log.WithError(err).Error("Could not process slots to get payload attribute")
-			return false, emptyAttri, 0
+			return emptyAttri
 		}
 	}
+	if e > stateEpoch {
+		emptyAttri := payloadattribute.EmptyWithVersion(st.Version())
+		val, ok = s.trackedProposer(st, slot)
+		if !ok {
+			return emptyAttri
+		}
+	}
+	// Get previous randao.
 	prevRando, err := helpers.RandaoMix(st, time.CurrentEpoch(st))
 	if err != nil {
 		log.WithError(err).Error("Could not get randao mix to get payload attribute")
-		return false, emptyAttri, 0
-	}
-
-	// Get fee recipient.
-	feeRecipient := params.BeaconConfig().DefaultFeeRecipient
-	recipient, err := s.cfg.BeaconDB.FeeRecipientByValidatorID(ctx, proposerID)
-	switch {
-	case errors.Is(err, kv.ErrNotFoundFeeRecipient):
-		if feeRecipient.String() == params.BeaconConfig().EthBurnAddressHex {
-			logrus.WithFields(logrus.Fields{
-				"validatorIndex": proposerID,
-				"burnAddress":    params.BeaconConfig().EthBurnAddressHex,
-			}).Warn("Fee recipient is currently using the burn address, " +
-				"you will not be rewarded transaction fees on this setting. " +
-				"Please set a different eth address as the fee recipient. " +
-				"Please refer to our documentation for instructions")
-		}
-	case err != nil:
-		log.WithError(err).Error("Could not get fee recipient to get payload attribute")
-		return false, emptyAttri, 0
-	default:
-		feeRecipient = recipient
+		return emptyAttri
 	}
 
 	// Get timestamp.
 	t, err := slots.ToTime(uint64(s.genesisTime.Unix()), slot)
 	if err != nil {
 		log.WithError(err).Error("Could not get timestamp to get payload attribute")
-		return false, emptyAttri, 0
+		return emptyAttri
 	}
 
 	var attr payloadattribute.Attributer
@@ -330,51 +328,51 @@ func (s *Service) getPayloadAttribute(ctx context.Context, st state.BeaconState,
 		withdrawals, err := st.ExpectedWithdrawals()
 		if err != nil {
 			log.WithError(err).Error("Could not get expected withdrawals to get payload attribute")
-			return false, emptyAttri, 0
+			return emptyAttri
 		}
 		attr, err = payloadattribute.New(&enginev1.PayloadAttributesV3{
 			Timestamp:             uint64(t.Unix()),
 			PrevRandao:            prevRando,
-			SuggestedFeeRecipient: feeRecipient.Bytes(),
+			SuggestedFeeRecipient: val.FeeRecipient[:],
 			Withdrawals:           withdrawals,
 			ParentBeaconBlockRoot: headRoot,
 		})
 		if err != nil {
 			log.WithError(err).Error("Could not get payload attribute")
-			return false, emptyAttri, 0
+			return emptyAttri
 		}
 	case version.Capella:
 		withdrawals, err := st.ExpectedWithdrawals()
 		if err != nil {
 			log.WithError(err).Error("Could not get expected withdrawals to get payload attribute")
-			return false, emptyAttri, 0
+			return emptyAttri
 		}
 		attr, err = payloadattribute.New(&enginev1.PayloadAttributesV2{
 			Timestamp:             uint64(t.Unix()),
 			PrevRandao:            prevRando,
-			SuggestedFeeRecipient: feeRecipient.Bytes(),
+			SuggestedFeeRecipient: val.FeeRecipient[:],
 			Withdrawals:           withdrawals,
 		})
 		if err != nil {
 			log.WithError(err).Error("Could not get payload attribute")
-			return false, emptyAttri, 0
+			return emptyAttri
 		}
 	case version.Bellatrix:
 		attr, err = payloadattribute.New(&enginev1.PayloadAttributes{
 			Timestamp:             uint64(t.Unix()),
 			PrevRandao:            prevRando,
-			SuggestedFeeRecipient: feeRecipient.Bytes(),
+			SuggestedFeeRecipient: val.FeeRecipient[:],
 		})
 		if err != nil {
 			log.WithError(err).Error("Could not get payload attribute")
-			return false, emptyAttri, 0
+			return emptyAttri
 		}
 	default:
 		log.WithField("version", st.Version()).Error("Could not get payload attribute due to unknown state version")
-		return false, emptyAttri, 0
+		return emptyAttri
 	}
 
-	return true, attr, proposerID
+	return attr
 }
 
 // removeInvalidBlockAndState removes the invalid block, blob and its corresponding state from the cache and DB.
@@ -389,10 +387,7 @@ func (s *Service) removeInvalidBlockAndState(ctx context.Context, blkRoots [][32
 			// This is an irreparable condition, it would me a justified or finalized block has become invalid.
 			return err
 		}
-		// No op if the sidecar does not exist.
-		if err := s.cfg.BeaconDB.DeleteBlobSidecar(ctx, root); err != nil {
-			return err
-		}
+		// TODO: delete blobs from storage
 	}
 	return nil
 }

beacon-chain/blockchain/execution_engine_test.go

@@ -26,11 +26,10 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/testing/assert"
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
 	"github.com/prysmaticlabs/prysm/v4/testing/util"
-	logTest "github.com/sirupsen/logrus/hooks/test"
 )
 
 func Test_NotifyForkchoiceUpdate_GetPayloadAttrErrorCanContinue(t *testing.T) {
-	service, tr := minimalTestService(t, WithProposerIdsCache(cache.NewProposerPayloadIDsCache()))
+	service, tr := minimalTestService(t, WithPayloadIDCache(cache.NewPayloadIDCache()))
 	ctx, beaconDB, fcs := tr.ctx, tr.db, tr.fcs
 
 	altairBlk := util.SaveBlock(t, ctx, beaconDB, util.NewBeaconBlockAltair())
@@ -57,11 +56,14 @@ func Test_NotifyForkchoiceUpdate_GetPayloadAttrErrorCanContinue(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, fcs.InsertNode(ctx, state, blkRoot))
 
-	b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlockBellatrix{
-		Body: &ethpb.BeaconBlockBodyBellatrix{
-			ExecutionPayload: &v1.ExecutionPayload{},
+	sb := &ethpb.SignedBeaconBlockBellatrix{
+		Block: &ethpb.BeaconBlockBellatrix{
+			Body: &ethpb.BeaconBlockBodyBellatrix{
+				ExecutionPayload: &v1.ExecutionPayload{},
+			},
 		},
-	})
+	}
+	b, err := consensusblocks.NewSignedBeaconBlock(sb)
 	require.NoError(t, err)
 
 	pid := &v1.PayloadIDBytes{1}
@@ -73,20 +75,20 @@ func Test_NotifyForkchoiceUpdate_GetPayloadAttrErrorCanContinue(t *testing.T) {
 	// Intentionally generate a bad state such that `hash_tree_root` fails during `process_slot`
 	s, err := state_native.InitializeFromProtoPhase0(&ethpb.BeaconState{})
 	require.NoError(t, err)
-	arg := &notifyForkchoiceUpdateArg{
+	arg := &fcuConfig{
 		headState: s,
 		headRoot:  [32]byte{},
 		headBlock: b,
 	}
 
-	service.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(1, 0, [8]byte{}, [32]byte{})
+	service.cfg.PayloadIDCache.Set(1, [32]byte{}, [8]byte{})
 	got, err := service.notifyForkchoiceUpdate(ctx, arg)
 	require.NoError(t, err)
 	require.DeepEqual(t, got, pid) // We still get a payload ID even though the state is bad. This means it returns until the end.
 }
 
 func Test_NotifyForkchoiceUpdate(t *testing.T) {
-	service, tr := minimalTestService(t, WithProposerIdsCache(cache.NewProposerPayloadIDsCache()))
+	service, tr := minimalTestService(t, WithPayloadIDCache(cache.NewPayloadIDCache()))
 	ctx, beaconDB, fcs := tr.ctx, tr.db, tr.fcs
 
 	altairBlk := util.SaveBlock(t, ctx, beaconDB, util.NewBeaconBlockAltair())
@@ -114,7 +116,7 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 
 	tests := []struct {
 		name             string
-		blk              interfaces.ReadOnlyBeaconBlock
+		blk              interfaces.ReadOnlySignedBeaconBlock
 		headRoot         [32]byte
 		finalizedRoot    [32]byte
 		justifiedRoot    [32]byte
@@ -123,24 +125,24 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 	}{
 		{
 			name: "phase0 block",
-			blk: func() interfaces.ReadOnlyBeaconBlock {
-				b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}})
+			blk: func() interfaces.ReadOnlySignedBeaconBlock {
+				b, err := consensusblocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}}})
 				require.NoError(t, err)
 				return b
 			}(),
 		},
 		{
 			name: "altair block",
-			blk: func() interfaces.ReadOnlyBeaconBlock {
-				b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlockAltair{Body: &ethpb.BeaconBlockBodyAltair{}})
+			blk: func() interfaces.ReadOnlySignedBeaconBlock {
+				b, err := consensusblocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlockAltair{Block: &ethpb.BeaconBlockAltair{Body: &ethpb.BeaconBlockBodyAltair{}}})
 				require.NoError(t, err)
 				return b
 			}(),
 		},
 		{
 			name: "not execution block",
-			blk: func() interfaces.ReadOnlyBeaconBlock {
-				b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlockBellatrix{
+			blk: func() interfaces.ReadOnlySignedBeaconBlock {
+				b, err := consensusblocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlockBellatrix{Block: &ethpb.BeaconBlockBellatrix{
 					Body: &ethpb.BeaconBlockBodyBellatrix{
 						ExecutionPayload: &v1.ExecutionPayload{
 							ParentHash:    make([]byte, fieldparams.RootLength),
@@ -153,19 +155,19 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 							BlockHash:     make([]byte, fieldparams.RootLength),
 						},
 					},
-				})
+				}})
 				require.NoError(t, err)
 				return b
 			}(),
 		},
 		{
 			name: "happy case: finalized root is altair block",
-			blk: func() interfaces.ReadOnlyBeaconBlock {
-				b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlockBellatrix{
+			blk: func() interfaces.ReadOnlySignedBeaconBlock {
+				b, err := consensusblocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlockBellatrix{Block: &ethpb.BeaconBlockBellatrix{
 					Body: &ethpb.BeaconBlockBodyBellatrix{
 						ExecutionPayload: &v1.ExecutionPayload{},
 					},
-				})
+				}})
 				require.NoError(t, err)
 				return b
 			}(),
@@ -174,12 +176,12 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 		},
 		{
 			name: "happy case: finalized root is bellatrix block",
-			blk: func() interfaces.ReadOnlyBeaconBlock {
-				b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlockBellatrix{
+			blk: func() interfaces.ReadOnlySignedBeaconBlock {
+				b, err := consensusblocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlockBellatrix{Block: &ethpb.BeaconBlockBellatrix{
 					Body: &ethpb.BeaconBlockBodyBellatrix{
 						ExecutionPayload: &v1.ExecutionPayload{},
 					},
-				})
+				}})
 				require.NoError(t, err)
 				return b
 			}(),
@@ -188,12 +190,12 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 		},
 		{
 			name: "forkchoice updated with optimistic block",
-			blk: func() interfaces.ReadOnlyBeaconBlock {
-				b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlockBellatrix{
+			blk: func() interfaces.ReadOnlySignedBeaconBlock {
+				b, err := consensusblocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlockBellatrix{Block: &ethpb.BeaconBlockBellatrix{
 					Body: &ethpb.BeaconBlockBodyBellatrix{
 						ExecutionPayload: &v1.ExecutionPayload{},
 					},
-				})
+				}})
 				require.NoError(t, err)
 				return b
 			}(),
@@ -203,12 +205,12 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 		},
 		{
 			name: "forkchoice updated with invalid block",
-			blk: func() interfaces.ReadOnlyBeaconBlock {
-				b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlockBellatrix{
+			blk: func() interfaces.ReadOnlySignedBeaconBlock {
+				b, err := consensusblocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlockBellatrix{Block: &ethpb.BeaconBlockBellatrix{
 					Body: &ethpb.BeaconBlockBodyBellatrix{
 						ExecutionPayload: &v1.ExecutionPayload{},
 					},
-				})
+				}})
 				require.NoError(t, err)
 				return b
 			}(),
@@ -226,7 +228,7 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 			st, _ := util.DeterministicGenesisState(t, 1)
 			require.NoError(t, beaconDB.SaveState(ctx, st, tt.finalizedRoot))
 			require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, tt.finalizedRoot))
-			arg := &notifyForkchoiceUpdateArg{
+			arg := &fcuConfig{
 				headState: st,
 				headRoot:  tt.headRoot,
 				headBlock: tt.blk,
@@ -246,7 +248,7 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 }
 
 func Test_NotifyForkchoiceUpdate_NIlLVH(t *testing.T) {
-	service, tr := minimalTestService(t, WithProposerIdsCache(cache.NewProposerPayloadIDsCache()))
+	service, tr := minimalTestService(t, WithPayloadIDCache(cache.NewPayloadIDCache()))
 	ctx, beaconDB, fcs := tr.ctx, tr.db, tr.fcs
 
 	// Prepare blocks
@@ -306,9 +308,9 @@ func Test_NotifyForkchoiceUpdate_NIlLVH(t *testing.T) {
 
 	require.NoError(t, beaconDB.SaveState(ctx, st, bra))
 	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, bra))
-	a := &notifyForkchoiceUpdateArg{
+	a := &fcuConfig{
 		headState: st,
-		headBlock: wbd.Block(),
+		headBlock: wbd,
 		headRoot:  brd,
 	}
 	_, err = service.notifyForkchoiceUpdate(ctx, a)
@@ -334,7 +336,7 @@ func Test_NotifyForkchoiceUpdate_NIlLVH(t *testing.T) {
 // 3. the blockchain package calls fcu to obtain heads G -> F -> D.
 
 func Test_NotifyForkchoiceUpdateRecursive_DoublyLinkedTree(t *testing.T) {
-	service, tr := minimalTestService(t, WithProposerIdsCache(cache.NewProposerPayloadIDsCache()))
+	service, tr := minimalTestService(t, WithPayloadIDCache(cache.NewPayloadIDCache()))
 	ctx, beaconDB, fcs := tr.ctx, tr.db, tr.fcs
 
 	// Prepare blocks
@@ -443,9 +445,9 @@ func Test_NotifyForkchoiceUpdateRecursive_DoublyLinkedTree(t *testing.T) {
 
 	require.NoError(t, beaconDB.SaveState(ctx, st, bra))
 	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, bra))
-	a := &notifyForkchoiceUpdateArg{
+	a := &fcuConfig{
 		headState: st,
-		headBlock: wbg.Block(),
+		headBlock: wbg,
 		headRoot:  brg,
 	}
 	_, err = service.notifyForkchoiceUpdate(ctx, a)
@@ -467,7 +469,7 @@ func Test_NotifyNewPayload(t *testing.T) {
 	cfg := params.BeaconConfig()
 	cfg.TerminalTotalDifficulty = "2"
 	params.OverrideBeaconConfig(cfg)
-	service, tr := minimalTestService(t, WithProposerIdsCache(cache.NewProposerPayloadIDsCache()))
+	service, tr := minimalTestService(t, WithPayloadIDCache(cache.NewPayloadIDCache()))
 	ctx, fcs := tr.ctx, tr.fcs
 
 	phase0State, _ := util.DeterministicGenesisState(t, 1)
@@ -709,7 +711,7 @@ func Test_NotifyNewPayload_SetOptimisticToValid(t *testing.T) {
 	cfg.TerminalTotalDifficulty = "2"
 	params.OverrideBeaconConfig(cfg)
 
-	service, tr := minimalTestService(t, WithProposerIdsCache(cache.NewProposerPayloadIDsCache()))
+	service, tr := minimalTestService(t, WithPayloadIDCache(cache.NewPayloadIDCache()))
 	ctx := tr.ctx
 
 	bellatrixState, _ := util.DeterministicGenesisStateBellatrix(t, 2)
@@ -777,83 +779,70 @@ func Test_reportInvalidBlock(t *testing.T) {
 }
 
 func Test_GetPayloadAttribute(t *testing.T) {
-	service, tr := minimalTestService(t, WithProposerIdsCache(cache.NewProposerPayloadIDsCache()))
+	service, tr := minimalTestService(t, WithPayloadIDCache(cache.NewPayloadIDCache()))
 	ctx := tr.ctx
 
 	st, _ := util.DeterministicGenesisStateBellatrix(t, 1)
-	hasPayload, _, vId := service.getPayloadAttribute(ctx, st, 0, []byte{})
-	require.Equal(t, false, hasPayload)
-	require.Equal(t, primitives.ValidatorIndex(0), vId)
+	attr := service.getPayloadAttribute(ctx, st, 0, []byte{})
+	require.Equal(t, true, attr.IsEmpty())
 
+	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, Index: 0})
 	// Cache hit, advance state, no fee recipient
-	suggestedVid := primitives.ValidatorIndex(1)
 	slot := primitives.Slot(1)
-	service.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(slot, suggestedVid, [8]byte{}, [32]byte{})
-	hook := logTest.NewGlobal()
-	hasPayload, attr, vId := service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
-	require.Equal(t, true, hasPayload)
-	require.Equal(t, suggestedVid, vId)
+	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
+	attr = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
+	require.Equal(t, false, attr.IsEmpty())
 	require.Equal(t, params.BeaconConfig().EthBurnAddressHex, common.BytesToAddress(attr.SuggestedFeeRecipient()).String())
-	require.LogsContain(t, hook, "Fee recipient is currently using the burn address")
 
 	// Cache hit, advance state, has fee recipient
 	suggestedAddr := common.HexToAddress("123")
-	require.NoError(t, service.cfg.BeaconDB.SaveFeeRecipientsByValidatorIDs(ctx, []primitives.ValidatorIndex{suggestedVid}, []common.Address{suggestedAddr}))
-	service.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(slot, suggestedVid, [8]byte{}, [32]byte{})
-	hasPayload, attr, vId = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
-	require.Equal(t, true, hasPayload)
-	require.Equal(t, suggestedVid, vId)
+	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, FeeRecipient: primitives.ExecutionAddress(suggestedAddr), Index: 0})
+	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
+	attr = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
+	require.Equal(t, false, attr.IsEmpty())
 	require.Equal(t, suggestedAddr, common.BytesToAddress(attr.SuggestedFeeRecipient()))
 }
 
 func Test_GetPayloadAttribute_PrepareAllPayloads(t *testing.T) {
-	hook := logTest.NewGlobal()
 	resetCfg := features.InitWithReset(&features.Flags{
 		PrepareAllPayloads: true,
 	})
 	defer resetCfg()
 
-	service, tr := minimalTestService(t, WithProposerIdsCache(cache.NewProposerPayloadIDsCache()))
+	service, tr := minimalTestService(t, WithPayloadIDCache(cache.NewPayloadIDCache()))
 	ctx := tr.ctx
 
 	st, _ := util.DeterministicGenesisStateBellatrix(t, 1)
-	hasPayload, attr, vId := service.getPayloadAttribute(ctx, st, 0, []byte{})
-	require.Equal(t, true, hasPayload)
-	require.Equal(t, primitives.ValidatorIndex(0), vId)
+	attr := service.getPayloadAttribute(ctx, st, 0, []byte{})
+	require.Equal(t, false, attr.IsEmpty())
 	require.Equal(t, params.BeaconConfig().EthBurnAddressHex, common.BytesToAddress(attr.SuggestedFeeRecipient()).String())
-	require.LogsContain(t, hook, "Fee recipient is currently using the burn address")
 }
 
 func Test_GetPayloadAttributeV2(t *testing.T) {
-	service, tr := minimalTestService(t, WithProposerIdsCache(cache.NewProposerPayloadIDsCache()))
+	service, tr := minimalTestService(t, WithPayloadIDCache(cache.NewPayloadIDCache()))
 	ctx := tr.ctx
 
 	st, _ := util.DeterministicGenesisStateCapella(t, 1)
-	hasPayload, _, vId := service.getPayloadAttribute(ctx, st, 0, []byte{})
-	require.Equal(t, false, hasPayload)
-	require.Equal(t, primitives.ValidatorIndex(0), vId)
+	attr := service.getPayloadAttribute(ctx, st, 0, []byte{})
+	require.Equal(t, true, attr.IsEmpty())
 
 	// Cache hit, advance state, no fee recipient
-	suggestedVid := primitives.ValidatorIndex(1)
+	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, Index: 0})
 	slot := primitives.Slot(1)
-	service.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(slot, suggestedVid, [8]byte{}, [32]byte{})
-	hook := logTest.NewGlobal()
-	hasPayload, attr, vId := service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
-	require.Equal(t, true, hasPayload)
-	require.Equal(t, suggestedVid, vId)
+	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
+	attr = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
+	require.Equal(t, false, attr.IsEmpty())
 	require.Equal(t, params.BeaconConfig().EthBurnAddressHex, common.BytesToAddress(attr.SuggestedFeeRecipient()).String())
-	require.LogsContain(t, hook, "Fee recipient is currently using the burn address")
 	a, err := attr.Withdrawals()
 	require.NoError(t, err)
 	require.Equal(t, 0, len(a))
 
 	// Cache hit, advance state, has fee recipient
 	suggestedAddr := common.HexToAddress("123")
-	require.NoError(t, service.cfg.BeaconDB.SaveFeeRecipientsByValidatorIDs(ctx, []primitives.ValidatorIndex{suggestedVid}, []common.Address{suggestedAddr}))
-	service.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(slot, suggestedVid, [8]byte{}, [32]byte{})
-	hasPayload, attr, vId = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
-	require.Equal(t, true, hasPayload)
-	require.Equal(t, suggestedVid, vId)
+	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, FeeRecipient: primitives.ExecutionAddress(suggestedAddr), Index: 0})
+	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
+	attr = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
+	require.Equal(t, false, attr.IsEmpty())
 	require.Equal(t, suggestedAddr, common.BytesToAddress(attr.SuggestedFeeRecipient()))
 	a, err = attr.Withdrawals()
 	require.NoError(t, err)
@@ -861,35 +850,30 @@ func Test_GetPayloadAttributeV2(t *testing.T) {
 }
 
 func Test_GetPayloadAttributeDeneb(t *testing.T) {
-	service, tr := minimalTestService(t, WithProposerIdsCache(cache.NewProposerPayloadIDsCache()))
+	service, tr := minimalTestService(t, WithPayloadIDCache(cache.NewPayloadIDCache()))
 	ctx := tr.ctx
 
 	st, _ := util.DeterministicGenesisStateDeneb(t, 1)
-	hasPayload, _, vId := service.getPayloadAttribute(ctx, st, 0, []byte{})
-	require.Equal(t, false, hasPayload)
-	require.Equal(t, primitives.ValidatorIndex(0), vId)
+	attr := service.getPayloadAttribute(ctx, st, 0, []byte{})
+	require.Equal(t, true, attr.IsEmpty())
 
 	// Cache hit, advance state, no fee recipient
-	suggestedVid := primitives.ValidatorIndex(1)
 	slot := primitives.Slot(1)
-	service.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(slot, suggestedVid, [8]byte{}, [32]byte{})
-	hook := logTest.NewGlobal()
-	hasPayload, attr, vId := service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
-	require.Equal(t, true, hasPayload)
-	require.Equal(t, suggestedVid, vId)
+	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, Index: 0})
+	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
+	attr = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
+	require.Equal(t, false, attr.IsEmpty())
 	require.Equal(t, params.BeaconConfig().EthBurnAddressHex, common.BytesToAddress(attr.SuggestedFeeRecipient()).String())
-	require.LogsContain(t, hook, "Fee recipient is currently using the burn address")
 	a, err := attr.Withdrawals()
 	require.NoError(t, err)
 	require.Equal(t, 0, len(a))
 
 	// Cache hit, advance state, has fee recipient
 	suggestedAddr := common.HexToAddress("123")
-	require.NoError(t, service.cfg.BeaconDB.SaveFeeRecipientsByValidatorIDs(ctx, []primitives.ValidatorIndex{suggestedVid}, []common.Address{suggestedAddr}))
-	service.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(slot, suggestedVid, [8]byte{}, [32]byte{})
-	hasPayload, attr, vId = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
-	require.Equal(t, true, hasPayload)
-	require.Equal(t, suggestedVid, vId)
+	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, FeeRecipient: primitives.ExecutionAddress(suggestedAddr), Index: 0})
+	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
+	attr = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
+	require.Equal(t, false, attr.IsEmpty())
 	require.Equal(t, suggestedAddr, common.BytesToAddress(attr.SuggestedFeeRecipient()))
 	a, err = attr.Withdrawals()
 	require.NoError(t, err)
@@ -1112,3 +1096,35 @@ func TestKZGCommitmentToVersionedHashes(t *testing.T) {
 	require.Equal(t, vhs[0].String(), vh0)
 	require.Equal(t, vhs[1].String(), vh1)
 }
+
+func TestComputePayloadAttribute(t *testing.T) {
+	service, tr := minimalTestService(t, WithPayloadIDCache(cache.NewPayloadIDCache()))
+	ctx := tr.ctx
+
+	st, _ := util.DeterministicGenesisStateBellatrix(t, 1)
+
+	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, Index: 0})
+	// Cache hit, advance state, no fee recipient
+	slot := primitives.Slot(1)
+	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
+	cfg := &postBlockProcessConfig{
+		ctx:       ctx,
+		blockRoot: [32]byte{'a'},
+	}
+	fcu := &fcuConfig{
+		headState:     st,
+		proposingSlot: slot,
+		headRoot:      [32]byte{},
+	}
+	require.NoError(t, service.computePayloadAttributes(cfg, fcu))
+	require.Equal(t, false, fcu.attributes.IsEmpty())
+	require.Equal(t, params.BeaconConfig().EthBurnAddressHex, common.BytesToAddress(fcu.attributes.SuggestedFeeRecipient()).String())
+
+	// Cache hit, advance state, has fee recipient
+	suggestedAddr := common.HexToAddress("123")
+	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, FeeRecipient: primitives.ExecutionAddress(suggestedAddr), Index: 0})
+	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
+	require.NoError(t, service.computePayloadAttributes(cfg, fcu))
+	require.Equal(t, false, fcu.attributes.IsEmpty())
+	require.Equal(t, suggestedAddr, common.BytesToAddress(fcu.attributes.SuggestedFeeRecipient()))
+}

beacon-chain/blockchain/forkchoice_update_execution.go

@@ -8,20 +8,15 @@ import (
 	"github.com/pkg/errors"
 	doublylinkedtree "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/doubly-linked-tree"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
-	"github.com/prysmaticlabs/prysm/v4/config/features"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
+	payloadattribute "github.com/prysmaticlabs/prysm/v4/consensus-types/payload-attribute"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/time/slots"
 	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
 
-func (s *Service) isNewProposer(slot primitives.Slot) bool {
-	_, _, ok := s.cfg.ProposerSlotIndexCache.GetProposerPayloadIDs(slot, [32]byte{} /* root */)
-	return ok || features.Get().PrepareAllPayloads
-}
-
 func (s *Service) isNewHead(r [32]byte) bool {
 	s.headLock.RLock()
 	defer s.headLock.RUnlock()
@@ -49,48 +44,69 @@ func (s *Service) getStateAndBlock(ctx context.Context, r [32]byte) (state.Beaco
 	return headState, newHeadBlock, nil
 }
 
+type fcuConfig struct {
+	headState     state.BeaconState
+	headBlock     interfaces.ReadOnlySignedBeaconBlock
+	headRoot      [32]byte
+	proposingSlot primitives.Slot
+	attributes    payloadattribute.Attributer
+}
+
+// sendFCU handles the logic to notify the engine of a forckhoice update
+// for the first time when processing an incoming block during regular sync. It
+// always updates the shuffling caches and handles epoch transitions when the
+// incoming block is late, preparing payload attributes in this case while it
+// only sends a message with empty attributes for early blocks.
+func (s *Service) sendFCU(cfg *postBlockProcessConfig, fcuArgs *fcuConfig) error {
+	if !s.isNewHead(cfg.headRoot) {
+		return nil
+	}
+	if fcuArgs.attributes != nil && !fcuArgs.attributes.IsEmpty() && s.shouldOverrideFCU(cfg.headRoot, s.CurrentSlot()+1) {
+		return nil
+	}
+	return s.forkchoiceUpdateWithExecution(cfg.ctx, fcuArgs)
+}
+
+// sendFCUWithAttributes computes the payload attributes and sends an FCU message
+// to the engine if needed
+func (s *Service) sendFCUWithAttributes(cfg *postBlockProcessConfig, fcuArgs *fcuConfig) {
+	slotCtx, cancel := context.WithTimeout(context.Background(), slotDeadline)
+	defer cancel()
+	cfg.ctx = slotCtx
+	if err := s.computePayloadAttributes(cfg, fcuArgs); err != nil {
+		log.WithError(err).Error("could not compute payload attributes")
+		return
+	}
+	if fcuArgs.attributes.IsEmpty() {
+		return
+	}
+	s.cfg.ForkChoiceStore.RLock()
+	defer s.cfg.ForkChoiceStore.RUnlock()
+	if _, err := s.notifyForkchoiceUpdate(cfg.ctx, fcuArgs); err != nil {
+		log.WithError(err).Error("could not update forkchoice with payload attributes for proposal")
+	}
+}
+
 // fockchoiceUpdateWithExecution is a wrapper around notifyForkchoiceUpdate. It decides whether a new call to FCU should be made.
-// it returns true if the new head is updated
-func (s *Service) forkchoiceUpdateWithExecution(ctx context.Context, newHeadRoot [32]byte, proposingSlot primitives.Slot) (bool, error) {
+func (s *Service) forkchoiceUpdateWithExecution(ctx context.Context, args *fcuConfig) error {
 	_, span := trace.StartSpan(ctx, "beacon-chain.blockchain.forkchoiceUpdateWithExecution")
 	defer span.End()
 	// Note: Use the service context here to avoid the parent context being ended during a forkchoice update.
 	ctx = trace.NewContext(s.ctx, span)
-
-	isNewHead := s.isNewHead(newHeadRoot)
-	if !isNewHead {
-		return false, nil
-	}
-	isNewProposer := s.isNewProposer(proposingSlot)
-	if isNewProposer && !features.Get().DisableReorgLateBlocks {
-		if s.shouldOverrideFCU(newHeadRoot, proposingSlot) {
-			return false, nil
-		}
-	}
-	headState, headBlock, err := s.getStateAndBlock(ctx, newHeadRoot)
+	_, err := s.notifyForkchoiceUpdate(ctx, args)
 	if err != nil {
-		log.WithError(err).Error("Could not get forkchoice update argument")
-		return false, nil
+		return errors.Wrap(err, "could not notify forkchoice update")
 	}
 
-	_, err = s.notifyForkchoiceUpdate(ctx, &notifyForkchoiceUpdateArg{
-		headState: headState,
-		headRoot:  newHeadRoot,
-		headBlock: headBlock.Block(),
-	})
-	if err != nil {
-		return false, errors.Wrap(err, "could not notify forkchoice update")
-	}
-
-	if err := s.saveHead(ctx, newHeadRoot, headBlock, headState); err != nil {
+	if err := s.saveHead(ctx, args.headRoot, args.headBlock, args.headState); err != nil {
 		log.WithError(err).Error("could not save head")
 	}
 
 	// Only need to prune attestations from pool if the head has changed.
-	if err := s.pruneAttsFromPool(headBlock); err != nil {
+	if err := s.pruneAttsFromPool(args.headBlock); err != nil {
 		log.WithError(err).Error("could not prune attestations from pool")
 	}
-	return true, nil
+	return nil
 }
 
 // shouldOverrideFCU checks whether the incoming block is still subject to being

beacon-chain/blockchain/forkchoice_update_execution_test.go

@@ -17,15 +17,6 @@ import (
 	logTest "github.com/sirupsen/logrus/hooks/test"
 )
 
-func TestService_isNewProposer(t *testing.T) {
-	beaconDB := testDB.SetupDB(t)
-	service := setupBeaconChain(t, beaconDB)
-	require.Equal(t, false, service.isNewProposer(service.CurrentSlot()+1))
-
-	service.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(service.CurrentSlot()+1, 0, [8]byte{}, [32]byte{} /* root */)
-	require.Equal(t, true, service.isNewProposer(service.CurrentSlot()+1))
-}
-
 func TestService_isNewHead(t *testing.T) {
 	beaconDB := testDB.SetupDB(t)
 	service := setupBeaconChain(t, beaconDB)
@@ -67,33 +58,14 @@ func TestService_getHeadStateAndBlock(t *testing.T) {
 }
 
 func TestService_forkchoiceUpdateWithExecution_exceptionalCases(t *testing.T) {
-	hook := logTest.NewGlobal()
 	ctx := context.Background()
 	opts := testServiceOptsWithDB(t)
 
 	service, err := NewService(ctx, opts...)
 	require.NoError(t, err)
-	service.cfg.ProposerSlotIndexCache = cache.NewProposerPayloadIDsCache()
-	_, err = service.forkchoiceUpdateWithExecution(ctx, service.headRoot(), service.CurrentSlot()+1)
-	require.NoError(t, err)
-	hookErr := "could not notify forkchoice update"
-	invalidStateErr := "could not get state summary: could not find block in DB"
-	require.LogsDoNotContain(t, hook, invalidStateErr)
-	require.LogsDoNotContain(t, hook, hookErr)
-	gb, err := blocks.NewSignedBeaconBlock(util.NewBeaconBlock())
-	require.NoError(t, err)
-	require.NoError(t, service.saveInitSyncBlock(ctx, [32]byte{'a'}, gb))
-	_, err = service.forkchoiceUpdateWithExecution(ctx, [32]byte{'a'}, service.CurrentSlot()+1)
-	require.NoError(t, err)
-	require.LogsContain(t, hook, invalidStateErr)
+	service.cfg.PayloadIDCache = cache.NewPayloadIDCache()
+	service.cfg.TrackedValidatorsCache = cache.NewTrackedValidatorsCache()
 
-	hook.Reset()
-	service.head = &head{
-		root:  [32]byte{'a'},
-		block: nil, /* should not panic if notify head uses correct head */
-	}
-
-	// Block in Cache
 	b := util.NewBeaconBlock()
 	b.Block.Slot = 2
 	wsb, err := blocks.NewSignedBeaconBlock(b)
@@ -107,13 +79,7 @@ func TestService_forkchoiceUpdateWithExecution_exceptionalCases(t *testing.T) {
 		block: wsb,
 		state: st,
 	}
-	service.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(2, 1, [8]byte{1}, [32]byte{2})
-	_, err = service.forkchoiceUpdateWithExecution(ctx, r1, service.CurrentSlot())
-	require.NoError(t, err)
-	require.LogsDoNotContain(t, hook, invalidStateErr)
-	require.LogsDoNotContain(t, hook, hookErr)
-
-	// Block in DB
+	service.cfg.PayloadIDCache.Set(2, [32]byte{2}, [8]byte{1})
 	b = util.NewBeaconBlock()
 	b.Block.Slot = 3
 	util.SaveBlock(t, ctx, service.cfg.BeaconDB, b)
@@ -125,25 +91,22 @@ func TestService_forkchoiceUpdateWithExecution_exceptionalCases(t *testing.T) {
 		block: wsb,
 		state: st,
 	}
-	service.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(2, 1, [8]byte{1}, [32]byte{2})
-	_, err = service.forkchoiceUpdateWithExecution(ctx, r1, service.CurrentSlot()+1)
-	require.NoError(t, err)
-	require.LogsDoNotContain(t, hook, invalidStateErr)
-	require.LogsDoNotContain(t, hook, hookErr)
-	vId, payloadID, has := service.cfg.ProposerSlotIndexCache.GetProposerPayloadIDs(2, [32]byte{2})
-	require.Equal(t, true, has)
-	require.Equal(t, primitives.ValidatorIndex(1), vId)
-	require.Equal(t, [8]byte{1}, payloadID)
+	service.cfg.PayloadIDCache.Set(2, [32]byte{2}, [8]byte{1})
+	args := &fcuConfig{
+		headState:     st,
+		headRoot:      r1,
+		headBlock:     wsb,
+		proposingSlot: service.CurrentSlot() + 1,
+	}
+	require.NoError(t, service.forkchoiceUpdateWithExecution(ctx, args))
 
-	// Test zero headRoot returns immediately.
-	headRoot := service.headRoot()
-	_, err = service.forkchoiceUpdateWithExecution(ctx, [32]byte{}, service.CurrentSlot()+1)
-	require.NoError(t, err)
-	require.Equal(t, service.headRoot(), headRoot)
+	payloadID, has := service.cfg.PayloadIDCache.PayloadID(2, [32]byte{2})
+	require.Equal(t, true, has)
+	require.Equal(t, primitives.PayloadID{1}, payloadID)
 }
 
 func TestService_forkchoiceUpdateWithExecution_SameHeadRootNewProposer(t *testing.T) {
-	service, tr := minimalTestService(t)
+	service, tr := minimalTestService(t, WithPayloadIDCache(cache.NewPayloadIDCache()))
 	ctx, beaconDB, fcs := tr.ctx, tr.db, tr.fcs
 
 	altairBlk := util.SaveBlock(t, ctx, beaconDB, util.NewBeaconBlockAltair())
@@ -182,10 +145,14 @@ func TestService_forkchoiceUpdateWithExecution_SameHeadRootNewProposer(t *testin
 	service.head.root = r
 	service.head.block = sb
 	service.head.state = st
-	service.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(service.CurrentSlot()+1, 0, [8]byte{}, [32]byte{} /* root */)
-	_, err = service.forkchoiceUpdateWithExecution(ctx, r, service.CurrentSlot()+1)
-	require.NoError(t, err)
-
+	service.cfg.PayloadIDCache.Set(service.CurrentSlot()+1, [32]byte{} /* root */, [8]byte{})
+	args := &fcuConfig{
+		headState:     st,
+		headBlock:     sb,
+		headRoot:      r,
+		proposingSlot: service.CurrentSlot() + 1,
+	}
+	require.NoError(t, service.forkchoiceUpdateWithExecution(ctx, args))
 }
 
 func TestShouldOverrideFCU(t *testing.T) {

beacon-chain/blockchain/head.go

@@ -276,7 +276,7 @@ func (s *Service) headBlock() (interfaces.ReadOnlySignedBeaconBlock, error) {
 // It does a full copy on head state for immutability.
 // This is a lock free version.
 func (s *Service) headState(ctx context.Context) state.BeaconState {
-	ctx, span := trace.StartSpan(ctx, "blockChain.headState")
+	_, span := trace.StartSpan(ctx, "blockChain.headState")
 	defer span.End()
 
 	return s.head.state.Copy()
@@ -286,7 +286,7 @@ func (s *Service) headState(ctx context.Context) state.BeaconState {
 // It does not perform a copy of the head state.
 // This is a lock free version.
 func (s *Service) headStateReadOnly(ctx context.Context) state.ReadOnlyBeaconState {
-	ctx, span := trace.StartSpan(ctx, "blockChain.headStateReadOnly")
+	_, span := trace.StartSpan(ctx, "blockChain.headStateReadOnly")
 	defer span.End()
 
 	return s.head.state

beacon-chain/blockchain/kzg/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain/kzg",
     visibility = ["//visibility:public"],
     deps = [
+        "//consensus-types/blocks:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "@com_github_crate_crypto_go_kzg_4844//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
@@ -26,6 +27,8 @@ go_test(
     deps = [
         "//proto/prysm/v1alpha1:go_default_library",
         "//testing/require:go_default_library",
+        "@com_github_consensys_gnark_crypto//ecc/bls12-381/fr:go_default_library",
         "@com_github_crate_crypto_go_kzg_4844//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
     ],
 )

beacon-chain/blockchain/kzg/validation.go

@@ -1,32 +1,28 @@
 package kzg
 
 import (
-	"fmt"
-
 	GoKZG "github.com/crate-crypto/go-kzg-4844"
-	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 )
 
-// IsDataAvailable checks that
-// - all blobs in the block are available
-// - Expected KZG commitments match the number of blobs in the block
-// - That the number of proofs match the number of blobs
-// - That the proofs are verified against the KZG commitments
-func IsDataAvailable(commitments [][]byte, sidecars []*ethpb.BlobSidecar) error {
-	if len(commitments) != len(sidecars) {
-		return fmt.Errorf("could not check data availability, expected %d commitments, obtained %d",
-			len(commitments), len(sidecars))
-	}
-	if len(commitments) == 0 {
+// Verify performs single or batch verification of commitments depending on the number of given BlobSidecars.
+func Verify(sidecars ...blocks.ROBlob) error {
+	if len(sidecars) == 0 {
 		return nil
 	}
-	blobs := make([]GoKZG.Blob, len(commitments))
-	proofs := make([]GoKZG.KZGProof, len(commitments))
-	cmts := make([]GoKZG.KZGCommitment, len(commitments))
+	if len(sidecars) == 1 {
+		return kzgContext.VerifyBlobKZGProof(
+			bytesToBlob(sidecars[0].Blob),
+			bytesToCommitment(sidecars[0].KzgCommitment),
+			bytesToKZGProof(sidecars[0].KzgProof))
+	}
+	blobs := make([]GoKZG.Blob, len(sidecars))
+	cmts := make([]GoKZG.KZGCommitment, len(sidecars))
+	proofs := make([]GoKZG.KZGProof, len(sidecars))
 	for i, sidecar := range sidecars {
 		blobs[i] = bytesToBlob(sidecar.Blob)
+		cmts[i] = bytesToCommitment(sidecar.KzgCommitment)
 		proofs[i] = bytesToKZGProof(sidecar.KzgProof)
-		cmts[i] = bytesToCommitment(commitments[i])
 	}
 	return kzgContext.VerifyBlobKZGProofBatch(blobs, cmts, proofs)
 }

beacon-chain/blockchain/kzg/validation_test.go

@@ -1,17 +1,66 @@
 package kzg
 
 import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
 	"testing"
 
+	"github.com/consensys/gnark-crypto/ecc/bls12-381/fr"
 	GoKZG "github.com/crate-crypto/go-kzg-4844"
-	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
+	"github.com/sirupsen/logrus"
 )
 
+func deterministicRandomness(seed int64) [32]byte {
+	// Converts an int64 to a byte slice
+	buf := new(bytes.Buffer)
+	err := binary.Write(buf, binary.BigEndian, seed)
+	if err != nil {
+		logrus.WithError(err).Error("Failed to write int64 to bytes buffer")
+		return [32]byte{}
+	}
+	bytes := buf.Bytes()
+
+	return sha256.Sum256(bytes)
+}
+
+// Returns a serialized random field element in big-endian
+func GetRandFieldElement(seed int64) [32]byte {
+	bytes := deterministicRandomness(seed)
+	var r fr.Element
+	r.SetBytes(bytes[:])
+
+	return GoKZG.SerializeScalar(r)
+}
+
+// Returns a random blob using the passed seed as entropy
+func GetRandBlob(seed int64) GoKZG.Blob {
+	var blob GoKZG.Blob
+	bytesPerBlob := GoKZG.ScalarsPerBlob * GoKZG.SerializedScalarSize
+	for i := 0; i < bytesPerBlob; i += GoKZG.SerializedScalarSize {
+		fieldElementBytes := GetRandFieldElement(seed + int64(i))
+		copy(blob[i:i+GoKZG.SerializedScalarSize], fieldElementBytes[:])
+	}
+	return blob
+}
+
+func GenerateCommitmentAndProof(blob GoKZG.Blob) (GoKZG.KZGCommitment, GoKZG.KZGProof, error) {
+	commitment, err := kzgContext.BlobToKZGCommitment(blob, 0)
+	if err != nil {
+		return GoKZG.KZGCommitment{}, GoKZG.KZGProof{}, err
+	}
+	proof, err := kzgContext.ComputeBlobKZGProof(blob, commitment, 0)
+	if err != nil {
+		return GoKZG.KZGCommitment{}, GoKZG.KZGProof{}, err
+	}
+	return commitment, proof, err
+}
+
 func TestIsDataAvailable(t *testing.T) {
-	sidecars := make([]*ethpb.BlobSidecar, 0)
-	commitments := make([][]byte, 0)
-	require.NoError(t, IsDataAvailable(commitments, sidecars))
+	sidecars := make([]blocks.ROBlob, 0)
+	require.NoError(t, Verify(sidecars[0]))
 }
 
 func TestBytesToAny(t *testing.T) {
@@ -23,3 +72,13 @@ func TestBytesToAny(t *testing.T) {
 	require.DeepEqual(t, commitment, bytesToCommitment(bytes))
 	require.DeepEqual(t, proof, bytesToKZGProof(bytes))
 }
+
+func TestGenerateCommitmentAndProof(t *testing.T) {
+	blob := GetRandBlob(123)
+	commitment, proof, err := GenerateCommitmentAndProof(blob)
+	require.NoError(t, err)
+	expectedCommitment := GoKZG.KZGCommitment{180, 218, 156, 194, 59, 20, 10, 189, 186, 254, 132, 93, 7, 127, 104, 172, 238, 240, 237, 70, 83, 89, 1, 152, 99, 0, 165, 65, 143, 62, 20, 215, 230, 14, 205, 95, 28, 245, 54, 25, 160, 16, 178, 31, 232, 207, 38, 85}
+	expectedProof := GoKZG.KZGProof{128, 110, 116, 170, 56, 111, 126, 87, 229, 234, 211, 42, 110, 150, 129, 206, 73, 142, 167, 243, 90, 149, 240, 240, 236, 204, 143, 182, 229, 249, 81, 27, 153, 171, 83, 70, 144, 250, 42, 1, 188, 215, 71, 235, 30, 7, 175, 86}
+	require.Equal(t, expectedCommitment, commitment)
+	require.Equal(t, expectedProof, proof)
+}

beacon-chain/blockchain/lightclient.go

@@ -8,7 +8,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
-	types "github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	ethpbv1 "github.com/prysmaticlabs/prysm/v4/proto/eth/v1"
 	ethpbv2 "github.com/prysmaticlabs/prysm/v4/proto/eth/v2"
@@ -63,7 +62,7 @@ func NewLightClientOptimisticUpdateFromBeaconState(
 	attestedState state.BeaconState) (*ethpbv2.LightClientUpdate, error) {
 	// assert compute_epoch_at_slot(attested_state.slot) >= ALTAIR_FORK_EPOCH
 	attestedEpoch := slots.ToEpoch(attestedState.Slot())
-	if attestedEpoch < types.Epoch(params.BeaconConfig().AltairForkEpoch) {
+	if attestedEpoch < params.BeaconConfig().AltairForkEpoch {
 		return nil, fmt.Errorf("invalid attested epoch %d", attestedEpoch)
 	}
 

beacon-chain/blockchain/log.go

@@ -73,7 +73,7 @@ func logStateTransitionData(b interfaces.ReadOnlyBeaconBlock) error {
 	return nil
 }
 
-func logBlockSyncStatus(block interfaces.ReadOnlyBeaconBlock, blockRoot [32]byte, justified, finalized *ethpb.Checkpoint, receivedTime time.Time, genesisTime uint64) error {
+func logBlockSyncStatus(block interfaces.ReadOnlyBeaconBlock, blockRoot [32]byte, justified, finalized *ethpb.Checkpoint, receivedTime time.Time, genesisTime uint64, daWaitedTime time.Duration) error {
 	startTime, err := slots.ToTime(genesisTime, block.Slot())
 	if err != nil {
 		return err
@@ -93,7 +93,7 @@ func logBlockSyncStatus(block interfaces.ReadOnlyBeaconBlock, blockRoot [32]byte
 			"parentRoot":                fmt.Sprintf("0x%s...", hex.EncodeToString(parentRoot[:])[:8]),
 			"version":                   version.String(block.Version()),
 			"sinceSlotStartTime":        prysmTime.Now().Sub(startTime),
-			"chainServiceProcessedTime": prysmTime.Now().Sub(receivedTime),
+			"chainServiceProcessedTime": prysmTime.Now().Sub(receivedTime) - daWaitedTime,
 			"deposits":                  len(block.Body().Deposits()),
 		}
 		log.WithFields(lf).Debug("Synced new block")
@@ -147,15 +147,3 @@ func logPayload(block interfaces.ReadOnlyBeaconBlock) error {
 	log.WithFields(fields).Debug("Synced new payload")
 	return nil
 }
-
-func logBlobSidecar(scs []*ethpb.BlobSidecar, startTime time.Time) {
-	if len(scs) == 0 {
-		return
-	}
-	log.WithFields(logrus.Fields{
-		"count":          len(scs),
-		"slot":           scs[0].Slot,
-		"block":          hex.EncodeToString(scs[0].BlockRoot),
-		"validationTime": time.Since(startTime),
-	}).Debug("Synced new blob sidecars")
-}

beacon-chain/blockchain/metrics.go

@@ -358,6 +358,7 @@ func reportEpochMetrics(ctx context.Context, postState, headState state.BeaconSt
 	for name, val := range refMap {
 		stateTrieReferences.WithLabelValues(name).Set(float64(val))
 	}
+	postState.RecordStateMetrics()
 
 	return nil
 }

beacon-chain/blockchain/options.go

@@ -5,6 +5,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
 	statefeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/execution"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/operations/attestations"
@@ -68,10 +69,18 @@ func WithDepositCache(c cache.DepositCache) Option {
 	}
 }
 
-// WithProposerIdsCache for proposer id cache.
-func WithProposerIdsCache(c *cache.ProposerPayloadIDsCache) Option {
+// WithPayloadIDCache for payload ID cache.
+func WithPayloadIDCache(c *cache.PayloadIDCache) Option {
 	return func(s *Service) error {
-		s.cfg.ProposerSlotIndexCache = c
+		s.cfg.PayloadIDCache = c
+		return nil
+	}
+}
+
+// WithTrackedValidatorsCache for tracked validators cache.
+func WithTrackedValidatorsCache(c *cache.TrackedValidatorsCache) Option {
+	return func(s *Service) error {
+		s.cfg.TrackedValidatorsCache = c
 		return nil
 	}
 }
@@ -164,6 +173,8 @@ func WithFinalizedStateAtStartUp(st state.BeaconState) Option {
 	}
 }
 
+// WithClockSynchronizer sets the ClockSetter/ClockWaiter values to be used by services that need to block until
+// the genesis timestamp is known (ClockWaiter) or which determine the genesis timestamp (ClockSetter).
 func WithClockSynchronizer(gs *startup.ClockSynchronizer) Option {
 	return func(s *Service) error {
 		s.clockSetter = gs
@@ -172,9 +183,18 @@ func WithClockSynchronizer(gs *startup.ClockSynchronizer) Option {
 	}
 }
 
+// WithSyncComplete sets a channel that is used to notify blockchain service that the node has synced to head.
 func WithSyncComplete(c chan struct{}) Option {
 	return func(s *Service) error {
 		s.syncComplete = c
 		return nil
 	}
 }
+
+// WithBlobStorage sets the blob storage backend for the blockchain service.
+func WithBlobStorage(b *filesystem.BlobStorage) Option {
+	return func(s *Service) error {
+		s.blobStorage = b
+		return nil
+	}
+}

beacon-chain/blockchain/process_attestation_test.go

@@ -2,6 +2,7 @@ package blockchain
 
 import (
 	"context"
+	"sync"
 	"testing"
 	"time"
 
@@ -145,6 +146,61 @@ func TestStore_OnAttestation_Ok_DoublyLinkedTree(t *testing.T) {
 	require.NoError(t, service.OnAttestation(ctx, att[0], 0))
 }
 
+func TestService_GetAttPreState_Concurrency(t *testing.T) {
+	service, _ := minimalTestService(t)
+	ctx := context.Background()
+
+	s, err := util.NewBeaconState()
+	require.NoError(t, err)
+	ckRoot := bytesutil.PadTo([]byte{'A'}, fieldparams.RootLength)
+	err = s.SetFinalizedCheckpoint(&ethpb.Checkpoint{Root: ckRoot})
+	require.NoError(t, err)
+	val := &ethpb.Validator{PublicKey: bytesutil.PadTo([]byte("foo"), 48), WithdrawalCredentials: bytesutil.PadTo([]byte("bar"), fieldparams.RootLength)}
+	err = s.SetValidators([]*ethpb.Validator{val})
+	require.NoError(t, err)
+	err = s.SetBalances([]uint64{0})
+	require.NoError(t, err)
+	r := [32]byte{'g'}
+	require.NoError(t, service.cfg.BeaconDB.SaveState(ctx, s, r))
+
+	cp1 := &ethpb.Checkpoint{Epoch: 1, Root: ckRoot}
+	require.NoError(t, service.cfg.BeaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'A'})))
+	require.NoError(t, service.cfg.BeaconDB.SaveStateSummary(ctx, &ethpb.StateSummary{Root: ckRoot}))
+
+	st, root, err := prepareForkchoiceState(ctx, 100, [32]byte(cp1.Root), [32]byte{}, [32]byte{'R'}, cp1, cp1)
+	require.NoError(t, err)
+	require.NoError(t, service.cfg.ForkChoiceStore.InsertNode(ctx, st, root))
+
+	var wg sync.WaitGroup
+	errChan := make(chan error, 1000)
+
+	for i := 0; i < 1000; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			cp1 := &ethpb.Checkpoint{Epoch: 1, Root: ckRoot}
+			_, err := service.getAttPreState(ctx, cp1)
+			if err != nil {
+				errChan <- err
+			}
+		}()
+	}
+
+	go func() {
+		wg.Wait()
+		close(errChan)
+	}()
+
+	select {
+	case <-time.After(10 * time.Second):
+		t.Fatal("Test timed out")
+	case err, ok := <-errChan:
+		if ok && err != nil {
+			require.ErrorContains(t, "not a checkpoint in forkchoice", err)
+		}
+	}
+}
+
 func TestStore_SaveCheckpointState(t *testing.T) {
 	service, tr := minimalTestService(t)
 	ctx := tr.ctx
@@ -179,14 +235,14 @@ func TestStore_SaveCheckpointState(t *testing.T) {
 	require.NoError(t, service.cfg.BeaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'B'})))
 	require.NoError(t, service.cfg.BeaconDB.SaveStateSummary(ctx, &ethpb.StateSummary{Root: bytesutil.PadTo([]byte{'B'}, fieldparams.RootLength)}))
 
-	s2, err := service.getAttPreState(ctx, cp2)
+	_, err = service.getAttPreState(ctx, cp2)
 	require.ErrorContains(t, "epoch 2 root 0x4200000000000000000000000000000000000000000000000000000000000000: not a checkpoint in forkchoice", err)
 
 	st, root, err = prepareForkchoiceState(ctx, 33, [32]byte(cp2.Root), [32]byte(cp1.Root), [32]byte{'R'}, cp2, cp2)
 	require.NoError(t, err)
 	require.NoError(t, service.cfg.ForkChoiceStore.InsertNode(ctx, st, root))
 
-	s2, err = service.getAttPreState(ctx, cp2)
+	s2, err := service.getAttPreState(ctx, cp2)
 	require.NoError(t, err)
 
 	assert.Equal(t, 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot(), "Unexpected state slot")

beacon-chain/blockchain/process_block.go

@@ -6,17 +6,18 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain/kzg"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/helpers"
 	coreTime "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/das"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/v4/config/features"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	consensusblocks "github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
@@ -28,7 +29,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1/attestation"
 	"github.com/prysmaticlabs/prysm/v4/runtime/version"
 	"github.com/prysmaticlabs/prysm/v4/time/slots"
-	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
 
@@ -41,106 +41,67 @@ const depositDeadline = 20 * time.Second
 // This defines size of the upper bound for initial sync block cache.
 var initialSyncBlockCacheSize = uint64(2 * params.BeaconConfig().SlotsPerEpoch)
 
+// postBlockProcessConfig is a structure that contains the data needed to
+// process the beacon block after validating the state transition function
+type postBlockProcessConfig struct {
+	ctx            context.Context
+	signed         interfaces.ReadOnlySignedBeaconBlock
+	blockRoot      [32]byte
+	headRoot       [32]byte
+	postState      state.BeaconState
+	isValidPayload bool
+}
+
 // postBlockProcess is called when a gossip block is received. This function performs
 // several duties most importantly informing the engine if head was updated,
 // saving the new head information to the blockchain package and
 // handling attestations, slashings and similar included in the block.
-func (s *Service) postBlockProcess(ctx context.Context, signed interfaces.ReadOnlySignedBeaconBlock, blockRoot [32]byte, postState state.BeaconState, isValidPayload bool) error {
-	ctx, span := trace.StartSpan(ctx, "blockChain.onBlock")
+func (s *Service) postBlockProcess(cfg *postBlockProcessConfig) error {
+	ctx, span := trace.StartSpan(cfg.ctx, "blockChain.onBlock")
 	defer span.End()
-	if err := consensusblocks.BeaconBlockIsNil(signed); err != nil {
+	cfg.ctx = ctx
+	if err := consensusblocks.BeaconBlockIsNil(cfg.signed); err != nil {
 		return invalidBlock{error: err}
 	}
 	startTime := time.Now()
-	b := signed.Block()
+	fcuArgs := &fcuConfig{}
 
-	if err := s.cfg.ForkChoiceStore.InsertNode(ctx, postState, blockRoot); err != nil {
-		return errors.Wrapf(err, "could not insert block %d to fork choice store", signed.Block().Slot())
+	defer s.handleSecondFCUCall(cfg, fcuArgs)
+	defer s.sendStateFeedOnBlock(cfg)
+	defer reportProcessingTime(startTime)
+	defer reportAttestationInclusion(cfg.signed.Block())
+
+	err := s.cfg.ForkChoiceStore.InsertNode(ctx, cfg.postState, cfg.blockRoot)
+	if err != nil {
+		return errors.Wrapf(err, "could not insert block %d to fork choice store", cfg.signed.Block().Slot())
 	}
-	if err := s.handleBlockAttestations(ctx, signed.Block(), postState); err != nil {
+	if err := s.handleBlockAttestations(ctx, cfg.signed.Block(), cfg.postState); err != nil {
 		return errors.Wrap(err, "could not handle block's attestations")
 	}
 
-	s.InsertSlashingsToForkChoiceStore(ctx, signed.Block().Body().AttesterSlashings())
-	if isValidPayload {
-		if err := s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, blockRoot); err != nil {
+	s.InsertSlashingsToForkChoiceStore(ctx, cfg.signed.Block().Body().AttesterSlashings())
+	if cfg.isValidPayload {
+		if err := s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, cfg.blockRoot); err != nil {
 			return errors.Wrap(err, "could not set optimistic block to valid")
 		}
 	}
-
 	start := time.Now()
-	headRoot, err := s.cfg.ForkChoiceStore.Head(ctx)
+	cfg.headRoot, err = s.cfg.ForkChoiceStore.Head(ctx)
 	if err != nil {
 		log.WithError(err).Warn("Could not update head")
 	}
-	if blockRoot != headRoot {
-		receivedWeight, err := s.cfg.ForkChoiceStore.Weight(blockRoot)
-		if err != nil {
-			log.WithField("root", fmt.Sprintf("%#x", blockRoot)).Warn("could not determine node weight")
-		}
-		headWeight, err := s.cfg.ForkChoiceStore.Weight(headRoot)
-		if err != nil {
-			log.WithField("root", fmt.Sprintf("%#x", headRoot)).Warn("could not determine node weight")
-		}
-		log.WithFields(logrus.Fields{
-			"receivedRoot":   fmt.Sprintf("%#x", blockRoot),
-			"receivedWeight": receivedWeight,
-			"headRoot":       fmt.Sprintf("%#x", headRoot),
-			"headWeight":     headWeight,
-		}).Debug("Head block is not the received block")
-	}
 	newBlockHeadElapsedTime.Observe(float64(time.Since(start).Milliseconds()))
-
-	// verify conditions for FCU, notifies FCU, and saves the new head.
-	// This function also prunes attestations, other similar operations happen in prunePostBlockOperationPools.
-	if _, err := s.forkchoiceUpdateWithExecution(ctx, headRoot, s.CurrentSlot()+1); err != nil {
-		return err
+	if cfg.headRoot != cfg.blockRoot {
+		s.logNonCanonicalBlockReceived(cfg.blockRoot, cfg.headRoot)
+		return nil
 	}
-
-	optimistic, err := s.cfg.ForkChoiceStore.IsOptimistic(blockRoot)
-	if err != nil {
-		log.WithError(err).Debug("Could not check if block is optimistic")
-		optimistic = true
+	if err := s.getFCUArgs(cfg, fcuArgs); err != nil {
+		log.WithError(err).Error("Could not get forkchoice update argument")
+		return nil
 	}
-
-	// Send notification of the processed block to the state feed.
-	s.cfg.StateNotifier.StateFeed().Send(&feed.Event{
-		Type: statefeed.BlockProcessed,
-		Data: &statefeed.BlockProcessedData{
-			Slot:        signed.Block().Slot(),
-			BlockRoot:   blockRoot,
-			SignedBlock: signed,
-			Verified:    true,
-			Optimistic:  optimistic,
-		},
-	})
-
-	defer reportAttestationInclusion(b)
-	if headRoot == blockRoot {
-		// Updating next slot state cache can happen in the background
-		// except in the epoch boundary in which case we lock to handle
-		// the shuffling and proposer caches updates.
-		// We handle these caches only on canonical
-		// blocks, otherwise this will be handled by lateBlockTasks
-		slot := postState.Slot()
-		if slots.IsEpochEnd(slot) {
-			if err := transition.UpdateNextSlotCache(ctx, blockRoot[:], postState); err != nil {
-				return errors.Wrap(err, "could not update next slot state cache")
-			}
-			if err := s.handleEpochBoundary(ctx, slot, postState, blockRoot[:]); err != nil {
-				return errors.Wrap(err, "could not handle epoch boundary")
-			}
-		} else {
-			go func() {
-				slotCtx, cancel := context.WithTimeout(context.Background(), slotDeadline)
-				defer cancel()
-				if err := transition.UpdateNextSlotCache(slotCtx, blockRoot[:], postState); err != nil {
-					log.WithError(err).Error("could not update next slot state cache")
-				}
-			}()
-		}
+	if err := s.sendFCU(cfg, fcuArgs); err != nil {
+		return errors.Wrap(err, "could not send FCU to engine")
 	}
-	onBlockProcessingTime.Observe(float64(time.Since(startTime).Milliseconds()))
 	return nil
 }
 
@@ -162,7 +123,7 @@ func getStateVersionAndPayload(st state.BeaconState) (int, interfaces.ExecutionD
 	return preStateVersion, preStateHeader, nil
 }
 
-func (s *Service) onBlockBatch(ctx context.Context, blks []consensusblocks.ROBlock) error {
+func (s *Service) onBlockBatch(ctx context.Context, blks []consensusblocks.ROBlock, avs das.AvailabilityStore) error {
 	ctx, span := trace.StartSpan(ctx, "blockChain.onBlockBatch")
 	defer span.End()
 
@@ -265,8 +226,8 @@ func (s *Service) onBlockBatch(ctx context.Context, blks []consensusblocks.ROBlo
 				return err
 			}
 		}
-		if err := s.databaseDACheck(ctx, b); err != nil {
-			return errors.Wrap(err, "could not validate blob data availability")
+		if err := avs.IsDataAvailable(ctx, s.CurrentSlot(), b); err != nil {
+			return errors.Wrapf(err, "could not validate blob data availability at slot %d", b.Block().Slot())
 		}
 		args := &forkchoicetypes.BlockAndCheckpoints{Block: b.Block(),
 			JustifiedCheckpoint: jCheckpoints[i],
@@ -322,10 +283,10 @@ func (s *Service) onBlockBatch(ctx context.Context, blks []consensusblocks.ROBlo
 			return errors.Wrap(err, "could not set optimistic block to valid")
 		}
 	}
-	arg := &notifyForkchoiceUpdateArg{
+	arg := &fcuConfig{
 		headState: preState,
 		headRoot:  lastBR,
-		headBlock: lastB.Block(),
+		headBlock: lastB,
 	}
 	if _, err := s.notifyForkchoiceUpdate(ctx, arg); err != nil {
 		return err
@@ -333,33 +294,6 @@ func (s *Service) onBlockBatch(ctx context.Context, blks []consensusblocks.ROBlo
 	return s.saveHeadNoDB(ctx, lastB, lastBR, preState, !isValidPayload)
 }
 
-func commitmentsToCheck(b consensusblocks.ROBlock, current primitives.Slot) [][]byte {
-	if b.Version() < version.Deneb {
-		return nil
-	}
-	// We are only required to check within MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS
-	if !params.WithinDAPeriod(slots.ToEpoch(b.Block().Slot()), slots.ToEpoch(current)) {
-		return nil
-	}
-	kzgCommitments, err := b.Block().Body().BlobKzgCommitments()
-	if err != nil {
-		return nil
-	}
-	return kzgCommitments
-}
-
-func (s *Service) databaseDACheck(ctx context.Context, b consensusblocks.ROBlock) error {
-	commitments := commitmentsToCheck(b, s.CurrentSlot())
-	if len(commitments) == 0 {
-		return nil
-	}
-	sidecars, err := s.cfg.BeaconDB.BlobSidecarsByRoot(ctx, b.Root())
-	if err != nil {
-		return errors.Wrap(err, "could not get blob sidecars")
-	}
-	return kzg.IsDataAvailable(commitments, sidecars)
-}
-
 func (s *Service) updateEpochBoundaryCaches(ctx context.Context, st state.BeaconState) error {
 	e := coreTime.CurrentEpoch(st)
 	if err := helpers.UpdateCommitteeCache(ctx, st, e); err != nil {
@@ -377,10 +311,24 @@ func (s *Service) updateEpochBoundaryCaches(ctx context.Context, st state.Beacon
 		if err := helpers.UpdateCommitteeCache(slotCtx, st, e+1); err != nil {
 			log.WithError(err).Warn("Could not update committee cache")
 		}
-		if err := helpers.UpdateProposerIndicesInCache(slotCtx, st, e+1); err != nil {
-			log.WithError(err).Warn("Failed to cache next epoch proposers")
-		}
 	}()
+	// The latest block header is from the previous epoch
+	r, err := st.LatestBlockHeader().HashTreeRoot()
+	if err != nil {
+		log.WithError(err).Error("could not update proposer index state-root map")
+		return nil
+	}
+	// The proposer indices cache takes the target root for the previous
+	// epoch as key
+	target, err := s.cfg.ForkChoiceStore.TargetRootForEpoch(r, e-1)
+	if err != nil {
+		log.WithError(err).Error("could not update proposer index state-root map")
+		return nil
+	}
+	err = helpers.UpdateCachedCheckpointToStateRoot(st, &forkchoicetypes.Checkpoint{Epoch: e, Root: target})
+	if err != nil {
+		log.WithError(err).Error("could not update proposer index state-root map")
+	}
 	return nil
 }
 
@@ -529,11 +477,43 @@ func (s *Service) runLateBlockTasks() {
 	}
 }
 
+// missingIndices uses the expected commitments from the block to determine
+// which BlobSidecar indices would need to be in the database for DA success.
+// It returns a map where each key represents a missing BlobSidecar index.
+// An empty map means we have all indices; a non-empty map can be used to compare incoming
+// BlobSidecars against the set of known missing sidecars.
+func missingIndices(bs *filesystem.BlobStorage, root [32]byte, expected [][]byte) (map[uint64]struct{}, error) {
+	if len(expected) == 0 {
+		return nil, nil
+	}
+	if len(expected) > fieldparams.MaxBlobsPerBlock {
+		return nil, errMaxBlobsExceeded
+	}
+	indices, err := bs.Indices(root)
+	if err != nil {
+		return nil, err
+	}
+	missing := make(map[uint64]struct{}, len(expected))
+	for i := range expected {
+		ui := uint64(i)
+		if len(expected[i]) > 0 {
+			if !indices[i] {
+				missing[ui] = struct{}{}
+			}
+		}
+	}
+	return missing, nil
+}
+
+// isDataAvailable blocks until all BlobSidecars committed to in the block are available,
+// or an error or context cancellation occurs. A nil result means that the data availability check is successful.
+// The function will first check the database to see if all sidecars have been persisted. If any
+// sidecars are missing, it will then read from the blobNotifier channel for the given root until the channel is
+// closed, the context hits cancellation/timeout, or notifications have been received for all the missing sidecars.
 func (s *Service) isDataAvailable(ctx context.Context, root [32]byte, signed interfaces.ReadOnlySignedBeaconBlock) error {
 	if signed.Version() < version.Deneb {
 		return nil
 	}
-	t := time.Now()
 
 	block := signed.Block()
 	if block == nil {
@@ -552,51 +532,35 @@ func (s *Service) isDataAvailable(ctx context.Context, root [32]byte, signed int
 	if err != nil {
 		return errors.Wrap(err, "could not get KZG commitments")
 	}
+	// expected is the number of kzg commitments observed in the block.
 	expected := len(kzgCommitments)
 	if expected == 0 {
 		return nil
 	}
-
-	// Read first from db in case we have the blobs
-	sidecars, err := s.cfg.BeaconDB.BlobSidecarsByRoot(ctx, root)
-	switch {
-	case err == nil:
-		if len(sidecars) >= expected {
-			s.blobNotifiers.delete(root)
-			if err := kzg.IsDataAvailable(kzgCommitments, sidecars); err != nil {
-				return err
-			}
-			logBlobSidecar(sidecars, t)
-			return nil
-		}
-	case errors.Is(err, db.ErrNotFound):
-		// If the blob sidecars haven't arrived yet, the subsequent code will wait for them.
-		// Note: The system will not exit with an error in this scenario.
-	default:
-		log.WithError(err).Error("could not get blob sidecars from DB")
+	// get a map of BlobSidecar indices that are not currently available.
+	missing, err := missingIndices(s.blobStorage, root, kzgCommitments)
+	if err != nil {
+		return err
+	}
+	// If there are no missing indices, all BlobSidecars are available.
+	if len(missing) == 0 {
+		return nil
 	}
 
-	found := map[uint64]struct{}{}
-	for _, sc := range sidecars {
-		found[sc.Index] = struct{}{}
-	}
+	// The gossip handler for blobs writes the index of each verified blob referencing the given
+	// root to the channel returned by blobNotifiers.forRoot.
 	nc := s.blobNotifiers.forRoot(root)
 	for {
 		select {
 		case idx := <-nc:
-			found[idx] = struct{}{}
-			if len(found) != expected {
+			// Delete each index seen in the notification channel.
+			delete(missing, idx)
+			// Read from the channel until there are no more missing sidecars.
+			if len(missing) > 0 {
 				continue
 			}
+			// Once all sidecars have been observed, clean up the notification channel.
 			s.blobNotifiers.delete(root)
-			sidecars, err := s.cfg.BeaconDB.BlobSidecarsByRoot(ctx, root)
-			if err != nil {
-				return errors.Wrap(err, "could not get blob sidecars")
-			}
-			if err := kzg.IsDataAvailable(kzgCommitments, sidecars); err != nil {
-				return err
-			}
-			logBlobSidecar(sidecars, t)
 			return nil
 		case <-ctx.Done():
 			return errors.Wrap(ctx.Err(), "context deadline waiting for blob sidecars")
@@ -631,16 +595,18 @@ func (s *Service) lateBlockTasks(ctx context.Context) {
 	if err := transition.UpdateNextSlotCache(ctx, lastRoot, lastState); err != nil {
 		log.WithError(err).Debug("could not update next slot state cache")
 	}
+	// handleEpochBoundary requires a forkchoice lock to obtain the target root.
+	s.cfg.ForkChoiceStore.RLock()
 	if err := s.handleEpochBoundary(ctx, currentSlot, headState, headRoot[:]); err != nil {
 		log.WithError(err).Error("lateBlockTasks: could not update epoch boundary caches")
 	}
-	// Head root should be empty when retrieving proposer index for the next slot.
-	_, id, has := s.cfg.ProposerSlotIndexCache.GetProposerPayloadIDs(s.CurrentSlot()+1, [32]byte{} /* head root */)
-	// There exists proposer for next slot, but we haven't called fcu w/ payload attribute yet.
-	if (!has && !features.Get().PrepareAllPayloads) || id != [8]byte{} {
+	s.cfg.ForkChoiceStore.RUnlock()
+	// return early if we already started building a block for the current
+	// head root
+	_, has := s.cfg.PayloadIDCache.PayloadID(s.CurrentSlot()+1, headRoot)
+	if has {
 		return
 	}
-
 	s.headLock.RLock()
 	headBlock, err := s.headBlock()
 	if err != nil {
@@ -649,12 +615,19 @@ func (s *Service) lateBlockTasks(ctx context.Context) {
 		return
 	}
 	s.headLock.RUnlock()
-	s.cfg.ForkChoiceStore.RLock()
-	_, err = s.notifyForkchoiceUpdate(ctx, &notifyForkchoiceUpdateArg{
+
+	fcuArgs := &fcuConfig{
 		headState: headState,
 		headRoot:  headRoot,
-		headBlock: headBlock.Block(),
-	})
+		headBlock: headBlock,
+	}
+	fcuArgs.attributes = s.getPayloadAttribute(ctx, headState, s.CurrentSlot()+1, headRoot[:])
+	// return early if we are not proposing next slot
+	if fcuArgs.attributes.IsEmpty() {
+		return
+	}
+	s.cfg.ForkChoiceStore.RLock()
+	_, err = s.notifyForkchoiceUpdate(ctx, fcuArgs)
 	s.cfg.ForkChoiceStore.RUnlock()
 	if err != nil {
 		log.WithError(err).Debug("could not perform late block tasks: failed to update forkchoice with engine")

beacon-chain/blockchain/process_block_helpers.go

@@ -3,9 +3,13 @@ package blockchain
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed"
+	statefeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/state"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
 	doublylinkedtree "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/doubly-linked-tree"
 	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
@@ -15,8 +19,8 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	mathutil "github.com/prysmaticlabs/prysm/v4/math"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
-	"github.com/prysmaticlabs/prysm/v4/time"
 	"github.com/prysmaticlabs/prysm/v4/time/slots"
+	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
 
@@ -25,6 +29,127 @@ func (s *Service) CurrentSlot() primitives.Slot {
 	return slots.CurrentSlot(uint64(s.genesisTime.Unix()))
 }
 
+// getFCUArgs returns the arguments to call forkchoice update
+func (s *Service) getFCUArgs(cfg *postBlockProcessConfig, fcuArgs *fcuConfig) error {
+	if err := s.getFCUArgsEarlyBlock(cfg, fcuArgs); err != nil {
+		return err
+	}
+	slot := cfg.signed.Block().Slot()
+	if slots.WithinVotingWindow(uint64(s.genesisTime.Unix()), slot) {
+		return nil
+	}
+	return s.computePayloadAttributes(cfg, fcuArgs)
+}
+
+func (s *Service) getFCUArgsEarlyBlock(cfg *postBlockProcessConfig, fcuArgs *fcuConfig) error {
+	if cfg.blockRoot == cfg.headRoot {
+		fcuArgs.headState = cfg.postState
+		fcuArgs.headBlock = cfg.signed
+		fcuArgs.headRoot = cfg.headRoot
+		fcuArgs.proposingSlot = s.CurrentSlot() + 1
+		return nil
+	}
+	return s.fcuArgsNonCanonicalBlock(cfg, fcuArgs)
+}
+
+// logNonCanonicalBlockReceived prints a message informing that the received
+// block is not the head of the chain. It requires the caller holds a lock on
+// Foprkchoice.
+func (s *Service) logNonCanonicalBlockReceived(blockRoot [32]byte, headRoot [32]byte) {
+	receivedWeight, err := s.cfg.ForkChoiceStore.Weight(blockRoot)
+	if err != nil {
+		log.WithField("root", fmt.Sprintf("%#x", blockRoot)).Warn("could not determine node weight")
+	}
+	headWeight, err := s.cfg.ForkChoiceStore.Weight(headRoot)
+	if err != nil {
+		log.WithField("root", fmt.Sprintf("%#x", headRoot)).Warn("could not determine node weight")
+	}
+	log.WithFields(logrus.Fields{
+		"receivedRoot":   fmt.Sprintf("%#x", blockRoot),
+		"receivedWeight": receivedWeight,
+		"headRoot":       fmt.Sprintf("%#x", headRoot),
+		"headWeight":     headWeight,
+	}).Debug("Head block is not the received block")
+}
+
+// fcuArgsNonCanonicalBlock returns the arguments to the FCU call when the
+// incoming block is non-canonical, that is, based on the head root.
+func (s *Service) fcuArgsNonCanonicalBlock(cfg *postBlockProcessConfig, fcuArgs *fcuConfig) error {
+	headState, headBlock, err := s.getStateAndBlock(cfg.ctx, cfg.headRoot)
+	if err != nil {
+		return err
+	}
+	fcuArgs.headState = headState
+	fcuArgs.headBlock = headBlock
+	fcuArgs.headRoot = cfg.headRoot
+	fcuArgs.proposingSlot = s.CurrentSlot() + 1
+	return nil
+}
+
+// sendStateFeedOnBlock sends an event that a new block has been synced
+func (s *Service) sendStateFeedOnBlock(cfg *postBlockProcessConfig) {
+	optimistic, err := s.cfg.ForkChoiceStore.IsOptimistic(cfg.blockRoot)
+	if err != nil {
+		log.WithError(err).Debug("Could not check if block is optimistic")
+		optimistic = true
+	}
+	// Send notification of the processed block to the state feed.
+	s.cfg.StateNotifier.StateFeed().Send(&feed.Event{
+		Type: statefeed.BlockProcessed,
+		Data: &statefeed.BlockProcessedData{
+			Slot:        cfg.signed.Block().Slot(),
+			BlockRoot:   cfg.blockRoot,
+			SignedBlock: cfg.signed,
+			Verified:    true,
+			Optimistic:  optimistic,
+		},
+	})
+}
+
+// updateCachesPostBlockProcessing updates the next slot cache and handles the epoch
+// boundary in order to compute the right proposer indices after processing
+// state transition. This function is called on late blocks while still locked,
+// before sending FCU to the engine.
+func (s *Service) updateCachesPostBlockProcessing(cfg *postBlockProcessConfig) error {
+	slot := cfg.postState.Slot()
+	if err := transition.UpdateNextSlotCache(cfg.ctx, cfg.blockRoot[:], cfg.postState); err != nil {
+		return errors.Wrap(err, "could not update next slot state cache")
+	}
+	if !slots.IsEpochEnd(slot) {
+		return nil
+	}
+	return s.handleEpochBoundary(cfg.ctx, slot, cfg.postState, cfg.blockRoot[:])
+}
+
+// handleSecondFCUCall handles a second call to FCU when syncing a new block.
+// This is useful when proposing in the next block and we want to defer the
+// computation of the next slot shuffling.
+func (s *Service) handleSecondFCUCall(cfg *postBlockProcessConfig, fcuArgs *fcuConfig) {
+	if (fcuArgs.attributes == nil || fcuArgs.attributes.IsEmpty()) && cfg.headRoot == cfg.blockRoot {
+		go s.sendFCUWithAttributes(cfg, fcuArgs)
+	}
+}
+
+// reportProcessingTime reports the metric of how long it took to process the
+// current block
+func reportProcessingTime(startTime time.Time) {
+	onBlockProcessingTime.Observe(float64(time.Since(startTime).Milliseconds()))
+}
+
+// computePayloadAttributes modifies the passed FCU arguments to
+// contain the right payload attributes with the tracked proposer. It gets
+// called on blocks that arrive after the attestation voting window, or in a
+// background routine after syncing early blocks.
+func (s *Service) computePayloadAttributes(cfg *postBlockProcessConfig, fcuArgs *fcuConfig) error {
+	if cfg.blockRoot == cfg.headRoot {
+		if err := s.updateCachesPostBlockProcessing(cfg); err != nil {
+			return err
+		}
+	}
+	fcuArgs.attributes = s.getPayloadAttribute(cfg.ctx, fcuArgs.headState, fcuArgs.proposingSlot, cfg.headRoot[:])
+	return nil
+}
+
 // getBlockPreState returns the pre state of an incoming block. It uses the parent root of the block
 // to retrieve the state in DB. It verifies the pre state's validity and the incoming block
 // is in the correct time window.
@@ -46,7 +171,7 @@ func (s *Service) getBlockPreState(ctx context.Context, b interfaces.ReadOnlyBea
 	}
 
 	// Verify block slot time is not from the future.
-	if err := slots.VerifyTime(uint64(s.genesisTime.Unix()), b.Slot(), params.BeaconNetworkConfig().MaximumGossipClockDisparity); err != nil {
+	if err := slots.VerifyTime(uint64(s.genesisTime.Unix()), b.Slot(), params.BeaconConfig().MaximumGossipClockDisparityDuration()); err != nil {
 		return nil, err
 	}
 

beacon-chain/blockchain/process_block_test.go

@@ -1,7 +1,6 @@
 package blockchain
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"math/big"
@@ -17,7 +16,9 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/signing"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/das"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	testDB "github.com/prysmaticlabs/prysm/v4/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/execution"
 	mockExecution "github.com/prysmaticlabs/prysm/v4/beacon-chain/execution/testing"
@@ -39,7 +40,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
 	"github.com/prysmaticlabs/prysm/v4/testing/util"
 	prysmTime "github.com/prysmaticlabs/prysm/v4/time"
-	"github.com/prysmaticlabs/prysm/v4/time/slots"
 	logTest "github.com/sirupsen/logrus/hooks/test"
 )
 
@@ -68,7 +68,7 @@ func TestStore_OnBlockBatch(t *testing.T) {
 		require.NoError(t, err)
 		blks = append(blks, rwsb)
 	}
-	err := service.onBlockBatch(ctx, blks)
+	err := service.onBlockBatch(ctx, blks, &das.MockAvailabilityStore{})
 	require.NoError(t, err)
 	jcp := service.CurrentJustifiedCheckpt()
 	jroot := bytesutil.ToBytes32(jcp.Root)
@@ -98,7 +98,7 @@ func TestStore_OnBlockBatch_NotifyNewPayload(t *testing.T) {
 		require.NoError(t, service.saveInitSyncBlock(ctx, rwsb.Root(), wsb))
 		blks = append(blks, rwsb)
 	}
-	require.NoError(t, service.onBlockBatch(ctx, blks))
+	require.NoError(t, service.onBlockBatch(ctx, blks, &das.MockAvailabilityStore{}))
 }
 
 func TestCachedPreState_CanGetFromStateSummary(t *testing.T) {
@@ -566,7 +566,7 @@ func TestOnBlock_CanFinalize_WithOnTick(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, r, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, r, postState, true))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, r, [32]byte{}, postState, true}))
 		require.NoError(t, service.updateJustificationOnBlock(ctx, preState, postState, currStoreJustifiedEpoch))
 		_, err = service.updateFinalizationOnBlock(ctx, preState, postState, currStoreFinalizedEpoch)
 		require.NoError(t, err)
@@ -614,7 +614,7 @@ func TestOnBlock_CanFinalize(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, r, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, r, postState, true))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, r, [32]byte{}, postState, true}))
 		require.NoError(t, service.updateJustificationOnBlock(ctx, preState, postState, currStoreJustifiedEpoch))
 		_, err = service.updateFinalizationOnBlock(ctx, preState, postState, currStoreFinalizedEpoch)
 		require.NoError(t, err)
@@ -640,7 +640,7 @@ func TestOnBlock_CanFinalize(t *testing.T) {
 
 func TestOnBlock_NilBlock(t *testing.T) {
 	service, tr := minimalTestService(t)
-	err := service.postBlockProcess(tr.ctx, nil, [32]byte{}, nil, true)
+	err := service.postBlockProcess(&postBlockProcessConfig{tr.ctx, nil, [32]byte{}, [32]byte{}, nil, true})
 	require.Equal(t, true, IsInvalidBlock(err))
 }
 
@@ -688,7 +688,7 @@ func TestOnBlock_CallNewPayloadAndForkchoiceUpdated(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, r, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, r, postState, false))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, r, [32]byte{}, postState, false}))
 		testState, err = service.cfg.StateGen.StateByRoot(ctx, r)
 		require.NoError(t, err)
 	}
@@ -894,7 +894,7 @@ func Test_validateMergeTransitionBlock(t *testing.T) {
 	cfg.TerminalBlockHash = params.BeaconConfig().ZeroHash
 	params.OverrideBeaconConfig(cfg)
 
-	service, tr := minimalTestService(t, WithProposerIdsCache(cache.NewProposerPayloadIDsCache()))
+	service, tr := minimalTestService(t, WithPayloadIDCache(cache.NewPayloadIDCache()))
 	ctx := tr.ctx
 
 	aHash := common.BytesToHash([]byte("a"))
@@ -1103,12 +1103,15 @@ func TestOnBlock_ProcessBlocksParallel(t *testing.T) {
 		require.NoError(t, service.cfg.ForkChoiceStore.InsertNode(ctx, st, blkRoot))
 		var wg sync.WaitGroup
 		wg.Add(4)
+		var lock sync.Mutex
 		go func() {
 			preState, err := service.getBlockPreState(ctx, wsb1.Block())
 			require.NoError(t, err)
 			postState, err := service.validateStateTransition(ctx, preState, wsb1)
 			require.NoError(t, err)
-			require.NoError(t, service.postBlockProcess(ctx, wsb1, r1, postState, true))
+			lock.Lock()
+			require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb1, r1, [32]byte{}, postState, true}))
+			lock.Unlock()
 			wg.Done()
 		}()
 		go func() {
@@ -1116,7 +1119,9 @@ func TestOnBlock_ProcessBlocksParallel(t *testing.T) {
 			require.NoError(t, err)
 			postState, err := service.validateStateTransition(ctx, preState, wsb2)
 			require.NoError(t, err)
-			require.NoError(t, service.postBlockProcess(ctx, wsb2, r2, postState, true))
+			lock.Lock()
+			require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb2, r2, [32]byte{}, postState, true}))
+			lock.Unlock()
 			wg.Done()
 		}()
 		go func() {
@@ -1124,7 +1129,9 @@ func TestOnBlock_ProcessBlocksParallel(t *testing.T) {
 			require.NoError(t, err)
 			postState, err := service.validateStateTransition(ctx, preState, wsb3)
 			require.NoError(t, err)
-			require.NoError(t, service.postBlockProcess(ctx, wsb3, r3, postState, true))
+			lock.Lock()
+			require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb3, r3, [32]byte{}, postState, true}))
+			lock.Unlock()
 			wg.Done()
 		}()
 		go func() {
@@ -1132,7 +1139,9 @@ func TestOnBlock_ProcessBlocksParallel(t *testing.T) {
 			require.NoError(t, err)
 			postState, err := service.validateStateTransition(ctx, preState, wsb4)
 			require.NoError(t, err)
-			require.NoError(t, service.postBlockProcess(ctx, wsb4, r4, postState, true))
+			lock.Lock()
+			require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb4, r4, [32]byte{}, postState, true}))
+			lock.Unlock()
 			wg.Done()
 		}()
 		wg.Wait()
@@ -1206,7 +1215,7 @@ func TestStore_NoViableHead_FCU(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, root, postState, false))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false}))
 	}
 
 	for i := 6; i < 12; i++ {
@@ -1224,7 +1233,7 @@ func TestStore_NoViableHead_FCU(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, root, postState, false)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false})
 		require.NoError(t, err)
 	}
 
@@ -1243,7 +1252,7 @@ func TestStore_NoViableHead_FCU(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, root, postState, false)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false})
 		require.NoError(t, err)
 	}
 	// Check that we haven't justified the second epoch yet
@@ -1265,7 +1274,7 @@ func TestStore_NoViableHead_FCU(t *testing.T) {
 	postState, err := service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, firstInvalidRoot, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, firstInvalidRoot, postState, false)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, firstInvalidRoot, [32]byte{}, postState, false})
 	require.NoError(t, err)
 	jc = service.cfg.ForkChoiceStore.JustifiedCheckpoint()
 	require.Equal(t, primitives.Epoch(2), jc.Epoch)
@@ -1293,7 +1302,7 @@ func TestStore_NoViableHead_FCU(t *testing.T) {
 	postState, err = service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, root, postState, false)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false})
 	require.ErrorContains(t, "received an INVALID payload from execution engine", err)
 	// Check that forkchoice's head is the last invalid block imported. The
 	// store's headroot is the previous head (since the invalid block did
@@ -1322,7 +1331,7 @@ func TestStore_NoViableHead_FCU(t *testing.T) {
 	postState, err = service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, root, postState, true)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, true})
 	require.NoError(t, err)
 	// Check the newly imported block is head, it justified the right
 	// checkpoint and the node is no longer optimistic
@@ -1384,7 +1393,7 @@ func TestStore_NoViableHead_NewPayload(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, root, postState, false))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false}))
 	}
 
 	for i := 6; i < 12; i++ {
@@ -1402,7 +1411,7 @@ func TestStore_NoViableHead_NewPayload(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, root, postState, false)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false})
 		require.NoError(t, err)
 	}
 
@@ -1422,7 +1431,7 @@ func TestStore_NoViableHead_NewPayload(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, root, postState, false)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false})
 		require.NoError(t, err)
 	}
 	// Check that we haven't justified the second epoch yet
@@ -1444,7 +1453,7 @@ func TestStore_NoViableHead_NewPayload(t *testing.T) {
 	postState, err := service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, firstInvalidRoot, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, firstInvalidRoot, postState, false)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, firstInvalidRoot, [32]byte{}, postState, false})
 	require.NoError(t, err)
 	jc = service.cfg.ForkChoiceStore.JustifiedCheckpoint()
 	require.Equal(t, primitives.Epoch(2), jc.Epoch)
@@ -1500,7 +1509,7 @@ func TestStore_NoViableHead_NewPayload(t *testing.T) {
 	postState, err = service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, root, postState, true)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, true})
 	require.NoError(t, err)
 	// Check the newly imported block is head, it justified the right
 	// checkpoint and the node is no longer optimistic
@@ -1564,7 +1573,7 @@ func TestStore_NoViableHead_Liveness(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, root, postState, false))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false}))
 	}
 
 	for i := 6; i < 12; i++ {
@@ -1583,7 +1592,7 @@ func TestStore_NoViableHead_Liveness(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, root, postState, false)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false})
 		require.NoError(t, err)
 	}
 
@@ -1602,7 +1611,7 @@ func TestStore_NoViableHead_Liveness(t *testing.T) {
 	postState, err := service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, lastValidRoot, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, lastValidRoot, postState, false)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, lastValidRoot, [32]byte{}, postState, false})
 	require.NoError(t, err)
 	// save the post state and the payload Hash of this block since it will
 	// be the LVH
@@ -1629,7 +1638,7 @@ func TestStore_NoViableHead_Liveness(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, invalidRoots[i-13], wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, invalidRoots[i-13], postState, false)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, invalidRoots[i-13], [32]byte{}, postState, false})
 		require.NoError(t, err)
 	}
 	// Check that we have justified the second epoch
@@ -1694,7 +1703,7 @@ func TestStore_NoViableHead_Liveness(t *testing.T) {
 	postState, err = service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-	require.NoError(t, service.postBlockProcess(ctx, wsb, root, postState, true))
+	require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, true}))
 	// Check that the head is still INVALID and the node is still optimistic
 	require.Equal(t, invalidHeadRoot, service.cfg.ForkChoiceStore.CachedHeadRoot())
 	optimistic, err = service.IsOptimistic(ctx)
@@ -1717,7 +1726,7 @@ func TestStore_NoViableHead_Liveness(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, root, postState, true)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, true})
 		require.NoError(t, err)
 		st, err = service.cfg.StateGen.StateByRoot(ctx, root)
 		require.NoError(t, err)
@@ -1743,7 +1752,7 @@ func TestStore_NoViableHead_Liveness(t *testing.T) {
 	postState, err = service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, root, postState, true)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, true})
 	require.NoError(t, err)
 	require.Equal(t, root, service.cfg.ForkChoiceStore.CachedHeadRoot())
 	sjc = service.CurrentJustifiedCheckpt()
@@ -1799,7 +1808,7 @@ func TestNoViableHead_Reboot(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, root, postState, false))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false}))
 	}
 
 	for i := 6; i < 12; i++ {
@@ -1817,7 +1826,7 @@ func TestNoViableHead_Reboot(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, root, postState, false)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false})
 		require.NoError(t, err)
 	}
 
@@ -1836,7 +1845,7 @@ func TestNoViableHead_Reboot(t *testing.T) {
 	postState, err := service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, lastValidRoot, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, lastValidRoot, postState, false)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, lastValidRoot, [32]byte{}, postState, false})
 	require.NoError(t, err)
 	// save the post state and the payload Hash of this block since it will
 	// be the LVH
@@ -1865,7 +1874,7 @@ func TestNoViableHead_Reboot(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, root, postState, false))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false}))
 		require.NoError(t, service.updateJustificationOnBlock(ctx, preState, postState, currStoreJustifiedEpoch))
 		_, err = service.updateFinalizationOnBlock(ctx, preState, postState, currStoreFinalizedEpoch)
 		require.NoError(t, err)
@@ -1936,7 +1945,7 @@ func TestNoViableHead_Reboot(t *testing.T) {
 	rwsb, err := consensusblocks.NewROBlock(wsb)
 	require.NoError(t, err)
 	// We use onBlockBatch here because the valid chain is missing in forkchoice
-	require.NoError(t, service.onBlockBatch(ctx, []consensusblocks.ROBlock{rwsb}))
+	require.NoError(t, service.onBlockBatch(ctx, []consensusblocks.ROBlock{rwsb}, &das.MockAvailabilityStore{}))
 	// Check that the head is now VALID and the node is not optimistic
 	require.Equal(t, genesisRoot, service.ensureRootNotZeros(service.cfg.ForkChoiceStore.CachedHeadRoot()))
 	headRoot, err = service.HeadRoot(ctx)
@@ -1980,7 +1989,7 @@ func TestOnBlock_HandleBlockAttestations(t *testing.T) {
 	postState, err := service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-	require.NoError(t, service.postBlockProcess(ctx, wsb, root, postState, false))
+	require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false}))
 
 	st, err = service.HeadState(ctx)
 	require.NoError(t, err)
@@ -2035,74 +2044,131 @@ func TestFillMissingBlockPayloadId_PrepareAllPayloads(t *testing.T) {
 // Helper function to simulate the block being on time or delayed for proposer
 // boost. It alters the genesisTime tracked by the store.
 func driftGenesisTime(s *Service, slot, delay int64) {
-	offset := slot*int64(params.BeaconConfig().SecondsPerSlot) - delay
+	offset := slot*int64(params.BeaconConfig().SecondsPerSlot) + delay
 	s.SetGenesisTime(time.Unix(time.Now().Unix()-offset, 0))
 }
 
-func Test_commitmentsToCheck(t *testing.T) {
-	windowSlots, err := slots.EpochEnd(params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest)
-	require.NoError(t, err)
-	commits := [][]byte{
-		bytesutil.PadTo([]byte("a"), 48),
-		bytesutil.PadTo([]byte("b"), 48),
-		bytesutil.PadTo([]byte("c"), 48),
-		bytesutil.PadTo([]byte("d"), 48),
-	}
+func TestMissingIndices(t *testing.T) {
 	cases := []struct {
-		name    string
-		commits [][]byte
-		block   func(*testing.T) consensusblocks.ROBlock
-		slot    primitives.Slot
+		name     string
+		expected [][]byte
+		present  []uint64
+		result   map[uint64]struct{}
+		root     [32]byte
+		err      error
 	}{
 		{
-			name: "pre deneb",
-			block: func(t *testing.T) consensusblocks.ROBlock {
-				bb := util.NewBeaconBlockBellatrix()
-				sb, err := consensusblocks.NewSignedBeaconBlock(bb)
-				require.NoError(t, err)
-				rb, err := consensusblocks.NewROBlock(sb)
-				require.NoError(t, err)
-				return rb
-			},
+			name: "zero len",
 		},
 		{
-			name: "commitments within da",
-			block: func(t *testing.T) consensusblocks.ROBlock {
-				d := util.NewBeaconBlockDeneb()
-				d.Block.Body.BlobKzgCommitments = commits
-				d.Block.Slot = 100
-				sb, err := consensusblocks.NewSignedBeaconBlock(d)
-				require.NoError(t, err)
-				rb, err := consensusblocks.NewROBlock(sb)
-				require.NoError(t, err)
-				return rb
-			},
-			commits: commits,
-			slot:    100,
+			name:     "expected exceeds max",
+			expected: fakeCommitments(fieldparams.MaxBlobsPerBlock + 1),
+			err:      errMaxBlobsExceeded,
 		},
 		{
-			name: "commitments outside da",
-			block: func(t *testing.T) consensusblocks.ROBlock {
-				d := util.NewBeaconBlockDeneb()
-				// block is from slot 0, "current slot" is window size +1 (so outside the window)
-				d.Block.Body.BlobKzgCommitments = commits
-				sb, err := consensusblocks.NewSignedBeaconBlock(d)
-				require.NoError(t, err)
-				rb, err := consensusblocks.NewROBlock(sb)
-				require.NoError(t, err)
-				return rb
-			},
-			slot: windowSlots + 1,
+			name:     "first missing",
+			expected: fakeCommitments(fieldparams.MaxBlobsPerBlock),
+			present:  []uint64{1, 2, 3, 4, 5},
+			result:   fakeResult([]uint64{0}),
+		},
+		{
+			name:     "all missing",
+			expected: fakeCommitments(fieldparams.MaxBlobsPerBlock),
+			result:   fakeResult([]uint64{0, 1, 2, 3, 4, 5}),
+		},
+		{
+			name:     "none missing",
+			expected: fakeCommitments(fieldparams.MaxBlobsPerBlock),
+			present:  []uint64{0, 1, 2, 3, 4, 5},
+			result:   fakeResult([]uint64{}),
+		},
+		{
+			name:     "one commitment, missing",
+			expected: fakeCommitments(1),
+			present:  []uint64{},
+			result:   fakeResult([]uint64{0}),
+		},
+		{
+			name:     "3 commitments, 1 missing",
+			expected: fakeCommitments(3),
+			present:  []uint64{1},
+			result:   fakeResult([]uint64{0, 2}),
+		},
+		{
+			name:     "3 commitments, none missing",
+			expected: fakeCommitments(3),
+			present:  []uint64{0, 1, 2},
+			result:   fakeResult([]uint64{}),
+		},
+		{
+			name:     "3 commitments, all missing",
+			expected: fakeCommitments(3),
+			present:  []uint64{},
+			result:   fakeResult([]uint64{0, 1, 2}),
 		},
 	}
+
 	for _, c := range cases {
+		bm, bs := filesystem.NewEphemeralBlobStorageWithMocker(t)
 		t.Run(c.name, func(t *testing.T) {
-			b := c.block(t)
-			co := commitmentsToCheck(b, c.slot)
-			require.Equal(t, len(c.commits), len(co))
-			for i := 0; i < len(c.commits); i++ {
-				require.Equal(t, true, bytes.Equal(c.commits[i], co[i]))
+			require.NoError(t, bm.CreateFakeIndices(c.root, c.present))
+			missing, err := missingIndices(bs, c.root, c.expected)
+			if c.err != nil {
+				require.ErrorIs(t, err, c.err)
+				return
+			}
+			require.NoError(t, err)
+			require.Equal(t, len(c.result), len(missing))
+			for key := range c.result {
+				m, ok := missing[key]
+				require.Equal(t, true, ok)
+				require.Equal(t, c.result[key], m)
 			}
 		})
 	}
 }
+
+func Test_getFCUArgs(t *testing.T) {
+	s, tr := minimalTestService(t)
+	ctx := tr.ctx
+	st, keys := util.DeterministicGenesisState(t, 64)
+	b, err := util.GenerateFullBlock(st, keys, util.DefaultBlockGenConfig(), 1)
+	require.NoError(t, err)
+	wsb, err := consensusblocks.NewSignedBeaconBlock(b)
+	require.NoError(t, err)
+
+	cfg := &postBlockProcessConfig{
+		ctx:            ctx,
+		signed:         wsb,
+		blockRoot:      [32]byte{'a'},
+		postState:      st,
+		isValidPayload: true,
+	}
+	// error branch
+	fcuArgs := &fcuConfig{}
+	err = s.getFCUArgs(cfg, fcuArgs)
+	require.ErrorContains(t, "block does not exist", err)
+
+	// canonical branch
+	cfg.headRoot = cfg.blockRoot
+	fcuArgs = &fcuConfig{}
+	err = s.getFCUArgs(cfg, fcuArgs)
+	require.NoError(t, err)
+	require.Equal(t, cfg.blockRoot, fcuArgs.headRoot)
+}
+
+func fakeCommitments(n int) [][]byte {
+	f := make([][]byte, n)
+	for i := range f {
+		f[i] = make([]byte, 48)
+	}
+	return f
+}
+
+func fakeResult(missing []uint64) map[uint64]struct{} {
+	r := make(map[uint64]struct{}, len(missing))
+	for i := range missing {
+		r[missing[i]] = struct{}{}
+	}
+	return r
+}

beacon-chain/blockchain/receive_attestation.go

@@ -9,7 +9,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
-	"github.com/prysmaticlabs/prysm/v4/config/features"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
@@ -53,15 +52,11 @@ func (s *Service) AttestationTargetState(ctx context.Context, target *ethpb.Chec
 
 // VerifyLmdFfgConsistency verifies that attestation's LMD and FFG votes are consistency to each other.
 func (s *Service) VerifyLmdFfgConsistency(ctx context.Context, a *ethpb.Attestation) error {
-	targetSlot, err := slots.EpochStart(a.Data.Target.Epoch)
+	r, err := s.TargetRootForEpoch([32]byte(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)
 	if err != nil {
 		return err
 	}
-	r, err := s.Ancestor(ctx, a.Data.BeaconBlockRoot, targetSlot)
-	if err != nil {
-		return err
-	}
-	if !bytes.Equal(a.Data.Target.Root, r) {
+	if !bytes.Equal(a.Data.Target.Root, r[:]) {
 		return fmt.Errorf("FFG and LMD votes are not consistent, block root: %#x, target root: %#x, canonical target root: %#x", a.Data.BeaconBlockRoot, a.Data.Target.Root, r)
 	}
 	return nil
@@ -125,36 +120,42 @@ func (s *Service) UpdateHead(ctx context.Context, proposingSlot primitives.Slot)
 	s.cfg.ForkChoiceStore.Lock()
 	defer s.cfg.ForkChoiceStore.Unlock()
 	// This function is only called at 10 seconds or 0 seconds into the slot
-	disparity := params.BeaconNetworkConfig().MaximumGossipClockDisparity
-	if !features.Get().DisableReorgLateBlocks {
-		disparity += reorgLateBlockCountAttestations
-	}
+	disparity := params.BeaconConfig().MaximumGossipClockDisparityDuration()
+	disparity += reorgLateBlockCountAttestations
+
 	s.processAttestations(ctx, disparity)
 
 	processAttsElapsedTime.Observe(float64(time.Since(start).Milliseconds()))
 
 	start = time.Now()
+	// return early if we haven't changed head
 	newHeadRoot, err := s.cfg.ForkChoiceStore.Head(ctx)
 	if err != nil {
 		log.WithError(err).Error("Could not compute head from new attestations")
-		// Fallback to our current head root in the event of a failure.
-		s.headLock.RLock()
-		newHeadRoot = s.headRoot()
-		s.headLock.RUnlock()
+		return
+	}
+	if !s.isNewHead(newHeadRoot) {
+		return
+	}
+	log.WithField("newHeadRoot", fmt.Sprintf("%#x", newHeadRoot)).Debug("Head changed due to attestations")
+	headState, headBlock, err := s.getStateAndBlock(ctx, newHeadRoot)
+	if err != nil {
+		log.WithError(err).Error("could not get head block")
+		return
 	}
 	newAttHeadElapsedTime.Observe(float64(time.Since(start).Milliseconds()))
-
-	changed, err := s.forkchoiceUpdateWithExecution(s.ctx, newHeadRoot, proposingSlot)
-	if err != nil {
-		log.WithError(err).Error("could not update forkchoice")
+	fcuArgs := &fcuConfig{
+		headState:     headState,
+		headRoot:      newHeadRoot,
+		headBlock:     headBlock,
+		proposingSlot: proposingSlot,
 	}
-	if changed {
-		s.headLock.RLock()
-		log.WithFields(logrus.Fields{
-			"oldHeadRoot": fmt.Sprintf("%#x", s.headRoot()),
-			"newHeadRoot": fmt.Sprintf("%#x", newHeadRoot),
-		}).Debug("Head changed due to attestations")
-		s.headLock.RUnlock()
+	fcuArgs.attributes = s.getPayloadAttribute(ctx, headState, proposingSlot, newHeadRoot[:])
+	if fcuArgs.attributes != nil && s.shouldOverrideFCU(newHeadRoot, proposingSlot) {
+		return
+	}
+	if err := s.forkchoiceUpdateWithExecution(s.ctx, fcuArgs); err != nil {
+		log.WithError(err).Error("could not update forkchoice")
 	}
 }
 

beacon-chain/blockchain/receive_attestation_test.go

@@ -36,50 +36,28 @@ func TestAttestationCheckPtState_FarFutureSlot(t *testing.T) {
 	require.ErrorContains(t, "exceeds max allowed value relative to the local clock", err)
 }
 
-func TestVerifyLMDFFGConsistent_NotOK(t *testing.T) {
+func TestVerifyLMDFFGConsistent(t *testing.T) {
 	service, tr := minimalTestService(t)
 	ctx := tr.ctx
 
-	b32 := util.NewBeaconBlock()
-	b32.Block.Slot = 32
-	util.SaveBlock(t, ctx, service.cfg.BeaconDB, b32)
-	r32, err := b32.Block.HashTreeRoot()
+	f := service.cfg.ForkChoiceStore
+	fc := &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	state, r32, err := prepareForkchoiceState(ctx, 32, [32]byte{'a'}, params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, fc, fc)
 	require.NoError(t, err)
-	b33 := util.NewBeaconBlock()
-	b33.Block.Slot = 33
-	b33.Block.ParentRoot = r32[:]
-	util.SaveBlock(t, ctx, service.cfg.BeaconDB, b33)
-	r33, err := b33.Block.HashTreeRoot()
+	require.NoError(t, f.InsertNode(ctx, state, r32))
+
+	state, r33, err := prepareForkchoiceState(ctx, 33, [32]byte{'b'}, r32, params.BeaconConfig().ZeroHash, fc, fc)
 	require.NoError(t, err)
+	require.NoError(t, f.InsertNode(ctx, state, r33))
 
 	wanted := "FFG and LMD votes are not consistent"
 	a := util.NewAttestation()
 	a.Data.Target.Epoch = 1
-	a.Data.Target.Root = []byte{'a'}
+	a.Data.Target.Root = []byte{'c'}
 	a.Data.BeaconBlockRoot = r33[:]
 	require.ErrorContains(t, wanted, service.VerifyLmdFfgConsistency(context.Background(), a))
-}
 
-func TestVerifyLMDFFGConsistent_OK(t *testing.T) {
-	service, tr := minimalTestService(t)
-	ctx := tr.ctx
-
-	b32 := util.NewBeaconBlock()
-	b32.Block.Slot = 32
-	util.SaveBlock(t, ctx, service.cfg.BeaconDB, b32)
-	r32, err := b32.Block.HashTreeRoot()
-	require.NoError(t, err)
-	b33 := util.NewBeaconBlock()
-	b33.Block.Slot = 33
-	b33.Block.ParentRoot = r32[:]
-	util.SaveBlock(t, ctx, service.cfg.BeaconDB, b33)
-	r33, err := b33.Block.HashTreeRoot()
-	require.NoError(t, err)
-
-	a := util.NewAttestation()
-	a.Data.Target.Epoch = 1
 	a.Data.Target.Root = r32[:]
-	a.Data.BeaconBlockRoot = r33[:]
 	err = service.VerifyLmdFfgConsistency(context.Background(), a)
 	require.NoError(t, err, "Could not verify LMD and FFG votes to be consistent")
 }
@@ -134,7 +112,7 @@ func TestService_ProcessAttestationsAndUpdateHead(t *testing.T) {
 	postState, err := service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, tRoot, wsb, postState))
-	require.NoError(t, service.postBlockProcess(ctx, wsb, tRoot, postState, false))
+	require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, tRoot, [32]byte{}, postState, false}))
 	copied, err = service.cfg.StateGen.StateByRoot(ctx, tRoot)
 	require.NoError(t, err)
 	require.Equal(t, 2, fcs.NodeCount())
@@ -190,7 +168,7 @@ func TestService_UpdateHead_NoAtts(t *testing.T) {
 	postState, err := service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, tRoot, wsb, postState))
-	require.NoError(t, service.postBlockProcess(ctx, wsb, tRoot, postState, false))
+	require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, tRoot, [32]byte{}, postState, false}))
 	require.Equal(t, 2, fcs.NodeCount())
 	require.NoError(t, service.cfg.BeaconDB.SaveBlock(ctx, wsb))
 	require.Equal(t, tRoot, service.head.root)

beacon-chain/blockchain/receive_blob.go

@@ -3,21 +3,21 @@ package blockchain
 import (
 	"context"
 
-	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 )
 
 // SendNewBlobEvent sends a message to the BlobNotifier channel that the blob
-// for the blocroot `root` is ready in the database
+// for the block root `root` is ready in the database
 func (s *Service) sendNewBlobEvent(root [32]byte, index uint64) {
-	s.blobNotifiers.forRoot(root) <- index
+	s.blobNotifiers.notifyIndex(root, index)
 }
 
 // ReceiveBlob saves the blob to database and sends the new event
-func (s *Service) ReceiveBlob(ctx context.Context, b *ethpb.BlobSidecar) error {
-	if err := s.cfg.BeaconDB.SaveBlobSidecar(ctx, []*ethpb.BlobSidecar{b}); err != nil {
+func (s *Service) ReceiveBlob(ctx context.Context, b blocks.VerifiedROBlob) error {
+	if err := s.blobStorage.Save(b); err != nil {
 		return err
 	}
 
-	s.sendNewBlobEvent([32]byte(b.BlockRoot), b.Index)
+	s.sendNewBlobEvent(b.BlockRoot(), b.Index)
 	return nil
 }

beacon-chain/blockchain/receive_block.go

@@ -3,6 +3,7 @@ package blockchain
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"time"
 
 	"github.com/pkg/errors"
@@ -11,6 +12,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/helpers"
 	coreTime "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/das"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/v4/config/features"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
@@ -32,8 +34,8 @@ var epochsSinceFinalitySaveHotStateDB = primitives.Epoch(100)
 
 // BlockReceiver interface defines the methods of chain service for receiving and processing new blocks.
 type BlockReceiver interface {
-	ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, blockRoot [32]byte) error
-	ReceiveBlockBatch(ctx context.Context, blocks []blocks.ROBlock) error
+	ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, blockRoot [32]byte, avs das.AvailabilityStore) error
+	ReceiveBlockBatch(ctx context.Context, blocks []blocks.ROBlock, avs das.AvailabilityStore) error
 	HasBlock(ctx context.Context, root [32]byte) bool
 	RecentBlockSlot(root [32]byte) (primitives.Slot, error)
 	BlockBeingSynced([32]byte) bool
@@ -42,7 +44,7 @@ type BlockReceiver interface {
 // BlobReceiver interface defines the methods of chain service for receiving new
 // blobs
 type BlobReceiver interface {
-	ReceiveBlob(context.Context, *ethpb.BlobSidecar) error
+	ReceiveBlob(context.Context, blocks.VerifiedROBlob) error
 }
 
 // SlashingReceiver interface defines the methods of chain service for receiving validated slashing over the wire.
@@ -55,9 +57,14 @@ type SlashingReceiver interface {
 //  1. Validate block, apply state transition and update checkpoints
 //  2. Apply fork choice to the processed block
 //  3. Save latest head info
-func (s *Service) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, blockRoot [32]byte) error {
+func (s *Service) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, blockRoot [32]byte, avs das.AvailabilityStore) error {
 	ctx, span := trace.StartSpan(ctx, "blockChain.ReceiveBlock")
 	defer span.End()
+	// Return early if the block has been synced
+	if s.InForkchoice(blockRoot) {
+		log.WithField("blockRoot", fmt.Sprintf("%#x", blockRoot)).Debug("Ignoring already synced block")
+		return nil
+	}
 	receivedTime := time.Now()
 	s.blockBeingSynced.set(blockRoot)
 	defer s.blockBeingSynced.unset(blockRoot)
@@ -66,6 +73,10 @@ func (s *Service) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySig
 	if err != nil {
 		return err
 	}
+	rob, err := blocks.NewROBlockWithRoot(block, blockRoot)
+	if err != nil {
+		return err
+	}
 
 	preState, err := s.getBlockPreState(ctx, blockCopy.Block())
 	if err != nil {
@@ -100,21 +111,37 @@ func (s *Service) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySig
 	if err := eg.Wait(); err != nil {
 		return err
 	}
-	if err := s.isDataAvailable(ctx, blockRoot, blockCopy); err != nil {
-		return errors.Wrap(err, "could not validate blob data availability")
+	daStartTime := time.Now()
+	if avs != nil {
+		if err := avs.IsDataAvailable(ctx, s.CurrentSlot(), rob); err != nil {
+			return errors.Wrap(err, "could not validate blob data availability (AvailabilityStore.IsDataAvailable)")
+		}
+	} else {
+		if err := s.isDataAvailable(ctx, blockRoot, blockCopy); err != nil {
+			return errors.Wrap(err, "could not validate blob data availability")
+		}
 	}
+	daWaitedTime := time.Since(daStartTime)
+
 	// The rest of block processing takes a lock on forkchoice.
 	s.cfg.ForkChoiceStore.Lock()
 	defer s.cfg.ForkChoiceStore.Unlock()
 	if err := s.savePostStateInfo(ctx, blockRoot, blockCopy, postState); err != nil {
 		return errors.Wrap(err, "could not save post state info")
 	}
-	if err := s.postBlockProcess(ctx, blockCopy, blockRoot, postState, isValidPayload); err != nil {
+	args := &postBlockProcessConfig{
+		ctx:            ctx,
+		signed:         blockCopy,
+		blockRoot:      blockRoot,
+		postState:      postState,
+		isValidPayload: isValidPayload,
+	}
+	if err := s.postBlockProcess(args); err != nil {
 		err := errors.Wrap(err, "could not process block")
 		tracing.AnnotateError(span, err)
 		return err
 	}
-	if coreTime.CurrentEpoch(postState) > currentEpoch {
+	if coreTime.CurrentEpoch(postState) > currentEpoch && s.cfg.ForkChoiceStore.IsCanonical(blockRoot) {
 		headSt, err := s.HeadState(ctx)
 		if err != nil {
 			return errors.Wrap(err, "could not get head state")
@@ -165,7 +192,7 @@ func (s *Service) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySig
 	// Log block sync status.
 	cp = s.cfg.ForkChoiceStore.JustifiedCheckpoint()
 	justified := &ethpb.Checkpoint{Epoch: cp.Epoch, Root: bytesutil.SafeCopyBytes(cp.Root[:])}
-	if err := logBlockSyncStatus(blockCopy.Block(), blockRoot, justified, finalized, receivedTime, uint64(s.genesisTime.Unix())); err != nil {
+	if err := logBlockSyncStatus(blockCopy.Block(), blockRoot, justified, finalized, receivedTime, uint64(s.genesisTime.Unix()), daWaitedTime); err != nil {
 		log.WithError(err).Error("Unable to log block sync status")
 	}
 	// Log payload data
@@ -177,7 +204,8 @@ func (s *Service) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySig
 		log.WithError(err).Error("Unable to log state transition data")
 	}
 
-	chainServiceProcessingTime.Observe(float64(time.Since(receivedTime).Milliseconds()))
+	timeWithoutDaWait := time.Since(receivedTime) - daWaitedTime
+	chainServiceProcessingTime.Observe(float64(timeWithoutDaWait.Milliseconds()))
 
 	return nil
 }
@@ -185,7 +213,7 @@ func (s *Service) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySig
 // ReceiveBlockBatch processes the whole block batch at once, assuming the block batch is linear ,transitioning
 // the state, performing batch verification of all collected signatures and then performing the appropriate
 // actions for a block post-transition.
-func (s *Service) ReceiveBlockBatch(ctx context.Context, blocks []blocks.ROBlock) error {
+func (s *Service) ReceiveBlockBatch(ctx context.Context, blocks []blocks.ROBlock, avs das.AvailabilityStore) error {
 	ctx, span := trace.StartSpan(ctx, "blockChain.ReceiveBlockBatch")
 	defer span.End()
 
@@ -193,7 +221,7 @@ func (s *Service) ReceiveBlockBatch(ctx context.Context, blocks []blocks.ROBlock
 	defer s.cfg.ForkChoiceStore.Unlock()
 
 	// Apply state transition on the incoming newly received block batches, one by one.
-	if err := s.onBlockBatch(ctx, blocks); err != nil {
+	if err := s.onBlockBatch(ctx, blocks, avs); err != nil {
 		err := errors.Wrap(err, "could not process block in batch")
 		tracing.AnnotateError(span, err)
 		return err
@@ -252,11 +280,6 @@ func (s *Service) HasBlock(ctx context.Context, root [32]byte) bool {
 	return s.hasBlockInInitSyncOrDB(ctx, root)
 }
 
-// RecentBlockSlot returns block slot form fork choice store
-func (s *Service) RecentBlockSlot(root [32]byte) (primitives.Slot, error) {
-	return s.cfg.ForkChoiceStore.Slot(root)
-}
-
 // ReceiveAttesterSlashing receives an attester slashing and inserts it to forkchoice
 func (s *Service) ReceiveAttesterSlashing(ctx context.Context, slashing *ethpb.AttesterSlashing) {
 	s.cfg.ForkChoiceStore.Lock()

beacon-chain/blockchain/receive_block_test.go

@@ -7,6 +7,8 @@ import (
 	"time"
 
 	blockchainTesting "github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain/testing"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/das"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/operations/voluntaryexits"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
@@ -130,7 +132,9 @@ func TestService_ReceiveBlock(t *testing.T) {
 			s, tr := minimalTestService(t,
 				WithFinalizedStateAtStartUp(genesis),
 				WithExitPool(voluntaryexits.NewPool()),
-				WithStateNotifier(&blockchainTesting.MockStateNotifier{RecordEvents: true}))
+				WithStateNotifier(&blockchainTesting.MockStateNotifier{RecordEvents: true}),
+				WithTrackedValidatorsCache(cache.NewTrackedValidatorsCache()),
+			)
 
 			beaconDB := tr.db
 			genesisBlockRoot := bytesutil.ToBytes32(nil)
@@ -143,7 +147,7 @@ func TestService_ReceiveBlock(t *testing.T) {
 			require.NoError(t, err)
 			wsb, err := blocks.NewSignedBeaconBlock(tt.args.block)
 			require.NoError(t, err)
-			err = s.ReceiveBlock(ctx, wsb, root)
+			err = s.ReceiveBlock(ctx, wsb, root, nil)
 			if tt.wantedErr != "" {
 				assert.ErrorContains(t, tt.wantedErr, err)
 			} else {
@@ -176,7 +180,7 @@ func TestService_ReceiveBlockUpdateHead(t *testing.T) {
 	go func() {
 		wsb, err := blocks.NewSignedBeaconBlock(b)
 		require.NoError(t, err)
-		require.NoError(t, s.ReceiveBlock(ctx, wsb, root))
+		require.NoError(t, s.ReceiveBlock(ctx, wsb, root, nil))
 		wg.Done()
 	}()
 	wg.Wait()
@@ -240,7 +244,7 @@ func TestService_ReceiveBlockBatch(t *testing.T) {
 			require.NoError(t, err)
 			rwsb, err := blocks.NewROBlock(wsb)
 			require.NoError(t, err)
-			err = s.ReceiveBlockBatch(ctx, []blocks.ROBlock{rwsb})
+			err = s.ReceiveBlockBatch(ctx, []blocks.ROBlock{rwsb}, &das.MockAvailabilityStore{})
 			if tt.wantedErr != "" {
 				assert.ErrorContains(t, tt.wantedErr, err)
 			} else {

beacon-chain/blockchain/service.go

@@ -20,6 +20,7 @@ import (
 	coreTime "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/execution"
 	f "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice"
 	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
@@ -63,6 +64,7 @@ type Service struct {
 	syncComplete         chan struct{}
 	blobNotifiers        *blobNotifierMap
 	blockBeingSynced     *currentlySyncingBlock
+	blobStorage          *filesystem.BlobStorage
 }
 
 // config options for the service.
@@ -71,7 +73,8 @@ type config struct {
 	ChainStartFetcher       execution.ChainStartFetcher
 	BeaconDB                db.HeadAccessDatabase
 	DepositCache            cache.DepositCache
-	ProposerSlotIndexCache  *cache.ProposerPayloadIDsCache
+	PayloadIDCache          *cache.PayloadIDCache
+	TrackedValidatorsCache  *cache.TrackedValidatorsCache
 	AttPool                 attestations.Pool
 	ExitPool                voluntaryexits.PoolManager
 	SlashingPool            slashings.PoolManager
@@ -94,6 +97,35 @@ var ErrMissingClockSetter = errors.New("blockchain Service initialized without a
 type blobNotifierMap struct {
 	sync.RWMutex
 	notifiers map[[32]byte]chan uint64
+	seenIndex map[[32]byte][fieldparams.MaxBlobsPerBlock]bool
+}
+
+// notifyIndex notifies a blob by its index for a given root.
+// It uses internal maps to keep track of seen indices and notifier channels.
+func (bn *blobNotifierMap) notifyIndex(root [32]byte, idx uint64) {
+	if idx >= fieldparams.MaxBlobsPerBlock {
+		return
+	}
+
+	bn.Lock()
+	seen := bn.seenIndex[root]
+	if seen[idx] {
+		bn.Unlock()
+		return
+	}
+	seen[idx] = true
+	bn.seenIndex[root] = seen
+
+	// Retrieve or create the notifier channel for the given root.
+	c, ok := bn.notifiers[root]
+	if !ok {
+		c = make(chan uint64, fieldparams.MaxBlobsPerBlock)
+		bn.notifiers[root] = c
+	}
+
+	bn.Unlock()
+
+	c <- idx
 }
 
 func (bn *blobNotifierMap) forRoot(root [32]byte) chan uint64 {
@@ -110,6 +142,7 @@ func (bn *blobNotifierMap) forRoot(root [32]byte) chan uint64 {
 func (bn *blobNotifierMap) delete(root [32]byte) {
 	bn.Lock()
 	defer bn.Unlock()
+	delete(bn.seenIndex, root)
 	delete(bn.notifiers, root)
 }
 
@@ -126,6 +159,7 @@ func NewService(ctx context.Context, opts ...Option) (*Service, error) {
 	ctx, cancel := context.WithCancel(ctx)
 	bn := &blobNotifierMap{
 		notifiers: make(map[[32]byte]chan uint64),
+		seenIndex: make(map[[32]byte][fieldparams.MaxBlobsPerBlock]bool),
 	}
 	srv := &Service{
 		ctx:                  ctx,
@@ -134,7 +168,7 @@ func NewService(ctx context.Context, opts ...Option) (*Service, error) {
 		checkpointStateCache: cache.NewCheckpointStateCache(),
 		initSyncBlocks:       make(map[[32]byte]interfaces.ReadOnlySignedBeaconBlock),
 		blobNotifiers:        bn,
-		cfg:                  &config{ProposerSlotIndexCache: cache.NewProposerPayloadIDsCache()},
+		cfg:                  &config{},
 		blockBeingSynced:     &currentlySyncingBlock{roots: make(map[[32]byte]struct{})},
 	}
 	for _, opt := range opts {

beacon-chain/blockchain/service_test.go

@@ -25,6 +25,7 @@ import (
 	state_native "github.com/prysmaticlabs/prysm/v4/beacon-chain/state/state-native"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state/stategen"
 	"github.com/prysmaticlabs/prysm/v4/config/features"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	consensusblocks "github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
@@ -99,7 +100,7 @@ func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
 		WithForkChoiceStore(fc),
 		WithAttestationService(attService),
 		WithStateGen(stateGen),
-		WithProposerIdsCache(cache.NewProposerPayloadIDsCache()),
+		WithPayloadIDCache(cache.NewPayloadIDCache()),
 		WithClockSynchronizer(startup.NewClockSynchronizer()),
 	}
 
@@ -445,11 +446,10 @@ func BenchmarkHasBlockForkChoiceStore_DoublyLinkedTree(b *testing.B) {
 	s := &Service{
 		cfg: &config{ForkChoiceStore: doublylinkedtree.New(), BeaconDB: beaconDB},
 	}
-	blk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}}}
+	blk := util.NewBeaconBlock()
 	r, err := blk.Block.HashTreeRoot()
 	require.NoError(b, err)
-	bs := &ethpb.BeaconState{FinalizedCheckpoint: &ethpb.Checkpoint{Root: make([]byte, 32)}, CurrentJustifiedCheckpoint: &ethpb.Checkpoint{Root: make([]byte, 32)}}
-	beaconState, err := state_native.InitializeFromProtoPhase0(bs)
+	beaconState, err := util.NewBeaconState()
 	require.NoError(b, err)
 	require.NoError(b, s.cfg.ForkChoiceStore.InsertNode(ctx, beaconState, r))
 
@@ -514,3 +514,48 @@ func (s *MockClockSetter) SetClock(g *startup.Clock) error {
 	s.G = g
 	return s.Err
 }
+
+func TestNotifyIndex(t *testing.T) {
+	// Initialize a blobNotifierMap
+	bn := &blobNotifierMap{
+		seenIndex: make(map[[32]byte][fieldparams.MaxBlobsPerBlock]bool),
+		notifiers: make(map[[32]byte]chan uint64),
+	}
+
+	// Sample root and index
+	var root [32]byte
+	copy(root[:], "exampleRoot")
+
+	// Test notifying a new index
+	bn.notifyIndex(root, 1)
+	if !bn.seenIndex[root][1] {
+		t.Errorf("Index was not marked as seen")
+	}
+
+	// Test that a new channel is created
+	if _, ok := bn.notifiers[root]; !ok {
+		t.Errorf("Notifier channel was not created")
+	}
+
+	// Test notifying an already seen index
+	bn.notifyIndex(root, 1)
+	if len(bn.notifiers[root]) > 1 {
+		t.Errorf("Notifier channel should not receive multiple messages for the same index")
+	}
+
+	// Test notifying a new index again
+	bn.notifyIndex(root, 2)
+	if !bn.seenIndex[root][2] {
+		t.Errorf("Index was not marked as seen")
+	}
+
+	// Test that the notifier channel receives the index
+	select {
+	case idx := <-bn.notifiers[root]:
+		if idx != 1 {
+			t.Errorf("Received index on channel is incorrect")
+		}
+	default:
+		t.Errorf("Notifier channel did not receive the index")
+	}
+}

beacon-chain/blockchain/setup_test.go

@@ -2,9 +2,11 @@ package blockchain
 
 import (
 	"context"
+	"sync"
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/v4/async/event"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache/depositcache"
 	statefeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
@@ -23,10 +25,13 @@ import (
 
 type mockBeaconNode struct {
 	stateFeed *event.Feed
+	mu        sync.Mutex
 }
 
 // StateFeed mocks the same method in the beacon node.
 func (mbn *mockBeaconNode) StateFeed() *event.Feed {
+	mbn.mu.Lock()
+	defer mbn.mu.Unlock()
 	if mbn.stateFeed == nil {
 		mbn.stateFeed = new(event.Feed)
 	}
@@ -52,7 +57,7 @@ func (mb *mockBroadcaster) BroadcastSyncCommitteeMessage(_ context.Context, _ ui
 	return nil
 }
 
-func (mb *mockBroadcaster) BroadcastBlob(_ context.Context, _ uint64, _ *ethpb.SignedBlobSidecar) error {
+func (mb *mockBroadcaster) BroadcastBlob(_ context.Context, _ uint64, _ *ethpb.BlobSidecar) error {
 	mb.broadcastCalled = true
 	return nil
 }
@@ -110,6 +115,7 @@ func minimalTestService(t *testing.T, opts ...Option) (*Service, *testServiceReq
 		WithAttestationService(req.attSrv),
 		WithBLSToExecPool(req.blsPool),
 		WithDepositCache(dc),
+		WithTrackedValidatorsCache(cache.NewTrackedValidatorsCache()),
 	}
 	// append the variadic opts so they override the defaults by being processed afterwards
 	opts = append(defOpts, opts...)

beacon-chain/blockchain/testing/BUILD.bazel

@@ -17,6 +17,7 @@ go_library(
         "//beacon-chain/core/feed/operation:go_default_library",
         "//beacon-chain/core/feed/state:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/das:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/forkchoice:go_default_library",
         "//beacon-chain/state:go_default_library",
@@ -24,11 +25,11 @@ go_library(
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//consensus-types/blocks:go_default_library",
+        "//consensus-types/forkchoice:go_default_library",
         "//consensus-types/interfaces:go_default_library",
         "//consensus-types/primitives:go_default_library",
         "//encoding/bytesutil:go_default_library",
         "//proto/engine/v1:go_default_library",
-        "//proto/eth/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",

beacon-chain/blockchain/testing/mock.go

@@ -16,6 +16,7 @@ import (
 	opfeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/operation"
 	statefeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/das"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
@@ -23,11 +24,11 @@ import (
 	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
+	forkchoice2 "github.com/prysmaticlabs/prysm/v4/consensus-types/forkchoice"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	enginev1 "github.com/prysmaticlabs/prysm/v4/proto/engine/v1"
-	ethpbv1 "github.com/prysmaticlabs/prysm/v4/proto/eth/v1"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/sirupsen/logrus"
 )
@@ -72,6 +73,8 @@ type ChainService struct {
 	OptimisticRoots             map[[32]byte]bool
 	BlockSlot                   primitives.Slot
 	SyncingRoot                 [32]byte
+	Blobs                       []blocks.VerifiedROBlob
+	TargetRoot                  [32]byte
 }
 
 func (s *ChainService) Ancestor(ctx context.Context, root []byte, slot primitives.Slot) ([]byte, error) {
@@ -206,7 +209,7 @@ func (s *ChainService) ReceiveBlockInitialSync(ctx context.Context, block interf
 }
 
 // ReceiveBlockBatch processes blocks in batches from initial-sync.
-func (s *ChainService) ReceiveBlockBatch(ctx context.Context, blks []blocks.ROBlock) error {
+func (s *ChainService) ReceiveBlockBatch(ctx context.Context, blks []blocks.ROBlock, _ das.AvailabilityStore) error {
 	if s.State == nil {
 		return ErrNilState
 	}
@@ -236,7 +239,7 @@ func (s *ChainService) ReceiveBlockBatch(ctx context.Context, blks []blocks.ROBl
 }
 
 // ReceiveBlock mocks ReceiveBlock method in chain service.
-func (s *ChainService) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, _ [32]byte) error {
+func (s *ChainService) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, _ [32]byte, _ das.AvailabilityStore) error {
 	if s.ReceiveBlockMockErr != nil {
 		return s.ReceiveBlockMockErr
 	}
@@ -318,7 +321,7 @@ func (s *ChainService) PreviousJustifiedCheckpt() *ethpb.Checkpoint {
 }
 
 // ReceiveAttestation mocks ReceiveAttestation method in chain service.
-func (_ *ChainService) ReceiveAttestation(_ context.Context, _ *ethpb.Attestation) error {
+func (*ChainService) ReceiveAttestation(_ context.Context, _ *ethpb.Attestation) error {
 	return nil
 }
 
@@ -398,12 +401,12 @@ func (s *ChainService) RecentBlockSlot([32]byte) (primitives.Slot, error) {
 }
 
 // HeadGenesisValidatorsRoot mocks HeadGenesisValidatorsRoot method in chain service.
-func (_ *ChainService) HeadGenesisValidatorsRoot() [32]byte {
+func (*ChainService) HeadGenesisValidatorsRoot() [32]byte {
 	return [32]byte{}
 }
 
 // VerifyLmdFfgConsistency mocks VerifyLmdFfgConsistency and always returns nil.
-func (_ *ChainService) VerifyLmdFfgConsistency(_ context.Context, a *ethpb.Attestation) error {
+func (*ChainService) VerifyLmdFfgConsistency(_ context.Context, a *ethpb.Attestation) error {
 	if !bytes.Equal(a.Data.BeaconBlockRoot, a.Data.Target.Root) {
 		return errors.New("LMD and FFG miss matched")
 	}
@@ -411,7 +414,7 @@ func (_ *ChainService) VerifyLmdFfgConsistency(_ context.Context, a *ethpb.Attes
 }
 
 // ChainHeads mocks ChainHeads and always return nil.
-func (_ *ChainService) ChainHeads() ([][32]byte, []primitives.Slot) {
+func (*ChainService) ChainHeads() ([][32]byte, []primitives.Slot) {
 	return [][32]byte{
 			bytesutil.ToBytes32(bytesutil.PadTo([]byte("foo"), 32)),
 			bytesutil.ToBytes32(bytesutil.PadTo([]byte("bar"), 32)),
@@ -420,7 +423,7 @@ func (_ *ChainService) ChainHeads() ([][32]byte, []primitives.Slot) {
 }
 
 // HeadPublicKeyToValidatorIndex mocks HeadPublicKeyToValidatorIndex and always return 0 and true.
-func (_ *ChainService) HeadPublicKeyToValidatorIndex(_ [fieldparams.BLSPubkeyLength]byte) (primitives.ValidatorIndex, bool) {
+func (*ChainService) HeadPublicKeyToValidatorIndex(_ [fieldparams.BLSPubkeyLength]byte) (primitives.ValidatorIndex, bool) {
 	return 0, true
 }
 
@@ -484,7 +487,7 @@ func (s *ChainService) UpdateHead(ctx context.Context, slot primitives.Slot) {
 }
 
 // ReceiveAttesterSlashing mocks the same method in the chain service.
-func (s *ChainService) ReceiveAttesterSlashing(context.Context, *ethpb.AttesterSlashing) {}
+func (*ChainService) ReceiveAttesterSlashing(context.Context, *ethpb.AttesterSlashing) {}
 
 // IsFinalized mocks the same method in the chain service.
 func (s *ChainService) IsFinalized(_ context.Context, blockRoot [32]byte) bool {
@@ -573,7 +576,7 @@ func (s *ChainService) InsertNode(ctx context.Context, st state.BeaconState, roo
 }
 
 // ForkChoiceDump mocks the same method in the chain service
-func (s *ChainService) ForkChoiceDump(ctx context.Context) (*ethpbv1.ForkChoiceDump, error) {
+func (s *ChainService) ForkChoiceDump(ctx context.Context) (*forkchoice2.Dump, error) {
 	if s.ForkChoiceStore != nil {
 		return s.ForkChoiceStore.ForkChoiceDump(ctx)
 	}
@@ -597,12 +600,12 @@ func (s *ChainService) ProposerBoost() [32]byte {
 }
 
 // FinalizedBlockHash mocks the same method in the chain service
-func (s *ChainService) FinalizedBlockHash() [32]byte {
+func (*ChainService) FinalizedBlockHash() [32]byte {
 	return [32]byte{}
 }
 
 // UnrealizedJustifiedPayloadBlockHash mocks the same method in the chain service
-func (s *ChainService) UnrealizedJustifiedPayloadBlockHash() [32]byte {
+func (*ChainService) UnrealizedJustifiedPayloadBlockHash() [32]byte {
 	return [32]byte{}
 }
 
@@ -612,6 +615,12 @@ func (c *ChainService) BlockBeingSynced(root [32]byte) bool {
 }
 
 // ReceiveBlob implements the same method in the chain service
-func (*ChainService) ReceiveBlob(_ context.Context, _ *ethpb.BlobSidecar) error {
+func (c *ChainService) ReceiveBlob(_ context.Context, b blocks.VerifiedROBlob) error {
+	c.Blobs = append(c.Blobs, b)
 	return nil
 }
+
+// TargetRootForEpoch mocks the same method in the chain service
+func (c *ChainService) TargetRootForEpoch(_ [32]byte, _ primitives.Epoch) ([32]byte, error) {
+	return c.TargetRoot, nil
+}

beacon-chain/blockchain/tracked_proposer.go

@@ -0,0 +1,27 @@
+package blockchain
+
+import (
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/v4/config/features"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+)
+
+// trackedProposer returns whether the beacon node was informed, via the
+// validators/prepare_proposer endpoint, of the proposer at the given slot.
+// It only returns true if the tracked proposer is present and active.
+func (s *Service) trackedProposer(st state.ReadOnlyBeaconState, slot primitives.Slot) (cache.TrackedValidator, bool) {
+	if features.Get().PrepareAllPayloads {
+		return cache.TrackedValidator{Active: true}, true
+	}
+	id, err := helpers.BeaconProposerIndexAtSlot(s.ctx, st, slot)
+	if err != nil {
+		return cache.TrackedValidator{}, false
+	}
+	val, ok := s.cfg.TrackedValidatorsCache.Validator(id)
+	if !ok {
+		return cache.TrackedValidator{}, false
+	}
+	return val, val.Active
+}

beacon-chain/builder/BUILD.bazel

@@ -15,7 +15,6 @@ go_library(
         "//beacon-chain/cache:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
-        "//config/fieldparams:go_default_library",
         "//consensus-types/interfaces:go_default_library",
         "//consensus-types/primitives:go_default_library",
         "//encoding/bytesutil:go_default_library",

beacon-chain/builder/service.go

@@ -2,7 +2,6 @@ package builder
 
 import (
 	"context"
-	"fmt"
 	"reflect"
 	"time"
 
@@ -11,7 +10,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
-	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
@@ -27,7 +25,7 @@ var ErrNoBuilder = errors.New("builder endpoint not configured")
 
 // BlockBuilder defines the interface for interacting with the block builder
 type BlockBuilder interface {
-	SubmitBlindedBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, blobs []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error)
+	SubmitBlindedBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error)
 	GetHeader(ctx context.Context, slot primitives.Slot, parentHash [32]byte, pubKey [48]byte) (builder.SignedBid, error)
 	RegisterValidator(ctx context.Context, reg []*ethpb.SignedValidatorRegistrationV1) error
 	RegistrationByValidatorID(ctx context.Context, id primitives.ValidatorIndex) (*ethpb.ValidatorRegistrationV1, error)
@@ -90,7 +88,7 @@ func (s *Service) Stop() error {
 }
 
 // SubmitBlindedBlock submits a blinded block to the builder relay network.
-func (s *Service) SubmitBlindedBlock(ctx context.Context, b interfaces.ReadOnlySignedBeaconBlock, blobs []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
+func (s *Service) SubmitBlindedBlock(ctx context.Context, b interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
 	ctx, span := trace.StartSpan(ctx, "builder.SubmitBlindedBlock")
 	defer span.End()
 	start := time.Now()
@@ -100,11 +98,8 @@ func (s *Service) SubmitBlindedBlock(ctx context.Context, b interfaces.ReadOnlyS
 	if s.c == nil {
 		return nil, nil, ErrNoBuilder
 	}
-	if uint64(len(blobs)) > fieldparams.MaxBlobsPerBlock {
-		return nil, nil, fmt.Errorf("blob count %d beyond max limit of %d", len(blobs), fieldparams.MaxBlobsPerBlock)
-	}
 
-	return s.c.SubmitBlindedBlock(ctx, b, blobs)
+	return s.c.SubmitBlindedBlock(ctx, b)
 }
 
 // GetHeader retrieves the header for a given slot and parent hash from the builder relay network.

beacon-chain/builder/service_test.go

@@ -62,7 +62,7 @@ func Test_BuilderMethodsWithouClient(t *testing.T) {
 	_, err = s.GetHeader(context.Background(), 0, [32]byte{}, [48]byte{})
 	assert.ErrorContains(t, ErrNoBuilder.Error(), err)
 
-	_, _, err = s.SubmitBlindedBlock(context.Background(), nil, nil)
+	_, _, err = s.SubmitBlindedBlock(context.Background(), nil)
 	assert.ErrorContains(t, ErrNoBuilder.Error(), err)
 
 	err = s.RegisterValidator(context.Background(), nil)

beacon-chain/builder/testing/mock.go

@@ -45,7 +45,7 @@ func (s *MockBuilderService) Configured() bool {
 }
 
 // SubmitBlindedBlock for mocking.
-func (s *MockBuilderService) SubmitBlindedBlock(_ context.Context, b interfaces.ReadOnlySignedBeaconBlock, _ []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
+func (s *MockBuilderService) SubmitBlindedBlock(_ context.Context, b interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
 	switch b.Version() {
 	case version.Bellatrix:
 		w, err := blocks.WrappedExecutionPayload(s.Payload)

beacon-chain/cache/BUILD.bazel

@@ -25,6 +25,7 @@ go_library(
         "sync_committee_disabled.go",  # keep
         "sync_committee_head_state.go",
         "sync_subnet_ids.go",
+        "tracked_validators.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/v4/beacon-chain/cache",
     visibility = [
@@ -33,6 +34,7 @@ go_library(
         "//tools:__subpackages__",
     ],
     deps = [
+        "//beacon-chain/forkchoice/types:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//cache/lru:go_default_library",
         "//config/fieldparams:go_default_library",
@@ -78,6 +80,7 @@ go_test(
     ],
     embed = [":go_default_library"],
     deps = [
+        "//beacon-chain/forkchoice/types:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/state-native:go_default_library",
         "//config/fieldparams:go_default_library",
@@ -90,6 +93,7 @@ go_test(
         "//testing/util:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_google_gofuzz//:go_default_library",
+        "@com_github_stretchr_testify//require:go_default_library",
         "@org_golang_google_protobuf//proto:go_default_library",
     ],
 )

beacon-chain/cache/attestation_data.go

@@ -1,171 +1,41 @@
 package cache
 
 import (
-	"context"
 	"errors"
-	"fmt"
-	"math"
 	"sync"
-	"time"
 
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
-	"k8s.io/client-go/tools/cache"
+	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 )
 
-var (
-	// Delay parameters
-	minDelay    = float64(10)        // 10 nanoseconds
-	maxDelay    = float64(100000000) // 0.1 second
-	delayFactor = 1.1
+type AttestationConsensusData struct {
+	Slot     primitives.Slot
+	HeadRoot []byte
+	Target   forkchoicetypes.Checkpoint
+	Source   forkchoicetypes.Checkpoint
+}
 
-	// Metrics
-	attestationCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "attestation_cache_miss",
-		Help: "The number of attestation data requests that aren't present in the cache.",
-	})
-	attestationCacheHit = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "attestation_cache_hit",
-		Help: "The number of attestation data requests that are present in the cache.",
-	})
-	attestationCacheSize = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "attestation_cache_size",
-		Help: "The number of attestation data in the attestations cache",
-	})
-)
-
-// ErrAlreadyInProgress appears when attempting to mark a cache as in progress while it is
-// already in progress. The client should handle this error and wait for the in progress
-// data to resolve via Get.
-var ErrAlreadyInProgress = errors.New("already in progress")
-
-// AttestationCache is used to store the cached results of an AttestationData request.
+// AttestationCache stores cached results of AttestationData requests.
 type AttestationCache struct {
-	cache      *cache.FIFO
-	lock       sync.RWMutex
-	inProgress map[string]bool
+	a *AttestationConsensusData
+	sync.RWMutex
 }
 
-// NewAttestationCache initializes the map and underlying cache.
+// NewAttestationCache creates a new instance of AttestationCache.
 func NewAttestationCache() *AttestationCache {
-	return &AttestationCache{
-		cache:      cache.NewFIFO(wrapperToKey),
-		inProgress: make(map[string]bool),
-	}
+	return &AttestationCache{}
 }
 
-// Get waits for any in progress calculation to complete before returning a
-// cached response, if any.
-func (c *AttestationCache) Get(ctx context.Context, req *ethpb.AttestationDataRequest) (*ethpb.AttestationData, error) {
-	if req == nil {
-		return nil, errors.New("nil attestation data request")
-	}
-
-	s, e := reqToKey(req)
-	if e != nil {
-		return nil, e
-	}
-
-	delay := minDelay
-
-	// Another identical request may be in progress already. Let's wait until
-	// any in progress request resolves or our timeout is exceeded.
-	for {
-		if ctx.Err() != nil {
-			return nil, ctx.Err()
-		}
-
-		c.lock.RLock()
-		if !c.inProgress[s] {
-			c.lock.RUnlock()
-			break
-		}
-		c.lock.RUnlock()
-
-		// This increasing backoff is to decrease the CPU cycles while waiting
-		// for the in progress boolean to flip to false.
-		time.Sleep(time.Duration(delay) * time.Nanosecond)
-		delay *= delayFactor
-		delay = math.Min(delay, maxDelay)
-	}
-
-	item, exists, err := c.cache.GetByKey(s)
-	if err != nil {
-		return nil, err
-	}
-
-	if exists && item != nil && item.(*attestationReqResWrapper).res != nil {
-		attestationCacheHit.Inc()
-		return ethpb.CopyAttestationData(item.(*attestationReqResWrapper).res), nil
-	}
-	attestationCacheMiss.Inc()
-	return nil, nil
+// Get retrieves cached attestation data, recording a cache hit or miss. This method is lock free.
+func (c *AttestationCache) Get() *AttestationConsensusData {
+	return c.a
 }
 
-// MarkInProgress a request so that any other similar requests will block on
-// Get until MarkNotInProgress is called.
-func (c *AttestationCache) MarkInProgress(req *ethpb.AttestationDataRequest) error {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	s, e := reqToKey(req)
-	if e != nil {
-		return e
+// Put adds a response to the cache. This method is lock free.
+func (c *AttestationCache) Put(a *AttestationConsensusData) error {
+	if a == nil {
+		return errors.New("attestation cannot be nil")
 	}
-	if c.inProgress[s] {
-		return ErrAlreadyInProgress
-	}
-	c.inProgress[s] = true
+	c.a = a
 	return nil
 }
-
-// MarkNotInProgress will release the lock on a given request. This should be
-// called after put.
-func (c *AttestationCache) MarkNotInProgress(req *ethpb.AttestationDataRequest) error {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	s, e := reqToKey(req)
-	if e != nil {
-		return e
-	}
-	delete(c.inProgress, s)
-	return nil
-}
-
-// Put the response in the cache.
-func (c *AttestationCache) Put(_ context.Context, req *ethpb.AttestationDataRequest, res *ethpb.AttestationData) error {
-	data := &attestationReqResWrapper{
-		req,
-		res,
-	}
-	if err := c.cache.AddIfNotPresent(data); err != nil {
-		return err
-	}
-	trim(c.cache, maxCacheSize)
-
-	attestationCacheSize.Set(float64(len(c.cache.List())))
-	return nil
-}
-
-func wrapperToKey(i interface{}) (string, error) {
-	w, ok := i.(*attestationReqResWrapper)
-	if !ok {
-		return "", errors.New("key is not of type *attestationReqResWrapper")
-	}
-	if w == nil {
-		return "", errors.New("nil wrapper")
-	}
-	if w.req == nil {
-		return "", errors.New("nil wrapper.request")
-	}
-	return reqToKey(w.req)
-}
-
-func reqToKey(req *ethpb.AttestationDataRequest) (string, error) {
-	return fmt.Sprintf("%d", req.Slot), nil
-}
-
-type attestationReqResWrapper struct {
-	req *ethpb.AttestationDataRequest
-	res *ethpb.AttestationData
-}

beacon-chain/cache/attestation_data_test.go

@@ -1,41 +1,53 @@
 package cache_test
 
 import (
-	"context"
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
-	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
-	"github.com/prysmaticlabs/prysm/v4/testing/assert"
-	"google.golang.org/protobuf/proto"
+	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
+	"github.com/stretchr/testify/require"
 )
 
 func TestAttestationCache_RoundTrip(t *testing.T) {
-	ctx := context.Background()
 	c := cache.NewAttestationCache()
 
-	req := &ethpb.AttestationDataRequest{
-		CommitteeIndex: 0,
-		Slot:           1,
+	a := c.Get()
+	require.Nil(t, a)
+
+	insert := &cache.AttestationConsensusData{
+		Slot:     1,
+		HeadRoot: []byte{1},
+		Target: forkchoicetypes.Checkpoint{
+			Epoch: 2,
+			Root:  [32]byte{3},
+		},
+		Source: forkchoicetypes.Checkpoint{
+			Epoch: 4,
+			Root:  [32]byte{5},
+		},
+	}
+	err := c.Put(insert)
+	require.NoError(t, err)
+
+	a = c.Get()
+	require.Equal(t, insert, a)
+
+	insert = &cache.AttestationConsensusData{
+		Slot:     6,
+		HeadRoot: []byte{7},
+		Target: forkchoicetypes.Checkpoint{
+			Epoch: 8,
+			Root:  [32]byte{9},
+		},
+		Source: forkchoicetypes.Checkpoint{
+			Epoch: 10,
+			Root:  [32]byte{11},
+		},
 	}
 
-	response, err := c.Get(ctx, req)
-	assert.NoError(t, err)
-	assert.Equal(t, (*ethpb.AttestationData)(nil), response)
+	err = c.Put(insert)
+	require.NoError(t, err)
 
-	assert.NoError(t, c.MarkInProgress(req))
-
-	res := &ethpb.AttestationData{
-		Target: &ethpb.Checkpoint{Epoch: 5, Root: make([]byte, 32)},
-	}
-
-	assert.NoError(t, c.Put(ctx, req, res))
-	assert.NoError(t, c.MarkNotInProgress(req))
-
-	response, err = c.Get(ctx, req)
-	assert.NoError(t, err)
-
-	if !proto.Equal(response, res) {
-		t.Error("Expected equal protos to return from cache")
-	}
+	a = c.Get()
+	require.Equal(t, insert, a)
 }

beacon-chain/cache/checkpoint_state.go

@@ -42,7 +42,7 @@ func NewCheckpointStateCache() *CheckpointStateCache {
 // StateByCheckpoint fetches state by checkpoint. Returns true with a
 // reference to the CheckpointState info, if exists. Otherwise returns false, nil.
 func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (state.BeaconState, error) {
-	h, err := hash.HashProto(cp)
+	h, err := hash.Proto(cp)
 	if err != nil {
 		return nil, err
 	}
@@ -62,7 +62,7 @@ func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (state.Be
 // AddCheckpointState adds CheckpointState object to the cache. This method also trims the least
 // recently added CheckpointState object if the cache size has ready the max cache size limit.
 func (c *CheckpointStateCache) AddCheckpointState(cp *ethpb.Checkpoint, s state.ReadOnlyBeaconState) error {
-	h, err := hash.HashProto(cp)
+	h, err := hash.Proto(cp)
 	if err != nil {
 		return err
 	}

beacon-chain/cache/common.go

@@ -1,16 +1,9 @@
 package cache
 
 import (
-	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"k8s.io/client-go/tools/cache"
 )
 
-var (
-	// maxCacheSize is 4x of the epoch length for additional cache padding.
-	// Requests should be only accessing committees within defined epoch length.
-	maxCacheSize = uint64(4 * params.BeaconConfig().SlotsPerEpoch)
-)
-
 // trim the FIFO queue to the maxSize.
 func trim(queue *cache.FIFO, maxSize uint64) {
 	for s := uint64(len(queue.ListKeys())); s > maxSize; s-- {

beacon-chain/cache/depositcache/deposits_cache.go

@@ -70,7 +70,7 @@ func New() (*DepositCache, error) {
 // InsertDeposit into the database. If deposit or block number are nil
 // then this method does nothing.
 func (dc *DepositCache) InsertDeposit(ctx context.Context, d *ethpb.Deposit, blockNum uint64, index int64, depositRoot [32]byte) error {
-	ctx, span := trace.StartSpan(ctx, "DepositsCache.InsertDeposit")
+	_, span := trace.StartSpan(ctx, "DepositsCache.InsertDeposit")
 	defer span.End()
 	if d == nil {
 		log.WithFields(logrus.Fields{
@@ -104,7 +104,7 @@ func (dc *DepositCache) InsertDeposit(ctx context.Context, d *ethpb.Deposit, blo
 
 // InsertDepositContainers inserts a set of deposit containers into our deposit cache.
 func (dc *DepositCache) InsertDepositContainers(ctx context.Context, ctrs []*ethpb.DepositContainer) {
-	ctx, span := trace.StartSpan(ctx, "DepositsCache.InsertDepositContainers")
+	_, span := trace.StartSpan(ctx, "DepositsCache.InsertDepositContainers")
 	defer span.End()
 	dc.depositsLock.Lock()
 	defer dc.depositsLock.Unlock()
@@ -124,7 +124,7 @@ func (dc *DepositCache) InsertDepositContainers(ctx context.Context, ctrs []*eth
 // InsertFinalizedDeposits inserts deposits up to eth1DepositIndex (inclusive) into the finalized deposits cache.
 func (dc *DepositCache) InsertFinalizedDeposits(ctx context.Context,
 	eth1DepositIndex int64, _ common.Hash, _ uint64) error {
-	ctx, span := trace.StartSpan(ctx, "DepositsCache.InsertFinalizedDeposits")
+	_, span := trace.StartSpan(ctx, "DepositsCache.InsertFinalizedDeposits")
 	defer span.End()
 	dc.depositsLock.Lock()
 	defer dc.depositsLock.Unlock()
@@ -176,7 +176,7 @@ func (dc *DepositCache) InsertFinalizedDeposits(ctx context.Context,
 
 // AllDepositContainers returns all historical deposit containers.
 func (dc *DepositCache) AllDepositContainers(ctx context.Context) []*ethpb.DepositContainer {
-	ctx, span := trace.StartSpan(ctx, "DepositsCache.AllDepositContainers")
+	_, span := trace.StartSpan(ctx, "DepositsCache.AllDepositContainers")
 	defer span.End()
 	dc.depositsLock.RLock()
 	defer dc.depositsLock.RUnlock()
@@ -220,7 +220,7 @@ func (dc *DepositCache) allDeposits(untilBlk *big.Int) []*ethpb.Deposit {
 // DepositsNumberAndRootAtHeight returns number of deposits made up to blockheight and the
 // root that corresponds to the latest deposit at that blockheight.
 func (dc *DepositCache) DepositsNumberAndRootAtHeight(ctx context.Context, blockHeight *big.Int) (uint64, [32]byte) {
-	ctx, span := trace.StartSpan(ctx, "DepositsCache.DepositsNumberAndRootAtHeight")
+	_, span := trace.StartSpan(ctx, "DepositsCache.DepositsNumberAndRootAtHeight")
 	defer span.End()
 	dc.depositsLock.RLock()
 	defer dc.depositsLock.RUnlock()
@@ -236,7 +236,7 @@ func (dc *DepositCache) DepositsNumberAndRootAtHeight(ctx context.Context, block
 // DepositByPubkey looks through historical deposits and finds one which contains
 // a certain public key within its deposit data.
 func (dc *DepositCache) DepositByPubkey(ctx context.Context, pubKey []byte) (*ethpb.Deposit, *big.Int) {
-	ctx, span := trace.StartSpan(ctx, "DepositsCache.DepositByPubkey")
+	_, span := trace.StartSpan(ctx, "DepositsCache.DepositByPubkey")
 	defer span.End()
 	dc.depositsLock.RLock()
 	defer dc.depositsLock.RUnlock()
@@ -257,7 +257,7 @@ func (dc *DepositCache) DepositByPubkey(ctx context.Context, pubKey []byte) (*et
 
 // FinalizedDeposits returns the finalized deposits trie.
 func (dc *DepositCache) FinalizedDeposits(ctx context.Context) (cache.FinalizedDeposits, error) {
-	ctx, span := trace.StartSpan(ctx, "DepositsCache.FinalizedDeposits")
+	_, span := trace.StartSpan(ctx, "DepositsCache.FinalizedDeposits")
 	defer span.End()
 	dc.depositsLock.RLock()
 	defer dc.depositsLock.RUnlock()
@@ -271,7 +271,7 @@ func (dc *DepositCache) FinalizedDeposits(ctx context.Context) (cache.FinalizedD
 // NonFinalizedDeposits returns the list of non-finalized deposits until the given block number (inclusive).
 // If no block is specified then this method returns all non-finalized deposits.
 func (dc *DepositCache) NonFinalizedDeposits(ctx context.Context, lastFinalizedIndex int64, untilBlk *big.Int) []*ethpb.Deposit {
-	ctx, span := trace.StartSpan(ctx, "DepositsCache.NonFinalizedDeposits")
+	_, span := trace.StartSpan(ctx, "DepositsCache.NonFinalizedDeposits")
 	defer span.End()
 	dc.depositsLock.RLock()
 	defer dc.depositsLock.RUnlock()
@@ -292,7 +292,7 @@ func (dc *DepositCache) NonFinalizedDeposits(ctx context.Context, lastFinalizedI
 
 // PruneProofs removes proofs from all deposits whose index is equal or less than untilDepositIndex.
 func (dc *DepositCache) PruneProofs(ctx context.Context, untilDepositIndex int64) error {
-	ctx, span := trace.StartSpan(ctx, "DepositsCache.PruneProofs")
+	_, span := trace.StartSpan(ctx, "DepositsCache.PruneProofs")
 	defer span.End()
 	dc.depositsLock.Lock()
 	defer dc.depositsLock.Unlock()
@@ -314,7 +314,10 @@ func (dc *DepositCache) PruneProofs(ctx context.Context, untilDepositIndex int64
 
 // Deposits returns the cached internal deposit tree.
 func (fd *FinalizedDeposits) Deposits() cache.MerkleTree {
-	return fd.deposits
+	if fd.deposits != nil {
+		return fd.deposits
+	}
+	return nil
 }
 
 // MerkleTrieIndex represents the last finalized index in

beacon-chain/cache/depositcache/pending_deposits.go

@@ -29,7 +29,7 @@ type PendingDepositsFetcher interface {
 // InsertPendingDeposit into the database. If deposit or block number are nil
 // then this method does nothing.
 func (dc *DepositCache) InsertPendingDeposit(ctx context.Context, d *ethpb.Deposit, blockNum uint64, index int64, depositRoot [32]byte) {
-	ctx, span := trace.StartSpan(ctx, "DepositsCache.InsertPendingDeposit")
+	_, span := trace.StartSpan(ctx, "DepositsCache.InsertPendingDeposit")
 	defer span.End()
 	if d == nil {
 		log.WithFields(logrus.Fields{
@@ -66,7 +66,7 @@ func (dc *DepositCache) PendingDeposits(ctx context.Context, untilBlk *big.Int)
 // PendingContainers returns a list of deposit containers until the given block number
 // (inclusive).
 func (dc *DepositCache) PendingContainers(ctx context.Context, untilBlk *big.Int) []*ethpb.DepositContainer {
-	ctx, span := trace.StartSpan(ctx, "DepositsCache.PendingDeposits")
+	_, span := trace.StartSpan(ctx, "DepositsCache.PendingDeposits")
 	defer span.End()
 	dc.depositsLock.RLock()
 	defer dc.depositsLock.RUnlock()
@@ -90,7 +90,7 @@ func (dc *DepositCache) PendingContainers(ctx context.Context, untilBlk *big.Int
 // RemovePendingDeposit from the database. The deposit is indexed by the
 // Index. This method does nothing if deposit ptr is nil.
 func (dc *DepositCache) RemovePendingDeposit(ctx context.Context, d *ethpb.Deposit) {
-	ctx, span := trace.StartSpan(ctx, "DepositsCache.RemovePendingDeposit")
+	_, span := trace.StartSpan(ctx, "DepositsCache.RemovePendingDeposit")
 	defer span.End()
 
 	if d == nil {
@@ -98,7 +98,7 @@ func (dc *DepositCache) RemovePendingDeposit(ctx context.Context, d *ethpb.Depos
 		return
 	}
 
-	depRoot, err := hash.HashProto(d)
+	depRoot, err := hash.Proto(d)
 	if err != nil {
 		log.WithError(err).Error("Could not remove deposit")
 		return
@@ -109,7 +109,7 @@ func (dc *DepositCache) RemovePendingDeposit(ctx context.Context, d *ethpb.Depos
 
 	idx := -1
 	for i, ctnr := range dc.pendingDeposits {
-		h, err := hash.HashProto(ctnr.Deposit)
+		h, err := hash.Proto(ctnr.Deposit)
 		if err != nil {
 			log.WithError(err).Error("Could not hash deposit")
 			continue
@@ -128,7 +128,7 @@ func (dc *DepositCache) RemovePendingDeposit(ctx context.Context, d *ethpb.Depos
 
 // PrunePendingDeposits removes any deposit which is older than the given deposit merkle tree index.
 func (dc *DepositCache) PrunePendingDeposits(ctx context.Context, merkleTreeIndex int64) {
-	ctx, span := trace.StartSpan(ctx, "DepositsCache.PrunePendingDeposits")
+	_, span := trace.StartSpan(ctx, "DepositsCache.PrunePendingDeposits")
 	defer span.End()
 
 	if merkleTreeIndex == 0 {

beacon-chain/cache/depositsnapshot/deposit_cache_test.go

@@ -769,7 +769,7 @@ func TestFinalizedDeposits_ReturnsTrieCorrectly(t *testing.T) {
 		}
 	}
 
-	ctrs := []*ethpb.DepositContainer{}
+	var ctrs []*ethpb.DepositContainer
 	for i := 0; i < 2000; i++ {
 		ctrs = append(ctrs, generateCtr(uint64(10+(i/2)), int64(i)))
 	}

beacon-chain/cache/depositsnapshot/deposit_fetcher.go

@@ -76,7 +76,7 @@ func (c *Cache) allDeposits(untilBlk *big.Int) []*ethpb.Deposit {
 
 // AllDepositContainers returns all historical deposit containers.
 func (c *Cache) AllDepositContainers(ctx context.Context) []*ethpb.DepositContainer {
-	ctx, span := trace.StartSpan(ctx, "Cache.AllDepositContainers")
+	_, span := trace.StartSpan(ctx, "Cache.AllDepositContainers")
 	defer span.End()
 	c.depositsLock.RLock()
 	defer c.depositsLock.RUnlock()
@@ -101,7 +101,7 @@ func (c *Cache) AllDepositContainers(ctx context.Context) []*ethpb.DepositContai
 // DepositByPubkey looks through historical deposits and finds one which contains
 // a certain public key within its deposit data.
 func (c *Cache) DepositByPubkey(ctx context.Context, pubKey []byte) (*ethpb.Deposit, *big.Int) {
-	ctx, span := trace.StartSpan(ctx, "Cache.DepositByPubkey")
+	_, span := trace.StartSpan(ctx, "Cache.DepositByPubkey")
 	defer span.End()
 	c.depositsLock.RLock()
 	defer c.depositsLock.RUnlock()
@@ -123,7 +123,7 @@ func (c *Cache) DepositByPubkey(ctx context.Context, pubKey []byte) (*ethpb.Depo
 // DepositsNumberAndRootAtHeight returns number of deposits made up to blockheight and the
 // root that corresponds to the latest deposit at that blockheight.
 func (c *Cache) DepositsNumberAndRootAtHeight(ctx context.Context, blockHeight *big.Int) (uint64, [32]byte) {
-	ctx, span := trace.StartSpan(ctx, "Cache.DepositsNumberAndRootAtHeight")
+	_, span := trace.StartSpan(ctx, "Cache.DepositsNumberAndRootAtHeight")
 	defer span.End()
 	c.depositsLock.RLock()
 	defer c.depositsLock.RUnlock()
@@ -141,7 +141,7 @@ func (c *Cache) DepositsNumberAndRootAtHeight(ctx context.Context, blockHeight *
 
 // FinalizedDeposits returns the finalized deposits trie.
 func (c *Cache) FinalizedDeposits(ctx context.Context) (cache.FinalizedDeposits, error) {
-	ctx, span := trace.StartSpan(ctx, "Cache.FinalizedDeposits")
+	_, span := trace.StartSpan(ctx, "Cache.FinalizedDeposits")
 	defer span.End()
 	c.depositsLock.RLock()
 	defer c.depositsLock.RUnlock()
@@ -159,7 +159,7 @@ func (c *Cache) FinalizedDeposits(ctx context.Context) (cache.FinalizedDeposits,
 // NonFinalizedDeposits returns the list of non-finalized deposits until the given block number (inclusive).
 // If no block is specified then this method returns all non-finalized deposits.
 func (c *Cache) NonFinalizedDeposits(ctx context.Context, lastFinalizedIndex int64, untilBlk *big.Int) []*ethpb.Deposit {
-	ctx, span := trace.StartSpan(ctx, "Cache.NonFinalizedDeposits")
+	_, span := trace.StartSpan(ctx, "Cache.NonFinalizedDeposits")
 	defer span.End()
 	c.depositsLock.RLock()
 	defer c.depositsLock.RUnlock()
@@ -180,7 +180,7 @@ func (c *Cache) NonFinalizedDeposits(ctx context.Context, lastFinalizedIndex int
 
 // PruneProofs removes proofs from all deposits whose index is equal or less than untilDepositIndex.
 func (c *Cache) PruneProofs(ctx context.Context, untilDepositIndex int64) error {
-	ctx, span := trace.StartSpan(ctx, "Cache.PruneProofs")
+	_, span := trace.StartSpan(ctx, "Cache.PruneProofs")
 	defer span.End()
 	c.depositsLock.Lock()
 	defer c.depositsLock.Unlock()
@@ -202,7 +202,7 @@ func (c *Cache) PruneProofs(ctx context.Context, untilDepositIndex int64) error
 
 // PrunePendingDeposits removes any deposit which is older than the given deposit merkle tree index.
 func (c *Cache) PrunePendingDeposits(ctx context.Context, merkleTreeIndex int64) {
-	ctx, span := trace.StartSpan(ctx, "Cache.PrunePendingDeposits")
+	_, span := trace.StartSpan(ctx, "Cache.PrunePendingDeposits")
 	defer span.End()
 
 	if merkleTreeIndex == 0 {
@@ -227,7 +227,7 @@ func (c *Cache) PrunePendingDeposits(ctx context.Context, merkleTreeIndex int64)
 // InsertPendingDeposit into the database. If deposit or block number are nil
 // then this method does nothing.
 func (c *Cache) InsertPendingDeposit(ctx context.Context, d *ethpb.Deposit, blockNum uint64, index int64, depositRoot [32]byte) {
-	ctx, span := trace.StartSpan(ctx, "Cache.InsertPendingDeposit")
+	_, span := trace.StartSpan(ctx, "Cache.InsertPendingDeposit")
 	defer span.End()
 	if d == nil {
 		log.WithFields(logrus.Fields{
@@ -282,7 +282,7 @@ func (c *Cache) PendingDeposits(ctx context.Context, untilBlk *big.Int) []*ethpb
 // PendingContainers returns a list of deposit containers until the given block number
 // (inclusive).
 func (c *Cache) PendingContainers(ctx context.Context, untilBlk *big.Int) []*ethpb.DepositContainer {
-	ctx, span := trace.StartSpan(ctx, "Cache.PendingContainers")
+	_, span := trace.StartSpan(ctx, "Cache.PendingContainers")
 	defer span.End()
 	c.depositsLock.RLock()
 	defer c.depositsLock.RUnlock()

beacon-chain/cache/depositsnapshot/deposit_inserter.go

@@ -28,6 +28,9 @@ var (
 func (c *Cache) InsertDeposit(ctx context.Context, d *ethpb.Deposit, blockNum uint64, index int64, depositRoot [32]byte) error {
 	ctx, span := trace.StartSpan(ctx, "Cache.InsertDeposit")
 	defer span.End()
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 	if d == nil {
 		log.WithFields(logrus.Fields{
 			"block":        blockNum,
@@ -60,7 +63,7 @@ func (c *Cache) InsertDeposit(ctx context.Context, d *ethpb.Deposit, blockNum ui
 
 // InsertDepositContainers inserts a set of deposit containers into our deposit cache.
 func (c *Cache) InsertDepositContainers(ctx context.Context, ctrs []*ethpb.DepositContainer) {
-	ctx, span := trace.StartSpan(ctx, "Cache.InsertDepositContainers")
+	_, span := trace.StartSpan(ctx, "Cache.InsertDepositContainers")
 	defer span.End()
 	c.depositsLock.Lock()
 	defer c.depositsLock.Unlock()
@@ -89,6 +92,10 @@ func (c *Cache) InsertFinalizedDeposits(ctx context.Context, eth1DepositIndex in
 	c.depositsLock.Lock()
 	defer c.depositsLock.Unlock()
 
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+
 	depositTrie := c.finalizedDeposits.depositTree
 	insertIndex := int(c.finalizedDeposits.MerkleTrieIndex() + 1)
 

beacon-chain/cache/error.go

@@ -14,4 +14,9 @@ var (
 	errNotSyncCommitteeIndexPosition = errors.New("not syncCommitteeIndexPosition struct")
 	// ErrNotFoundRegistration when validator registration does not exist in cache.
 	ErrNotFoundRegistration = errors.Wrap(ErrNotFound, "no validator registered")
+
+	// ErrAlreadyInProgress appears when attempting to mark a cache as in progress while it is
+	// already in progress. The client should handle this error and wait for the in progress
+	// data to resolve via Get.
+	ErrAlreadyInProgress = errors.New("already in progress")
 )

beacon-chain/cache/payload_id.go

@@ -1,94 +1,63 @@
 package cache
 
 import (
-	"bytes"
 	"sync"
 
-	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
-	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 )
 
-const keyLength = 40
-const vIdLength = 8
-const pIdLength = 8
-const vpIdsLength = vIdLength + pIdLength
+// RootToPayloadIDMap is a map with keys the head root and values the
+// corresponding PayloadID
+type RootToPayloadIDMap map[[32]byte]primitives.PayloadID
 
-// ProposerPayloadIDsCache is a cache of proposer payload IDs.
-// The key is the concatenation of the slot and the block root.
-// The value is the concatenation of the proposer and payload IDs, 8 bytes each.
-type ProposerPayloadIDsCache struct {
-	slotToProposerAndPayloadIDs map[[keyLength]byte][vpIdsLength]byte
-	sync.RWMutex
+// PayloadIDCache is a cache that keeps track of the prepared payload ID for the
+// given slot and with the given head root.
+type PayloadIDCache struct {
+	slotToPayloadID map[primitives.Slot]RootToPayloadIDMap
+	sync.Mutex
 }
 
-// NewProposerPayloadIDsCache creates a new proposer payload IDs cache.
-func NewProposerPayloadIDsCache() *ProposerPayloadIDsCache {
-	return &ProposerPayloadIDsCache{
-		slotToProposerAndPayloadIDs: make(map[[keyLength]byte][vpIdsLength]byte),
-	}
+// NewPayloadIDCache returns a new payload ID cache
+func NewPayloadIDCache() *PayloadIDCache {
+	return &PayloadIDCache{slotToPayloadID: make(map[primitives.Slot]RootToPayloadIDMap)}
 }
 
-// GetProposerPayloadIDs returns the proposer and payload IDs for the given slot and head root to build the block.
-func (f *ProposerPayloadIDsCache) GetProposerPayloadIDs(
-	slot primitives.Slot,
-	r [fieldparams.RootLength]byte,
-) (primitives.ValidatorIndex, [pIdLength]byte, bool) {
-	f.RLock()
-	defer f.RUnlock()
-	ids, ok := f.slotToProposerAndPayloadIDs[idKey(slot, r)]
+// PayloadID returns the payload ID for the given slot and parent block root
+func (p *PayloadIDCache) PayloadID(slot primitives.Slot, root [32]byte) (primitives.PayloadID, bool) {
+	p.Lock()
+	defer p.Unlock()
+	inner, ok := p.slotToPayloadID[slot]
 	if !ok {
-		return 0, [pIdLength]byte{}, false
+		return primitives.PayloadID{}, false
 	}
-	vId := ids[:vIdLength]
-
-	b := ids[vIdLength:]
-	var pId [pIdLength]byte
-	copy(pId[:], b)
-
-	return primitives.ValidatorIndex(bytesutil.BytesToUint64BigEndian(vId)), pId, true
+	pid, ok := inner[root]
+	if !ok {
+		return primitives.PayloadID{}, false
+	}
+	return pid, true
 }
 
-// SetProposerAndPayloadIDs sets the proposer and payload IDs for the given slot and head root to build block.
-func (f *ProposerPayloadIDsCache) SetProposerAndPayloadIDs(
-	slot primitives.Slot,
-	vId primitives.ValidatorIndex,
-	pId [pIdLength]byte,
-	r [fieldparams.RootLength]byte,
-) {
-	f.Lock()
-	defer f.Unlock()
-	var vIdBytes [vIdLength]byte
-	copy(vIdBytes[:], bytesutil.Uint64ToBytesBigEndian(uint64(vId)))
-
-	var bs [vpIdsLength]byte
-	copy(bs[:], append(vIdBytes[:], pId[:]...))
-
-	k := idKey(slot, r)
-	ids, ok := f.slotToProposerAndPayloadIDs[k]
-	// Ok to overwrite if the slot is already set but the cached payload ID is not set.
-	// This combats the re-org case where payload assignment could change at the start of the epoch.
-	var byte8 [vIdLength]byte
-	if !ok || (ok && bytes.Equal(ids[vIdLength:], byte8[:])) {
-		f.slotToProposerAndPayloadIDs[k] = bs
+// SetPayloadID updates the payload ID for the given slot and head root
+// it also prunes older entries in the cache
+func (p *PayloadIDCache) Set(slot primitives.Slot, root [32]byte, pid primitives.PayloadID) {
+	p.Lock()
+	defer p.Unlock()
+	if slot > 1 {
+		p.prune(slot - 2)
 	}
+	inner, ok := p.slotToPayloadID[slot]
+	if !ok {
+		inner = make(RootToPayloadIDMap)
+		p.slotToPayloadID[slot] = inner
+	}
+	inner[root] = pid
 }
 
-// PrunePayloadIDs removes the payload ID entries older than input slot.
-func (f *ProposerPayloadIDsCache) PrunePayloadIDs(slot primitives.Slot) {
-	f.Lock()
-	defer f.Unlock()
-
-	for k := range f.slotToProposerAndPayloadIDs {
-		s := primitives.Slot(bytesutil.BytesToUint64BigEndian(k[:8]))
-		if slot > s {
-			delete(f.slotToProposerAndPayloadIDs, k)
+// Prune prunes old payload IDs. Requires a Lock in the cache
+func (p *PayloadIDCache) prune(slot primitives.Slot) {
+	for key := range p.slotToPayloadID {
+		if key < slot {
+			delete(p.slotToPayloadID, key)
 		}
 	}
 }
-
-func idKey(slot primitives.Slot, r [fieldparams.RootLength]byte) [keyLength]byte {
-	var k [keyLength]byte
-	copy(k[:], append(bytesutil.Uint64ToBytesBigEndian(uint64(slot)), r[:]...))
-	return k
-}

beacon-chain/cache/payload_id_test.go

@@ -8,65 +8,54 @@ import (
 )
 
 func TestValidatorPayloadIDsCache_GetAndSaveValidatorPayloadIDs(t *testing.T) {
-	cache := NewProposerPayloadIDsCache()
+	cache := NewPayloadIDCache()
 	var r [32]byte
-	i, p, ok := cache.GetProposerPayloadIDs(0, r)
+	p, ok := cache.PayloadID(0, r)
 	require.Equal(t, false, ok)
-	require.Equal(t, primitives.ValidatorIndex(0), i)
-	require.Equal(t, [pIdLength]byte{}, p)
+	require.Equal(t, primitives.PayloadID{}, p)
 
 	slot := primitives.Slot(1234)
-	vid := primitives.ValidatorIndex(34234324)
-	pid := [8]byte{1, 2, 3, 3, 7, 8, 7, 8}
+	pid := primitives.PayloadID{1, 2, 3, 3, 7, 8, 7, 8}
 	r = [32]byte{1, 2, 3}
-	cache.SetProposerAndPayloadIDs(slot, vid, pid, r)
-	i, p, ok = cache.GetProposerPayloadIDs(slot, r)
+	cache.Set(slot, r, pid)
+	p, ok = cache.PayloadID(slot, r)
 	require.Equal(t, true, ok)
-	require.Equal(t, vid, i)
 	require.Equal(t, pid, p)
 
 	slot = primitives.Slot(9456456)
-	vid = primitives.ValidatorIndex(6786745)
 	r = [32]byte{4, 5, 6}
-	cache.SetProposerAndPayloadIDs(slot, vid, [pIdLength]byte{}, r)
-	i, p, ok = cache.GetProposerPayloadIDs(slot, r)
+	cache.Set(slot, r, primitives.PayloadID{})
+	p, ok = cache.PayloadID(slot, r)
 	require.Equal(t, true, ok)
-	require.Equal(t, vid, i)
-	require.Equal(t, [pIdLength]byte{}, p)
+	require.Equal(t, primitives.PayloadID{}, p)
 
 	// reset cache without pid
 	slot = primitives.Slot(9456456)
-	vid = primitives.ValidatorIndex(11111)
 	r = [32]byte{7, 8, 9}
 	pid = [8]byte{3, 2, 3, 33, 72, 8, 7, 8}
-	cache.SetProposerAndPayloadIDs(slot, vid, pid, r)
-	i, p, ok = cache.GetProposerPayloadIDs(slot, r)
+	cache.Set(slot, r, pid)
+	p, ok = cache.PayloadID(slot, r)
 	require.Equal(t, true, ok)
-	require.Equal(t, vid, i)
 	require.Equal(t, pid, p)
 
 	// Forked chain
 	r = [32]byte{1, 2, 3}
-	i, p, ok = cache.GetProposerPayloadIDs(slot, r)
+	p, ok = cache.PayloadID(slot, r)
 	require.Equal(t, false, ok)
-	require.Equal(t, primitives.ValidatorIndex(0), i)
-	require.Equal(t, [pIdLength]byte{}, p)
+	require.Equal(t, primitives.PayloadID{}, p)
 
-	// existing pid - no change in cache
+	// existing pid - change the cache
 	slot = primitives.Slot(9456456)
-	vid = primitives.ValidatorIndex(11111)
 	r = [32]byte{7, 8, 9}
-	newPid := [8]byte{1, 2, 3, 33, 72, 8, 7, 1}
-	cache.SetProposerAndPayloadIDs(slot, vid, newPid, r)
-	i, p, ok = cache.GetProposerPayloadIDs(slot, r)
+	newPid := primitives.PayloadID{1, 2, 3, 33, 72, 8, 7, 1}
+	cache.Set(slot, r, newPid)
+	p, ok = cache.PayloadID(slot, r)
 	require.Equal(t, true, ok)
-	require.Equal(t, vid, i)
-	require.Equal(t, pid, p)
+	require.Equal(t, newPid, p)
 
 	// remove cache entry
-	cache.PrunePayloadIDs(slot + 1)
-	i, p, ok = cache.GetProposerPayloadIDs(slot, r)
+	cache.prune(slot + 1)
+	p, ok = cache.PayloadID(slot, r)
 	require.Equal(t, false, ok)
-	require.Equal(t, primitives.ValidatorIndex(0), i)
-	require.Equal(t, [pIdLength]byte{}, p)
+	require.Equal(t, primitives.PayloadID{}, p)
 }

beacon-chain/cache/proposer_indices.go

@@ -7,15 +7,12 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
-	"k8s.io/client-go/tools/cache"
 )
 
 var (
-	// maxProposerIndicesCacheSize defines the max number of proposer indices on per block root basis can cache.
-	// Due to reorgs and long finality, it's good to keep the old cache around for quickly switch over.
-	maxProposerIndicesCacheSize = uint64(8)
-
 	// ProposerIndicesCacheMiss tracks the number of proposerIndices requests that aren't present in the cache.
 	ProposerIndicesCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "proposer_indices_cache_miss",
@@ -28,85 +25,97 @@ var (
 	})
 )
 
-// ProposerIndicesCache is a struct with 1 queue for looking up proposer indices by root.
+// ProposerIndicesCache keeps track of the proposer indices in the next two
+// epochs. It is keyed by the state root of the last epoch before. That is, for
+// blocks during epoch 2, for example slot 65, it will be keyed by the state
+// root of slot 63 (last slot in epoch 1).
+// The cache keeps two sets of indices computed, the "safe" set is computed
+// right before the epoch transition into the current epoch. For example for
+// epoch 2 we will compute this list after importing block 63. The "unsafe"
+// version is computed an epoch in advance, for example for epoch 3, it will be
+// computed after importing block 63.
+//
+// The cache also keeps a map from checkpoints to state roots so that one is
+// able to access the proposer indices list from a checkpoint instead. The
+// checkpoint is the checkpoint for the epoch previous to the requested
+// proposer indices. That is, for a slot in epoch 2 (eg. 65), the checkpoint
+// root would be for slot 32 if present.
 type ProposerIndicesCache struct {
-	proposerIndicesCache *cache.FIFO
-	lock                 sync.RWMutex
+	sync.Mutex
+	indices map[primitives.Epoch]map[[32]byte][fieldparams.SlotsPerEpoch]primitives.ValidatorIndex
+	rootMap map[forkchoicetypes.Checkpoint][32]byte // A map from checkpoint root to state root
 }
 
-// proposerIndicesKeyFn takes the block root as the key to retrieve proposer indices in a given epoch.
-func proposerIndicesKeyFn(obj interface{}) (string, error) {
-	info, ok := obj.(*ProposerIndices)
-	if !ok {
-		return "", ErrNotProposerIndices
-	}
-
-	return key(info.BlockRoot), nil
-}
-
-// NewProposerIndicesCache creates a new proposer indices cache for storing/accessing proposer index assignments of an epoch.
+// NewProposerIndicesCache returns a newly created cache
 func NewProposerIndicesCache() *ProposerIndicesCache {
-	c := &ProposerIndicesCache{}
-	c.Clear()
-	return c
-}
-
-// Clear resets the ProposerIndicesCache to its initial state
-func (c *ProposerIndicesCache) Clear() {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	c.proposerIndicesCache = cache.NewFIFO(proposerIndicesKeyFn)
-}
-
-// AddProposerIndices adds ProposerIndices object to the cache.
-// This method also trims the least recently list if the cache size has ready the max cache size limit.
-func (c *ProposerIndicesCache) AddProposerIndices(p *ProposerIndices) error {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	if err := c.proposerIndicesCache.AddIfNotPresent(p); err != nil {
-		return err
+	return &ProposerIndicesCache{
+		indices: make(map[primitives.Epoch]map[[32]byte][fieldparams.SlotsPerEpoch]primitives.ValidatorIndex),
+		rootMap: make(map[forkchoicetypes.Checkpoint][32]byte),
 	}
-	trim(c.proposerIndicesCache, maxProposerIndicesCacheSize)
-	return nil
 }
 
-// HasProposerIndices returns the proposer indices of a block root seed.
-func (c *ProposerIndicesCache) HasProposerIndices(r [32]byte) (bool, error) {
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	_, exists, err := c.proposerIndicesCache.GetByKey(key(r))
-	if err != nil {
-		return false, err
+// ProposerIndices returns the proposer indices (safe) for the given root
+func (p *ProposerIndicesCache) ProposerIndices(epoch primitives.Epoch, root [32]byte) ([fieldparams.SlotsPerEpoch]primitives.ValidatorIndex, bool) {
+	p.Lock()
+	defer p.Unlock()
+	inner, ok := p.indices[epoch]
+	if !ok {
+		ProposerIndicesCacheMiss.Inc()
+		return [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex{}, false
 	}
-	return exists, nil
-}
-
-// ProposerIndices returns the proposer indices of a block root seed.
-func (c *ProposerIndicesCache) ProposerIndices(r [32]byte) ([]primitives.ValidatorIndex, error) {
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	obj, exists, err := c.proposerIndicesCache.GetByKey(key(r))
-	if err != nil {
-		return nil, err
-	}
-
+	indices, exists := inner[root]
 	if exists {
 		ProposerIndicesCacheHit.Inc()
 	} else {
 		ProposerIndicesCacheMiss.Inc()
-		return nil, nil
 	}
+	return indices, exists
+}
 
-	item, ok := obj.(*ProposerIndices)
+// Prune resets the ProposerIndicesCache to its initial state
+func (p *ProposerIndicesCache) Prune(epoch primitives.Epoch) {
+	p.Lock()
+	defer p.Unlock()
+	for key := range p.indices {
+		if key < epoch {
+			delete(p.indices, key)
+		}
+	}
+	for key := range p.rootMap {
+		if key.Epoch+1 < epoch {
+			delete(p.rootMap, key)
+		}
+	}
+}
+
+// Set sets the proposer indices for the given root as key
+func (p *ProposerIndicesCache) Set(epoch primitives.Epoch, root [32]byte, indices [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex) {
+	p.Lock()
+	defer p.Unlock()
+
+	inner, ok := p.indices[epoch]
 	if !ok {
-		return nil, ErrNotProposerIndices
+		inner = make(map[[32]byte][fieldparams.SlotsPerEpoch]primitives.ValidatorIndex)
+		p.indices[epoch] = inner
 	}
-
-	return item.ProposerIndices, nil
+	inner[root] = indices
 }
 
-// Len returns the number of keys in the underlying cache.
-func (c *ProposerIndicesCache) Len() int {
-	return len(c.proposerIndicesCache.ListKeys())
+// SetCheckpoint updates the map from checkpoints to state roots
+func (p *ProposerIndicesCache) SetCheckpoint(c forkchoicetypes.Checkpoint, root [32]byte) {
+	p.Lock()
+	defer p.Unlock()
+	p.rootMap[c] = root
+}
+
+// IndicesFromCheckpoint returns the proposer indices from a checkpoint rather than the state root
+func (p *ProposerIndicesCache) IndicesFromCheckpoint(c forkchoicetypes.Checkpoint) ([fieldparams.SlotsPerEpoch]primitives.ValidatorIndex, bool) {
+	p.Lock()
+	emptyIndices := [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex{}
+	root, ok := p.rootMap[c]
+	p.Unlock()
+	if !ok {
+		return emptyIndices, ok
+	}
+	return p.ProposerIndices(c.Epoch+1, root)
 }

beacon-chain/cache/proposer_indices_disabled.go

@@ -3,7 +3,26 @@
 // This file is used in fuzzer builds to bypass proposer indices caches.
 package cache
 
-import "github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+)
+
+var (
+	// ProposerIndicesCacheMiss tracks the number of proposerIndices requests that aren't present in the cache.
+	ProposerIndicesCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "proposer_indices_cache_miss",
+		Help: "The number of proposer indices requests that aren't present in the cache.",
+	})
+	// ProposerIndicesCacheHit tracks the number of proposerIndices requests that are in the cache.
+	ProposerIndicesCacheHit = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "proposer_indices_cache_hit",
+		Help: "The number of proposer indices requests that are present in the cache.",
+	})
+)
 
 // FakeProposerIndicesCache is a struct with 1 queue for looking up proposer indices by root.
 type FakeProposerIndicesCache struct {
@@ -14,26 +33,31 @@ func NewProposerIndicesCache() *FakeProposerIndicesCache {
 	return &FakeProposerIndicesCache{}
 }
 
-// AddProposerIndices adds ProposerIndices object to the cache.
-// This method also trims the least recently list if the cache size has ready the max cache size limit.
-func (c *FakeProposerIndicesCache) AddProposerIndices(p *ProposerIndices) error {
-	return nil
+// ProposerIndices is a stub.
+func (c *FakeProposerIndicesCache) ProposerIndices(_ primitives.Epoch, _ [32]byte) ([fieldparams.SlotsPerEpoch]primitives.ValidatorIndex, bool) {
+	return [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex{}, false
 }
 
-// ProposerIndices returns the proposer indices of a block root seed.
-func (c *FakeProposerIndicesCache) ProposerIndices(r [32]byte) ([]primitives.ValidatorIndex, error) {
-	return nil, nil
+// UnsafeProposerIndices is a stub.
+func (c *FakeProposerIndicesCache) UnsafeProposerIndices(_ primitives.Epoch, _ [32]byte) ([fieldparams.SlotsPerEpoch]primitives.ValidatorIndex, bool) {
+	return [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex{}, false
 }
 
-// HasProposerIndices returns the proposer indices of a block root seed.
-func (c *FakeProposerIndicesCache) HasProposerIndices(r [32]byte) (bool, error) {
-	return false, nil
+// Prune is a stub.
+func (p *FakeProposerIndicesCache) Prune(epoch primitives.Epoch) {}
+
+// Set is a stub.
+func (p *FakeProposerIndicesCache) Set(epoch primitives.Epoch, root [32]byte, indices [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex) {
 }
 
-func (c *FakeProposerIndicesCache) Len() int {
-	return 0
+// SetUnsafe is a stub.
+func (p *FakeProposerIndicesCache) SetUnsafe(epoch primitives.Epoch, root [32]byte, indices [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex) {
 }
 
-// Clear is a stub.
-func (c *FakeProposerIndicesCache) Clear() {
+// SetCheckpoint is a stub.
+func (p *FakeProposerIndicesCache) SetCheckpoint(c forkchoicetypes.Checkpoint, root [32]byte) {}
+
+// IndicesFromCheckpoint is a stub.
+func (p *FakeProposerIndicesCache) IndicesFromCheckpoint(_ forkchoicetypes.Checkpoint) ([fieldparams.SlotsPerEpoch]primitives.ValidatorIndex, bool) {
+	return [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex{}, false
 }

beacon-chain/cache/proposer_indices_test.go

@@ -3,72 +3,104 @@
 package cache
 
 import (
-	"strconv"
 	"testing"
 
+	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
-	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
-	"github.com/prysmaticlabs/prysm/v4/testing/assert"
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
 )
 
-func TestProposerKeyFn_OK(t *testing.T) {
-	item := &ProposerIndices{
-		BlockRoot:       [32]byte{'A'},
-		ProposerIndices: []primitives.ValidatorIndex{1, 2, 3, 4, 5},
-	}
-
-	k, err := proposerIndicesKeyFn(item)
-	require.NoError(t, err)
-	assert.Equal(t, key(item.BlockRoot), k)
-}
-
-func TestProposerKeyFn_InvalidObj(t *testing.T) {
-	_, err := proposerIndicesKeyFn("bad")
-	assert.Equal(t, ErrNotProposerIndices, err)
-}
-
-func TestProposerCache_AddProposerIndicesList(t *testing.T) {
+func TestProposerCache_Set(t *testing.T) {
 	cache := NewProposerIndicesCache()
 	bRoot := [32]byte{'A'}
-	indices, err := cache.ProposerIndices(bRoot)
-	require.NoError(t, err)
-	if indices != nil {
-		t.Error("Expected committee count not to exist in empty cache")
-	}
-	has, err := cache.HasProposerIndices(bRoot)
-	require.NoError(t, err)
-	assert.Equal(t, false, has)
-	require.NoError(t, cache.AddProposerIndices(&ProposerIndices{
-		ProposerIndices: indices,
-		BlockRoot:       bRoot,
-	}))
+	indices, ok := cache.ProposerIndices(0, bRoot)
+	require.Equal(t, false, ok)
+	emptyIndices := [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex{}
+	require.Equal(t, indices, emptyIndices, "Expected committee count not to exist in empty cache")
+	emptyIndices[0] = 1
+	cache.Set(0, bRoot, emptyIndices)
 
-	received, err := cache.ProposerIndices(bRoot)
-	require.NoError(t, err)
-	assert.DeepEqual(t, received, indices)
-	has, err = cache.HasProposerIndices(bRoot)
-	require.NoError(t, err)
-	assert.Equal(t, true, has)
+	received, ok := cache.ProposerIndices(0, bRoot)
+	require.Equal(t, true, ok)
+	require.Equal(t, received, emptyIndices)
 
-	item := &ProposerIndices{BlockRoot: [32]byte{'B'}, ProposerIndices: []primitives.ValidatorIndex{1, 2, 3, 4, 5, 6}}
-	require.NoError(t, cache.AddProposerIndices(item))
-
-	received, err = cache.ProposerIndices(item.BlockRoot)
-	require.NoError(t, err)
-	assert.DeepEqual(t, item.ProposerIndices, received)
-	has, err = cache.HasProposerIndices(bRoot)
-	require.NoError(t, err)
-	assert.Equal(t, true, has)
+	newRoot := [32]byte{'B'}
+	copy(emptyIndices[3:], []primitives.ValidatorIndex{1, 2, 3, 4, 5, 6})
+	cache.Set(0, newRoot, emptyIndices)
 
+	received, ok = cache.ProposerIndices(0, newRoot)
+	require.Equal(t, true, ok)
+	require.Equal(t, emptyIndices, received)
 }
 
-func TestProposerCache_CanRotate(t *testing.T) {
+func TestProposerCache_CheckpointAndPrune(t *testing.T) {
 	cache := NewProposerIndicesCache()
-	for i := 0; i < int(maxProposerIndicesCacheSize)+1; i++ {
-		s := []byte(strconv.Itoa(i))
-		item := &ProposerIndices{BlockRoot: bytesutil.ToBytes32(s)}
-		require.NoError(t, cache.AddProposerIndices(item))
+	indices := [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex{}
+	root := [32]byte{'a'}
+	cpRoot := [32]byte{'b'}
+	copy(indices[3:], []primitives.ValidatorIndex{1, 2, 3, 4, 5, 6})
+	for i := 1; i < 10; i++ {
+		cache.Set(primitives.Epoch(i), root, indices)
+		cache.SetCheckpoint(forkchoicetypes.Checkpoint{Epoch: primitives.Epoch(i - 1), Root: cpRoot}, root)
 	}
-	assert.Equal(t, int(maxProposerIndicesCacheSize), cache.Len())
+	received, ok := cache.ProposerIndices(1, root)
+	require.Equal(t, true, ok)
+	require.Equal(t, indices, received)
+
+	received, ok = cache.ProposerIndices(4, root)
+	require.Equal(t, true, ok)
+	require.Equal(t, indices, received)
+
+	received, ok = cache.ProposerIndices(9, root)
+	require.Equal(t, true, ok)
+	require.Equal(t, indices, received)
+
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 0, Root: cpRoot})
+	require.Equal(t, true, ok)
+	require.Equal(t, indices, received)
+
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 3, Root: cpRoot})
+	require.Equal(t, true, ok)
+	require.Equal(t, indices, received)
+
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 4, Root: cpRoot})
+	require.Equal(t, true, ok)
+	require.Equal(t, indices, received)
+
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 8, Root: cpRoot})
+	require.Equal(t, true, ok)
+	require.Equal(t, indices, received)
+
+	cache.Prune(5)
+
+	emptyIndices := [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex{}
+	received, ok = cache.ProposerIndices(1, root)
+	require.Equal(t, false, ok)
+	require.Equal(t, emptyIndices, received)
+
+	received, ok = cache.ProposerIndices(4, root)
+	require.Equal(t, false, ok)
+	require.Equal(t, emptyIndices, received)
+
+	received, ok = cache.ProposerIndices(9, root)
+	require.Equal(t, true, ok)
+	require.Equal(t, indices, received)
+
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 0, Root: cpRoot})
+	require.Equal(t, false, ok)
+	require.Equal(t, emptyIndices, received)
+
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 3, Root: cpRoot})
+	require.Equal(t, false, ok)
+	require.Equal(t, emptyIndices, received)
+
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 4, Root: cpRoot})
+	require.Equal(t, true, ok)
+	require.Equal(t, indices, received)
+
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 8, Root: cpRoot})
+	require.Equal(t, true, ok)
+	require.Equal(t, indices, received)
+
 }

beacon-chain/cache/skip_slot_cache.go

@@ -15,6 +15,11 @@ import (
 )
 
 var (
+	// Delay parameters
+	minDelay    = float64(10)        // 10 nanoseconds
+	maxDelay    = float64(100000000) // 0.1 second
+	delayFactor = 1.1
+
 	// Metrics
 	skipSlotCacheHit = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "skip_slot_cache_hit",