Ignore blocks if their parent is older than justified

* scaffolding for verification package

* WIP blob verification methods

* lock wrapper for safer forkchoice sharing

* more solid cache and verification designs; adding tests

* more test coverage, adding missing cache files

* clearer func name

* remove forkchoice borrower (it's in another PR)

* revert temporary interface experiment

* lint

* nishant feedback

* add comments with spec text to all verifications

* some comments on public methods

* invert confusing verification name

* deep source

* remove cache from ProposerCache + gaz

* more consistently early return on error paths

* messed up the test with the wrong config value

* terence naming feedback

* tests on BeginsAt

* lint

* deep source...

* name errors after failure, not expectation

* deep sooource

* check len()==0 instead of nil so empty lists work

* update test for EIP-7044

---------

Co-authored-by: Kasey Kirkham <kasey@users.noreply.github.com>

.bazelrc

@@ -27,6 +27,7 @@ build:minimal --@io_bazel_rules_go//go/config:tags=minimal
 # Release flags
 build:release --compilation_mode=opt
 build:release --stamp
+build:release --define pgo_enabled=1
 
 # Build binary with cgo symbolizer for debugging / profiling.
 build:cgo_symbolizer --copt=-g

.bazelversion

@@ -1 +1 @@
-6.3.2
+6.4.0

.buildkite-bazelrc

@@ -9,8 +9,8 @@
 #build:remote-cache --strategy=Genrule=standalone
 
 # Prysm specific remote-cache properties.
-#build:remote-cache --disk_cache=
-build:remote-cache --remote_download_toplevel
+build:remote-cache --remote_download_minimal
+build:remote-cache --experimental_remote_build_event_upload=minimal
 build:remote-cache --remote_cache=grpc://bazel-remote-cache:9092
 # Does not work with rules_oci. See https://github.com/bazel-contrib/rules_oci/issues/292
 #build:remote-cache --experimental_remote_downloader=grpc://bazel-remote-cache:9092
@@ -29,7 +29,10 @@ build --experimental_use_hermetic_linux_sandbox
 # Import workspace options.
 import %workspace%/.bazelrc
 
-startup --host_jvm_args=-Xmx4g --host_jvm_args=-Xms2g
+# Enable blake3 once it is supported in remote cache. See: https://github.com/buchgr/bazel-remote/issues/710
+# startup --digest_function=blake3
+
+startup --host_jvm_args=-Xmx8g --host_jvm_args=-Xms4g
 build --experimental_strict_action_env
 build --sandbox_tmpfs_path=/tmp
 build --verbose_failures
@@ -39,6 +42,7 @@ build --curses=no --color=no
 build --keep_going
 build --test_output=errors
 build --flaky_test_attempts=5
+build --build_runfile_links=false # Only build runfile symlink forest when required by local action, test, or run command.
 # Disabled race detection due to unstable test results under constrained environment build kite
 # build --features=race
 

.github/workflows/go.yml

@@ -5,6 +5,8 @@ on:
     branches: [ master ]
   pull_request:
     branches: [ '*' ]
+  merge_group: 
+    types: [checks_requested]
 
 jobs:
   formatting:
@@ -34,7 +36,7 @@ jobs:
         run: | # https://github.com/securego/gosec/issues/469
           export PATH=$PATH:$(go env GOPATH)/bin
           go install github.com/securego/gosec/v2/cmd/gosec@v2.15.0
-          gosec -exclude=G307 -exclude-dir=crypto/bls/herumi ./...
+          gosec -exclude-generated -exclude=G307 -exclude-dir=crypto/bls/herumi ./...
 
   lint:
     name: Lint

BUILD.bazel

@@ -144,6 +144,11 @@ config_setting(
     values = {"define": "coverage_enabled=1"},
 )
 
+config_setting(
+    name = "pgo_enabled",
+    values = {"define": "pgo_enabled=1"},
+)
+
 common_files = {
     "//:LICENSE.md": "LICENSE.md",
     "//:README.md": "README.md",

TERMS_OF_SERVICE.md

@@ -1,45 +1,53 @@
-## Terms of Use
+# Terms of Use
+Effective as of November 2, 2023
 
-Effective as of Oct 14, 2020
+By downloading, accessing or using the Prysm implementation (“Prysm”), you (referenced herein as “you” or the “user”) certify that you have read and agreed to the terms and conditions below (the “Terms”) which form a binding contract between you and Offchain Labs, Inc. (as successor in interest to Prysmatic Labs LLC) (referenced herein as “Offchain Labs”, “we” or “us”). If you do not agree to the Terms, do not download or use Prysm.  Additionally, the Terms of Use available at https://arbitrum.io/tos (or any successor site, the “OCL Terms of Use”) are hereby incorporated by reference into these Terms.  In the event of any conflict between provisions set forth herein and those set forth in the OCL Terms of Use, the provisions set forth herein shall control.
 
-By downloading, accessing or using the Prysm implementation (“Prysm”), you (referenced herein as “you” or the “user”) certify that you have read and agreed to the terms and conditions below (the “Terms”) which form a binding contract between you and Prysmatic Labs (referenced herein as “we” or “us”). If you do not agree to the Terms, do not download or use Prysm.
+## About Prysm
 
-### About Prysm
-Prysm is a client implementation for Ethereum consensus protocol for a proof-of-stake blockchain. To participate in the network, a user must send ETH from the Eth1.0 chain into a validator deposit contract, which will queue in the user as a validator in the system. Validators participate in proposing and voting on blocks in the protocol, and the network applies rewards/penalties based on their behavior. A detailed set of installation and usage instructions as well as breakdowns of each individual component are available in the official documentation portal, however, we do not warrant the accuracy, completeness or usefulness of this documentation. Any reliance you place on such information is strictly at your own risk.
+Prysm is a client implementation for the Ethereum blockchain’s consensus protocol. To participate in the network, a user must send ETH from the Ethereum mainnet blockchain to a validator deposit smart contract on Ethereum mainnet. Validators participate in proposing and voting on blocks in the protocol, and the network applies rewards/penalties based on their behavior. A detailed set of installation and usage instructions as well as breakdowns of each individual component are available in the official documentation portal, however, we do not warrant the accuracy, completeness or usefulness of this documentation. Any reliance you place on such information is strictly at your own risk.
 
-### Licensing Terms
-Prysm is a fully open-source software program licensed pursuant to the GNU General Public License v3.0.
+## Licensing Terms
+Prysm is an open-source software program licensed pursuant to the GNU General Public License v3.0.
+The Offchain Labs name, the term “Prysm” and all related names, logos, product and service names, designs and slogans are trademarks of Offchain Labs or its affiliates and/or licensors. You must not use such marks without our prior written permission.
+PLEASE READ THESE TERMS CAREFULLY, AS THE OCL TERMS OF USE INCORPORATED BY REFERENCE HEREIN CONTAIN AN AGREEMENT TO ARBITRATE AND OTHER IMPORTANT INFORMATION REGARDING YOUR LEGAL RIGHTS, REMEDIES, AND OBLIGATIONS. THE AGREEMENT TO ARBITRATE REQUIRES (WITH LIMITED EXCEPTION) THAT YOU SUBMIT CLAIMS YOU HAVE AGAINST US TO BINDING AND FINAL ARBITRATION, AND FURTHER (1) YOU WILL ONLY BE PERMITTED TO PURSUE CLAIMS AGAINST OFFCHAIN LABS ON AN INDIVIDUAL BASIS, NOT AS A PLAINTIFF OR CLASS MEMBER IN ANY CLASS OR REPRESENTATIVE ACTION OR PROCEEDING, (2) YOU WILL ONLY BE PERMITTED TO SEEK RELIEF (INCLUDING MONETARY, INJUNCTIVE, AND DECLARATORY RELIEF) ON AN INDIVIDUAL BASIS, AND (3) YOU MAY NOT BE ABLE TO HAVE ANY CLAIMS YOU HAVE AGAINST US RESOLVED BY A JURY OR IN A COURT OF LAW.
 
-The Prysmatic Labs name, the term “Prysm” and all related names, logos, product and service names, designs and slogans are trademarks of Prysmatic Labs or its affiliates and/or licensors. You must not use such marks without our prior written permission.
+## Risks of Operating Prysm
 
-### Risks of Operating Prysm
-The use of Prysm and acting as a validator on the Ethereum network can lead to loss of money. Ethereum is still an experimental system and ETH remains a risky investment. You alone are responsible for your actions on Prysm including the security of your ETH and meeting any applicable minimum system requirements.
+The use of Prysm and acting as a validator on the Ethereum network can lead to loss of money, tokens and value. Ethereum is still an experimental system and ETH remains a risky investment. You alone are responsible for your actions on Prysm, including the security of your ETH and meeting any applicable minimum system requirements.
 
 Use of Prysm and the ability to receive rewards or penalties may be affected at any time by mistakes made by the user or other users, software problems such as bugs, errors, incorrectly constructed transactions, unsafe cryptographic libraries or malware affecting the network, technical failures in the hardware of a user, security problems experienced by a user and/or actions or inactions of third parties and/or events experienced by third parties, among other risks. We cannot and do not guarantee that any user of Prysm will make money, that the Prysm network will operate in accordance with the documentation or that transactions will be effective or secure.
 
-We make no claims that Prysm is appropriate or permitted for use in any specific jurisdiction. Access to Prysm may not be legal by certain persons or in certain jurisdictions or countries. If you access Prysm, you do so on your own initiative and are responsible for compliance with local laws.
+YOU ACKNOWLEDGE THAT WE ARE NOT RESPONSIBLE FOR ANY RISKS ASSOCIATED WITH YOUR USE OF PRYSM, AND CANNOT BE HELD LIABLE FOR ANY RESULTING LOSSES THAT YOU EXPERIENCE WHILE ACCESSING OR USING PRYSM.
 
-Some Internet plans will charge an additional amount for any excess upload bandwidth used that isn’t included in the plan and may terminate your connection without warning because of overuse. We advise that you check whether your Internet connection is subjected to such limitations and monitor your bandwidth use so that you can stop Prysm before you reach your upload limit.
+BY ACCESSING AND USING PRYSM, YOU REPRESENT AND WARRANT THAT YOU UNDERSTAND THE INHERENT RISKS ASSOCIATED WITH USING CRYPTOGRAPHIC AND BLOCKCHAIN-BASED SYSTEMS, AND THAT YOU HAVE A WORKING KNOWLEDGE OF THE USAGE AND INTRICACIES OF DIGITAL ASSETS, SUCH AS THOSE FOLLOWING THE ETHEREUM TOKEN STANDARD (ERC-20). YOU FURTHER UNDERSTAND THAT THE MARKETS FOR DIGITAL ASSETS ARE HIGHLY VOLATILE DUE TO VARIOUS FACTORS, INCLUDING ADOPTION, SPECULATION, TECHNOLOGY, SECURITY, AND REGULATION. YOU ACKNOWLEDGE AND ACCEPT THAT THE COST AND SPEED OF TRANSACTING WITH CRYPTOGRAPHIC AND BLOCKCHAIN-BASED SYSTEMS SUCH AS ETHEREUM ARE VARIABLE AND MAY INCREASE DRAMATICALLY AT ANY TIME. YOU UNDERSTAND THAT ANYONE CAN CREATE A TOKEN, INCLUDING FAKE VERSIONS OF EXISTING TOKENS AND TOKENS THAT FALSELY CLAIM TO REPRESENT PROJECTS, AND ACKNOWLEDGE AND ACCEPT THE RISK THAT YOU MAY MISTAKENLY INTERACT WITH THOSE OR OTHER TOKENS. YOU FURTHER ACKNOWLEDGE THAT WE ARE NOT RESPONSIBLE FOR ANY OF THE VARIABLES OR RISKS DESCRIBED IN THESE TERMS. YOU UNDERSTAND AND AGREE TO ASSUME FULL RESPONSIBILITY FOR ALL OF THE RISKS OF ACCESSING AND USING PRYSM. YOU ARE SOLELY RESPONSIBLE FOR YOUR WALLETS, FOR SAFEGUARDING THE ASSOCIATED PRIVATE KEY AND FOR ANY ACTIVITY THAT OCCURS USING YOUR WALLET. WITHOUT LIMITING THE FOREGOING, YOU ALSO UNDERSTAND THAT THERE MAY BE TAX AND REGULATORY RISKS RELATED TO USING PRYSM. IT IS YOUR SOLE RESPONSIBILITY TO DETERMINE WHETHER, AND TO WHAT EXTENT, ANY TAXES APPLY TO ANY TRANSACTIONS YOU CONDUCT IN CONNECTION WITH YOUR USE OF PRYSM, AND TO WITHHOLD, COLLECT, REPORT AND REMIT THE CORRECT AMOUNTS OF TAXES TO THE APPROPRIATE TAX AUTHORITIES. DIGITAL ASSETS, BLOCKCHAIN TECHNOLOGY, AND ANY RELATED SOFTWARE AND SERVICES ARE ALSO SUBJECT TO LEGAL AND REGULATORY UNCERTAINTY IN THE UNITED STATES AND OTHER JURISDICTIONS. YOU UNDERSTAND THAT LEGISLATIVE AND REGULATORY CHANGES OR ACTIONS MAY ADVERSELY AFFECT THE USAGE, TRANSFERABILITY, TRANSACTABILITY AND ACCESSIBILITY RELATED TO PRYSM.
 
-### Warranty Disclaimer
-PRYSM IS PROVIDED ON AN “AS-IS” BASIS AND MAY INCLUDE ERRORS, OMISSIONS, OR OTHER INACCURACIES. PRYSMATIC LABS AND ITS CONTRIBUTORS MAKE NO REPRESENTATIONS OR WARRANTIES ABOUT PRYSM FOR ANY PURPOSE, AND HEREBY EXPRESSLY DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT OR ANY OTHER IMPLIED WARRANTY UNDER THE UNIFORM COMPUTER INFORMATION TRANSACTIONS ACT AS ENACTED BY ANY STATE. WE ALSO MAKE NO REPRESENTATIONS OR WARRANTIES THAT PRYSM WILL OPERATE ERROR-FREE, UNINTERRUPTED, OR IN A MANNER THAT WILL MEET YOUR REQUIREMENTS AND/OR NEEDS. THEREFORE, YOU ASSUME THE ENTIRE RISK REGARDING THE QUALITY AND/OR PERFORMANCE OF PRYSM AND ANY TRANSACTIONS ENTERED INTO THEREON.
+We make no claims that Prysm is appropriate or permitted for use in any specific jurisdiction. Access to Prysm may not be legal by certain persons or in certain jurisdictions or countries. If you access Prysm, you do so on your own initiative and are responsible for compliance with all Applicable Law (as defined below), including, without limitation, for the avoidance of doubt, local laws.
 
-### Limitation of Liability
-In no event will Prysmatic Labs or any of its contributors be liable, whether in contract, warranty, tort (including negligence, whether active, passive or imputed), product liability, strict liability or other theory, breach of statutory duty or otherwise arising out of, or in connection with, your use of Prysm, for any direct, indirect, incidental, special or consequential damages (including any loss of profits or data, business interruption or other pecuniary loss, or damage, loss or other compromise of data, in each case whether direct, indirect, incidental, special or consequential) arising out of use Prysm, even if we or other users have been advised of the possibility of such damages. The foregoing limitations and disclaimers shall apply to the maximum extent permitted by applicable law, even if any remedy fails of its essential purpose. You acknowledge and agree that the limitations of liability afforded us hereunder constitute a material and actual inducement and condition to entering into these Terms, and are reasonable, fair and equitable in scope to protect our legitimate interests in light of the fact that we are not receiving consideration from you for providing Prysm.
+Some Internet plans will charge additional amounts for bandwidth or any excess upload bandwidth used that isn’t included in the plan and may terminate your connection without warning because of overuse. We advise that you check whether your Internet connection is subjected to any such limitations and monitor your bandwidth use and upload volumes.
 
-### Indemnification
-To the maximum extent permitted by law, you will defend, indemnify and hold Prysmatic Labs and its contributors harmless from and against any and all claims, actions, suits, investigations, or proceedings by any third party (including any party or purported party to or beneficiary or purported beneficiary of any transaction on Prysm), as well as any and all losses, liabilities,
-damages, costs, and expenses (including reasonable attorneys’ fees) arising out of, accruing from, or in any way related to (i) your breach of the terms of this Agreement, (ii) any transaction, or the failure to occur of any transaction on Prysm, and (iii) your negligence, fraud, or willful misconduct.
+## Warranty Disclaimer
 
-### Compliance with Laws and Tax Obligations
-Your use of Prysm is subject to all applicable laws of any governmental authority, including, without limitation, federal, state and foreign securities laws, tax laws, tariff and trade laws, ordinances, judgments, decrees, injunctions, writs and orders or like actions of any governmental authority and rules, regulations, orders, interpretations, licenses, and permits of any federal,
-regional, state, county, municipal or other governmental authority and you agree to comply with all such laws in your use of Prysm. The users of Prysm are solely responsible to determinate what, if any, taxes apply to their ETH transactions. The owners of, or contributors to, Prysm are not responsible for determining the taxes that apply to ETH transactions.
+PRYSM IS PROVIDED ON AN “AS-IS” BASIS AND MAY INCLUDE ERRORS, OMISSIONS, OR OTHER INACCURACIES. WITHOUT LIMITING ANYTHING SET FORTH ELSEWHERE IN THESE TERMS, OFFCHAIN LABS AND ITS CONTRIBUTORS MAKE NO REPRESENTATIONS OR WARRANTIES ABOUT PRYSM FOR ANY PURPOSE, AND HEREBY EXPRESSLY DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT OR ANY OTHER IMPLIED WARRANTY UNDER THE UNIFORM COMPUTER INFORMATION TRANSACTIONS ACT AS ENACTED BY ANY STATE OR OTHER GOVERNMENTAL AUTHORITY. WE ALSO MAKE NO REPRESENTATIONS OR WARRANTIES THAT PRYSM WILL OPERATE ERROR-FREE, UNINTERRUPTED, OR IN A MANNER THAT WILL MEET YOUR REQUIREMENTS AND/OR NEEDS. THEREFORE, YOU ASSUME THE ENTIRE RISK REGARDING THE QUALITY AND/OR PERFORMANCE OF PRYSM AND ANY TRANSACTIONS ENTERED INTO THEREON.
 
-### Miscellaneous
-These Terms will be construed and enforced in accordance with the laws of the state of Illinois as applied to agreements entered into and completely performed in Illinois. You agree to the personal jurisdiction by and venue in Illinois and waive any objection to such jurisdiction or venue.
+## Limitation of Liability
 
-We reserve the right to revise these Terms, and your rights and obligations are at all times subject to the then-current Terms provided on Prysm. Your continued use of Prysm constitutes acceptance of such revised Terms.
+IN NO EVENT WILL OFFCHAIN LABS OR ANY OF ITS AFFILIATES OR ITS OR ANY SUCH AFFILIATE’S DIRECTORS, OFFICERS, EMPLOYEES, AGENTS, OR REPRESENTATIVES OR ANY CONTRIBUTORS (COLLECTIVELY, THE “OCL PARTIES”) BE LIABLE, WHETHER IN CONTRACT, WARRANTY, TORT (INCLUDING NEGLIGENCE, WHETHER ACTIVE, PASSIVE OR IMPUTED), PRODUCT LIABILITY, STRICT LIABILITY OR OTHER THEORY, BREACH OF STATUTORY DUTY OR OTHERWISE ARISING OUT OF, OR IN CONNECTION WITH, YOUR USE OF PRYSM, FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL OR CONSEQUENTIAL DAMAGES (INCLUDING ANY LOSS OF PROFITS OR DATA, BUSINESS INTERRUPTION OR OTHER PECUNIARY LOSS, OR DAMAGE, LOSS OR OTHER COMPROMISE OF DATA, IN EACH CASE WHETHER DIRECT, INDIRECT, INCIDENTAL, SPECIAL OR CONSEQUENTIAL) ARISING OUT OF USE PRYSM, EVEN IF WE OR OTHER USERS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. The foregoing limitations and disclaimers shall apply to the maximum extent permitted by Applicable Law, even if any remedy fails of its essential purpose. You acknowledge and agree that the limitations of liability afforded us hereunder constitute a material and actual inducement and condition to entering into these Terms, and are reasonable, fair and equitable in scope to protect our legitimate interests in light of the fact that we are not receiving consideration from you for providing Prysm.
 
-These Terms constitute the entire agreement between you and Prysmatic Labs regarding use of Prysm and will supersede all prior agreements whether, written or oral. No usage of trade or other regular practice or method of dealing between the parties will be used to modify, interpret, supplement, or alter the terms of these Terms.
+## Indemnification
+
+To the maximum extent permitted by Applicable Law, you will defend, indemnify and hold each OCL Party harmless from and against any and all claims, actions, suits, investigations, or proceedings by any third party (including any party or purported party to or beneficiary or purported beneficiary of any transaction or other activity on Prysm), as well as any and all losses, liabilities, damages, costs, and expenses (including reasonable attorneys’ fees and costs) arising out of, accruing from, or in any way related to (i) your breach of the terms of this Agreement, (ii) any transaction, or the failure to occur of any transaction on Prysm, and (iii) your negligence, fraud, or willful misconduct.
+
+## Compliance with Laws
+
+Your use of Prysm is subject to all applicable laws of any governmental authority, including, without limitation, federal, state and foreign securities laws, tax laws, tariff and trade laws, ordinances, judgments, decrees, injunctions, writs and orders or like actions of any governmental authority and rules, regulations, orders, interpretations, licenses, and permits of any federal, regional, state, county, municipal or other governmental authority (collectively, “Applicable Law”) and you agree to comply with all such Applicable Law in your use of Prysm. The users of Prysm are solely responsible to determinate what, if any, taxes apply to their ETH transactions. The owners of, or contributors to, Prysm are not responsible for determining the taxes that apply to ETH transactions.
+
+## Miscellaneous
+
+These Terms will be governed by the laws of the State of Delaware without regard to its conflict of law provisions. With respect to any disputes or claims not subject to arbitration, as set forth in the OCL Terms of Use, you and Offchain Labs submit to the personal and exclusive jurisdiction of the state and federal courts located within New York, New York and waive any objection to such jurisdiction and venue. The failure of Offchain Labs to exercise or enforce any right or provision of these Terms will not constitute a waiver of such right or provision.
+We reserve the right to revise these Terms, and your rights and obligations are at all times subject to the then-current Terms provided on Prysm. Your  use of Prysm following any such revision to these Terms constitutes acceptance of such revised Terms.
+
+These Terms constitute the entire agreement between you and Offchain Labs regarding use of Prysm and will supersede all prior agreements whether, written or oral. No usage of trade or other regular practice or method of dealing between the parties will be used to modify, interpret, supplement, or alter the terms of these Terms.
+
+If any portion of these Terms is held invalid or unenforceable, such invalidity or enforceability will not affect the other provisions of these Terms, which will remain in full force and effect, and the invalid or unenforceable portion will be given effect to the greatest extent possible. The failure of a party to require performance of any provision will not affect that party’s right to require performance at any time thereafter, nor will a waiver of any breach or default of these Terms or any provision of these Terms constitute a waiver of any subsequent breach or default or a waiver of the provision itself.
 
-If any portion of these Terms is held invalid or unenforceable, such invalidity or enforceability will not affect the other provisions of these Terms, which will remain in full force and effect, and the invalid or unenforceable portion will be given effect to the greatest extent possible. The failure of a party to require performance of any provision will not affect that party’s right to require performance at any time thereafter, nor will a waiver of any breach or default of these Terms or any provision of these Terms constitute a waiver of any subsequent breach or default or a waiver of the provision itself.

WORKSPACE

@@ -27,7 +27,23 @@ http_archive(
 
 load("@hermetic_cc_toolchain//toolchain:defs.bzl", zig_toolchains = "toolchains")
 
-zig_toolchains()
+# Temporarily use a nightly build until 0.12.0 is released.
+# See: https://github.com/prysmaticlabs/prysm/issues/13130
+zig_toolchains(
+    host_platform_sha256 = {
+        "linux-aarch64": "45afb8e32adde825165f4f293fcea9ecea503f7f9ec0e9bf4435afe70e67fb70",
+        "linux-x86_64": "f136c6a8a0f6adcb057d73615fbcd6f88281b3593f7008d5f7ed514ff925c02e",
+        "macos-aarch64": "05d995853c05243151deff47b60bdc2674f1e794a939eaeca0f42312da031cee",
+        "macos-x86_64": "721754ba5a50f31e8a1f0e1a74cace26f8246576878ac4a8591b0ee7b6db1fc1",
+        "windows-x86_64": "93f5248b2ea8c5ee8175e15b1384e133edc1cd49870b3ea259062a2e04164343",
+    },
+    url_formats = [
+        "https://ziglang.org/builds/zig-{host_platform}-{version}.{_ext}",
+        "https://mirror.bazel.build/ziglang.org/builds/zig-{host_platform}-{version}.{_ext}",
+        "https://prysmaticlabs.com/mirror/ziglang.org/builds/zig-{host_platform}-{version}.{_ext}",
+    ],
+    version = "0.12.0-dev.1349+fa022d1ec",
+)
 
 # Register zig sdk toolchains with support for Ubuntu 20.04 (Focal Fossa) which has an EOL date of April, 2025.
 # For ubuntu glibc support, see https://launchpad.net/ubuntu/+source/glibc
@@ -65,10 +81,10 @@ bazel_skylib_workspace()
 
 http_archive(
     name = "bazel_gazelle",
-    sha256 = "29d5dafc2a5582995488c6735115d1d366fcd6a0fc2e2a153f02988706349825",
+    sha256 = "d3fa66a39028e97d76f9e2db8f1b0c11c099e8e01bf363a923074784e451f809",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.31.0/bazel-gazelle-v0.31.0.tar.gz",
-        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.31.0/bazel-gazelle-v0.31.0.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.33.0/bazel-gazelle-v0.33.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.33.0/bazel-gazelle-v0.33.0.tar.gz",
     ],
 )
 
@@ -110,10 +126,10 @@ http_archive(
         # Expose internals of go_test for custom build transitions.
         "//third_party:io_bazel_rules_go_test.patch",
     ],
-    sha256 = "bfc5ce70b9d1634ae54f4e7b495657a18a04e0d596785f672d35d5f505ab491a",
+    sha256 = "91585017debb61982f7054c9688857a2ad1fd823fc3f9cb05048b0025c47d023",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.40.0/rules_go-v0.40.0.zip",
-        "https://github.com/bazelbuild/rules_go/releases/download/v0.40.0/rules_go-v0.40.0.zip",
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.42.0/rules_go-v0.42.0.zip",
+        "https://github.com/bazelbuild/rules_go/releases/download/v0.42.0/rules_go-v0.42.0.zip",
     ],
 )
 
@@ -206,7 +222,7 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe
 go_rules_dependencies()
 
 go_register_toolchains(
-    go_version = "1.20.7",
+    go_version = "1.20.10",
     nogo = "@//:nogo",
 )
 
@@ -228,8 +244,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "91434d5fd5e1c6eb7b0174fed2afe25e09bddf00e1e4c431db931b2cee4e7773",
-    url = "https://github.com/eth-clients/slashing-protection-interchange-tests/archive/b8413ca42dc92308019d0d4db52c87e9e125c4e9.tar.gz",
+    sha256 = "516d551cfb3e50e4ac2f42db0992f4ceb573a7cb1616d727a725c8161485329f",
+    url = "https://github.com/eth-clients/slashing-protection-interchange-tests/archive/refs/tags/v5.3.0.tar.gz",
 )
 
 http_archive(
@@ -247,9 +263,7 @@ filegroup(
     url = "https://github.com/ethereum/EIPs/archive/5480440fe51742ed23342b68cf106cefd427e39d.tar.gz",
 )
 
-consensus_spec_test_version = "v1.4.0-beta.2-hotfix"
-
-consensus_spec_version = "v1.4.0-beta.2"
+consensus_spec_version = "v1.4.0-beta.3"
 
 bls_test_version = "v0.1.1"
 
@@ -265,8 +279,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "99770a001189f66204a4ef79161c8002bcbbcbd8236f1c6479bd5b83a3c68d42",
-    url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/general.tar.gz" % consensus_spec_test_version,
+    sha256 = "67ae5b8fc368853da23d4297e480a4b7f4722fb970d1c7e2b6a5b7faef9cb907",
+    url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/general.tar.gz" % consensus_spec_version,
 )
 
 http_archive(
@@ -281,8 +295,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "56763f6492ee137108271007d62feef60d8e3f1698e53dee4bc4b07e55f7326b",
-    url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/minimal.tar.gz" % consensus_spec_test_version,
+    sha256 = "82474f29fff4abd09fb1e71bafa98827e2573cf0ad02cf119610961831dc3bb5",
+    url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/minimal.tar.gz" % consensus_spec_version,
 )
 
 http_archive(
@@ -297,8 +311,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "bc1cac1a991cdc7426efea14385dcf215df85ed3f0572b824ad6a1d7ca0c89ad",
-    url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/mainnet.tar.gz" % consensus_spec_test_version,
+    sha256 = "60e4b6eb6c341daab7ee5614a8e3f28567247c504c593b951bfe919622c8ef8f",
+    url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/mainnet.tar.gz" % consensus_spec_version,
 )
 
 http_archive(
@@ -312,7 +326,7 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "c5898001aaab2a5bb38a39ff9d17a52f1f9befcc26e63752cbf556040f0c884e",
+    sha256 = "fdab9756c93a250219ff6a10d5a9faee1e2e6878a14508410409e307362c6991",
     strip_prefix = "consensus-specs-" + consensus_spec_version[1:],
     url = "https://github.com/ethereum/consensus-specs/archive/refs/tags/%s.tar.gz" % consensus_spec_version,
 )
@@ -374,6 +388,22 @@ http_archive(
 )
 
 # External dependencies
+http_archive(
+    name = "googleapis",
+    sha256 = "9d1a930e767c93c825398b8f8692eca3fe353b9aaadedfbcf1fca2282c85df88",
+    strip_prefix = "googleapis-64926d52febbf298cb82a8f472ade4a3969ba922",
+    urls = [
+        "https://github.com/googleapis/googleapis/archive/64926d52febbf298cb82a8f472ade4a3969ba922.zip",
+    ],
+)
+
+load("@googleapis//:repository_rules.bzl", "switched_rules_by_language")
+
+switched_rules_by_language(
+    name = "com_google_googleapis_imports",
+    go = True,
+)
+
 load("//:deps.bzl", "prysm_deps")
 
 # gazelle:repository_macro deps.bzl%prysm_deps

api/BUILD.bazel

@@ -2,7 +2,10 @@ load("@prysm//tools/go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
-    srcs = ["headers.go"],
+    srcs = [
+        "constants.go",
+        "headers.go",
+    ],
     importpath = "github.com/prysmaticlabs/prysm/v4/api",
     visibility = ["//visibility:public"],
 )

api/client/beacon/BUILD.bazel

@@ -12,8 +12,10 @@ go_library(
     deps = [
         "//api/client:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/rpc/apimiddleware:go_default_library",
+        "//beacon-chain/rpc/eth/beacon:go_default_library",
+        "//beacon-chain/rpc/eth/config:go_default_library",
         "//beacon-chain/rpc/eth/shared:go_default_library",
+        "//beacon-chain/rpc/prysm/beacon:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//consensus-types/interfaces:go_default_library",
         "//consensus-types/primitives:go_default_library",
@@ -21,7 +23,6 @@ go_library(
         "//encoding/ssz/detect:go_default_library",
         "//io/file:go_default_library",
         "//network/forks:go_default_library",
-        "//proto/eth/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//runtime/version:go_default_library",
         "//time/slots:go_default_library",

api/client/beacon/client.go

@@ -13,16 +13,16 @@ import (
 	"strconv"
 	"text/template"
 
-	"github.com/prysmaticlabs/prysm/v4/api/client"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
-	"github.com/prysmaticlabs/prysm/v4/network/forks"
-	v1 "github.com/prysmaticlabs/prysm/v4/proto/eth/v1"
-
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/apimiddleware"
+	"github.com/prysmaticlabs/prysm/v4/api/client"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/beacon"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/config"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
+	apibeacon "github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/prysm/beacon"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
+	"github.com/prysmaticlabs/prysm/v4/network/forks"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	log "github.com/sirupsen/logrus"
 )
@@ -31,7 +31,7 @@ const (
 	getSignedBlockPath       = "/eth/v2/beacon/blocks"
 	getBlockRootPath         = "/eth/v1/beacon/blocks/{{.Id}}/root"
 	getForkForStatePath      = "/eth/v1/beacon/states/{{.Id}}/fork"
-	getWeakSubjectivityPath  = "/eth/v1/beacon/weak_subjectivity"
+	getWeakSubjectivityPath  = "/prysm/v1/beacon/weak_subjectivity"
 	getForkSchedulePath      = "/eth/v1/config/fork_schedule"
 	getConfigSpecPath        = "/eth/v1/config/spec"
 	getStatePath             = "/eth/v2/debug/beacon/states"
@@ -178,12 +178,12 @@ func (c *Client) GetForkSchedule(ctx context.Context) (forks.OrderedSchedule, er
 }
 
 // GetConfigSpec retrieve the current configs of the network used by the beacon node.
-func (c *Client) GetConfigSpec(ctx context.Context) (*v1.SpecResponse, error) {
+func (c *Client) GetConfigSpec(ctx context.Context) (*config.GetSpecResponse, error) {
 	body, err := c.Get(ctx, getConfigSpecPath)
 	if err != nil {
 		return nil, errors.Wrap(err, "error requesting configSpecPath")
 	}
-	fsr := &v1.SpecResponse{}
+	fsr := &config.GetSpecResponse{}
 	err = json.Unmarshal(body, fsr)
 	if err != nil {
 		return nil, err
@@ -258,16 +258,16 @@ func (c *Client) GetWeakSubjectivity(ctx context.Context) (*WeakSubjectivityData
 	if err != nil {
 		return nil, err
 	}
-	v := &apimiddleware.WeakSubjectivityResponse{}
+	v := &apibeacon.GetWeakSubjectivityResponse{}
 	err = json.Unmarshal(body, v)
 	if err != nil {
 		return nil, err
 	}
-	epoch, err := strconv.ParseUint(v.Data.Checkpoint.Epoch, 10, 64)
+	epoch, err := strconv.ParseUint(v.Data.WsCheckpoint.Epoch, 10, 64)
 	if err != nil {
 		return nil, err
 	}
-	blockRoot, err := hexutil.Decode(v.Data.Checkpoint.Root)
+	blockRoot, err := hexutil.Decode(v.Data.WsCheckpoint.Root)
 	if err != nil {
 		return nil, err
 	}
@@ -284,7 +284,7 @@ func (c *Client) GetWeakSubjectivity(ctx context.Context) (*WeakSubjectivityData
 
 // SubmitChangeBLStoExecution calls a beacon API endpoint to set the withdrawal addresses based on the given signed messages.
 // If the API responds with something other than OK there will be failure messages associated to the corresponding request message.
-func (c *Client) SubmitChangeBLStoExecution(ctx context.Context, request []*apimiddleware.SignedBLSToExecutionChangeJson) error {
+func (c *Client) SubmitChangeBLStoExecution(ctx context.Context, request []*shared.SignedBLSToExecutionChange) error {
 	u := c.BaseURL().ResolveReference(&url.URL{Path: changeBLStoExecutionPath})
 	body, err := json.Marshal(request)
 	if err != nil {
@@ -305,7 +305,7 @@ func (c *Client) SubmitChangeBLStoExecution(ctx context.Context, request []*apim
 	if resp.StatusCode != http.StatusOK {
 		decoder := json.NewDecoder(resp.Body)
 		decoder.DisallowUnknownFields()
-		errorJson := &apimiddleware.IndexedVerificationFailureErrorJson{}
+		errorJson := &shared.IndexedVerificationFailureError{}
 		if err := decoder.Decode(errorJson); err != nil {
 			return errors.Wrapf(err, "failed to decode error JSON for %s", resp.Request.URL)
 		}
@@ -323,12 +323,12 @@ func (c *Client) SubmitChangeBLStoExecution(ctx context.Context, request []*apim
 
 // GetBLStoExecutionChanges gets all the set withdrawal messages in the node's operation pool.
 // Returns a struct representation of json response.
-func (c *Client) GetBLStoExecutionChanges(ctx context.Context) (*apimiddleware.BLSToExecutionChangesPoolResponseJson, error) {
+func (c *Client) GetBLStoExecutionChanges(ctx context.Context) (*beacon.BLSToExecutionChangesPoolResponse, error) {
 	body, err := c.Get(ctx, changeBLStoExecutionPath)
 	if err != nil {
 		return nil, err
 	}
-	poolResponse := &apimiddleware.BLSToExecutionChangesPoolResponseJson{}
+	poolResponse := &beacon.BLSToExecutionChangesPoolResponse{}
 	err = json.Unmarshal(body, poolResponse)
 	if err != nil {
 		return nil, err

api/client/builder/BUILD.bazel

@@ -20,8 +20,6 @@ go_library(
         "//encoding/bytesutil:go_default_library",
         "//math:go_default_library",
         "//monitoring/tracing:go_default_library",
-        "//network:go_default_library",
-        "//network/authorization:go_default_library",
         "//proto/engine/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//runtime/version:go_default_library",

api/client/builder/bid.go

@@ -6,7 +6,6 @@ import (
 	consensus_types "github.com/prysmaticlabs/prysm/v4/consensus-types"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
-	enginev1 "github.com/prysmaticlabs/prysm/v4/proto/engine/v1"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/runtime/version"
 )
@@ -22,7 +21,7 @@ type SignedBid interface {
 // Bid is an interface describing the method set of a builder bid.
 type Bid interface {
 	Header() (interfaces.ExecutionData, error)
-	BlindedBlobsBundle() (*enginev1.BlindedBlobsBundle, error)
+	BlobKzgCommitments() ([][]byte, error)
 	Value() []byte
 	Pubkey() []byte
 	Version() int
@@ -115,9 +114,9 @@ func (b builderBid) Header() (interfaces.ExecutionData, error) {
 	return blocks.WrappedExecutionPayloadHeader(b.p.Header)
 }
 
-// BlindedBlobsBundle --
-func (b builderBid) BlindedBlobsBundle() (*enginev1.BlindedBlobsBundle, error) {
-	return nil, errors.New("blinded blobs bundle not available before Deneb")
+// BlobKzgCommitments --
+func (b builderBid) BlobKzgCommitments() ([][]byte, error) {
+	return [][]byte{}, errors.New("blob kzg commitments not available before Deneb")
 }
 
 // Version --
@@ -169,9 +168,9 @@ func (b builderBidCapella) Header() (interfaces.ExecutionData, error) {
 	return blocks.WrappedExecutionPayloadHeaderCapella(b.p.Header, blocks.PayloadValueToGwei(b.p.Value))
 }
 
-// BlindedBlobsBundle --
-func (b builderBidCapella) BlindedBlobsBundle() (*enginev1.BlindedBlobsBundle, error) {
-	return nil, errors.New("blinded blobs bundle not available before Deneb")
+// BlobKzgCommitments --
+func (b builderBidCapella) BlobKzgCommitments() ([][]byte, error) {
+	return [][]byte{}, errors.New("blob kzg commitments not available before Deneb")
 }
 
 // Version --
@@ -253,9 +252,9 @@ func (b builderBidDeneb) Header() (interfaces.ExecutionData, error) {
 	return blocks.WrappedExecutionPayloadHeaderDeneb(b.p.Header, blocks.PayloadValueToGwei(b.p.Value))
 }
 
-// BlindedBlobsBundle --
-func (b builderBidDeneb) BlindedBlobsBundle() (*enginev1.BlindedBlobsBundle, error) {
-	return b.p.BlindedBlobsBundle, nil
+// BlobKzgCommitments --
+func (b builderBidDeneb) BlobKzgCommitments() ([][]byte, error) {
+	return b.p.BlobKzgCommitments, nil
 }
 
 type signedBuilderBidDeneb struct {

api/client/builder/client.go

@@ -19,8 +19,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	"github.com/prysmaticlabs/prysm/v4/monitoring/tracing"
-	"github.com/prysmaticlabs/prysm/v4/network"
-	"github.com/prysmaticlabs/prysm/v4/network/authorization"
 	v1 "github.com/prysmaticlabs/prysm/v4/proto/engine/v1"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/runtime/version"
@@ -89,7 +87,7 @@ type BuilderClient interface {
 	NodeURL() string
 	GetHeader(ctx context.Context, slot primitives.Slot, parentHash [32]byte, pubkey [48]byte) (SignedBid, error)
 	RegisterValidator(ctx context.Context, svr []*ethpb.SignedValidatorRegistrationV1) error
-	SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlySignedBeaconBlock, blobs []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error)
+	SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error)
 	Status(ctx context.Context) error
 }
 
@@ -104,8 +102,7 @@ type Client struct {
 // `host` is the base host + port used to construct request urls. This value can be
 // a URL string, or NewClient will assume an http endpoint if just `host:port` is used.
 func NewClient(host string, opts ...ClientOpt) (*Client, error) {
-	endpoint := covertEndPoint(host)
-	u, err := urlForHost(endpoint.Url)
+	u, err := urlForHost(host)
 	if err != nil {
 		return nil, err
 	}
@@ -121,8 +118,7 @@ func NewClient(host string, opts ...ClientOpt) (*Client, error) {
 
 func urlForHost(h string) (*url.URL, error) {
 	// try to parse as url (being permissive)
-	u, err := url.Parse(h)
-	if err == nil && u.Host != "" {
+	if u, err := url.Parse(h); err == nil && u.Host != "" {
 		return u, nil
 	}
 	// try to parse as host:port
@@ -140,7 +136,7 @@ func (c *Client) NodeURL() string {
 
 type reqOption func(*http.Request)
 
-// do is a generic, opinionated request function to reduce boilerplate amongst the methods in this package api/client/builder/types.go.
+// do is a generic, opinionated request function to reduce boilerplate amongst the methods in this package api/client/builder.
 func (c *Client) do(ctx context.Context, method string, path string, body io.Reader, opts ...reqOption) (res []byte, err error) {
 	ctx, span := trace.StartSpan(ctx, "builder.client.do")
 	defer func() {
@@ -291,7 +287,7 @@ func (c *Client) RegisterValidator(ctx context.Context, svr []*ethpb.SignedValid
 
 // SubmitBlindedBlock calls the builder API endpoint that binds the validator to the builder and submits the block.
 // The response is the full execution payload used to create the blinded block.
-func (c *Client) SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlySignedBeaconBlock, blobs []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
+func (c *Client) SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
 	if !sb.IsBlinded() {
 		return nil, nil, errNotBlinded
 	}
@@ -375,9 +371,9 @@ func (c *Client) SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlyS
 		if err != nil {
 			return nil, nil, errors.Wrapf(err, "could not get protobuf block")
 		}
-		b, err := shared.SignedBlindedBeaconBlockContentsDenebFromConsensus(&ethpb.SignedBlindedBeaconBlockAndBlobsDeneb{SignedBlindedBlock: psb, SignedBlindedBlobSidecars: blobs})
+		b, err := shared.SignedBlindedBeaconBlockDenebFromConsensus(&ethpb.SignedBlindedBeaconBlockDeneb{Message: psb.Message, Signature: bytesutil.SafeCopyBytes(psb.Signature)})
 		if err != nil {
-			return nil, nil, errors.Wrapf(err, "could not convert SignedBlindedBeaconBlockContentsDeneb to json marshalable type")
+			return nil, nil, errors.Wrapf(err, "could not convert SignedBlindedBeaconBlockDeneb to json marshalable type")
 		}
 		body, err := json.Marshal(b)
 		if err != nil {
@@ -457,12 +453,3 @@ func non200Err(response *http.Response) error {
 		return errors.Wrap(ErrNotOK, fmt.Sprintf("unsupported error code: %d", response.StatusCode))
 	}
 }
-
-func covertEndPoint(ep string) network.Endpoint {
-	return network.Endpoint{
-		Url: ep,
-		Auth: network.AuthorizationData{ // Auth is not used for builder.
-			Method: authorization.None,
-			Value:  "",
-		}}
-}

api/client/builder/client_test.go

@@ -12,10 +12,8 @@ import (
 	"strconv"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/prysmaticlabs/go-bitfield"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
-	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	types "github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
@@ -270,55 +268,12 @@ func TestClient_GetHeader(t *testing.T) {
 		bidValue := bytesutil.ReverseByteOrder(bid.Value())
 		require.DeepEqual(t, bidValue, value.Bytes())
 		require.DeepEqual(t, big.NewInt(0).SetBytes(bidValue), value.Int)
-		bundle, err := bid.BlindedBlobsBundle()
+		kcgCommitments, err := bid.BlobKzgCommitments()
 		require.NoError(t, err)
-		require.Equal(t, len(bundle.BlobRoots) <= fieldparams.MaxBlobsPerBlock && len(bundle.BlobRoots) > 0, true)
-		for i := range bundle.BlobRoots {
-			require.Equal(t, len(bundle.BlobRoots[i]) == fieldparams.RootLength, true)
+		require.Equal(t, len(kcgCommitments) > 0, true)
+		for i := range kcgCommitments {
+			require.Equal(t, len(kcgCommitments[i]) == 48, true)
 		}
-		require.Equal(t, len(bundle.KzgCommitments) > 0, true)
-		for i := range bundle.KzgCommitments {
-			require.Equal(t, len(bundle.KzgCommitments[i]) == 48, true)
-		}
-		require.Equal(t, len(bundle.Proofs) > 0, true)
-		for i := range bundle.Proofs {
-			require.Equal(t, len(bundle.Proofs[i]) == 48, true)
-		}
-	})
-	t.Run("deneb, no bundle", func(t *testing.T) {
-		hc := &http.Client{
-			Transport: roundtrip(func(r *http.Request) (*http.Response, error) {
-				require.Equal(t, expectedPath, r.URL.Path)
-				return &http.Response{
-					StatusCode: http.StatusOK,
-					Body:       io.NopCloser(bytes.NewBufferString(testExampleHeaderResponseDenebNoBundle)),
-					Request:    r.Clone(ctx),
-				}, nil
-			}),
-		}
-		c := &Client{
-			hc:      hc,
-			baseURL: &url.URL{Host: "localhost:3500", Scheme: "http"},
-		}
-		h, err := c.GetHeader(ctx, slot, bytesutil.ToBytes32(parentHash), bytesutil.ToBytes48(pubkey))
-		require.NoError(t, err)
-		expectedWithdrawalsRoot := ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2")
-		bid, err := h.Message()
-		require.NoError(t, err)
-		bidHeader, err := bid.Header()
-		require.NoError(t, err)
-		withdrawalsRoot, err := bidHeader.WithdrawalsRoot()
-		require.NoError(t, err)
-		require.Equal(t, true, bytes.Equal(expectedWithdrawalsRoot, withdrawalsRoot))
-		value, err := stringToUint256("652312848583266388373324160190187140051835877600158453279131187530910662656")
-		require.NoError(t, err)
-		require.Equal(t, fmt.Sprintf("%#x", value.SSZBytes()), fmt.Sprintf("%#x", bid.Value()))
-		bidValue := bytesutil.ReverseByteOrder(bid.Value())
-		require.DeepEqual(t, bidValue, value.Bytes())
-		require.DeepEqual(t, big.NewInt(0).SetBytes(bidValue), value.Int)
-		bundle, err := bid.BlindedBlobsBundle()
-		require.NoError(t, err)
-		require.Equal(t, (*v1.BlindedBlobsBundle)(nil), bundle)
 	})
 
 	t.Run("unsupported version", func(t *testing.T) {
@@ -362,7 +317,7 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		}
 		sbbb, err := blocks.NewSignedBeaconBlock(testSignedBlindedBeaconBlockBellatrix(t))
 		require.NoError(t, err)
-		ep, _, err := c.SubmitBlindedBlock(ctx, sbbb, nil)
+		ep, _, err := c.SubmitBlindedBlock(ctx, sbbb)
 		require.NoError(t, err)
 		require.Equal(t, true, bytes.Equal(ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"), ep.ParentHash()))
 		bfpg, err := stringToUint256("452312848583266388373324160190187140051835877600158453279131187530910662656")
@@ -388,7 +343,7 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		}
 		sbb, err := blocks.NewSignedBeaconBlock(testSignedBlindedBeaconBlockCapella(t))
 		require.NoError(t, err)
-		ep, _, err := c.SubmitBlindedBlock(ctx, sbb, nil)
+		ep, _, err := c.SubmitBlindedBlock(ctx, sbb)
 		require.NoError(t, err)
 		withdrawals, err := ep.Withdrawals()
 		require.NoError(t, err)
@@ -399,18 +354,17 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		assert.Equal(t, uint64(1), withdrawals[0].Amount)
 	})
 	t.Run("deneb", func(t *testing.T) {
-
-		test := testSignedBlindedBeaconBlockAndBlobsDeneb(t)
+		test := testSignedBlindedBeaconBlockDeneb(t)
 		hc := &http.Client{
 			Transport: roundtrip(func(r *http.Request) (*http.Response, error) {
 				require.Equal(t, postBlindedBeaconBlockPath, r.URL.Path)
 				require.Equal(t, "deneb", r.Header.Get("Eth-Consensus-Version"))
-				var req shared.SignedBlindedBeaconBlockContentsDeneb
+				var req shared.SignedBlindedBeaconBlockDeneb
 				err := json.NewDecoder(r.Body).Decode(&req)
 				require.NoError(t, err)
-				block, err := req.SignedBlindedBlock.ToConsensus()
+				block, err := req.ToConsensus()
 				require.NoError(t, err)
-				require.DeepEqual(t, block, test.SignedBlindedBlock)
+				require.DeepEqual(t, block, test)
 				return &http.Response{
 					StatusCode: http.StatusOK,
 					Body:       io.NopCloser(bytes.NewBufferString(testExampleExecutionPayloadDeneb)),
@@ -423,10 +377,10 @@ func TestSubmitBlindedBlock(t *testing.T) {
 			baseURL: &url.URL{Host: "localhost:3500", Scheme: "http"},
 		}
 
-		sbb, err := blocks.NewSignedBeaconBlock(test.SignedBlindedBlock)
+		sbb, err := blocks.NewSignedBeaconBlock(test)
 		require.NoError(t, err)
 
-		ep, blobBundle, err := c.SubmitBlindedBlock(ctx, sbb, test.SignedBlindedBlobSidecars)
+		ep, blobBundle, err := c.SubmitBlindedBlock(ctx, sbb)
 		require.NoError(t, err)
 		withdrawals, err := ep.Withdrawals()
 		require.NoError(t, err)
@@ -436,9 +390,6 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		assert.DeepEqual(t, ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943"), withdrawals[0].Address)
 		assert.Equal(t, uint64(1), withdrawals[0].Amount)
 		require.NotNil(t, blobBundle)
-		require.Equal(t, hexutil.Encode(blobBundle.Blobs[0]), hexutil.Encode(make([]byte, fieldparams.BlobLength)))
-		require.Equal(t, hexutil.Encode(blobBundle.KzgCommitments[0]), "0x8dab030c51e16e84be9caab84ee3d0b8bbec1db4a0e4de76439da8424d9b957370a10a78851f97e4b54d2ce1ab0d686f")
-		require.Equal(t, hexutil.Encode(blobBundle.Proofs[0]), "0xb4021b0de10f743893d4f71e1bf830c019e832958efd6795baf2f83b8699a9eccc5dc99015d8d4d8ec370d0cc333c06a")
 	})
 	t.Run("mismatched versions, expected bellatrix got capella", func(t *testing.T) {
 		hc := &http.Client{
@@ -457,13 +408,13 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		}
 		sbbb, err := blocks.NewSignedBeaconBlock(testSignedBlindedBeaconBlockBellatrix(t))
 		require.NoError(t, err)
-		_, _, err = c.SubmitBlindedBlock(ctx, sbbb, nil)
+		_, _, err = c.SubmitBlindedBlock(ctx, sbbb)
 		require.ErrorContains(t, "not a bellatrix payload", err)
 	})
 	t.Run("not blinded", func(t *testing.T) {
 		sbb, err := blocks.NewSignedBeaconBlock(&eth.SignedBeaconBlockBellatrix{Block: &eth.BeaconBlockBellatrix{Body: &eth.BeaconBlockBodyBellatrix{}}})
 		require.NoError(t, err)
-		_, _, err = (&Client{}).SubmitBlindedBlock(ctx, sbb, nil)
+		_, _, err = (&Client{}).SubmitBlindedBlock(ctx, sbb)
 		require.ErrorIs(t, err, errNotBlinded)
 	})
 }
@@ -753,91 +704,70 @@ func testSignedBlindedBeaconBlockCapella(t *testing.T) *eth.SignedBlindedBeaconB
 	}
 }
 
-func testSignedBlindedBeaconBlockAndBlobsDeneb(t *testing.T) *eth.SignedBlindedBeaconBlockAndBlobsDeneb {
+func testSignedBlindedBeaconBlockDeneb(t *testing.T) *eth.SignedBlindedBeaconBlockDeneb {
 	basebytes, err := shared.Uint256ToSSZBytes("14074904626401341155369551180448584754667373453244490859944217516317499064576")
 	if err != nil {
 		log.Error(err)
 	}
-	return &eth.SignedBlindedBeaconBlockAndBlobsDeneb{
-		SignedBlindedBlock: &eth.SignedBlindedBeaconBlockDeneb{
-			Message: &eth.BlindedBeaconBlockDeneb{
-				Slot:          1,
-				ProposerIndex: 1,
-				ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-				StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-				Body: &eth.BlindedBeaconBlockBodyDeneb{
-					RandaoReveal: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
-					Eth1Data: &eth.Eth1Data{
-						DepositRoot:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						DepositCount: 1,
-						BlockHash:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-					},
-					Graffiti: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-					ProposerSlashings: []*eth.ProposerSlashing{
-						{
-							Header_1: &eth.SignedBeaconBlockHeader{
-								Header: &eth.BeaconBlockHeader{
-									Slot:          1,
-									ProposerIndex: 1,
-									ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									BodyRoot:      ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-								},
-								Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+	return &eth.SignedBlindedBeaconBlockDeneb{
+		Message: &eth.BlindedBeaconBlockDeneb{
+			Slot:          1,
+			ProposerIndex: 1,
+			ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+			StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+			Body: &eth.BlindedBeaconBlockBodyDeneb{
+				RandaoReveal: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+				Eth1Data: &eth.Eth1Data{
+					DepositRoot:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					DepositCount: 1,
+					BlockHash:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+				},
+				Graffiti: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+				ProposerSlashings: []*eth.ProposerSlashing{
+					{
+						Header_1: &eth.SignedBeaconBlockHeader{
+							Header: &eth.BeaconBlockHeader{
+								Slot:          1,
+								ProposerIndex: 1,
+								ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+								StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+								BodyRoot:      ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
 							},
-							Header_2: &eth.SignedBeaconBlockHeader{
-								Header: &eth.BeaconBlockHeader{
-									Slot:          1,
-									ProposerIndex: 1,
-									ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									BodyRoot:      ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-								},
-								Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+							Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+						},
+						Header_2: &eth.SignedBeaconBlockHeader{
+							Header: &eth.BeaconBlockHeader{
+								Slot:          1,
+								ProposerIndex: 1,
+								ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+								StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+								BodyRoot:      ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
 							},
+							Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 						},
 					},
-					AttesterSlashings: []*eth.AttesterSlashing{
-						{
-							Attestation_1: &eth.IndexedAttestation{
-								AttestingIndices: []uint64{1},
-								Data: &eth.AttestationData{
-									Slot:            1,
-									CommitteeIndex:  1,
-									BeaconBlockRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									Source: &eth.Checkpoint{
-										Epoch: 1,
-										Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									},
-									Target: &eth.Checkpoint{
-										Epoch: 1,
-										Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									},
+				},
+				AttesterSlashings: []*eth.AttesterSlashing{
+					{
+						Attestation_1: &eth.IndexedAttestation{
+							AttestingIndices: []uint64{1},
+							Data: &eth.AttestationData{
+								Slot:            1,
+								CommitteeIndex:  1,
+								BeaconBlockRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+								Source: &eth.Checkpoint{
+									Epoch: 1,
+									Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
 								},
-								Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
-							},
-							Attestation_2: &eth.IndexedAttestation{
-								AttestingIndices: []uint64{1},
-								Data: &eth.AttestationData{
-									Slot:            1,
-									CommitteeIndex:  1,
-									BeaconBlockRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									Source: &eth.Checkpoint{
-										Epoch: 1,
-										Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									},
-									Target: &eth.Checkpoint{
-										Epoch: 1,
-										Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									},
+								Target: &eth.Checkpoint{
+									Epoch: 1,
+									Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
 								},
-								Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 							},
+							Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 						},
-					},
-					Attestations: []*eth.Attestation{
-						{
-							AggregationBits: bitfield.Bitlist{0x01},
+						Attestation_2: &eth.IndexedAttestation{
+							AttestingIndices: []uint64{1},
 							Data: &eth.AttestationData{
 								Slot:            1,
 								CommitteeIndex:  1,
@@ -854,68 +784,72 @@ func testSignedBlindedBeaconBlockAndBlobsDeneb(t *testing.T) *eth.SignedBlindedB
 							Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 						},
 					},
-					Deposits: []*eth.Deposit{
-						{
-							Proof: [][]byte{ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2")},
-							Data: &eth.Deposit_Data{
-								PublicKey:             ezDecode(t, "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a"),
-								WithdrawalCredentials: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-								Amount:                1,
-								Signature:             ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+				},
+				Attestations: []*eth.Attestation{
+					{
+						AggregationBits: bitfield.Bitlist{0x01},
+						Data: &eth.AttestationData{
+							Slot:            1,
+							CommitteeIndex:  1,
+							BeaconBlockRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+							Source: &eth.Checkpoint{
+								Epoch: 1,
+								Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+							},
+							Target: &eth.Checkpoint{
+								Epoch: 1,
+								Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
 							},
 						},
+						Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 					},
-					VoluntaryExits: []*eth.SignedVoluntaryExit{
-						{
-							Exit: &eth.VoluntaryExit{
-								Epoch:          1,
-								ValidatorIndex: 1,
-							},
-							Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+				},
+				Deposits: []*eth.Deposit{
+					{
+						Proof: [][]byte{ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2")},
+						Data: &eth.Deposit_Data{
+							PublicKey:             ezDecode(t, "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a"),
+							WithdrawalCredentials: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+							Amount:                1,
+							Signature:             ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 						},
 					},
-					SyncAggregate: &eth.SyncAggregate{
-						SyncCommitteeSignature: make([]byte, 96),
-						SyncCommitteeBits:      bitfield.Bitvector512(ezDecode(t, "0x6451e9f951ebf05edc01de67e593484b672877054f055903ff0df1a1a945cf30ca26bb4d4b154f94a1bc776bcf5d0efb3603e1f9b8ee2499ccdcfe2a18cef458")),
-					},
-					ExecutionPayloadHeader: &v1.ExecutionPayloadHeaderDeneb{
-						ParentHash:       ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						FeeRecipient:     ezDecode(t, "0xabcf8e0d4e9587369b2301d0790347320302cc09"),
-						StateRoot:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						ReceiptsRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						LogsBloom:        ezDecode(t, "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"),
-						PrevRandao:       ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						BlockNumber:      1,
-						GasLimit:         1,
-						GasUsed:          1,
-						Timestamp:        1,
-						ExtraData:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						BaseFeePerGas:    basebytes,
-						BlockHash:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						TransactionsRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						WithdrawalsRoot:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						BlobGasUsed:      1,
-						ExcessBlobGas:    2,
+				},
+				VoluntaryExits: []*eth.SignedVoluntaryExit{
+					{
+						Exit: &eth.VoluntaryExit{
+							Epoch:          1,
+							ValidatorIndex: 1,
+						},
+						Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 					},
 				},
-			},
-			Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
-		},
-		SignedBlindedBlobSidecars: []*eth.SignedBlindedBlobSidecar{
-			{
-				Message: &eth.BlindedBlobSidecar{
-					BlockRoot:       ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-					Index:           0,
-					Slot:            1,
-					BlockParentRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-					ProposerIndex:   1,
-					BlobRoot:        ezDecode(t, "0x24564723180fcb3d994104538d351c8dcbde12d541676bb736cf678018ca4739"),
-					KzgCommitment:   ezDecode(t, "0x8dab030c51e16e84be9caab84ee3d0b8bbec1db4a0e4de76439da8424d9b957370a10a78851f97e4b54d2ce1ab0d686f"),
-					KzgProof:        ezDecode(t, "0xb4021b0de10f743893d4f71e1bf830c019e832958efd6795baf2f83b8699a9eccc5dc99015d8d4d8ec370d0cc333c06a"),
+				SyncAggregate: &eth.SyncAggregate{
+					SyncCommitteeSignature: make([]byte, 96),
+					SyncCommitteeBits:      ezDecode(t, "0x6451e9f951ebf05edc01de67e593484b672877054f055903ff0df1a1a945cf30ca26bb4d4b154f94a1bc776bcf5d0efb3603e1f9b8ee2499ccdcfe2a18cef458"),
+				},
+				ExecutionPayloadHeader: &v1.ExecutionPayloadHeaderDeneb{
+					ParentHash:       ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					FeeRecipient:     ezDecode(t, "0xabcf8e0d4e9587369b2301d0790347320302cc09"),
+					StateRoot:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					ReceiptsRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					LogsBloom:        ezDecode(t, "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"),
+					PrevRandao:       ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					BlockNumber:      1,
+					GasLimit:         1,
+					GasUsed:          1,
+					Timestamp:        1,
+					ExtraData:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					BaseFeePerGas:    basebytes,
+					BlockHash:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					TransactionsRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					WithdrawalsRoot:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					BlobGasUsed:      1,
+					ExcessBlobGas:    2,
 				},
-				Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 			},
 		},
+		Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 	}
 }
 

api/client/builder/testing/mock.go

@@ -41,7 +41,7 @@ func (m MockClient) RegisterValidator(_ context.Context, svr []*ethpb.SignedVali
 }
 
 // SubmitBlindedBlock --
-func (MockClient) SubmitBlindedBlock(_ context.Context, _ interfaces.ReadOnlySignedBeaconBlock, _ []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
+func (MockClient) SubmitBlindedBlock(_ context.Context, _ interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
 	return nil, nil, nil
 }
 

api/client/builder/types.go

@@ -117,7 +117,7 @@ type VersionResponse struct {
 	Version string `json:"version"`
 }
 
-// ExecHeaderResponse is a JSON representation of  the builder API header response for Bellatrix.
+// ExecHeaderResponse is a JSON representation of the builder API header response for Bellatrix.
 type ExecHeaderResponse struct {
 	Version string `json:"version"`
 	Data    struct {
@@ -869,16 +869,16 @@ func (bb *BuilderBidDeneb) ToProto() (*eth.BuilderBidDeneb, error) {
 	if err != nil {
 		return nil, err
 	}
-	var bundle *v1.BlindedBlobsBundle
-	if bb.BlindedBlobsBundle != nil {
-		bundle, err = bb.BlindedBlobsBundle.ToProto()
-		if err != nil {
-			return nil, err
+	kzgCommitments := make([][]byte, len(bb.BlobKzgCommitments))
+	for i, commit := range bb.BlobKzgCommitments {
+		if len(commit) != fieldparams.BLSPubkeyLength {
+			return nil, fmt.Errorf("commitment length %d is not %d", len(commit), fieldparams.BLSPubkeyLength)
 		}
+		kzgCommitments[i] = bytesutil.SafeCopyBytes(commit)
 	}
 	return &eth.BuilderBidDeneb{
 		Header:             header,
-		BlindedBlobsBundle: bundle,
+		BlobKzgCommitments: kzgCommitments,
 		Value:              bytesutil.SafeCopyBytes(bb.Value.SSZBytes()),
 		Pubkey:             bytesutil.SafeCopyBytes(bb.Pubkey),
 	}, nil
@@ -887,42 +887,11 @@ func (bb *BuilderBidDeneb) ToProto() (*eth.BuilderBidDeneb, error) {
 // BuilderBidDeneb is a field of ExecHeaderResponseDeneb.
 type BuilderBidDeneb struct {
 	Header             *ExecutionPayloadHeaderDeneb `json:"header"`
-	BlindedBlobsBundle *BlindedBlobsBundle          `json:"blinded_blobs_bundle"`
+	BlobKzgCommitments []hexutil.Bytes              `json:"blob_kzg_commitments"`
 	Value              Uint256                      `json:"value"`
 	Pubkey             hexutil.Bytes                `json:"pubkey"`
 }
 
-// BlindedBlobsBundle is a field of BuilderBidDeneb and represents the blinded blobs of the associated header.
-type BlindedBlobsBundle struct {
-	KzgCommitments []hexutil.Bytes `json:"commitments"`
-	Proofs         []hexutil.Bytes `json:"proofs"`
-	BlobRoots      []hexutil.Bytes `json:"blob_roots"`
-}
-
-// ToProto creates a BlindedBlobsBundle Proto from BlindedBlobsBundle.
-func (r *BlindedBlobsBundle) ToProto() (*v1.BlindedBlobsBundle, error) {
-	kzg := make([][]byte, len(r.KzgCommitments))
-	for i := range kzg {
-		kzg[i] = bytesutil.SafeCopyBytes(r.KzgCommitments[i])
-	}
-
-	proofs := make([][]byte, len(r.Proofs))
-	for i := range proofs {
-		proofs[i] = bytesutil.SafeCopyBytes(r.Proofs[i])
-	}
-
-	blobRoots := make([][]byte, len(r.BlobRoots))
-	for i := range blobRoots {
-		blobRoots[i] = bytesutil.SafeCopyBytes(r.BlobRoots[i])
-	}
-
-	return &v1.BlindedBlobsBundle{
-		KzgCommitments: kzg,
-		Proofs:         proofs,
-		BlobRoots:      blobRoots,
-	}, nil
-}
-
 // ExecutionPayloadHeaderDeneb a field part of the BuilderBidDeneb.
 type ExecutionPayloadHeaderDeneb struct {
 	ParentHash       hexutil.Bytes `json:"parent_hash"`

api/client/builder/types_test.go

@@ -142,17 +142,9 @@ var testExampleHeaderResponseDeneb = `{
 		"blob_gas_used": "1",	
 		"excess_blob_gas": "2"	
       },
-      "blinded_blobs_bundle": {
-        "commitments": [
+      "blob_kzg_commitments": [
           "0x8dab030c51e16e84be9caab84ee3d0b8bbec1db4a0e4de76439da8424d9b957370a10a78851f97e4b54d2ce1ab0d686f"
-        ],
-        "proofs": [
-          "0xb4021b0de10f743893d4f71e1bf830c019e832958efd6795baf2f83b8699a9eccc5dc99015d8d4d8ec370d0cc333c06a"
-        ],
-        "blob_roots": [
-          "0x24564723180fcb3d994104538d351c8dcbde12d541676bb736cf678018ca4739"
-        ]
-      },
+	  ],
       "value": "652312848583266388373324160190187140051835877600158453279131187530910662656",
       "pubkey": "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a"
     },

api/client/validator/BUILD.bazel

@@ -7,7 +7,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//api/client:go_default_library",
-        "//validator/rpc/apimiddleware:go_default_library",
+        "//validator/rpc:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
     ],
 )

api/client/validator/client.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v4/api/client"
-	"github.com/prysmaticlabs/prysm/v4/validator/rpc/apimiddleware"
+	"github.com/prysmaticlabs/prysm/v4/validator/rpc"
 )
 
 const (
@@ -41,17 +41,17 @@ func (c *Client) GetValidatorPubKeys(ctx context.Context) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	if len(jsonlocal.Keystores) == 0 && len(jsonremote.Keystores) == 0 {
+	if len(jsonlocal.Data) == 0 && len(jsonremote.Data) == 0 {
 		return nil, errors.New("there are no local keys or remote keys on the validator")
 	}
 
 	hexKeys := make(map[string]bool)
 
-	for index := range jsonlocal.Keystores {
-		hexKeys[jsonlocal.Keystores[index].ValidatingPubkey] = true
+	for index := range jsonlocal.Data {
+		hexKeys[jsonlocal.Data[index].ValidatingPubkey] = true
 	}
-	for index := range jsonremote.Keystores {
-		hexKeys[jsonremote.Keystores[index].Pubkey] = true
+	for index := range jsonremote.Data {
+		hexKeys[jsonremote.Data[index].Pubkey] = true
 	}
 	keys := make([]string, 0)
 	for k := range hexKeys {
@@ -61,12 +61,12 @@ func (c *Client) GetValidatorPubKeys(ctx context.Context) ([]string, error) {
 }
 
 // GetLocalValidatorKeys calls the keymanager APIs for local validator keys
-func (c *Client) GetLocalValidatorKeys(ctx context.Context) (*apimiddleware.ListKeystoresResponseJson, error) {
+func (c *Client) GetLocalValidatorKeys(ctx context.Context) (*rpc.ListKeystoresResponse, error) {
 	localBytes, err := c.Get(ctx, localKeysPath, client.WithAuthorizationToken(c.Token()))
 	if err != nil {
 		return nil, err
 	}
-	jsonlocal := &apimiddleware.ListKeystoresResponseJson{}
+	jsonlocal := &rpc.ListKeystoresResponse{}
 	if err := json.Unmarshal(localBytes, jsonlocal); err != nil {
 		return nil, errors.Wrap(err, "failed to parse local keystore list")
 	}
@@ -74,14 +74,14 @@ func (c *Client) GetLocalValidatorKeys(ctx context.Context) (*apimiddleware.List
 }
 
 // GetRemoteValidatorKeys calls the keymanager APIs for web3signer validator keys
-func (c *Client) GetRemoteValidatorKeys(ctx context.Context) (*apimiddleware.ListRemoteKeysResponseJson, error) {
+func (c *Client) GetRemoteValidatorKeys(ctx context.Context) (*rpc.ListRemoteKeysResponse, error) {
 	remoteBytes, err := c.Get(ctx, remoteKeysPath, client.WithAuthorizationToken(c.Token()))
 	if err != nil {
 		if !strings.Contains(err.Error(), "Prysm Wallet is not of type Web3Signer") {
 			return nil, err
 		}
 	}
-	jsonremote := &apimiddleware.ListRemoteKeysResponseJson{}
+	jsonremote := &rpc.ListRemoteKeysResponse{}
 	if len(remoteBytes) != 0 {
 		if err := json.Unmarshal(remoteBytes, jsonremote); err != nil {
 			return nil, errors.Wrap(err, "failed to parse remote keystore list")
@@ -107,13 +107,13 @@ func (c *Client) GetFeeRecipientAddresses(ctx context.Context, validators []stri
 }
 
 // GetFeeRecipientAddress takes a public key and calls the keymanager API to return its fee recipient.
-func (c *Client) GetFeeRecipientAddress(ctx context.Context, pubkey string) (*apimiddleware.GetFeeRecipientByPubkeyResponseJson, error) {
+func (c *Client) GetFeeRecipientAddress(ctx context.Context, pubkey string) (*rpc.GetFeeRecipientByPubkeyResponse, error) {
 	path := strings.Replace(feeRecipientPath, "{pubkey}", pubkey, 1)
 	b, err := c.Get(ctx, path, client.WithAuthorizationToken(c.Token()))
 	if err != nil {
 		return nil, err
 	}
-	feejson := &apimiddleware.GetFeeRecipientByPubkeyResponseJson{}
+	feejson := &rpc.GetFeeRecipientByPubkeyResponse{}
 	if err := json.Unmarshal(b, feejson); err != nil {
 		return nil, errors.Wrap(err, "failed to parse fee recipient")
 	}

api/constants.go

@@ -0,0 +1,3 @@
+package api
+
+const WebUrlPrefix = "/v2/validator/"

api/gateway/BUILD.bazel

@@ -14,7 +14,6 @@ go_library(
         "//validator:__subpackages__",
     ],
     deps = [
-        "//api/gateway/apimiddleware:go_default_library",
         "//runtime:go_default_library",
         "@com_github_gorilla_mux//:go_default_library",
         "@com_github_grpc_ecosystem_grpc_gateway_v2//runtime:go_default_library",
@@ -24,6 +23,7 @@ go_library(
         "@org_golang_google_grpc//:go_default_library",
         "@org_golang_google_grpc//connectivity:go_default_library",
         "@org_golang_google_grpc//credentials:go_default_library",
+        "@org_golang_google_grpc//credentials/insecure:go_default_library",
         "@org_golang_google_protobuf//proto:go_default_library",
     ],
 )
@@ -33,7 +33,6 @@ go_test(
     srcs = ["gateway_test.go"],
     embed = [":go_default_library"],
     deps = [
-        "//api/gateway/apimiddleware:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/require:go_default_library",

api/gateway/apimiddleware/BUILD.bazel

@@ -1,43 +0,0 @@
-load("@prysm//tools/go:def.bzl", "go_library", "go_test")
-
-go_library(
-    name = "go_default_library",
-    srcs = [
-        "api_middleware.go",
-        "log.go",
-        "param_handling.go",
-        "process_field.go",
-        "process_request.go",
-        "structs.go",
-    ],
-    importpath = "github.com/prysmaticlabs/prysm/v4/api/gateway/apimiddleware",
-    visibility = ["//visibility:public"],
-    deps = [
-        "//api:go_default_library",
-        "//api/grpc:go_default_library",
-        "//encoding/bytesutil:go_default_library",
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
-        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
-        "@com_github_gorilla_mux//:go_default_library",
-        "@com_github_pkg_errors//:go_default_library",
-        "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_wealdtech_go_bytesutil//:go_default_library",
-    ],
-)
-
-go_test(
-    name = "go_default_test",
-    srcs = [
-        "param_handling_test.go",
-        "process_request_test.go",
-    ],
-    embed = [":go_default_library"],
-    deps = [
-        "//api:go_default_library",
-        "//api/grpc:go_default_library",
-        "//testing/assert:go_default_library",
-        "//testing/require:go_default_library",
-        "@com_github_gorilla_mux//:go_default_library",
-        "@com_github_sirupsen_logrus//hooks/test:go_default_library",
-    ],
-)

api/gateway/apimiddleware/api_middleware.go

@@ -1,265 +0,0 @@
-package apimiddleware
-
-import (
-	"net/http"
-	"reflect"
-	"time"
-
-	"github.com/gorilla/mux"
-)
-
-// ApiProxyMiddleware is a proxy between an Ethereum consensus API HTTP client and grpc-gateway.
-// The purpose of the proxy is to handle HTTP requests and gRPC responses in such a way that:
-//   - Ethereum consensus API requests can be handled by grpc-gateway correctly
-//   - gRPC responses can be returned as spec-compliant Ethereum consensus API responses
-type ApiProxyMiddleware struct {
-	GatewayAddress  string
-	EndpointCreator EndpointFactory
-	Timeout         time.Duration
-	router          *mux.Router
-}
-
-// EndpointFactory is responsible for creating new instances of Endpoint values.
-type EndpointFactory interface {
-	Create(path string) (*Endpoint, error)
-	Paths() []string
-	IsNil() bool
-}
-
-// Endpoint is a representation of an API HTTP endpoint that should be proxied by the middleware.
-type Endpoint struct {
-	Path               string          // The path of the HTTP endpoint.
-	GetResponse        interface{}     // The struct corresponding to the JSON structure used in a GET response.
-	PostRequest        interface{}     // The struct corresponding to the JSON structure used in a POST request.
-	PostResponse       interface{}     // The struct corresponding to the JSON structure used in a POST response.
-	DeleteRequest      interface{}     // The struct corresponding to the JSON structure used in a DELETE request.
-	DeleteResponse     interface{}     // The struct corresponding to the JSON structure used in a DELETE response.
-	RequestURLLiterals []string        // Names of URL parameters that should not be base64-encoded.
-	RequestQueryParams []QueryParam    // Query parameters of the request.
-	Err                ErrorJson       // The struct corresponding to the error that should be returned in case of a request failure.
-	Hooks              HookCollection  // A collection of functions that can be invoked at various stages of the request/response cycle.
-	CustomHandlers     []CustomHandler // Functions that will be executed instead of the default request/response behaviour.
-}
-
-// RunDefault expresses whether the default processing logic should be carried out after running a pre hook.
-type RunDefault bool
-
-// DefaultEndpoint returns an Endpoint with default configuration, e.g. DefaultErrorJson for error handling.
-func DefaultEndpoint() Endpoint {
-	return Endpoint{
-		Err: &DefaultErrorJson{},
-	}
-}
-
-// QueryParam represents a single query parameter's metadata.
-type QueryParam struct {
-	Name string
-	Hex  bool
-	Enum bool
-}
-
-// CustomHandler is a function that can be invoked at the very beginning of the request,
-// essentially replacing the whole default request/response logic with custom logic for a specific endpoint.
-type CustomHandler = func(m *ApiProxyMiddleware, endpoint Endpoint, w http.ResponseWriter, req *http.Request) (handled bool)
-
-// HookCollection contains hooks that can be used to amend the default request/response cycle with custom logic for a specific endpoint.
-type HookCollection struct {
-	OnPreDeserializeRequestBodyIntoContainer      func(endpoint *Endpoint, w http.ResponseWriter, req *http.Request) (RunDefault, ErrorJson)
-	OnPostDeserializeRequestBodyIntoContainer     func(endpoint *Endpoint, w http.ResponseWriter, req *http.Request) ErrorJson
-	OnPreDeserializeGrpcResponseBodyIntoContainer func([]byte, interface{}) (RunDefault, ErrorJson)
-	OnPreSerializeMiddlewareResponseIntoJson      func(interface{}) (RunDefault, []byte, ErrorJson)
-}
-
-// fieldProcessor applies the processing function f to a value when the tag is present on the field.
-type fieldProcessor struct {
-	tag string
-	f   func(value reflect.Value) error
-}
-
-// Run starts the proxy, registering all proxy endpoints.
-func (m *ApiProxyMiddleware) Run(gatewayRouter *mux.Router) {
-	for _, path := range m.EndpointCreator.Paths() {
-		gatewayRouter.HandleFunc(path, m.WithMiddleware(path))
-	}
-	m.router = gatewayRouter
-}
-
-// ServeHTTP for the proxy middleware.
-func (m *ApiProxyMiddleware) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	m.router.ServeHTTP(w, req)
-}
-
-// WithMiddleware wraps the given endpoint handler with the middleware logic.
-func (m *ApiProxyMiddleware) WithMiddleware(path string) http.HandlerFunc {
-	return func(w http.ResponseWriter, req *http.Request) {
-		endpoint, err := m.EndpointCreator.Create(path)
-		if err != nil {
-			log.WithError(err).Errorf("Could not create endpoint for path: %s", path)
-			return
-		}
-
-		for _, handler := range endpoint.CustomHandlers {
-			if handler(m, *endpoint, w, req) {
-				return
-			}
-		}
-
-		if req.Method == "POST" {
-			if errJson := handlePostRequestForEndpoint(endpoint, w, req); errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-		}
-
-		if req.Method == "DELETE" && req.Body != http.NoBody {
-			if errJson := handleDeleteRequestForEndpoint(endpoint, req); errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-		}
-
-		if errJson := m.PrepareRequestForProxying(*endpoint, req); errJson != nil {
-			WriteError(w, errJson, nil)
-			return
-		}
-		grpcResp, errJson := m.ProxyRequest(req)
-		if errJson != nil {
-			WriteError(w, errJson, nil)
-			return
-		}
-		grpcRespBody, errJson := ReadGrpcResponseBody(grpcResp.Body)
-		if errJson != nil {
-			WriteError(w, errJson, nil)
-			return
-		}
-
-		var respJson []byte
-		if !GrpcResponseIsEmpty(grpcRespBody) {
-			respHasError, errJson := HandleGrpcResponseError(endpoint.Err, grpcResp, grpcRespBody, w)
-			if errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-			if respHasError {
-				return
-			}
-
-			var resp interface{}
-			if req.Method == "GET" {
-				resp = endpoint.GetResponse
-			} else if req.Method == "DELETE" {
-				resp = endpoint.DeleteResponse
-			} else {
-				resp = endpoint.PostResponse
-			}
-			if errJson := deserializeGrpcResponseBodyIntoContainerWrapped(endpoint, grpcRespBody, resp); errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-			if errJson := ProcessMiddlewareResponseFields(resp); errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-
-			respJson, errJson = serializeMiddlewareResponseIntoJsonWrapped(endpoint, respJson, resp)
-			if errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-		}
-
-		if errJson := WriteMiddlewareResponseHeadersAndBody(grpcResp, respJson, w); errJson != nil {
-			WriteError(w, errJson, nil)
-			return
-		}
-		if errJson := Cleanup(grpcResp.Body); errJson != nil {
-			WriteError(w, errJson, nil)
-			return
-		}
-	}
-}
-
-func handlePostRequestForEndpoint(endpoint *Endpoint, w http.ResponseWriter, req *http.Request) ErrorJson {
-	if errJson := deserializeRequestBodyIntoContainerWrapped(endpoint, req, w); errJson != nil {
-		return errJson
-	}
-	if errJson := ProcessRequestContainerFields(endpoint.PostRequest); errJson != nil {
-		return errJson
-	}
-	return SetRequestBodyToRequestContainer(endpoint.PostRequest, req)
-}
-
-func handleDeleteRequestForEndpoint(endpoint *Endpoint, req *http.Request) ErrorJson {
-	if errJson := DeserializeRequestBodyIntoContainer(req.Body, endpoint.DeleteRequest); errJson != nil {
-		return errJson
-	}
-	if errJson := ProcessRequestContainerFields(endpoint.DeleteRequest); errJson != nil {
-		return errJson
-	}
-	return SetRequestBodyToRequestContainer(endpoint.DeleteRequest, req)
-}
-
-func deserializeRequestBodyIntoContainerWrapped(endpoint *Endpoint, req *http.Request, w http.ResponseWriter) ErrorJson {
-	runDefault := true
-	if endpoint.Hooks.OnPreDeserializeRequestBodyIntoContainer != nil {
-		run, errJson := endpoint.Hooks.OnPreDeserializeRequestBodyIntoContainer(endpoint, w, req)
-		if errJson != nil {
-			return errJson
-		}
-		if !run {
-			runDefault = false
-		}
-	}
-	if runDefault {
-		if errJson := DeserializeRequestBodyIntoContainer(req.Body, endpoint.PostRequest); errJson != nil {
-			return errJson
-		}
-	}
-	if endpoint.Hooks.OnPostDeserializeRequestBodyIntoContainer != nil {
-		if errJson := endpoint.Hooks.OnPostDeserializeRequestBodyIntoContainer(endpoint, w, req); errJson != nil {
-			return errJson
-		}
-	}
-	return nil
-}
-
-func deserializeGrpcResponseBodyIntoContainerWrapped(endpoint *Endpoint, grpcResponseBody []byte, resp interface{}) ErrorJson {
-	runDefault := true
-	if endpoint.Hooks.OnPreDeserializeGrpcResponseBodyIntoContainer != nil {
-		run, errJson := endpoint.Hooks.OnPreDeserializeGrpcResponseBodyIntoContainer(grpcResponseBody, resp)
-		if errJson != nil {
-			return errJson
-		}
-		if !run {
-			runDefault = false
-		}
-	}
-	if runDefault {
-		if errJson := DeserializeGrpcResponseBodyIntoContainer(grpcResponseBody, resp); errJson != nil {
-			return errJson
-		}
-	}
-	return nil
-}
-
-func serializeMiddlewareResponseIntoJsonWrapped(endpoint *Endpoint, respJson []byte, resp interface{}) ([]byte, ErrorJson) {
-	runDefault := true
-	var errJson ErrorJson
-	if endpoint.Hooks.OnPreSerializeMiddlewareResponseIntoJson != nil {
-		var run RunDefault
-		run, respJson, errJson = endpoint.Hooks.OnPreSerializeMiddlewareResponseIntoJson(resp)
-		if errJson != nil {
-			return nil, errJson
-		}
-		if !run {
-			runDefault = false
-		}
-	}
-	if runDefault {
-		respJson, errJson = SerializeMiddlewareResponseIntoJson(resp)
-		if errJson != nil {
-			return nil, errJson
-		}
-	}
-	return respJson, nil
-}

api/gateway/apimiddleware/log.go

@@ -1,5 +0,0 @@
-package apimiddleware
-
-import "github.com/sirupsen/logrus"
-
-var log = logrus.WithField("prefix", "apimiddleware")

api/gateway/apimiddleware/param_handling.go

@@ -1,103 +0,0 @@
-package apimiddleware
-
-import (
-	"encoding/base64"
-	"net/http"
-	"net/url"
-	"strings"
-
-	"github.com/gorilla/mux"
-	butil "github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
-	"github.com/wealdtech/go-bytesutil"
-)
-
-// HandleURLParameters processes URL parameters, allowing parameterized URLs to be safely and correctly proxied to grpc-gateway.
-func HandleURLParameters(url string, req *http.Request, literals []string) ErrorJson {
-	segments := strings.Split(url, "/")
-
-segmentsLoop:
-	for i, s := range segments {
-		// We only care about segments which are parameterized.
-		if isRequestParam(s) {
-			// Don't do anything with parameters which should be forwarded literally to gRPC.
-			for _, l := range literals {
-				if s == "{"+l+"}" {
-					continue segmentsLoop
-				}
-			}
-
-			routeVar := mux.Vars(req)[s[1:len(s)-1]]
-			bRouteVar := []byte(routeVar)
-			if butil.IsHex(bRouteVar) {
-				var err error
-				bRouteVar, err = bytesutil.FromHexString(string(bRouteVar))
-				if err != nil {
-					return InternalServerErrorWithMessage(err, "could not process URL parameter")
-				}
-			}
-			// Converting hex to base64 may result in a value which malforms the URL.
-			// We use URLEncoding to safely escape such values.
-			base64RouteVar := base64.URLEncoding.EncodeToString(bRouteVar)
-
-			// Merge segments back into the full URL.
-			splitPath := strings.Split(req.URL.Path, "/")
-			splitPath[i] = base64RouteVar
-			req.URL.Path = strings.Join(splitPath, "/")
-		}
-	}
-	return nil
-}
-
-// HandleQueryParameters processes query parameters, allowing them to be safely and correctly proxied to grpc-gateway.
-func HandleQueryParameters(req *http.Request, params []QueryParam) ErrorJson {
-	queryParams := req.URL.Query()
-
-	normalizeQueryValues(queryParams)
-
-	for key, vals := range queryParams {
-		for _, p := range params {
-			if key == p.Name {
-				if p.Hex {
-					queryParams.Del(key)
-					for _, v := range vals {
-						b := []byte(v)
-						if butil.IsHex(b) {
-							var err error
-							b, err = bytesutil.FromHexString(v)
-							if err != nil {
-								return InternalServerErrorWithMessage(err, "could not process query parameter")
-							}
-						}
-						queryParams.Add(key, base64.URLEncoding.EncodeToString(b))
-					}
-				}
-				if p.Enum {
-					queryParams.Del(key)
-					for _, v := range vals {
-						// gRPC expects uppercase enum values.
-						queryParams.Add(key, strings.ToUpper(v))
-					}
-				}
-			}
-		}
-	}
-	req.URL.RawQuery = queryParams.Encode()
-	return nil
-}
-
-// isRequestParam verifies whether the passed string is a request parameter.
-// Request parameters are enclosed in { and }.
-func isRequestParam(s string) bool {
-	return len(s) > 2 && s[0] == '{' && s[len(s)-1] == '}'
-}
-
-func normalizeQueryValues(queryParams url.Values) {
-	// Replace comma-separated values with individual values.
-	for key, vals := range queryParams {
-		splitVals := make([]string, 0)
-		for _, v := range vals {
-			splitVals = append(splitVals, strings.Split(v, ",")...)
-		}
-		queryParams[key] = splitVals
-	}
-}

api/gateway/apimiddleware/param_handling_test.go

@@ -1,124 +0,0 @@
-package apimiddleware
-
-import (
-	"bytes"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/gorilla/mux"
-	"github.com/prysmaticlabs/prysm/v4/testing/assert"
-	"github.com/prysmaticlabs/prysm/v4/testing/require"
-)
-
-func TestHandleURLParameters(t *testing.T) {
-	var body bytes.Buffer
-
-	t.Run("no_params", func(t *testing.T) {
-		request := httptest.NewRequest("GET", "http://foo.example/bar", &body)
-
-		errJson := HandleURLParameters("/not_param", request, []string{})
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "/bar", request.URL.Path)
-	})
-
-	t.Run("with_params", func(t *testing.T) {
-		muxVars := make(map[string]string)
-		muxVars["bar_param"] = "bar"
-		muxVars["quux_param"] = "quux"
-		request := httptest.NewRequest("GET", "http://foo.example/bar/baz/quux", &body)
-		request = mux.SetURLVars(request, muxVars)
-
-		errJson := HandleURLParameters("/{bar_param}/not_param/{quux_param}", request, []string{})
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "/YmFy/baz/cXV1eA==", request.URL.Path)
-	})
-
-	t.Run("with_literal", func(t *testing.T) {
-		muxVars := make(map[string]string)
-		muxVars["bar_param"] = "bar"
-		request := httptest.NewRequest("GET", "http://foo.example/bar/baz", &body)
-		request = mux.SetURLVars(request, muxVars)
-
-		errJson := HandleURLParameters("/{bar_param}/not_param/", request, []string{"bar_param"})
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "/bar/baz", request.URL.Path)
-	})
-
-	t.Run("with_hex", func(t *testing.T) {
-		muxVars := make(map[string]string)
-		muxVars["hex_param"] = "0x626172"
-		request := httptest.NewRequest("GET", "http://foo.example/0x626172/baz", &body)
-		request = mux.SetURLVars(request, muxVars)
-
-		errJson := HandleURLParameters("/{hex_param}/not_param/", request, []string{})
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "/YmFy/baz", request.URL.Path)
-	})
-}
-
-func TestHandleQueryParameters(t *testing.T) {
-	var body bytes.Buffer
-
-	t.Run("regular_params", func(t *testing.T) {
-		request := httptest.NewRequest("GET", "http://foo.example?bar=bar&baz=baz", &body)
-
-		errJson := HandleQueryParameters(request, []QueryParam{{Name: "bar"}, {Name: "baz"}})
-		require.Equal(t, true, errJson == nil)
-		query := request.URL.Query()
-		v, ok := query["bar"]
-		require.Equal(t, true, ok, "query param not found")
-		require.Equal(t, 1, len(v), "wrong number of query param values")
-		assert.Equal(t, "bar", v[0])
-		v, ok = query["baz"]
-		require.Equal(t, true, ok, "query param not found")
-		require.Equal(t, 1, len(v), "wrong number of query param values")
-		assert.Equal(t, "baz", v[0])
-	})
-
-	t.Run("hex_and_enum_params", func(t *testing.T) {
-		request := httptest.NewRequest("GET", "http://foo.example?hex=0x626172&baz=baz", &body)
-
-		errJson := HandleQueryParameters(request, []QueryParam{{Name: "hex", Hex: true}, {Name: "baz", Enum: true}})
-		require.Equal(t, true, errJson == nil)
-		query := request.URL.Query()
-		v, ok := query["hex"]
-		require.Equal(t, true, ok, "query param not found")
-		require.Equal(t, 1, len(v), "wrong number of query param values")
-		assert.Equal(t, "YmFy", v[0])
-		v, ok = query["baz"]
-		require.Equal(t, true, ok, "query param not found")
-		require.Equal(t, 1, len(v), "wrong number of query param values")
-		assert.Equal(t, "BAZ", v[0])
-	})
-}
-
-func TestIsRequestParam(t *testing.T) {
-	tests := []struct {
-		s string
-		b bool
-	}{
-		{"", false},
-		{"{", false},
-		{"}", false},
-		{"{}", false},
-		{"{x}", true},
-		{"{very_long_parameter_name_with_underscores}", true},
-	}
-	for _, tt := range tests {
-		b := isRequestParam(tt.s)
-		assert.Equal(t, tt.b, b)
-	}
-}
-
-func TestNormalizeQueryValues(t *testing.T) {
-	input := make(map[string][]string)
-	input["key"] = []string{"value1", "value2,value3,value4", "value5"}
-
-	normalizeQueryValues(input)
-	require.Equal(t, 5, len(input["key"]))
-	assert.Equal(t, "value1", input["key"][0])
-	assert.Equal(t, "value2", input["key"][1])
-	assert.Equal(t, "value3", input["key"][2])
-	assert.Equal(t, "value4", input["key"][3])
-	assert.Equal(t, "value5", input["key"][4])
-}

api/gateway/apimiddleware/process_field.go

@@ -1,179 +0,0 @@
-package apimiddleware
-
-import (
-	"encoding/base64"
-	"fmt"
-	"math/big"
-	"reflect"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/pkg/errors"
-	"github.com/wealdtech/go-bytesutil"
-)
-
-// processField calls each processor function on any field that has the matching tag set.
-// It is a recursive function.
-func processField(s interface{}, processors []fieldProcessor) error {
-	kind := reflect.TypeOf(s).Kind()
-	if kind != reflect.Ptr && kind != reflect.Slice && kind != reflect.Array {
-		return fmt.Errorf("processing fields of kind '%v' is unsupported", kind)
-	}
-
-	t := reflect.TypeOf(s).Elem()
-	v := reflect.Indirect(reflect.ValueOf(s))
-
-	for i := 0; i < t.NumField(); i++ {
-		switch v.Field(i).Kind() {
-		case reflect.Slice:
-			sliceElem := t.Field(i).Type.Elem()
-			kind := sliceElem.Kind()
-			// Recursively process slices to struct pointers.
-			switch {
-			case kind == reflect.Ptr && sliceElem.Elem().Kind() == reflect.Struct:
-				for j := 0; j < v.Field(i).Len(); j++ {
-					if err := processField(v.Field(i).Index(j).Interface(), processors); err != nil {
-						return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
-					}
-				}
-			// Process each string in string slices.
-			case kind == reflect.String:
-				for _, proc := range processors {
-					_, hasTag := t.Field(i).Tag.Lookup(proc.tag)
-					if !hasTag {
-						continue
-					}
-					for j := 0; j < v.Field(i).Len(); j++ {
-						if err := proc.f(v.Field(i).Index(j)); err != nil {
-							return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
-						}
-					}
-				}
-			}
-		// Recursively process struct pointers.
-		case reflect.Ptr:
-			if v.Field(i).Elem().Kind() == reflect.Struct {
-				if err := processField(v.Field(i).Interface(), processors); err != nil {
-					return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
-				}
-			}
-		default:
-			field := t.Field(i)
-			for _, proc := range processors {
-				if _, hasTag := field.Tag.Lookup(proc.tag); hasTag {
-					if err := proc.f(v.Field(i)); err != nil {
-						return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
-					}
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func hexToBase64Processor(v reflect.Value) error {
-	if v.String() == "0x" {
-		v.SetString("")
-		return nil
-	}
-	b, err := bytesutil.FromHexString(v.String())
-	if err != nil {
-		return err
-	}
-	v.SetString(base64.StdEncoding.EncodeToString(b))
-	return nil
-}
-
-func base64ToHexProcessor(v reflect.Value) error {
-	if v.String() == "" {
-		// Empty hex values are represented as "0x".
-		v.SetString("0x")
-		return nil
-	}
-	b, err := base64.StdEncoding.DecodeString(v.String())
-	if err != nil {
-		return err
-	}
-	v.SetString(hexutil.Encode(b))
-	return nil
-}
-
-func base64ToChecksumAddressProcessor(v reflect.Value) error {
-	if v.String() == "" {
-		// Empty hex values are represented as "0x".
-		v.SetString("0x")
-		return nil
-	}
-	b, err := base64.StdEncoding.DecodeString(v.String())
-	if err != nil {
-		return err
-	}
-	v.SetString(common.BytesToAddress(b).Hex())
-	return nil
-}
-
-func base64ToUint256Processor(v reflect.Value) error {
-	if v.String() == "" {
-		return nil
-	}
-	littleEndian, err := base64.StdEncoding.DecodeString(v.String())
-	if err != nil {
-		return err
-	}
-	if len(littleEndian) != 32 {
-		return errors.New("invalid length for Uint256")
-	}
-
-	// Integers are stored as little-endian, but
-	// big.Int expects big-endian. So we need to reverse
-	// the byte order before decoding.
-	var bigEndian [32]byte
-	for i := 0; i < len(littleEndian); i++ {
-		bigEndian[i] = littleEndian[len(littleEndian)-1-i]
-	}
-	var uint256 big.Int
-	uint256.SetBytes(bigEndian[:])
-	v.SetString(uint256.String())
-	return nil
-}
-
-func uint256ToBase64Processor(v reflect.Value) error {
-	if v.String() == "" {
-		return nil
-	}
-	uint256, ok := new(big.Int).SetString(v.String(), 10)
-	if !ok {
-		return fmt.Errorf("could not parse Uint256")
-	}
-	bigEndian := uint256.Bytes()
-	if len(bigEndian) > 32 {
-		return fmt.Errorf("number too big for Uint256")
-	}
-
-	// Integers are stored as little-endian, but
-	// big.Int gives big-endian. So we need to reverse
-	// the byte order before encoding.
-	var littleEndian [32]byte
-	for i := 0; i < len(bigEndian); i++ {
-		littleEndian[i] = bigEndian[len(bigEndian)-1-i]
-	}
-	v.SetString(base64.StdEncoding.EncodeToString(littleEndian[:]))
-	return nil
-}
-
-func enumToLowercaseProcessor(v reflect.Value) error {
-	v.SetString(strings.ToLower(v.String()))
-	return nil
-}
-
-func timeToUnixProcessor(v reflect.Value) error {
-	t, err := time.Parse(time.RFC3339, v.String())
-	if err != nil {
-		return err
-	}
-	v.SetString(strconv.FormatUint(uint64(t.Unix()), 10))
-	return nil
-}

api/gateway/apimiddleware/process_request.go

@@ -1,283 +0,0 @@
-package apimiddleware
-
-import (
-	"bytes"
-	"encoding/json"
-	"io"
-	"net"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/v4/api"
-	"github.com/prysmaticlabs/prysm/v4/api/grpc"
-)
-
-// DeserializeRequestBodyIntoContainer deserializes the request's body into an endpoint-specific struct.
-func DeserializeRequestBodyIntoContainer(body io.Reader, requestContainer interface{}) ErrorJson {
-	decoder := json.NewDecoder(body)
-	decoder.DisallowUnknownFields()
-	if err := decoder.Decode(&requestContainer); err != nil {
-		if strings.Contains(err.Error(), "json: unknown field") {
-			e := errors.Wrap(err, "could not decode request body")
-			return &DefaultErrorJson{
-				Message: e.Error(),
-				Code:    http.StatusBadRequest,
-			}
-		}
-		return InternalServerErrorWithMessage(err, "could not decode request body")
-	}
-	return nil
-}
-
-// ProcessRequestContainerFields processes fields of an endpoint-specific container according to field tags.
-func ProcessRequestContainerFields(requestContainer interface{}) ErrorJson {
-	if err := processField(requestContainer, []fieldProcessor{
-		{
-			tag: "hex",
-			f:   hexToBase64Processor,
-		},
-		{
-			tag: "uint256",
-			f:   uint256ToBase64Processor,
-		},
-	}); err != nil {
-		return InternalServerErrorWithMessage(err, "could not process request data")
-	}
-	return nil
-}
-
-// SetRequestBodyToRequestContainer makes the endpoint-specific container the new body of the request.
-func SetRequestBodyToRequestContainer(requestContainer interface{}, req *http.Request) ErrorJson {
-	// Serialize the struct, which now includes a base64-encoded value, into JSON.
-	j, err := json.Marshal(requestContainer)
-	if err != nil {
-		return InternalServerErrorWithMessage(err, "could not marshal request")
-	}
-	// Set the body to the new JSON.
-	req.Body = io.NopCloser(bytes.NewReader(j))
-	req.Header.Set("Content-Length", strconv.Itoa(len(j)))
-	req.ContentLength = int64(len(j))
-	return nil
-}
-
-// PrepareRequestForProxying applies additional logic to the request so that it can be correctly proxied to grpc-gateway.
-func (m *ApiProxyMiddleware) PrepareRequestForProxying(endpoint Endpoint, req *http.Request) ErrorJson {
-	req.URL.Scheme = "http"
-	req.URL.Host = m.GatewayAddress
-	req.RequestURI = ""
-	if errJson := HandleURLParameters(endpoint.Path, req, endpoint.RequestURLLiterals); errJson != nil {
-		return errJson
-	}
-	if errJson := HandleQueryParameters(req, endpoint.RequestQueryParams); errJson != nil {
-		return errJson
-	}
-	// We have to add the prefix after handling parameters because adding the prefix changes URL segment indexing.
-	req.URL.Path = "/internal" + req.URL.Path
-	return nil
-}
-
-// ProxyRequest proxies the request to grpc-gateway.
-func (m *ApiProxyMiddleware) ProxyRequest(req *http.Request) (*http.Response, ErrorJson) {
-	// We do not use http.DefaultClient because it does not have any timeout.
-	netClient := &http.Client{Timeout: m.Timeout}
-	grpcResp, err := netClient.Do(req)
-	if err != nil {
-		if err, ok := err.(net.Error); ok && err.Timeout() {
-			return nil, TimeoutError()
-		}
-		return nil, InternalServerErrorWithMessage(err, "could not proxy request")
-	}
-	if grpcResp == nil {
-		return nil, &DefaultErrorJson{Message: "nil response from gRPC-gateway", Code: http.StatusInternalServerError}
-	}
-	return grpcResp, nil
-}
-
-// ReadGrpcResponseBody reads the body from the grpc-gateway's response.
-func ReadGrpcResponseBody(r io.Reader) ([]byte, ErrorJson) {
-	body, err := io.ReadAll(r)
-	if err != nil {
-		return nil, InternalServerErrorWithMessage(err, "could not read response body")
-	}
-	return body, nil
-}
-
-// HandleGrpcResponseError acts on an error that resulted from a grpc-gateway's response.
-// Whether there was an error is indicated by the bool return value. In case of an error,
-// there is no need to write to the response because it's taken care of by the function.
-func HandleGrpcResponseError(errJson ErrorJson, resp *http.Response, respBody []byte, w http.ResponseWriter) (bool, ErrorJson) {
-	responseHasError := false
-	if err := json.Unmarshal(respBody, errJson); err != nil {
-		return false, InternalServerErrorWithMessage(err, "could not unmarshal error")
-	}
-	if errJson.Msg() != "" {
-		responseHasError = true
-		// Something went wrong, but the request completed, meaning we can write headers and the error message.
-		for h, vs := range resp.Header {
-			for _, v := range vs {
-				if strings.HasSuffix(h, api.VersionHeader) {
-					w.Header().Set(api.VersionHeader, v)
-				} else {
-					w.Header().Set(h, v)
-				}
-			}
-		}
-		// Handle gRPC timeout.
-		if resp.StatusCode == http.StatusGatewayTimeout {
-			WriteError(w, TimeoutError(), resp.Header)
-		} else {
-			// Set code to HTTP code because unmarshalled body contained gRPC code.
-			errJson.SetCode(resp.StatusCode)
-			WriteError(w, errJson, resp.Header)
-		}
-	}
-	return responseHasError, nil
-}
-
-// GrpcResponseIsEmpty determines whether the grpc-gateway's response body contains no data.
-func GrpcResponseIsEmpty(grpcResponseBody []byte) bool {
-	return len(grpcResponseBody) == 0 || string(grpcResponseBody) == "{}"
-}
-
-// DeserializeGrpcResponseBodyIntoContainer deserializes the grpc-gateway's response body into an endpoint-specific struct.
-func DeserializeGrpcResponseBodyIntoContainer(body []byte, responseContainer interface{}) ErrorJson {
-	if err := json.Unmarshal(body, &responseContainer); err != nil {
-		return InternalServerErrorWithMessage(err, "could not unmarshal response")
-	}
-	return nil
-}
-
-// ProcessMiddlewareResponseFields processes fields of an endpoint-specific container according to field tags.
-func ProcessMiddlewareResponseFields(responseContainer interface{}) ErrorJson {
-	if err := processField(responseContainer, []fieldProcessor{
-		{
-			tag: "hex",
-			f:   base64ToHexProcessor,
-		},
-		{
-			tag: "address",
-			f:   base64ToChecksumAddressProcessor,
-		},
-		{
-			tag: "enum",
-			f:   enumToLowercaseProcessor,
-		},
-		{
-			tag: "time",
-			f:   timeToUnixProcessor,
-		},
-		{
-			tag: "uint256",
-			f:   base64ToUint256Processor,
-		},
-	}); err != nil {
-		return InternalServerErrorWithMessage(err, "could not process response data")
-	}
-	return nil
-}
-
-// SerializeMiddlewareResponseIntoJson serializes the endpoint-specific response struct into a JSON representation.
-func SerializeMiddlewareResponseIntoJson(responseContainer interface{}) (jsonResponse []byte, errJson ErrorJson) {
-	j, err := json.Marshal(responseContainer)
-	if err != nil {
-		return nil, InternalServerErrorWithMessage(err, "could not marshal response")
-	}
-	return j, nil
-}
-
-// WriteMiddlewareResponseHeadersAndBody populates headers and the body of the final response.
-func WriteMiddlewareResponseHeadersAndBody(grpcResp *http.Response, responseJson []byte, w http.ResponseWriter) ErrorJson {
-	var statusCodeHeader string
-	for h, vs := range grpcResp.Header {
-		// We don't want to expose any gRPC metadata in the HTTP response, so we skip forwarding metadata headers.
-		if strings.HasPrefix(h, grpc.MetadataPrefix) {
-			if h == grpc.WithPrefix(grpc.HttpCodeMetadataKey) {
-				statusCodeHeader = vs[0]
-			} else if strings.HasSuffix(h, api.VersionHeader) {
-				w.Header().Set(api.VersionHeader, vs[0])
-			}
-		} else {
-			for _, v := range vs {
-				w.Header().Set(h, v)
-			}
-		}
-	}
-	if !GrpcResponseIsEmpty(responseJson) {
-		w.Header().Set("Content-Length", strconv.Itoa(len(responseJson)))
-		if statusCodeHeader != "" {
-			code, err := strconv.Atoi(statusCodeHeader)
-			if err != nil {
-				return InternalServerErrorWithMessage(err, "could not parse status code")
-			}
-			w.WriteHeader(code)
-		} else {
-			w.WriteHeader(grpcResp.StatusCode)
-		}
-		if _, err := io.Copy(w, io.NopCloser(bytes.NewReader(responseJson))); err != nil {
-			return InternalServerErrorWithMessage(err, "could not write response message")
-		}
-	} else {
-		w.Header().Set("Content-Length", "0")
-		w.WriteHeader(grpcResp.StatusCode)
-	}
-	return nil
-}
-
-// WriteError writes the error by manipulating headers and the body of the final response.
-func WriteError(w http.ResponseWriter, errJson ErrorJson, responseHeader http.Header) {
-	// Include custom error in the error JSON.
-	hasCustomError := false
-	if responseHeader != nil {
-		customError, ok := responseHeader[grpc.WithPrefix(grpc.CustomErrorMetadataKey)]
-		if ok {
-			hasCustomError = true
-			// Assume header has only one value and read the 0 index.
-			if err := json.Unmarshal([]byte(customError[0]), errJson); err != nil {
-				log.WithError(err).Error("Could not unmarshal custom error message")
-				return
-			}
-		}
-	}
-
-	var j []byte
-	if hasCustomError {
-		var err error
-		j, err = json.Marshal(errJson)
-		if err != nil {
-			log.WithError(err).Error("Could not marshal error message")
-			return
-		}
-	} else {
-		var err error
-		// We marshal the response body into a DefaultErrorJson if the custom error is not present.
-		// This is because the ErrorJson argument is the endpoint's error definition, which may contain custom fields.
-		// In such a scenario marhaling the endpoint's error would populate the resulting JSON
-		// with these fields even if they are not present in the gRPC header.
-		d := &DefaultErrorJson{
-			Message: errJson.Msg(),
-			Code:    errJson.StatusCode(),
-		}
-		j, err = json.Marshal(d)
-		if err != nil {
-			log.WithError(err).Error("Could not marshal error message")
-			return
-		}
-	}
-
-	w.Header().Set("Content-Length", strconv.Itoa(len(j)))
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(errJson.StatusCode())
-	if _, err := io.Copy(w, io.NopCloser(bytes.NewReader(j))); err != nil {
-		log.WithError(err).Error("Could not write error message")
-	}
-}
-
-// Cleanup performs final cleanup on the initial response from grpc-gateway.
-func Cleanup(grpcResponseBody io.ReadCloser) ErrorJson {
-	if err := grpcResponseBody.Close(); err != nil {
-		return InternalServerErrorWithMessage(err, "could not close response body")
-	}
-	return nil
-}

api/gateway/apimiddleware/process_request_test.go

@@ -1,435 +0,0 @@
-package apimiddleware
-
-import (
-	"bytes"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/prysmaticlabs/prysm/v4/api"
-	"github.com/prysmaticlabs/prysm/v4/api/grpc"
-	"github.com/prysmaticlabs/prysm/v4/testing/assert"
-	"github.com/prysmaticlabs/prysm/v4/testing/require"
-	"github.com/sirupsen/logrus/hooks/test"
-)
-
-type testRequestContainer struct {
-	TestString         string
-	TestHexString      string `hex:"true"`
-	TestEmptyHexString string `hex:"true"`
-	TestUint256String  string `uint256:"true"`
-}
-
-func defaultRequestContainer() *testRequestContainer {
-	return &testRequestContainer{
-		TestString:         "test string",
-		TestHexString:      "0x666F6F", // hex encoding of "foo"
-		TestEmptyHexString: "0x",
-		TestUint256String:  "4196",
-	}
-}
-
-type testResponseContainer struct {
-	TestString       string
-	TestHex          string `hex:"true"`
-	TestEmptyHex     string `hex:"true"`
-	TestAddress      string `address:"true"`
-	TestEmptyAddress string `address:"true"`
-	TestUint256      string `uint256:"true"`
-	TestEnum         string `enum:"true"`
-	TestTime         string `time:"true"`
-}
-
-func defaultResponseContainer() *testResponseContainer {
-	return &testResponseContainer{
-		TestString:       "test string",
-		TestHex:          "Zm9v", // base64 encoding of "foo"
-		TestEmptyHex:     "",
-		TestAddress:      "Zm9v",
-		TestEmptyAddress: "",
-		TestEnum:         "Test Enum",
-		TestTime:         "2006-01-02T15:04:05Z",
-
-		// base64 encoding of 4196 in little-endian
-		TestUint256: "ZBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
-	}
-}
-
-type testErrorJson struct {
-	Message     string
-	Code        int
-	CustomField string
-}
-
-// StatusCode returns the error's underlying error code.
-func (e *testErrorJson) StatusCode() int {
-	return e.Code
-}
-
-// Msg returns the error's underlying message.
-func (e *testErrorJson) Msg() string {
-	return e.Message
-}
-
-// SetCode sets the error's underlying error code.
-func (e *testErrorJson) SetCode(code int) {
-	e.Code = code
-}
-
-// SetMsg sets the error's underlying message.
-func (e *testErrorJson) SetMsg(msg string) {
-	e.Message = msg
-}
-
-func TestDeserializeRequestBodyIntoContainer(t *testing.T) {
-	t.Run("ok", func(t *testing.T) {
-		var bodyJson bytes.Buffer
-		err := json.NewEncoder(&bodyJson).Encode(defaultRequestContainer())
-		require.NoError(t, err)
-
-		container := &testRequestContainer{}
-		errJson := DeserializeRequestBodyIntoContainer(&bodyJson, container)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "test string", container.TestString)
-	})
-
-	t.Run("error", func(t *testing.T) {
-		var bodyJson bytes.Buffer
-		bodyJson.Write([]byte("foo"))
-		errJson := DeserializeRequestBodyIntoContainer(&bodyJson, &testRequestContainer{})
-		require.NotNil(t, errJson)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not decode request body"))
-		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
-	})
-
-	t.Run("unknown field", func(t *testing.T) {
-		var bodyJson bytes.Buffer
-		bodyJson.Write([]byte("{\"foo\":\"foo\"}"))
-		errJson := DeserializeRequestBodyIntoContainer(&bodyJson, &testRequestContainer{})
-		require.NotNil(t, errJson)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not decode request body"))
-		assert.Equal(t, http.StatusBadRequest, errJson.StatusCode())
-	})
-}
-
-func TestProcessRequestContainerFields(t *testing.T) {
-	t.Run("ok", func(t *testing.T) {
-		container := defaultRequestContainer()
-
-		errJson := ProcessRequestContainerFields(container)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "Zm9v", container.TestHexString)
-		assert.Equal(t, "", container.TestEmptyHexString)
-		assert.Equal(t, "ZBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", container.TestUint256String)
-	})
-
-	t.Run("error", func(t *testing.T) {
-		errJson := ProcessRequestContainerFields("foo")
-		require.NotNil(t, errJson)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not process request data"))
-		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
-	})
-}
-
-func TestSetRequestBodyToRequestContainer(t *testing.T) {
-	var body bytes.Buffer
-	request := httptest.NewRequest("GET", "http://foo.example", &body)
-
-	errJson := SetRequestBodyToRequestContainer(defaultRequestContainer(), request)
-	require.Equal(t, true, errJson == nil)
-	container := &testRequestContainer{}
-	require.NoError(t, json.NewDecoder(request.Body).Decode(container))
-	assert.Equal(t, "test string", container.TestString)
-	contentLengthHeader, ok := request.Header["Content-Length"]
-	require.Equal(t, true, ok)
-	require.Equal(t, 1, len(contentLengthHeader), "wrong number of header values")
-	assert.Equal(t, "108", contentLengthHeader[0])
-	assert.Equal(t, int64(108), request.ContentLength)
-}
-
-func TestPrepareRequestForProxying(t *testing.T) {
-	middleware := &ApiProxyMiddleware{
-		GatewayAddress: "http://gateway.example",
-	}
-	// We will set some params to make the request more interesting.
-	endpoint := Endpoint{
-		Path:               "/{url_param}",
-		RequestURLLiterals: []string{"url_param"},
-		RequestQueryParams: []QueryParam{{Name: "query_param"}},
-	}
-	var body bytes.Buffer
-	request := httptest.NewRequest("GET", "http://foo.example?query_param=bar", &body)
-
-	errJson := middleware.PrepareRequestForProxying(endpoint, request)
-	require.Equal(t, true, errJson == nil)
-	assert.Equal(t, "http", request.URL.Scheme)
-	assert.Equal(t, middleware.GatewayAddress, request.URL.Host)
-	assert.Equal(t, "", request.RequestURI)
-}
-
-func TestReadGrpcResponseBody(t *testing.T) {
-	var b bytes.Buffer
-	b.Write([]byte("foo"))
-
-	body, jsonErr := ReadGrpcResponseBody(&b)
-	require.Equal(t, true, jsonErr == nil)
-	assert.Equal(t, "foo", string(body))
-}
-
-func TestHandleGrpcResponseError(t *testing.T) {
-	response := &http.Response{
-		StatusCode: 400,
-		Header: http.Header{
-			"Foo": []string{"foo"},
-			"Bar": []string{"bar"},
-		},
-	}
-	writer := httptest.NewRecorder()
-	errJson := &testErrorJson{
-		Message: "foo",
-		Code:    400,
-	}
-	b, err := json.Marshal(errJson)
-	require.NoError(t, err)
-
-	hasError, e := HandleGrpcResponseError(errJson, response, b, writer)
-	require.Equal(t, true, e == nil)
-	assert.Equal(t, true, hasError)
-	v, ok := writer.Header()["Foo"]
-	require.Equal(t, true, ok, "header not found")
-	require.Equal(t, 1, len(v), "wrong number of header values")
-	assert.Equal(t, "foo", v[0])
-	v, ok = writer.Header()["Bar"]
-	require.Equal(t, true, ok, "header not found")
-	require.Equal(t, 1, len(v), "wrong number of header values")
-	assert.Equal(t, "bar", v[0])
-	assert.Equal(t, 400, errJson.StatusCode())
-}
-
-func TestGrpcResponseIsEmpty(t *testing.T) {
-	t.Run("nil", func(t *testing.T) {
-		assert.Equal(t, true, GrpcResponseIsEmpty(nil))
-	})
-	t.Run("empty_slice", func(t *testing.T) {
-		assert.Equal(t, true, GrpcResponseIsEmpty(make([]byte, 0)))
-	})
-	t.Run("empty_brackets", func(t *testing.T) {
-		assert.Equal(t, true, GrpcResponseIsEmpty([]byte("{}")))
-	})
-	t.Run("non_empty", func(t *testing.T) {
-		assert.Equal(t, false, GrpcResponseIsEmpty([]byte("{\"foo\":\"bar\"})")))
-	})
-}
-
-func TestDeserializeGrpcResponseBodyIntoContainer(t *testing.T) {
-	t.Run("ok", func(t *testing.T) {
-		body, err := json.Marshal(defaultRequestContainer())
-		require.NoError(t, err)
-
-		container := &testRequestContainer{}
-		errJson := DeserializeGrpcResponseBodyIntoContainer(body, container)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "test string", container.TestString)
-	})
-
-	t.Run("error", func(t *testing.T) {
-		var bodyJson bytes.Buffer
-		bodyJson.Write([]byte("foo"))
-		errJson := DeserializeGrpcResponseBodyIntoContainer(bodyJson.Bytes(), &testRequestContainer{})
-		require.NotNil(t, errJson)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not unmarshal response"))
-		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
-	})
-}
-
-func TestProcessMiddlewareResponseFields(t *testing.T) {
-	t.Run("Ok", func(t *testing.T) {
-		container := defaultResponseContainer()
-
-		errJson := ProcessMiddlewareResponseFields(container)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "0x666f6f", container.TestHex)
-		assert.Equal(t, "0x", container.TestEmptyHex)
-		assert.Equal(t, "0x0000000000000000000000000000000000666F6f", container.TestAddress)
-		assert.Equal(t, "0x", container.TestEmptyAddress)
-		assert.Equal(t, "4196", container.TestUint256)
-		assert.Equal(t, "test enum", container.TestEnum)
-		assert.Equal(t, "1136214245", container.TestTime)
-	})
-
-	t.Run("error", func(t *testing.T) {
-		errJson := ProcessMiddlewareResponseFields("foo")
-		require.NotNil(t, errJson)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not process response data"))
-		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
-	})
-}
-
-func TestSerializeMiddlewareResponseIntoJson(t *testing.T) {
-	container := defaultResponseContainer()
-	j, errJson := SerializeMiddlewareResponseIntoJson(container)
-	assert.Equal(t, true, errJson == nil)
-	cToDeserialize := &testResponseContainer{}
-	require.NoError(t, json.Unmarshal(j, cToDeserialize))
-	assert.Equal(t, "test string", cToDeserialize.TestString)
-}
-
-func TestWriteMiddlewareResponseHeadersAndBody(t *testing.T) {
-	t.Run("GET", func(t *testing.T) {
-		response := &http.Response{
-			Header: http.Header{
-				"Foo": []string{"foo"},
-				grpc.WithPrefix(grpc.HttpCodeMetadataKey): []string{"204"},
-				grpc.WithPrefix(api.VersionHeader):        []string{"capella"},
-			},
-		}
-		container := defaultResponseContainer()
-		responseJson, err := json.Marshal(container)
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-		writer.Body = &bytes.Buffer{}
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, true, errJson == nil)
-		v, ok := writer.Header()["Foo"]
-		require.Equal(t, true, ok, "header not found")
-		require.Equal(t, 1, len(v), "wrong number of header values")
-		assert.Equal(t, "foo", v[0])
-		v, ok = writer.Header()["Content-Length"]
-		require.Equal(t, true, ok, "header not found")
-		require.Equal(t, 1, len(v), "wrong number of header values")
-		assert.Equal(t, "224", v[0])
-		v, ok = writer.Header()["Eth-Consensus-Version"]
-		require.Equal(t, true, ok, "header not found")
-		assert.Equal(t, "capella", v[0])
-		assert.Equal(t, 204, writer.Code)
-		assert.DeepEqual(t, responseJson, writer.Body.Bytes())
-	})
-
-	t.Run("GET_no_grpc_status_code_header", func(t *testing.T) {
-		response := &http.Response{
-			Header:     http.Header{},
-			StatusCode: 204,
-		}
-		container := defaultResponseContainer()
-		responseJson, err := json.Marshal(container)
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, 204, writer.Code)
-	})
-
-	t.Run("GET_invalid_status_code", func(t *testing.T) {
-		response := &http.Response{
-			Header: http.Header{"Grpc-Metadata-Eth-Consensus-Version": []string{"capella"}},
-		}
-
-		// Set invalid status code.
-		response.Header[grpc.WithPrefix(grpc.HttpCodeMetadataKey)] = []string{"invalid"}
-		response.Header[grpc.WithPrefix(api.VersionHeader)] = []string{"capella"}
-
-		container := defaultResponseContainer()
-		responseJson, err := json.Marshal(container)
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, false, errJson == nil)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not parse status code"))
-		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
-	})
-
-	t.Run("POST", func(t *testing.T) {
-		response := &http.Response{
-			Header:     http.Header{},
-			StatusCode: 204,
-		}
-		container := defaultResponseContainer()
-		responseJson, err := json.Marshal(container)
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, 204, writer.Code)
-	})
-
-	t.Run("POST_with_response_body", func(t *testing.T) {
-		response := &http.Response{
-			Header:     http.Header{},
-			StatusCode: 204,
-		}
-		container := defaultResponseContainer()
-		responseJson, err := json.Marshal(container)
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-		writer.Body = &bytes.Buffer{}
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, 204, writer.Code)
-		assert.DeepEqual(t, responseJson, writer.Body.Bytes())
-	})
-
-	t.Run("POST_with_empty_json_body", func(t *testing.T) {
-		response := &http.Response{
-			Header:     http.Header{},
-			StatusCode: 204,
-		}
-		responseJson, err := json.Marshal(struct{}{})
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-		writer.Body = &bytes.Buffer{}
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, 204, writer.Code)
-		assert.DeepEqual(t, []byte(nil), writer.Body.Bytes())
-		assert.Equal(t, "0", writer.Header()["Content-Length"][0])
-	})
-}
-
-func TestWriteError(t *testing.T) {
-	t.Run("ok", func(t *testing.T) {
-		responseHeader := http.Header{
-			grpc.WithPrefix(grpc.CustomErrorMetadataKey): []string{"{\"CustomField\":\"bar\"}"},
-		}
-		errJson := &testErrorJson{
-			Message: "foo",
-			Code:    500,
-		}
-		writer := httptest.NewRecorder()
-		writer.Body = &bytes.Buffer{}
-
-		WriteError(writer, errJson, responseHeader)
-		v, ok := writer.Header()["Content-Length"]
-		require.Equal(t, true, ok, "header not found")
-		require.Equal(t, 1, len(v), "wrong number of header values")
-		assert.Equal(t, "48", v[0])
-		v, ok = writer.Header()["Content-Type"]
-		require.Equal(t, true, ok, "header not found")
-		require.Equal(t, 1, len(v), "wrong number of header values")
-		assert.Equal(t, "application/json", v[0])
-		assert.Equal(t, 500, writer.Code)
-		eDeserialize := &testErrorJson{}
-		require.NoError(t, json.Unmarshal(writer.Body.Bytes(), eDeserialize))
-		assert.Equal(t, "foo", eDeserialize.Message)
-		assert.Equal(t, 500, eDeserialize.Code)
-		assert.Equal(t, "bar", eDeserialize.CustomField)
-	})
-
-	t.Run("invalid_custom_error_header", func(t *testing.T) {
-		logHook := test.NewGlobal()
-
-		responseHeader := http.Header{
-			grpc.WithPrefix(grpc.CustomErrorMetadataKey): []string{"invalid"},
-		}
-
-		WriteError(httptest.NewRecorder(), &testErrorJson{}, responseHeader)
-		assert.LogsContain(t, logHook, "Could not unmarshal custom error message")
-	})
-}

api/gateway/apimiddleware/structs.go

@@ -1,69 +0,0 @@
-package apimiddleware
-
-import (
-	"net/http"
-
-	"github.com/pkg/errors"
-)
-
-// ---------------
-// Error handling.
-// ---------------
-
-// ErrorJson describes common functionality of all JSON error representations.
-type ErrorJson interface {
-	StatusCode() int
-	SetCode(code int)
-	Msg() string
-	SetMsg(msg string)
-}
-
-// DefaultErrorJson is a JSON representation of a simple error value, containing only a message and an error code.
-type DefaultErrorJson struct {
-	Message string `json:"message"`
-	Code    int    `json:"code"`
-}
-
-// InternalServerErrorWithMessage returns a DefaultErrorJson with 500 code and a custom message.
-func InternalServerErrorWithMessage(err error, message string) *DefaultErrorJson {
-	e := errors.Wrapf(err, message)
-	return &DefaultErrorJson{
-		Message: e.Error(),
-		Code:    http.StatusInternalServerError,
-	}
-}
-
-// InternalServerError returns a DefaultErrorJson with 500 code.
-func InternalServerError(err error) *DefaultErrorJson {
-	return &DefaultErrorJson{
-		Message: err.Error(),
-		Code:    http.StatusInternalServerError,
-	}
-}
-
-func TimeoutError() *DefaultErrorJson {
-	return &DefaultErrorJson{
-		Message: "Request timeout",
-		Code:    http.StatusRequestTimeout,
-	}
-}
-
-// StatusCode returns the error's underlying error code.
-func (e *DefaultErrorJson) StatusCode() int {
-	return e.Code
-}
-
-// Msg returns the error's underlying message.
-func (e *DefaultErrorJson) Msg() string {
-	return e.Message
-}
-
-// SetCode sets the error's underlying error code.
-func (e *DefaultErrorJson) SetCode(code int) {
-	e.Code = code
-}
-
-// SetMsg sets the error's underlying message.
-func (e *DefaultErrorJson) SetMsg(msg string) {
-	e.Message = msg
-}

api/gateway/gateway.go

@@ -6,19 +6,17 @@ import (
 	"fmt"
 	"net"
 	"net/http"
-	"path"
-	"strings"
 	"time"
 
 	"github.com/gorilla/mux"
 	gwruntime "github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/v4/api/gateway/apimiddleware"
 	"github.com/prysmaticlabs/prysm/v4/runtime"
 	"github.com/rs/cors"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
 )
 
 var _ runtime.Service = (*Gateway)(nil)
@@ -35,7 +33,6 @@ type PbHandlerRegistration func(context.Context, *gwruntime.ServeMux, *grpc.Clie
 
 // MuxHandler is a function that implements the mux handler functionality.
 type MuxHandler func(
-	apiMiddlewareHandler *apimiddleware.ApiProxyMiddleware,
 	h http.HandlerFunc,
 	w http.ResponseWriter,
 	req *http.Request,
@@ -43,16 +40,15 @@ type MuxHandler func(
 
 // Config parameters for setting up the gateway service.
 type config struct {
-	maxCallRecvMsgSize           uint64
-	remoteCert                   string
-	gatewayAddr                  string
-	remoteAddr                   string
-	allowedOrigins               []string
-	apiMiddlewareEndpointFactory apimiddleware.EndpointFactory
-	muxHandler                   MuxHandler
-	pbHandlers                   []*PbMux
-	router                       *mux.Router
-	timeout                      time.Duration
+	maxCallRecvMsgSize uint64
+	remoteCert         string
+	gatewayAddr        string
+	remoteAddr         string
+	allowedOrigins     []string
+	muxHandler         MuxHandler
+	pbHandlers         []*PbMux
+	router             *mux.Router
+	timeout            time.Duration
 }
 
 // Gateway is the gRPC gateway to serve HTTP JSON traffic as a proxy and forward it to the gRPC server.
@@ -61,7 +57,6 @@ type Gateway struct {
 	conn         *grpc.ClientConn
 	server       *http.Server
 	cancel       context.CancelFunc
-	proxy        *apimiddleware.ApiProxyMiddleware
 	ctx          context.Context
 	startFailure error
 }
@@ -111,13 +106,9 @@ func (g *Gateway) Start() {
 
 	corsMux := g.corsMiddleware(g.cfg.router)
 
-	if g.cfg.apiMiddlewareEndpointFactory != nil && !g.cfg.apiMiddlewareEndpointFactory.IsNil() {
-		g.registerApiMiddleware()
-	}
-
 	if g.cfg.muxHandler != nil {
 		g.cfg.router.PathPrefix("/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			g.cfg.muxHandler(g.proxy, corsMux.ServeHTTP, w, r)
+			g.cfg.muxHandler(corsMux.ServeHTTP, w, r)
 		})
 	}
 
@@ -178,24 +169,6 @@ func (g *Gateway) corsMiddleware(h http.Handler) http.Handler {
 	return c.Handler(h)
 }
 
-const swaggerDir = "proto/prysm/v1alpha1/"
-
-// SwaggerServer returns swagger specification files located under "/swagger/"
-func SwaggerServer() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		if !strings.HasSuffix(r.URL.Path, ".swagger.json") {
-			log.Debugf("Not found: %s", r.URL.Path)
-			http.NotFound(w, r)
-			return
-		}
-
-		log.Debugf("Serving %s\n", r.URL.Path)
-		p := strings.TrimPrefix(r.URL.Path, "/swagger/")
-		p = path.Join(swaggerDir, p)
-		http.ServeFile(w, r, p)
-	}
-}
-
 // dial the gRPC server.
 func (g *Gateway) dial(ctx context.Context, network, addr string) (*grpc.ClientConn, error) {
 	switch network {
@@ -211,19 +184,21 @@ func (g *Gateway) dial(ctx context.Context, network, addr string) (*grpc.ClientC
 // dialTCP creates a client connection via TCP.
 // "addr" must be a valid TCP address with a port number.
 func (g *Gateway) dialTCP(ctx context.Context, addr string) (*grpc.ClientConn, error) {
-	security := grpc.WithInsecure()
+	var security grpc.DialOption
 	if len(g.cfg.remoteCert) > 0 {
 		creds, err := credentials.NewClientTLSFromFile(g.cfg.remoteCert, "")
 		if err != nil {
 			return nil, err
 		}
 		security = grpc.WithTransportCredentials(creds)
+	} else {
+		// Use insecure credentials when there's no remote cert provided.
+		security = grpc.WithTransportCredentials(insecure.NewCredentials())
 	}
 	opts := []grpc.DialOption{
 		security,
 		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(g.cfg.maxCallRecvMsgSize))),
 	}
-
 	return grpc.DialContext(ctx, addr, opts...)
 }
 
@@ -240,19 +215,9 @@ func (g *Gateway) dialUnix(ctx context.Context, addr string) (*grpc.ClientConn,
 		return d(addr, 0)
 	}
 	opts := []grpc.DialOption{
-		grpc.WithInsecure(),
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
 		grpc.WithContextDialer(f),
 		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(g.cfg.maxCallRecvMsgSize))),
 	}
 	return grpc.DialContext(ctx, addr, opts...)
 }
-
-func (g *Gateway) registerApiMiddleware() {
-	g.proxy = &apimiddleware.ApiProxyMiddleware{
-		GatewayAddress:  g.cfg.gatewayAddr,
-		EndpointCreator: g.cfg.apiMiddlewareEndpointFactory,
-		Timeout:         g.cfg.timeout,
-	}
-	log.Info("Starting API middleware")
-	g.proxy.Run(g.cfg.router)
-}

api/gateway/gateway_test.go

@@ -10,7 +10,6 @@ import (
 	"testing"
 
 	"github.com/gorilla/mux"
-	"github.com/prysmaticlabs/prysm/v4/api/gateway/apimiddleware"
 	"github.com/prysmaticlabs/prysm/v4/cmd/beacon-chain/flags"
 	"github.com/prysmaticlabs/prysm/v4/testing/assert"
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
@@ -18,36 +17,18 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
-type mockEndpointFactory struct {
-}
-
-func (*mockEndpointFactory) Paths() []string {
-	return []string{}
-}
-
-func (*mockEndpointFactory) Create(_ string) (*apimiddleware.Endpoint, error) {
-	return nil, nil
-}
-
-func (*mockEndpointFactory) IsNil() bool {
-	return false
-}
-
 func TestGateway_Customized(t *testing.T) {
 	r := mux.NewRouter()
 	cert := "cert"
 	origins := []string{"origin"}
 	size := uint64(100)
-	endpointFactory := &mockEndpointFactory{}
 
 	opts := []Option{
 		WithRouter(r),
 		WithRemoteCert(cert),
 		WithAllowedOrigins(origins),
 		WithMaxCallRecvMsgSize(size),
-		WithApiMiddleware(endpointFactory),
 		WithMuxHandler(func(
-			_ *apimiddleware.ApiProxyMiddleware,
 			_ http.HandlerFunc,
 			_ http.ResponseWriter,
 			_ *http.Request,
@@ -63,7 +44,6 @@ func TestGateway_Customized(t *testing.T) {
 	require.Equal(t, 1, len(g.cfg.allowedOrigins))
 	assert.Equal(t, origins[0], g.cfg.allowedOrigins[0])
 	assert.Equal(t, size, g.cfg.maxCallRecvMsgSize)
-	assert.Equal(t, endpointFactory, g.cfg.apiMiddlewareEndpointFactory)
 }
 
 func TestGateway_StartStop(t *testing.T) {
@@ -83,7 +63,6 @@ func TestGateway_StartStop(t *testing.T) {
 		WithGatewayAddr(gatewayAddress),
 		WithRemoteAddr(selfAddress),
 		WithMuxHandler(func(
-			_ *apimiddleware.ApiProxyMiddleware,
 			_ http.HandlerFunc,
 			_ http.ResponseWriter,
 			_ *http.Request,

api/gateway/options.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/gorilla/mux"
 	gwruntime "github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
-	"github.com/prysmaticlabs/prysm/v4/api/gateway/apimiddleware"
 )
 
 type Option func(g *Gateway) error
@@ -70,14 +69,6 @@ func WithMaxCallRecvMsgSize(size uint64) Option {
 	}
 }
 
-// WithApiMiddleware allows adding an API middleware proxy to the gateway.
-func WithApiMiddleware(endpointFactory apimiddleware.EndpointFactory) Option {
-	return func(g *Gateway) error {
-		g.cfg.apiMiddlewareEndpointFactory = endpointFactory
-		return nil
-	}
-}
-
 // WithTimeout allows changing the timeout value for API calls.
 func WithTimeout(seconds uint64) Option {
 	return func(g *Gateway) error {

api/headers.go

@@ -4,6 +4,7 @@ const (
 	VersionHeader                 = "Eth-Consensus-Version"
 	ExecutionPayloadBlindedHeader = "Eth-Execution-Payload-Blinded"
 	ExecutionPayloadValueHeader   = "Eth-Execution-Payload-Value"
+	ConsensusBlockValueHeader     = "Eth-Consensus-Block-Value"
 	JsonMediaType                 = "application/json"
 	OctetStreamMediaType          = "application/octet-stream"
 )

api/server/BUILD.bazel

@@ -0,0 +1,24 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "middleware.go",
+        "util.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/v4/api/server",
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "middleware_test.go",
+        "util_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = [
+        "//testing/assert:go_default_library",
+        "//testing/require:go_default_library",
+    ],
+)

api/server/middleware.go

@@ -1,15 +1,13 @@
-package node
+package server
 
 import (
 	"net/http"
-
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/helpers"
 )
 
-func middleware(next http.Handler) http.Handler {
+func NormalizeQueryValuesHandler(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		query := r.URL.Query()
-		helpers.NormalizeQueryValues(query)
+		NormalizeQueryValues(query)
 		r.URL.RawQuery = query.Encode()
 
 		next.ServeHTTP(w, r)

api/server/middleware_test.go

@@ -0,0 +1,54 @@
+package server
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/v4/testing/require"
+)
+
+func TestNormalizeQueryValuesHandler(t *testing.T) {
+	nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, err := w.Write([]byte("next handler"))
+		require.NoError(t, err)
+	})
+
+	handler := NormalizeQueryValuesHandler(nextHandler)
+
+	tests := []struct {
+		name          string
+		inputQuery    string
+		expectedQuery string
+	}{
+		{
+			name:          "3 values",
+			inputQuery:    "key=value1,value2,value3",
+			expectedQuery: "key=value1&key=value2&key=value3", // replace with expected normalized value
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			req, err := http.NewRequest("GET", "/test?"+test.inputQuery, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			rr := httptest.NewRecorder()
+			handler.ServeHTTP(rr, req)
+
+			if rr.Code != http.StatusOK {
+				t.Errorf("handler returned wrong status code: got %v want %v", rr.Code, http.StatusOK)
+			}
+
+			if req.URL.RawQuery != test.expectedQuery {
+				t.Errorf("query not normalized: got %v want %v", req.URL.RawQuery, test.expectedQuery)
+			}
+
+			if rr.Body.String() != "next handler" {
+				t.Errorf("next handler was not executed")
+			}
+		})
+	}
+}

api/server/util.go

@@ -1,4 +1,4 @@
-package helpers
+package server
 
 import (
 	"net/url"

api/server/util_test.go

@@ -1,4 +1,4 @@
-package helpers
+package server
 
 import (
 	"testing"

beacon-chain/BUILD.bazel

@@ -1,19 +0,0 @@
-load("//tools:target_migration.bzl", "moved_targets")
-
-moved_targets(
-    [
-        ":push_images_debug",
-        ":push_images_alpine",
-        ":push_images",
-        ":image_bundle_debug",
-        ":image_debug",
-        ":image_bundle_alpine",
-        ":image_bundle",
-        ":image_with_creation_time",
-        ":image_alpine",
-        ":image",
-        ":go_default_test",
-        ":beacon-chain",
-    ],
-    "//cmd/beacon-chain",
-)

beacon-chain/blockchain/BUILD.bazel

@@ -50,6 +50,7 @@ go_library(
         "//beacon-chain/core/time:go_default_library",
         "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/db/filesystem:go_default_library",
         "//beacon-chain/db/filters:go_default_library",
         "//beacon-chain/db/kv:go_default_library",
         "//beacon-chain/execution:go_default_library",
@@ -69,6 +70,7 @@ go_library(
         "//config/params:go_default_library",
         "//consensus-types:go_default_library",
         "//consensus-types/blocks:go_default_library",
+        "//consensus-types/forkchoice:go_default_library",
         "//consensus-types/interfaces:go_default_library",
         "//consensus-types/payload-attribute:go_default_library",
         "//consensus-types/primitives:go_default_library",
@@ -97,26 +99,22 @@ go_library(
     ],
 )
 
-test_suite(
-    name = "go_default_test",
-    tests = [
-        ":go_raceoff_test",
-        ":go_raceon_test",
-    ],
-)
-
 go_test(
-    name = "go_raceoff_test",
+    name = "go_default_test",
     size = "medium",
     srcs = [
         "blockchain_test.go",
+        "chain_info_norace_test.go",
         "chain_info_test.go",
         "checktags_test.go",
+        "error_test.go",
         "execution_engine_test.go",
         "forkchoice_update_execution_test.go",
         "head_sync_committee_info_test.go",
         "head_test.go",
+        "init_sync_process_block_test.go",
         "init_test.go",
+        "lightclient_test.go",
         "log_test.go",
         "metrics_test.go",
         "mock_test.go",
@@ -125,98 +123,66 @@ go_test(
         "process_block_test.go",
         "receive_attestation_test.go",
         "receive_block_test.go",
+        "service_norace_test.go",
         "service_test.go",
         "setup_test.go",
         "weak_subjectivity_checks_test.go",
     ],
     embed = [":go_default_library"],
     gotags = ["develop"],
+    tags = ["CI_race_detection"],
     deps = [
         "//async/event:go_default_library",
         "//beacon-chain/blockchain/testing:go_default_library",
+        "//beacon-chain/cache:go_default_library",
         "//beacon-chain/cache/depositcache:go_default_library",
         "//beacon-chain/core/blocks:go_default_library",
+        "//beacon-chain/core/feed/state:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/core/signing:go_default_library",
         "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/db/filesystem:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/execution:go_default_library",
         "//beacon-chain/execution/testing:go_default_library",
+        "//beacon-chain/forkchoice:go_default_library",
+        "//beacon-chain/forkchoice/doubly-linked-tree:go_default_library",
         "//beacon-chain/forkchoice/types:go_default_library",
+        "//beacon-chain/operations/attestations:go_default_library",
+        "//beacon-chain/operations/blstoexec:go_default_library",
+        "//beacon-chain/operations/slashings:go_default_library",
+        "//beacon-chain/operations/voluntaryexits:go_default_library",
         "//beacon-chain/p2p:go_default_library",
+        "//beacon-chain/startup:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/state-native:go_default_library",
-        "//beacon-chain/state/stateutil:go_default_library",
+        "//beacon-chain/state/stategen:go_default_library",
+        "//config/features:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//consensus-types/blocks:go_default_library",
-        "//consensus-types/blocks/testing:go_default_library",
+        "//consensus-types/interfaces:go_default_library",
+        "//consensus-types/primitives:go_default_library",
         "//container/trie:go_default_library",
+        "//crypto/bls:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//proto/engine/v1:go_default_library",
+        "//proto/eth/v1:go_default_library",
+        "//proto/eth/v2:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
+        "//runtime/version:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/require:go_default_library",
         "//testing/util:go_default_library",
-        "@com_github_ethereum_go_ethereum//:go_default_library",
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
-        "@com_github_ethereum_go_ethereum//core/types:go_default_library",
-        "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_sirupsen_logrus//hooks/test:go_default_library",
-        "@in_gopkg_d4l3k_messagediff_v1//:go_default_library",
-        "@org_golang_google_protobuf//proto:go_default_library",
-        "@org_golang_x_net//context:go_default_library",
-    ],
-)
-
-go_test(
-    name = "go_raceon_test",
-    srcs = [
-        "chain_info_norace_test.go",
-        "checktags_test.go",
-        "init_test.go",
-        "mock_test.go",
-        "receive_block_test.go",
-        "service_norace_test.go",
-        "setup_test.go",
-    ],
-    embed = [":go_default_library"],
-    gc_goopts = [
-        # Go 1.14 enables checkptr by default when building with -race or -msan. There is a pointer
-        # issue in boltdb, so must disable checkptr at compile time. This flag can be removed once
-        # the project is migrated to etcd's version of boltdb and the issue has been fixed.
-        # See: https://github.com/etcd-io/bbolt/issues/187.
-        "-d=checkptr=0",
-    ],
-    gotags = ["develop"],
-    race = "on",
-    tags = ["race_on"],
-    deps = [
-        "//async/event:go_default_library",
-        "//beacon-chain/blockchain/testing:go_default_library",
-        "//beacon-chain/cache/depositcache:go_default_library",
-        "//beacon-chain/core/blocks:go_default_library",
-        "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/core/transition:go_default_library",
-        "//beacon-chain/db:go_default_library",
-        "//beacon-chain/db/testing:go_default_library",
-        "//beacon-chain/execution:go_default_library",
-        "//beacon-chain/execution/testing:go_default_library",
-        "//beacon-chain/forkchoice/types:go_default_library",
-        "//beacon-chain/p2p:go_default_library",
-        "//config/params:go_default_library",
-        "//consensus-types/blocks:go_default_library",
-        "//consensus-types/blocks/testing:go_default_library",
-        "//container/trie:go_default_library",
-        "//encoding/bytesutil:go_default_library",
-        "//proto/prysm/v1alpha1:go_default_library",
-        "//testing/assert:go_default_library",
-        "//testing/require:go_default_library",
-        "//testing/util:go_default_library",
-        "@com_github_ethereum_go_ethereum//:go_default_library",
+        "//time:go_default_library",
+        "//time/slots:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
+        "@com_github_holiman_uint256//:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
         "@org_golang_google_protobuf//proto:go_default_library",
-        "@org_golang_x_net//context:go_default_library",
     ],
 )

beacon-chain/blockchain/chain_info.go

@@ -12,10 +12,10 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
 	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/forkchoice"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
-	ethpbv1 "github.com/prysmaticlabs/prysm/v4/proto/eth/v1"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/time/slots"
 	"go.opencensus.io/trace"
@@ -45,7 +45,7 @@ type ForkchoiceFetcher interface {
 	HighestReceivedBlockSlot() primitives.Slot
 	ReceivedBlocksLastEpoch() (uint64, error)
 	InsertNode(context.Context, state.BeaconState, [32]byte) error
-	ForkChoiceDump(context.Context) (*ethpbv1.ForkChoiceDump, error)
+	ForkChoiceDump(context.Context) (*forkchoice.Dump, error)
 	NewSlot(context.Context, primitives.Slot) error
 	ProposerBoost() [32]byte
 }
@@ -464,6 +464,13 @@ func (s *Service) IsOptimisticForRoot(ctx context.Context, root [32]byte) (bool,
 	return !isCanonical, nil
 }
 
+// TargetRootForEpoch wraps the corresponding method in forkchoice
+func (s *Service) TargetRootForEpoch(root [32]byte, epoch primitives.Epoch) ([32]byte, error) {
+	s.cfg.ForkChoiceStore.RLock()
+	defer s.cfg.ForkChoiceStore.RUnlock()
+	return s.cfg.ForkChoiceStore.TargetRootForEpoch(root, epoch)
+}
+
 // Ancestor returns the block root of an ancestry block from the input block root.
 //
 // Spec pseudocode definition:
@@ -530,3 +537,10 @@ func (s *Service) recoverStateSummary(ctx context.Context, blockRoot [32]byte) (
 func (s *Service) BlockBeingSynced(root [32]byte) bool {
 	return s.blockBeingSynced.isSyncing(root)
 }
+
+// RecentBlockSlot returns block slot form fork choice store
+func (s *Service) RecentBlockSlot(root [32]byte) (primitives.Slot, error) {
+	s.cfg.ForkChoiceStore.RLock()
+	defer s.cfg.ForkChoiceStore.RUnlock()
+	return s.cfg.ForkChoiceStore.Slot(root)
+}

beacon-chain/blockchain/chain_info_forkchoice.go

@@ -4,8 +4,8 @@ import (
 	"context"
 
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/forkchoice"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
-	ethpbv1 "github.com/prysmaticlabs/prysm/v4/proto/eth/v1"
 )
 
 // CachedHeadRoot returns the corresponding value from Forkchoice
@@ -51,7 +51,7 @@ func (s *Service) InsertNode(ctx context.Context, st state.BeaconState, root [32
 }
 
 // ForkChoiceDump returns the corresponding value from forkchoice
-func (s *Service) ForkChoiceDump(ctx context.Context) (*ethpbv1.ForkChoiceDump, error) {
+func (s *Service) ForkChoiceDump(ctx context.Context) (*forkchoice.Dump, error) {
 	s.cfg.ForkChoiceStore.RLock()
 	defer s.cfg.ForkChoiceStore.RUnlock()
 	return s.cfg.ForkChoiceStore.ForkChoiceDump(ctx)

beacon-chain/blockchain/error.go

@@ -30,6 +30,8 @@ var (
 	ErrNotCheckpoint = errors.New("not a checkpoint in forkchoice")
 )
 
+var errMaxBlobsExceeded = errors.New("Expected commitments in block exceeds MAX_BLOBS_PER_BLOCK")
+
 // An invalid block is the block that fails state transition based on the core protocol rules.
 // The beacon node shall not be accepting nor building blocks that branch off from an invalid block.
 // Some examples of invalid blocks are:
@@ -70,11 +72,8 @@ func IsInvalidBlock(e error) bool {
 	if e == nil {
 		return false
 	}
-	_, ok := e.(invalidBlockError)
-	if !ok {
-		return IsInvalidBlock(errors.Unwrap(e))
-	}
-	return true
+	var d invalidBlockError
+	return errors.As(e, &d)
 }
 
 // InvalidBlockLVH returns the invalid block last valid hash root. If the error
@@ -83,7 +82,8 @@ func InvalidBlockLVH(e error) [32]byte {
 	if e == nil {
 		return [32]byte{}
 	}
-	d, ok := e.(invalidBlockError)
+	var d invalidBlockError
+	ok := errors.As(e, &d)
 	if !ok {
 		return [32]byte{}
 	}
@@ -96,7 +96,8 @@ func InvalidBlockRoot(e error) [32]byte {
 	if e == nil {
 		return [32]byte{}
 	}
-	d, ok := e.(invalidBlockError)
+	var d invalidBlockError
+	ok := errors.As(e, &d)
 	if !ok {
 		return [32]byte{}
 	}
@@ -108,7 +109,8 @@ func InvalidAncestorRoots(e error) [][32]byte {
 	if e == nil {
 		return [][32]byte{}
 	}
-	d, ok := e.(invalidBlockError)
+	var d invalidBlockError
+	ok := errors.As(e, &d)
 	if !ok {
 		return [][32]byte{}
 	}

beacon-chain/blockchain/error_test.go

@@ -24,6 +24,9 @@ func TestInvalidBlockRoot(t *testing.T) {
 	err := invalidBlock{error: ErrInvalidPayload, root: [32]byte{'a'}}
 	require.Equal(t, [32]byte{'a'}, InvalidBlockRoot(err))
 	require.DeepEqual(t, [][32]byte(nil), InvalidAncestorRoots(err))
+
+	newErr := errors.Wrap(err, "wrap me")
+	require.Equal(t, [32]byte{'a'}, InvalidBlockRoot(newErr))
 }
 
 func TestInvalidRoots(t *testing.T) {
@@ -33,4 +36,9 @@ func TestInvalidRoots(t *testing.T) {
 	require.Equal(t, true, IsInvalidBlock(err))
 	require.Equal(t, [32]byte{'a'}, InvalidBlockRoot(err))
 	require.DeepEqual(t, roots, InvalidAncestorRoots(err))
+
+	newErr := errors.Wrap(err, "wrap me")
+	require.Equal(t, true, IsInvalidBlock(err))
+	require.Equal(t, [32]byte{'a'}, InvalidBlockRoot(newErr))
+	require.DeepEqual(t, roots, InvalidAncestorRoots(newErr))
 }

beacon-chain/blockchain/execution_engine.go

@@ -157,6 +157,11 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *notifyForkcho
 	if hasAttr && payloadID != nil {
 		var pId [8]byte
 		copy(pId[:], payloadID[:])
+		log.WithFields(logrus.Fields{
+			"blockRoot": fmt.Sprintf("%#x", bytesutil.Trunc(arg.headRoot[:])),
+			"headSlot":  headBlk.Slot(),
+			"payloadID": fmt.Sprintf("%#x", bytesutil.Trunc(payloadID[:])),
+		}).Info("Forkchoice updated with payload attributes for proposal")
 		s.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(nextSlot, proposerId, pId, arg.headRoot)
 	} else if hasAttr && payloadID == nil && !features.Get().PrepareAllPayloads {
 		log.WithFields(logrus.Fields{
@@ -390,7 +395,7 @@ func (s *Service) removeInvalidBlockAndState(ctx context.Context, blkRoots [][32
 			return err
 		}
 		// No op if the sidecar does not exist.
-		if err := s.cfg.BeaconDB.DeleteBlobSidecar(ctx, root); err != nil {
+		if err := s.cfg.BeaconDB.DeleteBlobSidecars(ctx, root); err != nil {
 			return err
 		}
 	}

beacon-chain/blockchain/kzg/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain/kzg",
     visibility = ["//visibility:public"],
     deps = [
+        "//consensus-types/blocks:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "@com_github_crate_crypto_go_kzg_4844//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
@@ -26,6 +27,8 @@ go_test(
     deps = [
         "//proto/prysm/v1alpha1:go_default_library",
         "//testing/require:go_default_library",
+        "@com_github_consensys_gnark_crypto//ecc/bls12-381/fr:go_default_library",
         "@com_github_crate_crypto_go_kzg_4844//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
     ],
 )

beacon-chain/blockchain/kzg/validation.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	GoKZG "github.com/crate-crypto/go-kzg-4844"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 )
 
@@ -12,7 +13,7 @@ import (
 // - Expected KZG commitments match the number of blobs in the block
 // - That the number of proofs match the number of blobs
 // - That the proofs are verified against the KZG commitments
-func IsDataAvailable(commitments [][]byte, sidecars []*ethpb.BlobSidecar) error {
+func IsDataAvailable(commitments [][]byte, sidecars []*ethpb.DeprecatedBlobSidecar) error {
 	if len(commitments) != len(sidecars) {
 		return fmt.Errorf("could not check data availability, expected %d commitments, obtained %d",
 			len(commitments), len(sidecars))
@@ -31,6 +32,11 @@ func IsDataAvailable(commitments [][]byte, sidecars []*ethpb.BlobSidecar) error
 	return kzgContext.VerifyBlobKZGProofBatch(blobs, cmts, proofs)
 }
 
+// VerifyROBlobCommitment is a helper that massages the fields of an ROBlob into the types needed to call VerifyBlobKZGProof.
+func VerifyROBlobCommitment(sc blocks.ROBlob) error {
+	return kzgContext.VerifyBlobKZGProof(bytesToBlob(sc.Blob), bytesToCommitment(sc.KzgCommitment), bytesToKZGProof(sc.KzgProof))
+}
+
 func bytesToBlob(blob []byte) (ret GoKZG.Blob) {
 	copy(ret[:], blob)
 	return

beacon-chain/blockchain/kzg/validation_test.go

@@ -1,15 +1,65 @@
 package kzg
 
 import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
 	"testing"
 
+	"github.com/consensys/gnark-crypto/ecc/bls12-381/fr"
 	GoKZG "github.com/crate-crypto/go-kzg-4844"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
+	"github.com/sirupsen/logrus"
 )
 
+func deterministicRandomness(seed int64) [32]byte {
+	// Converts an int64 to a byte slice
+	buf := new(bytes.Buffer)
+	err := binary.Write(buf, binary.BigEndian, seed)
+	if err != nil {
+		logrus.WithError(err).Error("Failed to write int64 to bytes buffer")
+		return [32]byte{}
+	}
+	bytes := buf.Bytes()
+
+	return sha256.Sum256(bytes)
+}
+
+// Returns a serialized random field element in big-endian
+func GetRandFieldElement(seed int64) [32]byte {
+	bytes := deterministicRandomness(seed)
+	var r fr.Element
+	r.SetBytes(bytes[:])
+
+	return GoKZG.SerializeScalar(r)
+}
+
+// Returns a random blob using the passed seed as entropy
+func GetRandBlob(seed int64) GoKZG.Blob {
+	var blob GoKZG.Blob
+	bytesPerBlob := GoKZG.ScalarsPerBlob * GoKZG.SerializedScalarSize
+	for i := 0; i < bytesPerBlob; i += GoKZG.SerializedScalarSize {
+		fieldElementBytes := GetRandFieldElement(seed + int64(i))
+		copy(blob[i:i+GoKZG.SerializedScalarSize], fieldElementBytes[:])
+	}
+	return blob
+}
+
+func GenerateCommitmentAndProof(blob GoKZG.Blob) (GoKZG.KZGCommitment, GoKZG.KZGProof, error) {
+	commitment, err := kzgContext.BlobToKZGCommitment(blob, 0)
+	if err != nil {
+		return GoKZG.KZGCommitment{}, GoKZG.KZGProof{}, err
+	}
+	proof, err := kzgContext.ComputeBlobKZGProof(blob, commitment, 0)
+	if err != nil {
+		return GoKZG.KZGCommitment{}, GoKZG.KZGProof{}, err
+	}
+	return commitment, proof, err
+}
+
 func TestIsDataAvailable(t *testing.T) {
-	sidecars := make([]*ethpb.BlobSidecar, 0)
+	sidecars := make([]*ethpb.DeprecatedBlobSidecar, 0)
 	commitments := make([][]byte, 0)
 	require.NoError(t, IsDataAvailable(commitments, sidecars))
 }
@@ -23,3 +73,13 @@ func TestBytesToAny(t *testing.T) {
 	require.DeepEqual(t, commitment, bytesToCommitment(bytes))
 	require.DeepEqual(t, proof, bytesToKZGProof(bytes))
 }
+
+func TestGenerateCommitmentAndProof(t *testing.T) {
+	blob := GetRandBlob(123)
+	commitment, proof, err := GenerateCommitmentAndProof(blob)
+	require.NoError(t, err)
+	expectedCommitment := GoKZG.KZGCommitment{180, 218, 156, 194, 59, 20, 10, 189, 186, 254, 132, 93, 7, 127, 104, 172, 238, 240, 237, 70, 83, 89, 1, 152, 99, 0, 165, 65, 143, 62, 20, 215, 230, 14, 205, 95, 28, 245, 54, 25, 160, 16, 178, 31, 232, 207, 38, 85}
+	expectedProof := GoKZG.KZGProof{128, 110, 116, 170, 56, 111, 126, 87, 229, 234, 211, 42, 110, 150, 129, 206, 73, 142, 167, 243, 90, 149, 240, 240, 236, 204, 143, 182, 229, 249, 81, 27, 153, 171, 83, 70, 144, 250, 42, 1, 188, 215, 71, 235, 30, 7, 175, 86}
+	require.Equal(t, expectedCommitment, commitment)
+	require.Equal(t, expectedProof, proof)
+}

beacon-chain/blockchain/lightclient.go

@@ -8,7 +8,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
-	types "github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	ethpbv1 "github.com/prysmaticlabs/prysm/v4/proto/eth/v1"
 	ethpbv2 "github.com/prysmaticlabs/prysm/v4/proto/eth/v2"
@@ -63,7 +62,7 @@ func NewLightClientOptimisticUpdateFromBeaconState(
 	attestedState state.BeaconState) (*ethpbv2.LightClientUpdate, error) {
 	// assert compute_epoch_at_slot(attested_state.slot) >= ALTAIR_FORK_EPOCH
 	attestedEpoch := slots.ToEpoch(attestedState.Slot())
-	if attestedEpoch < types.Epoch(params.BeaconConfig().AltairForkEpoch) {
+	if attestedEpoch < params.BeaconConfig().AltairForkEpoch {
 		return nil, fmt.Errorf("invalid attested epoch %d", attestedEpoch)
 	}
 

beacon-chain/blockchain/log.go

@@ -147,15 +147,3 @@ func logPayload(block interfaces.ReadOnlyBeaconBlock) error {
 	log.WithFields(fields).Debug("Synced new payload")
 	return nil
 }
-
-func logBlobSidecar(scs []*ethpb.BlobSidecar, startTime time.Time) {
-	if len(scs) == 0 {
-		return
-	}
-	log.WithFields(logrus.Fields{
-		"count":          len(scs),
-		"slot":           scs[0].Slot,
-		"block":          hex.EncodeToString(scs[0].BlockRoot),
-		"validationTime": time.Since(startTime),
-	}).Debug("Synced new blob sidecars")
-}

beacon-chain/blockchain/metrics_test.go

@@ -16,7 +16,7 @@ func TestReportEpochMetrics_BadHeadState(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, h.SetValidators(nil))
 	err = reportEpochMetrics(context.Background(), s, h)
-	require.ErrorContains(t, "failed to initialize precompute: nil validators in state", err)
+	require.ErrorContains(t, "failed to initialize precompute: state has nil validator slice", err)
 }
 
 func TestReportEpochMetrics_BadAttestation(t *testing.T) {

beacon-chain/blockchain/options.go

@@ -5,6 +5,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
 	statefeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/execution"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/operations/attestations"
@@ -164,6 +165,8 @@ func WithFinalizedStateAtStartUp(st state.BeaconState) Option {
 	}
 }
 
+// WithClockSychronizer sets the ClockSetter/ClockWaiter values to be used by services that need to block until
+// the genesis timestamp is known (ClockWaiter) or which determine the genesis timestamp (ClockSetter).
 func WithClockSynchronizer(gs *startup.ClockSynchronizer) Option {
 	return func(s *Service) error {
 		s.clockSetter = gs
@@ -172,9 +175,18 @@ func WithClockSynchronizer(gs *startup.ClockSynchronizer) Option {
 	}
 }
 
+// WithSyncComplete sets a channel that is used to notify blockchain service that the node has synced to head.
 func WithSyncComplete(c chan struct{}) Option {
 	return func(s *Service) error {
 		s.syncComplete = c
 		return nil
 	}
 }
+
+// WithBlobStorage sets the blob storage backend for the blockchain service.
+func WithBlobStorage(b *filesystem.BlobStorage) Option {
+	return func(s *Service) error {
+		s.blobStorage = b
+		return nil
+	}
+}

beacon-chain/blockchain/process_attestation_test.go

@@ -2,6 +2,7 @@ package blockchain
 
 import (
 	"context"
+	"sync"
 	"testing"
 	"time"
 
@@ -145,6 +146,61 @@ func TestStore_OnAttestation_Ok_DoublyLinkedTree(t *testing.T) {
 	require.NoError(t, service.OnAttestation(ctx, att[0], 0))
 }
 
+func TestService_GetAttPreState_Concurrency(t *testing.T) {
+	service, _ := minimalTestService(t)
+	ctx := context.Background()
+
+	s, err := util.NewBeaconState()
+	require.NoError(t, err)
+	ckRoot := bytesutil.PadTo([]byte{'A'}, fieldparams.RootLength)
+	err = s.SetFinalizedCheckpoint(&ethpb.Checkpoint{Root: ckRoot})
+	require.NoError(t, err)
+	val := &ethpb.Validator{PublicKey: bytesutil.PadTo([]byte("foo"), 48), WithdrawalCredentials: bytesutil.PadTo([]byte("bar"), fieldparams.RootLength)}
+	err = s.SetValidators([]*ethpb.Validator{val})
+	require.NoError(t, err)
+	err = s.SetBalances([]uint64{0})
+	require.NoError(t, err)
+	r := [32]byte{'g'}
+	require.NoError(t, service.cfg.BeaconDB.SaveState(ctx, s, r))
+
+	cp1 := &ethpb.Checkpoint{Epoch: 1, Root: ckRoot}
+	require.NoError(t, service.cfg.BeaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'A'})))
+	require.NoError(t, service.cfg.BeaconDB.SaveStateSummary(ctx, &ethpb.StateSummary{Root: ckRoot}))
+
+	st, root, err := prepareForkchoiceState(ctx, 100, [32]byte(cp1.Root), [32]byte{}, [32]byte{'R'}, cp1, cp1)
+	require.NoError(t, err)
+	require.NoError(t, service.cfg.ForkChoiceStore.InsertNode(ctx, st, root))
+
+	var wg sync.WaitGroup
+	errChan := make(chan error, 1000)
+
+	for i := 0; i < 1000; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			cp1 := &ethpb.Checkpoint{Epoch: 1, Root: ckRoot}
+			_, err := service.getAttPreState(ctx, cp1)
+			if err != nil {
+				errChan <- err
+			}
+		}()
+	}
+
+	go func() {
+		wg.Wait()
+		close(errChan)
+	}()
+
+	select {
+	case <-time.After(10 * time.Second):
+		t.Fatal("Test timed out")
+	case err, ok := <-errChan:
+		if ok && err != nil {
+			require.ErrorContains(t, "not a checkpoint in forkchoice", err)
+		}
+	}
+}
+
 func TestStore_SaveCheckpointState(t *testing.T) {
 	service, tr := minimalTestService(t)
 	ctx := tr.ctx

beacon-chain/blockchain/process_block.go

@@ -6,17 +6,17 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain/kzg"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/helpers"
 	coreTime "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/v4/config/features"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	consensusblocks "github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
@@ -353,11 +353,15 @@ func (s *Service) databaseDACheck(ctx context.Context, b consensusblocks.ROBlock
 	if len(commitments) == 0 {
 		return nil
 	}
-	sidecars, err := s.cfg.BeaconDB.BlobSidecarsByRoot(ctx, b.Root())
+	missing, err := missingIndices(s.blobStorage, b.Root(), commitments)
 	if err != nil {
-		return errors.Wrap(err, "could not get blob sidecars")
+		return err
 	}
-	return kzg.IsDataAvailable(commitments, sidecars)
+	if len(missing) == 0 {
+		return nil
+	}
+	// TODO: don't worry that this error isn't informative, it will be superceded by a detailed sidecar cache error.
+	return errors.New("not all kzg commitments are available")
 }
 
 func (s *Service) updateEpochBoundaryCaches(ctx context.Context, st state.BeaconState) error {
@@ -529,11 +533,43 @@ func (s *Service) runLateBlockTasks() {
 	}
 }
 
+// missingIndices uses the expected commitments from the block to determine
+// which BlobSidecar indices would need to be in the database for DA success.
+// It returns a map where each key represents a missing BlobSidecar index.
+// An empty map means we have all indices; a non-empty map can be used to compare incoming
+// BlobSidecars against the set of known missing sidecars.
+func missingIndices(bs *filesystem.BlobStorage, root [32]byte, expected [][]byte) (map[uint64]struct{}, error) {
+	if len(expected) == 0 {
+		return nil, nil
+	}
+	if len(expected) > fieldparams.MaxBlobsPerBlock {
+		return nil, errMaxBlobsExceeded
+	}
+	indices, err := bs.Indices(root)
+	if err != nil {
+		return nil, err
+	}
+	missing := make(map[uint64]struct{}, len(expected))
+	for i := range expected {
+		ui := uint64(i)
+		if len(expected[i]) > 0 {
+			if !indices[i] {
+				missing[ui] = struct{}{}
+			}
+		}
+	}
+	return missing, nil
+}
+
+// isDataAvailable blocks until all BlobSidecars committed to in the block are available,
+// or an error or context cancellation occurs. A nil result means that the data availability check is successful.
+// The function will first check the database to see if all sidecars have been persisted. If any
+// sidecars are missing, it will then read from the blobNotifier channel for the given root until the channel is
+// closed, the context hits cancellation/timeout, or notifications have been received for all the missing sidecars.
 func (s *Service) isDataAvailable(ctx context.Context, root [32]byte, signed interfaces.ReadOnlySignedBeaconBlock) error {
 	if signed.Version() < version.Deneb {
 		return nil
 	}
-	t := time.Now()
 
 	block := signed.Block()
 	if block == nil {
@@ -552,51 +588,35 @@ func (s *Service) isDataAvailable(ctx context.Context, root [32]byte, signed int
 	if err != nil {
 		return errors.Wrap(err, "could not get KZG commitments")
 	}
+	// expected is the number of kzg commitments observed in the block.
 	expected := len(kzgCommitments)
 	if expected == 0 {
 		return nil
 	}
-
-	// Read first from db in case we have the blobs
-	sidecars, err := s.cfg.BeaconDB.BlobSidecarsByRoot(ctx, root)
-	switch {
-	case err == nil:
-		if len(sidecars) >= expected {
-			s.blobNotifiers.delete(root)
-			if err := kzg.IsDataAvailable(kzgCommitments, sidecars); err != nil {
-				return err
-			}
-			logBlobSidecar(sidecars, t)
-			return nil
-		}
-	case errors.Is(err, db.ErrNotFound):
-		// If the blob sidecars haven't arrived yet, the subsequent code will wait for them.
-		// Note: The system will not exit with an error in this scenario.
-	default:
-		log.WithError(err).Error("could not get blob sidecars from DB")
+	// get a map of BlobSidecar indices that are not currently available.
+	missing, err := missingIndices(s.blobStorage, root, kzgCommitments)
+	if err != nil {
+		return err
+	}
+	// If there are no missing indices, all BlobSidecars are available.
+	if len(missing) == 0 {
+		return nil
 	}
 
-	found := map[uint64]struct{}{}
-	for _, sc := range sidecars {
-		found[sc.Index] = struct{}{}
-	}
+	// The gossip handler for blobs writes the index of each verified blob referencing the given
+	// root to the channel returned by blobNotifiers.forRoot.
 	nc := s.blobNotifiers.forRoot(root)
 	for {
 		select {
 		case idx := <-nc:
-			found[idx] = struct{}{}
-			if len(found) != expected {
+			// Delete each index seen in the notification channel.
+			delete(missing, idx)
+			// Read from the channel until there are no more missing sidecars.
+			if len(missing) > 0 {
 				continue
 			}
+			// Once all sidecars have been observed, clean up the notification channel.
 			s.blobNotifiers.delete(root)
-			sidecars, err := s.cfg.BeaconDB.BlobSidecarsByRoot(ctx, root)
-			if err != nil {
-				return errors.Wrap(err, "could not get blob sidecars")
-			}
-			if err := kzg.IsDataAvailable(kzgCommitments, sidecars); err != nil {
-				return err
-			}
-			logBlobSidecar(sidecars, t)
 			return nil
 		case <-ctx.Done():
 			return errors.Wrap(ctx.Err(), "context deadline waiting for blob sidecars")

beacon-chain/blockchain/process_block_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/signing"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	testDB "github.com/prysmaticlabs/prysm/v4/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/execution"
 	mockExecution "github.com/prysmaticlabs/prysm/v4/beacon-chain/execution/testing"
@@ -1103,12 +1104,15 @@ func TestOnBlock_ProcessBlocksParallel(t *testing.T) {
 		require.NoError(t, service.cfg.ForkChoiceStore.InsertNode(ctx, st, blkRoot))
 		var wg sync.WaitGroup
 		wg.Add(4)
+		var lock sync.Mutex
 		go func() {
 			preState, err := service.getBlockPreState(ctx, wsb1.Block())
 			require.NoError(t, err)
 			postState, err := service.validateStateTransition(ctx, preState, wsb1)
 			require.NoError(t, err)
+			lock.Lock()
 			require.NoError(t, service.postBlockProcess(ctx, wsb1, r1, postState, true))
+			lock.Unlock()
 			wg.Done()
 		}()
 		go func() {
@@ -1116,7 +1120,9 @@ func TestOnBlock_ProcessBlocksParallel(t *testing.T) {
 			require.NoError(t, err)
 			postState, err := service.validateStateTransition(ctx, preState, wsb2)
 			require.NoError(t, err)
+			lock.Lock()
 			require.NoError(t, service.postBlockProcess(ctx, wsb2, r2, postState, true))
+			lock.Unlock()
 			wg.Done()
 		}()
 		go func() {
@@ -1124,7 +1130,9 @@ func TestOnBlock_ProcessBlocksParallel(t *testing.T) {
 			require.NoError(t, err)
 			postState, err := service.validateStateTransition(ctx, preState, wsb3)
 			require.NoError(t, err)
+			lock.Lock()
 			require.NoError(t, service.postBlockProcess(ctx, wsb3, r3, postState, true))
+			lock.Unlock()
 			wg.Done()
 		}()
 		go func() {
@@ -1132,7 +1140,9 @@ func TestOnBlock_ProcessBlocksParallel(t *testing.T) {
 			require.NoError(t, err)
 			postState, err := service.validateStateTransition(ctx, preState, wsb4)
 			require.NoError(t, err)
+			lock.Lock()
 			require.NoError(t, service.postBlockProcess(ctx, wsb4, r4, postState, true))
+			lock.Unlock()
 			wg.Done()
 		}()
 		wg.Wait()
@@ -2106,3 +2116,99 @@ func Test_commitmentsToCheck(t *testing.T) {
 		})
 	}
 }
+
+func TestMissingIndices(t *testing.T) {
+	cases := []struct {
+		name     string
+		expected [][]byte
+		present  []uint64
+		result   map[uint64]struct{}
+		root     [32]byte
+		err      error
+	}{
+		{
+			name: "zero len",
+		},
+		{
+			name:     "expected exceeds max",
+			expected: fakeCommitments(fieldparams.MaxBlobsPerBlock + 1),
+			err:      errMaxBlobsExceeded,
+		},
+		{
+			name:     "first missing",
+			expected: fakeCommitments(fieldparams.MaxBlobsPerBlock),
+			present:  []uint64{1, 2, 3, 4, 5},
+			result:   fakeResult([]uint64{0}),
+		},
+		{
+			name:     "all missing",
+			expected: fakeCommitments(fieldparams.MaxBlobsPerBlock),
+			result:   fakeResult([]uint64{0, 1, 2, 3, 4, 5}),
+		},
+		{
+			name:     "none missing",
+			expected: fakeCommitments(fieldparams.MaxBlobsPerBlock),
+			present:  []uint64{0, 1, 2, 3, 4, 5},
+			result:   fakeResult([]uint64{}),
+		},
+		{
+			name:     "one commitment, missing",
+			expected: fakeCommitments(1),
+			present:  []uint64{},
+			result:   fakeResult([]uint64{0}),
+		},
+		{
+			name:     "3 commitments, 1 missing",
+			expected: fakeCommitments(3),
+			present:  []uint64{1},
+			result:   fakeResult([]uint64{0, 2}),
+		},
+		{
+			name:     "3 commitments, none missing",
+			expected: fakeCommitments(3),
+			present:  []uint64{0, 1, 2},
+			result:   fakeResult([]uint64{}),
+		},
+		{
+			name:     "3 commitments, all missing",
+			expected: fakeCommitments(3),
+			present:  []uint64{},
+			result:   fakeResult([]uint64{0, 1, 2}),
+		},
+	}
+
+	for _, c := range cases {
+		bm, bs := filesystem.NewEphemeralBlobStorageWithMocker(t)
+		t.Run(c.name, func(t *testing.T) {
+			require.NoError(t, bm.CreateFakeIndices(c.root, c.present))
+			missing, err := missingIndices(bs, c.root, c.expected)
+			if c.err != nil {
+				require.ErrorIs(t, err, c.err)
+				return
+			}
+			require.NoError(t, err)
+			require.Equal(t, len(c.result), len(missing))
+			for key := range c.result {
+				m, ok := missing[key]
+				require.Equal(t, true, ok)
+				require.Equal(t, c.result[key], m)
+			}
+		})
+	}
+}
+
+func fakeCommitments(n int) [][]byte {
+	f := make([][]byte, n)
+	for i := range f {
+		f[i] = make([]byte, 48)
+	}
+	return f
+}
+
+func fakeResult(missing []uint64) map[uint64]struct{} {
+	r := make(map[uint64]struct{}, len(missing))
+	for i := range missing {
+		r[missing[i]] = struct{}{}
+	}
+	return r
+}

beacon-chain/blockchain/receive_attestation.go

@@ -53,15 +53,11 @@ func (s *Service) AttestationTargetState(ctx context.Context, target *ethpb.Chec
 
 // VerifyLmdFfgConsistency verifies that attestation's LMD and FFG votes are consistency to each other.
 func (s *Service) VerifyLmdFfgConsistency(ctx context.Context, a *ethpb.Attestation) error {
-	targetSlot, err := slots.EpochStart(a.Data.Target.Epoch)
+	r, err := s.TargetRootForEpoch([32]byte(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)
 	if err != nil {
 		return err
 	}
-	r, err := s.Ancestor(ctx, a.Data.BeaconBlockRoot, targetSlot)
-	if err != nil {
-		return err
-	}
-	if !bytes.Equal(a.Data.Target.Root, r) {
+	if !bytes.Equal(a.Data.Target.Root, r[:]) {
 		return fmt.Errorf("FFG and LMD votes are not consistent, block root: %#x, target root: %#x, canonical target root: %#x", a.Data.BeaconBlockRoot, a.Data.Target.Root, r)
 	}
 	return nil

beacon-chain/blockchain/receive_attestation_test.go

@@ -36,50 +36,28 @@ func TestAttestationCheckPtState_FarFutureSlot(t *testing.T) {
 	require.ErrorContains(t, "exceeds max allowed value relative to the local clock", err)
 }
 
-func TestVerifyLMDFFGConsistent_NotOK(t *testing.T) {
+func TestVerifyLMDFFGConsistent(t *testing.T) {
 	service, tr := minimalTestService(t)
 	ctx := tr.ctx
 
-	b32 := util.NewBeaconBlock()
-	b32.Block.Slot = 32
-	util.SaveBlock(t, ctx, service.cfg.BeaconDB, b32)
-	r32, err := b32.Block.HashTreeRoot()
+	f := service.cfg.ForkChoiceStore
+	fc := &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	state, r32, err := prepareForkchoiceState(ctx, 32, [32]byte{'a'}, params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, fc, fc)
 	require.NoError(t, err)
-	b33 := util.NewBeaconBlock()
-	b33.Block.Slot = 33
-	b33.Block.ParentRoot = r32[:]
-	util.SaveBlock(t, ctx, service.cfg.BeaconDB, b33)
-	r33, err := b33.Block.HashTreeRoot()
+	require.NoError(t, f.InsertNode(ctx, state, r32))
+
+	state, r33, err := prepareForkchoiceState(ctx, 33, [32]byte{'b'}, r32, params.BeaconConfig().ZeroHash, fc, fc)
 	require.NoError(t, err)
+	require.NoError(t, f.InsertNode(ctx, state, r33))
 
 	wanted := "FFG and LMD votes are not consistent"
 	a := util.NewAttestation()
 	a.Data.Target.Epoch = 1
-	a.Data.Target.Root = []byte{'a'}
+	a.Data.Target.Root = []byte{'c'}
 	a.Data.BeaconBlockRoot = r33[:]
 	require.ErrorContains(t, wanted, service.VerifyLmdFfgConsistency(context.Background(), a))
-}
 
-func TestVerifyLMDFFGConsistent_OK(t *testing.T) {
-	service, tr := minimalTestService(t)
-	ctx := tr.ctx
-
-	b32 := util.NewBeaconBlock()
-	b32.Block.Slot = 32
-	util.SaveBlock(t, ctx, service.cfg.BeaconDB, b32)
-	r32, err := b32.Block.HashTreeRoot()
-	require.NoError(t, err)
-	b33 := util.NewBeaconBlock()
-	b33.Block.Slot = 33
-	b33.Block.ParentRoot = r32[:]
-	util.SaveBlock(t, ctx, service.cfg.BeaconDB, b33)
-	r33, err := b33.Block.HashTreeRoot()
-	require.NoError(t, err)
-
-	a := util.NewAttestation()
-	a.Data.Target.Epoch = 1
 	a.Data.Target.Root = r32[:]
-	a.Data.BeaconBlockRoot = r33[:]
 	err = service.VerifyLmdFfgConsistency(context.Background(), a)
 	require.NoError(t, err, "Could not verify LMD and FFG votes to be consistent")
 }

beacon-chain/blockchain/receive_blob.go

@@ -3,21 +3,21 @@ package blockchain
 import (
 	"context"
 
-	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 )
 
 // SendNewBlobEvent sends a message to the BlobNotifier channel that the blob
 // for the blocroot `root` is ready in the database
 func (s *Service) sendNewBlobEvent(root [32]byte, index uint64) {
-	s.blobNotifiers.forRoot(root) <- index
+	s.blobNotifiers.notifyIndex(root, index)
 }
 
 // ReceiveBlob saves the blob to database and sends the new event
-func (s *Service) ReceiveBlob(ctx context.Context, b *ethpb.BlobSidecar) error {
-	if err := s.cfg.BeaconDB.SaveBlobSidecar(ctx, []*ethpb.BlobSidecar{b}); err != nil {
+func (s *Service) ReceiveBlob(ctx context.Context, b blocks.VerifiedROBlob) error {
+	if err := s.blobStorage.Save(b); err != nil {
 		return err
 	}
 
-	s.sendNewBlobEvent([32]byte(b.BlockRoot), b.Index)
+	s.sendNewBlobEvent(b.BlockRoot(), b.Index)
 	return nil
 }

beacon-chain/blockchain/receive_block.go

@@ -3,6 +3,7 @@ package blockchain
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"time"
 
 	"github.com/pkg/errors"
@@ -42,7 +43,7 @@ type BlockReceiver interface {
 // BlobReceiver interface defines the methods of chain service for receiving new
 // blobs
 type BlobReceiver interface {
-	ReceiveBlob(context.Context, *ethpb.BlobSidecar) error
+	ReceiveBlob(context.Context, blocks.VerifiedROBlob) error
 }
 
 // SlashingReceiver interface defines the methods of chain service for receiving validated slashing over the wire.
@@ -58,6 +59,11 @@ type SlashingReceiver interface {
 func (s *Service) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, blockRoot [32]byte) error {
 	ctx, span := trace.StartSpan(ctx, "blockChain.ReceiveBlock")
 	defer span.End()
+	// Return early if the block has been synced
+	if s.InForkchoice(blockRoot) {
+		log.WithField("blockRoot", fmt.Sprintf("%#x", blockRoot)).Debug("Ignoring already synced block")
+		return nil
+	}
 	receivedTime := time.Now()
 	s.blockBeingSynced.set(blockRoot)
 	defer s.blockBeingSynced.unset(blockRoot)
@@ -252,11 +258,6 @@ func (s *Service) HasBlock(ctx context.Context, root [32]byte) bool {
 	return s.hasBlockInInitSyncOrDB(ctx, root)
 }
 
-// RecentBlockSlot returns block slot form fork choice store
-func (s *Service) RecentBlockSlot(root [32]byte) (primitives.Slot, error) {
-	return s.cfg.ForkChoiceStore.Slot(root)
-}
-
 // ReceiveAttesterSlashing receives an attester slashing and inserts it to forkchoice
 func (s *Service) ReceiveAttesterSlashing(ctx context.Context, slashing *ethpb.AttesterSlashing) {
 	s.cfg.ForkChoiceStore.Lock()

beacon-chain/blockchain/service.go

@@ -20,6 +20,7 @@ import (
 	coreTime "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/execution"
 	f "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice"
 	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
@@ -63,6 +64,7 @@ type Service struct {
 	syncComplete         chan struct{}
 	blobNotifiers        *blobNotifierMap
 	blockBeingSynced     *currentlySyncingBlock
+	blobStorage          *filesystem.BlobStorage
 }
 
 // config options for the service.
@@ -94,6 +96,35 @@ var ErrMissingClockSetter = errors.New("blockchain Service initialized without a
 type blobNotifierMap struct {
 	sync.RWMutex
 	notifiers map[[32]byte]chan uint64
+	seenIndex map[[32]byte][fieldparams.MaxBlobsPerBlock]bool
+}
+
+// notifyIndex notifies a blob by its index for a given root.
+// It uses internal maps to keep track of seen indices and notifier channels.
+func (bn *blobNotifierMap) notifyIndex(root [32]byte, idx uint64) {
+	if idx >= fieldparams.MaxBlobsPerBlock {
+		return
+	}
+
+	bn.Lock()
+	seen := bn.seenIndex[root]
+	if seen[idx] {
+		bn.Unlock()
+		return
+	}
+	seen[idx] = true
+	bn.seenIndex[root] = seen
+
+	// Retrieve or create the notifier channel for the given root.
+	c, ok := bn.notifiers[root]
+	if !ok {
+		c = make(chan uint64, fieldparams.MaxBlobsPerBlock)
+		bn.notifiers[root] = c
+	}
+
+	bn.Unlock()
+
+	c <- idx
 }
 
 func (bn *blobNotifierMap) forRoot(root [32]byte) chan uint64 {
@@ -110,6 +141,7 @@ func (bn *blobNotifierMap) forRoot(root [32]byte) chan uint64 {
 func (bn *blobNotifierMap) delete(root [32]byte) {
 	bn.Lock()
 	defer bn.Unlock()
+	delete(bn.seenIndex, root)
 	delete(bn.notifiers, root)
 }
 
@@ -126,6 +158,7 @@ func NewService(ctx context.Context, opts ...Option) (*Service, error) {
 	ctx, cancel := context.WithCancel(ctx)
 	bn := &blobNotifierMap{
 		notifiers: make(map[[32]byte]chan uint64),
+		seenIndex: make(map[[32]byte][fieldparams.MaxBlobsPerBlock]bool),
 	}
 	srv := &Service{
 		ctx:                  ctx,

beacon-chain/blockchain/service_test.go

@@ -25,6 +25,7 @@ import (
 	state_native "github.com/prysmaticlabs/prysm/v4/beacon-chain/state/state-native"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state/stategen"
 	"github.com/prysmaticlabs/prysm/v4/config/features"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	consensusblocks "github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
@@ -445,11 +446,10 @@ func BenchmarkHasBlockForkChoiceStore_DoublyLinkedTree(b *testing.B) {
 	s := &Service{
 		cfg: &config{ForkChoiceStore: doublylinkedtree.New(), BeaconDB: beaconDB},
 	}
-	blk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}}}
+	blk := util.NewBeaconBlock()
 	r, err := blk.Block.HashTreeRoot()
 	require.NoError(b, err)
-	bs := &ethpb.BeaconState{FinalizedCheckpoint: &ethpb.Checkpoint{Root: make([]byte, 32)}, CurrentJustifiedCheckpoint: &ethpb.Checkpoint{Root: make([]byte, 32)}}
-	beaconState, err := state_native.InitializeFromProtoPhase0(bs)
+	beaconState, err := util.NewBeaconState()
 	require.NoError(b, err)
 	require.NoError(b, s.cfg.ForkChoiceStore.InsertNode(ctx, beaconState, r))
 
@@ -514,3 +514,48 @@ func (s *MockClockSetter) SetClock(g *startup.Clock) error {
 	s.G = g
 	return s.Err
 }
+
+func TestNotifyIndex(t *testing.T) {
+	// Initialize a blobNotifierMap
+	bn := &blobNotifierMap{
+		seenIndex: make(map[[32]byte][fieldparams.MaxBlobsPerBlock]bool),
+		notifiers: make(map[[32]byte]chan uint64),
+	}
+
+	// Sample root and index
+	var root [32]byte
+	copy(root[:], "exampleRoot")
+
+	// Test notifying a new index
+	bn.notifyIndex(root, 1)
+	if !bn.seenIndex[root][1] {
+		t.Errorf("Index was not marked as seen")
+	}
+
+	// Test that a new channel is created
+	if _, ok := bn.notifiers[root]; !ok {
+		t.Errorf("Notifier channel was not created")
+	}
+
+	// Test notifying an already seen index
+	bn.notifyIndex(root, 1)
+	if len(bn.notifiers[root]) > 1 {
+		t.Errorf("Notifier channel should not receive multiple messages for the same index")
+	}
+
+	// Test notifying a new index again
+	bn.notifyIndex(root, 2)
+	if !bn.seenIndex[root][2] {
+		t.Errorf("Index was not marked as seen")
+	}
+
+	// Test that the notifier channel receives the index
+	select {
+	case idx := <-bn.notifiers[root]:
+		if idx != 1 {
+			t.Errorf("Received index on channel is incorrect")
+		}
+	default:
+		t.Errorf("Notifier channel did not receive the index")
+	}
+}

beacon-chain/blockchain/setup_test.go

@@ -2,6 +2,7 @@ package blockchain
 
 import (
 	"context"
+	"sync"
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/v4/async/event"
@@ -23,10 +24,13 @@ import (
 
 type mockBeaconNode struct {
 	stateFeed *event.Feed
+	mu        sync.Mutex
 }
 
 // StateFeed mocks the same method in the beacon node.
 func (mbn *mockBeaconNode) StateFeed() *event.Feed {
+	mbn.mu.Lock()
+	defer mbn.mu.Unlock()
 	if mbn.stateFeed == nil {
 		mbn.stateFeed = new(event.Feed)
 	}
@@ -52,7 +56,7 @@ func (mb *mockBroadcaster) BroadcastSyncCommitteeMessage(_ context.Context, _ ui
 	return nil
 }
 
-func (mb *mockBroadcaster) BroadcastBlob(_ context.Context, _ uint64, _ *ethpb.SignedBlobSidecar) error {
+func (mb *mockBroadcaster) BroadcastBlob(_ context.Context, _ uint64, _ *ethpb.BlobSidecar) error {
 	mb.broadcastCalled = true
 	return nil
 }

beacon-chain/blockchain/testing/BUILD.bazel

@@ -24,11 +24,11 @@ go_library(
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//consensus-types/blocks:go_default_library",
+        "//consensus-types/forkchoice:go_default_library",
         "//consensus-types/interfaces:go_default_library",
         "//consensus-types/primitives:go_default_library",
         "//encoding/bytesutil:go_default_library",
         "//proto/engine/v1:go_default_library",
-        "//proto/eth/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",

beacon-chain/blockchain/testing/mock.go

@@ -23,11 +23,11 @@ import (
 	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
+	forkchoice2 "github.com/prysmaticlabs/prysm/v4/consensus-types/forkchoice"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	enginev1 "github.com/prysmaticlabs/prysm/v4/proto/engine/v1"
-	ethpbv1 "github.com/prysmaticlabs/prysm/v4/proto/eth/v1"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/sirupsen/logrus"
 )
@@ -72,6 +72,7 @@ type ChainService struct {
 	OptimisticRoots             map[[32]byte]bool
 	BlockSlot                   primitives.Slot
 	SyncingRoot                 [32]byte
+	Blobs                       []blocks.VerifiedROBlob
 }
 
 func (s *ChainService) Ancestor(ctx context.Context, root []byte, slot primitives.Slot) ([]byte, error) {
@@ -573,7 +574,7 @@ func (s *ChainService) InsertNode(ctx context.Context, st state.BeaconState, roo
 }
 
 // ForkChoiceDump mocks the same method in the chain service
-func (s *ChainService) ForkChoiceDump(ctx context.Context) (*ethpbv1.ForkChoiceDump, error) {
+func (s *ChainService) ForkChoiceDump(ctx context.Context) (*forkchoice2.Dump, error) {
 	if s.ForkChoiceStore != nil {
 		return s.ForkChoiceStore.ForkChoiceDump(ctx)
 	}
@@ -612,6 +613,7 @@ func (c *ChainService) BlockBeingSynced(root [32]byte) bool {
 }
 
 // ReceiveBlob implements the same method in the chain service
-func (*ChainService) ReceiveBlob(_ context.Context, _ *ethpb.BlobSidecar) error {
+func (c *ChainService) ReceiveBlob(_ context.Context, b blocks.VerifiedROBlob) error {
+	c.Blobs = append(c.Blobs, b)
 	return nil
 }

beacon-chain/builder/BUILD.bazel

@@ -15,7 +15,6 @@ go_library(
         "//beacon-chain/cache:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
-        "//config/fieldparams:go_default_library",
         "//consensus-types/interfaces:go_default_library",
         "//consensus-types/primitives:go_default_library",
         "//encoding/bytesutil:go_default_library",

beacon-chain/builder/service.go

@@ -2,7 +2,6 @@ package builder
 
 import (
 	"context"
-	"fmt"
 	"reflect"
 	"time"
 
@@ -11,7 +10,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
-	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
@@ -27,7 +25,7 @@ var ErrNoBuilder = errors.New("builder endpoint not configured")
 
 // BlockBuilder defines the interface for interacting with the block builder
 type BlockBuilder interface {
-	SubmitBlindedBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, blobs []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error)
+	SubmitBlindedBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error)
 	GetHeader(ctx context.Context, slot primitives.Slot, parentHash [32]byte, pubKey [48]byte) (builder.SignedBid, error)
 	RegisterValidator(ctx context.Context, reg []*ethpb.SignedValidatorRegistrationV1) error
 	RegistrationByValidatorID(ctx context.Context, id primitives.ValidatorIndex) (*ethpb.ValidatorRegistrationV1, error)
@@ -90,7 +88,7 @@ func (s *Service) Stop() error {
 }
 
 // SubmitBlindedBlock submits a blinded block to the builder relay network.
-func (s *Service) SubmitBlindedBlock(ctx context.Context, b interfaces.ReadOnlySignedBeaconBlock, blobs []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
+func (s *Service) SubmitBlindedBlock(ctx context.Context, b interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
 	ctx, span := trace.StartSpan(ctx, "builder.SubmitBlindedBlock")
 	defer span.End()
 	start := time.Now()
@@ -100,11 +98,8 @@ func (s *Service) SubmitBlindedBlock(ctx context.Context, b interfaces.ReadOnlyS
 	if s.c == nil {
 		return nil, nil, ErrNoBuilder
 	}
-	if uint64(len(blobs)) > fieldparams.MaxBlobsPerBlock {
-		return nil, nil, fmt.Errorf("blob count %d beyond max limit of %d", len(blobs), fieldparams.MaxBlobsPerBlock)
-	}
 
-	return s.c.SubmitBlindedBlock(ctx, b, blobs)
+	return s.c.SubmitBlindedBlock(ctx, b)
 }
 
 // GetHeader retrieves the header for a given slot and parent hash from the builder relay network.

beacon-chain/builder/service_test.go

@@ -62,7 +62,7 @@ func Test_BuilderMethodsWithouClient(t *testing.T) {
 	_, err = s.GetHeader(context.Background(), 0, [32]byte{}, [48]byte{})
 	assert.ErrorContains(t, ErrNoBuilder.Error(), err)
 
-	_, _, err = s.SubmitBlindedBlock(context.Background(), nil, nil)
+	_, _, err = s.SubmitBlindedBlock(context.Background(), nil)
 	assert.ErrorContains(t, ErrNoBuilder.Error(), err)
 
 	err = s.RegisterValidator(context.Background(), nil)

beacon-chain/builder/testing/mock.go

@@ -45,7 +45,7 @@ func (s *MockBuilderService) Configured() bool {
 }
 
 // SubmitBlindedBlock for mocking.
-func (s *MockBuilderService) SubmitBlindedBlock(_ context.Context, b interfaces.ReadOnlySignedBeaconBlock, _ []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
+func (s *MockBuilderService) SubmitBlindedBlock(_ context.Context, b interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
 	switch b.Version() {
 	case version.Bellatrix:
 		w, err := blocks.WrappedExecutionPayload(s.Payload)

beacon-chain/cache/checkpoint_state.go

@@ -42,7 +42,7 @@ func NewCheckpointStateCache() *CheckpointStateCache {
 // StateByCheckpoint fetches state by checkpoint. Returns true with a
 // reference to the CheckpointState info, if exists. Otherwise returns false, nil.
 func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (state.BeaconState, error) {
-	h, err := hash.HashProto(cp)
+	h, err := hash.Proto(cp)
 	if err != nil {
 		return nil, err
 	}
@@ -62,7 +62,7 @@ func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (state.Be
 // AddCheckpointState adds CheckpointState object to the cache. This method also trims the least
 // recently added CheckpointState object if the cache size has ready the max cache size limit.
 func (c *CheckpointStateCache) AddCheckpointState(cp *ethpb.Checkpoint, s state.ReadOnlyBeaconState) error {
-	h, err := hash.HashProto(cp)
+	h, err := hash.Proto(cp)
 	if err != nil {
 		return err
 	}

beacon-chain/cache/depositcache/pending_deposits.go

@@ -98,7 +98,7 @@ func (dc *DepositCache) RemovePendingDeposit(ctx context.Context, d *ethpb.Depos
 		return
 	}
 
-	depRoot, err := hash.HashProto(d)
+	depRoot, err := hash.Proto(d)
 	if err != nil {
 		log.WithError(err).Error("Could not remove deposit")
 		return
@@ -109,7 +109,7 @@ func (dc *DepositCache) RemovePendingDeposit(ctx context.Context, d *ethpb.Depos
 
 	idx := -1
 	for i, ctnr := range dc.pendingDeposits {
-		h, err := hash.HashProto(ctnr.Deposit)
+		h, err := hash.Proto(ctnr.Deposit)
 		if err != nil {
 			log.WithError(err).Error("Could not hash deposit")
 			continue

beacon-chain/cache/depositsnapshot/deposit_cache_test.go

@@ -769,7 +769,7 @@ func TestFinalizedDeposits_ReturnsTrieCorrectly(t *testing.T) {
 		}
 	}
 
-	ctrs := []*ethpb.DepositContainer{}
+	var ctrs []*ethpb.DepositContainer
 	for i := 0; i < 2000; i++ {
 		ctrs = append(ctrs, generateCtr(uint64(10+(i/2)), int64(i)))
 	}

beacon-chain/core/altair/reward_test.go

@@ -31,7 +31,7 @@ func Test_BaseReward(t *testing.T) {
 			valIdx:    2,
 			st:        genState(1),
 			want:      0,
-			errString: "index 2 out of range",
+			errString: "validator index 2 does not exist",
 		},
 		{
 			name:      "active balance is 32eth",
@@ -89,7 +89,7 @@ func Test_BaseRewardWithTotalBalance(t *testing.T) {
 			valIdx:        2,
 			activeBalance: 1,
 			want:          0,
-			errString:     "index 2 out of range",
+			errString:     "validator index 2 does not exist",
 		},
 		{
 			name:          "active balance is 1",

beacon-chain/core/altair/sync_committee_test.go

@@ -28,10 +28,10 @@ func TestSyncCommitteeIndices_CanGet(t *testing.T) {
 			}
 		}
 		st, err := state_native.InitializeFromProtoAltair(&ethpb.BeaconStateAltair{
-			Validators:  validators,
 			RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
 		})
 		require.NoError(t, err)
+		require.NoError(t, st.SetValidators(validators))
 		return st
 	}
 
@@ -69,6 +69,15 @@ func TestSyncCommitteeIndices_CanGet(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "no active validators, epoch 100",
+			args: args{
+				state: getState(t, 0), // Regression test for divide by zero. Issue #13051.
+				epoch: 100,
+			},
+			wantErr:   true,
+			errString: "no active validator indices",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

beacon-chain/core/blocks/BUILD.bazel

@@ -42,7 +42,6 @@ go_library(
         "//math:go_default_library",
         "//network/forks:go_default_library",
         "//proto/engine/v1:go_default_library",
-        "//proto/eth/v2:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/attestation:go_default_library",
         "//proto/prysm/v1alpha1/slashings:go_default_library",
@@ -100,7 +99,6 @@ go_test(
         "//encoding/bytesutil:go_default_library",
         "//encoding/ssz:go_default_library",
         "//proto/engine/v1:go_default_library",
-        "//proto/migration:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/attestation:go_default_library",
         "//proto/prysm/v1alpha1/attestation/aggregation:go_default_library",

beacon-chain/core/blocks/exit.go

@@ -55,7 +55,7 @@ func ProcessVoluntaryExits(
 	if len(exits) == 0 {
 		return beaconState, nil
 	}
-	maxExitEpoch, churn := v.ValidatorsMaxExitEpochAndChurn(beaconState)
+	maxExitEpoch, churn := v.MaxExitEpochAndChurn(beaconState)
 	var exitEpoch primitives.Epoch
 	for idx, exit := range exits {
 		if exit == nil || exit.Exit == nil {

beacon-chain/core/blocks/exit_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/testing/assert"
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
 	"github.com/prysmaticlabs/prysm/v4/testing/util"
+	"github.com/prysmaticlabs/prysm/v4/time/slots"
 )
 
 func TestProcessVoluntaryExits_NotActiveLongEnoughToExit(t *testing.T) {
@@ -134,6 +135,10 @@ func TestProcessVoluntaryExits_AppliesCorrectStatus(t *testing.T) {
 }
 
 func TestVerifyExitAndSignature(t *testing.T) {
+	undo := util.HackDenebMaxuint(t)
+	defer undo()
+	denebSlot, err := slots.EpochStart(params.BeaconConfig().DenebForkEpoch)
+	require.NoError(t, err)
 	tests := []struct {
 		name    string
 		setup   func() (*ethpb.Validator, *ethpb.SignedVoluntaryExit, state.ReadOnlyBeaconState, error)
@@ -241,11 +246,11 @@ func TestVerifyExitAndSignature(t *testing.T) {
 				fork := &ethpb.Fork{
 					PreviousVersion: params.BeaconConfig().CapellaForkVersion,
 					CurrentVersion:  params.BeaconConfig().DenebForkVersion,
-					Epoch:           primitives.Epoch(2),
+					Epoch:           params.BeaconConfig().DenebForkEpoch,
 				}
 				signedExit := &ethpb.SignedVoluntaryExit{
 					Exit: &ethpb.VoluntaryExit{
-						Epoch:          2,
+						Epoch:          params.BeaconConfig().CapellaForkEpoch,
 						ValidatorIndex: 0,
 					},
 				}
@@ -253,7 +258,7 @@ func TestVerifyExitAndSignature(t *testing.T) {
 				bs, err := state_native.InitializeFromProtoUnsafeDeneb(&ethpb.BeaconStateDeneb{
 					GenesisValidatorsRoot: bs.GenesisValidatorsRoot(),
 					Fork:                  fork,
-					Slot:                  (params.BeaconConfig().SlotsPerEpoch * 2) + 1,
+					Slot:                  denebSlot,
 					Validators:            bs.Validators(),
 				})
 				if err != nil {

beacon-chain/core/blocks/randao.go

@@ -82,7 +82,7 @@ func ProcessRandaoNoVerify(
 	for i, x := range blockRandaoReveal {
 		latestMixSlice[i] ^= x
 	}
-	if err := beaconState.UpdateRandaoMixesAtIndex(uint64(currentEpoch%latestMixesLength), latestMixSlice); err != nil {
+	if err := beaconState.UpdateRandaoMixesAtIndex(uint64(currentEpoch%latestMixesLength), [32]byte(latestMixSlice)); err != nil {
 		return nil, err
 	}
 	return beaconState, nil

beacon-chain/core/blocks/withdrawals.go

@@ -16,7 +16,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/crypto/hash"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	"github.com/prysmaticlabs/prysm/v4/encoding/ssz"
-	ethpbv2 "github.com/prysmaticlabs/prysm/v4/proto/eth/v2"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/runtime/version"
 )
@@ -236,7 +235,7 @@ func BLSChangesSignatureBatch(
 			return nil, errors.Wrap(err, "could not convert bytes to public key")
 		}
 		batch.PublicKeys[i] = publicKey
-		htr, err := signing.SigningData(change.Message.HashTreeRoot, domain)
+		htr, err := signing.Data(change.Message.HashTreeRoot, domain)
 		if err != nil {
 			return nil, errors.Wrap(err, "could not compute BLSToExecutionChange signing data")
 		}
@@ -251,7 +250,7 @@ func BLSChangesSignatureBatch(
 // is from a previous fork.
 func VerifyBLSChangeSignature(
 	st state.ReadOnlyBeaconState,
-	change *ethpbv2.SignedBLSToExecutionChange,
+	change *ethpb.SignedBLSToExecutionChange,
 ) error {
 	c := params.BeaconConfig()
 	domain, err := signing.ComputeDomain(c.DomainBLSToExecutionChange, c.GenesisForkVersion, st.GenesisValidatorsRoot())

beacon-chain/core/blocks/withdrawals_test.go

@@ -18,7 +18,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/crypto/hash"
 	"github.com/prysmaticlabs/prysm/v4/encoding/ssz"
 	enginev1 "github.com/prysmaticlabs/prysm/v4/proto/engine/v1"
-	"github.com/prysmaticlabs/prysm/v4/proto/migration"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
 	"github.com/prysmaticlabs/prysm/v4/time/slots"
@@ -152,7 +151,7 @@ func TestProcessBLSToExecutionChange(t *testing.T) {
 		}
 
 		_, err = blocks.ProcessBLSToExecutionChange(st, signed)
-		require.ErrorContains(t, "out of range", err)
+		require.ErrorContains(t, "out of bounds", err)
 	})
 
 	t.Run("signature does not verify", func(t *testing.T) {
@@ -1209,8 +1208,7 @@ func TestBLSChangesSignatureBatch(t *testing.T) {
 	require.Equal(t, true, verify)
 
 	// Verify a single change
-	change := migration.V1Alpha1SignedBLSToExecChangeToV2(signedChanges[0])
-	require.NoError(t, blocks.VerifyBLSChangeSignature(st, change))
+	require.NoError(t, blocks.VerifyBLSChangeSignature(st, signedChanges[0]))
 }
 
 func TestBLSChangesSignatureBatchWrongFork(t *testing.T) {
@@ -1274,8 +1272,7 @@ func TestBLSChangesSignatureBatchWrongFork(t *testing.T) {
 	require.Equal(t, false, verify)
 
 	// Verify a single change
-	change := migration.V1Alpha1SignedBLSToExecChangeToV2(signedChanges[0])
-	require.ErrorIs(t, signing.ErrSigFailedToVerify, blocks.VerifyBLSChangeSignature(st, change))
+	require.ErrorIs(t, signing.ErrSigFailedToVerify, blocks.VerifyBLSChangeSignature(st, signedChanges[0]))
 }
 
 func TestBLSChangesSignatureBatchFromBellatrix(t *testing.T) {
@@ -1362,7 +1359,6 @@ func TestBLSChangesSignatureBatchFromBellatrix(t *testing.T) {
 	require.Equal(t, true, verify)
 
 	// Verify a single change
-	change := migration.V1Alpha1SignedBLSToExecChangeToV2(signedChanges[0])
-	require.NoError(t, blocks.VerifyBLSChangeSignature(st, change))
+	require.NoError(t, blocks.VerifyBLSChangeSignature(st, signedChanges[0]))
 	params.OverrideBeaconConfig(savedConfig)
 }

beacon-chain/core/epoch/epoch_processing.go

@@ -112,7 +112,7 @@ func ProcessRegistryUpdates(ctx context.Context, state state.BeaconState) (state
 		if isActive && belowEjectionBalance {
 			// Here is fine to do a quadratic loop since this should
 			// barely happen
-			maxExitEpoch, churn := validators.ValidatorsMaxExitEpochAndChurn(state)
+			maxExitEpoch, churn := validators.MaxExitEpochAndChurn(state)
 			state, _, err = validators.InitiateValidatorExit(ctx, state, primitives.ValidatorIndex(idx), maxExitEpoch, churn)
 			if err != nil && !errors.Is(err, validators.ValidatorAlreadyExitedErr) {
 				return nil, errors.Wrapf(err, "could not initiate exit for validator %d", idx)
@@ -349,7 +349,7 @@ func ProcessRandaoMixesReset(state state.BeaconState) (state.BeaconState, error)
 	if err != nil {
 		return nil, err
 	}
-	if err := state.UpdateRandaoMixesAtIndex(uint64(nextEpoch%randaoMixLength), mix); err != nil {
+	if err := state.UpdateRandaoMixesAtIndex(uint64(nextEpoch%randaoMixLength), [32]byte(mix)); err != nil {
 		return nil, err
 	}
 

beacon-chain/core/helpers/BUILD.bazel

@@ -58,8 +58,8 @@ go_test(
         "weak_subjectivity_test.go",
     ],
     embed = [":go_default_library"],
-    race = "on",
     shard_count = 2,
+    tags = ["CI_race_detection"],
     deps = [
         "//beacon-chain/cache:go_default_library",
         "//beacon-chain/core/time:go_default_library",

beacon-chain/core/helpers/sync_committee_test.go

@@ -99,7 +99,7 @@ func TestIsCurrentEpochSyncCommittee_DoesNotExist(t *testing.T) {
 	require.NoError(t, state.SetNextSyncCommittee(syncCommittee))
 
 	ok, err := IsCurrentPeriodSyncCommittee(state, 12390192)
-	require.ErrorContains(t, "index 12390192 out of range", err)
+	require.ErrorContains(t, "validator index 12390192 does not exist", err)
 	require.Equal(t, false, ok)
 }
 
@@ -182,7 +182,7 @@ func TestIsNextEpochSyncCommittee_DoesNotExist(t *testing.T) {
 	require.NoError(t, state.SetNextSyncCommittee(syncCommittee))
 
 	ok, err := IsNextPeriodSyncCommittee(state, 120391029)
-	require.ErrorContains(t, "index 120391029 out of range", err)
+	require.ErrorContains(t, "validator index 120391029 does not exist", err)
 	require.Equal(t, false, ok)
 }
 
@@ -282,7 +282,7 @@ func TestCurrentEpochSyncSubcommitteeIndices_DoesNotExist(t *testing.T) {
 	require.NoError(t, state.SetNextSyncCommittee(syncCommittee))
 
 	index, err := CurrentPeriodSyncSubcommitteeIndices(state, 129301923)
-	require.ErrorContains(t, "index 129301923 out of range", err)
+	require.ErrorContains(t, "validator index 129301923 does not exist", err)
 	require.DeepEqual(t, []primitives.CommitteeIndex(nil), index)
 }
 
@@ -367,7 +367,7 @@ func TestNextEpochSyncSubcommitteeIndices_DoesNotExist(t *testing.T) {
 	require.NoError(t, state.SetNextSyncCommittee(syncCommittee))
 
 	index, err := NextPeriodSyncSubcommitteeIndices(state, 21093019)
-	require.ErrorContains(t, "index 21093019 out of range", err)
+	require.ErrorContains(t, "validator index 21093019 does not exist", err)
 	require.DeepEqual(t, []primitives.CommitteeIndex(nil), index)
 }
 

beacon-chain/core/helpers/validators.go

@@ -136,6 +136,10 @@ func ActiveValidatorIndices(ctx context.Context, s state.ReadOnlyBeaconState, ep
 		return nil, err
 	}
 
+	if len(indices) == 0 {
+		return nil, errors.New("no active validator indices")
+	}
+
 	if err := UpdateCommitteeCache(ctx, s, epoch); err != nil {
 		return nil, errors.Wrap(err, "could not update committee cache")
 	}

beacon-chain/core/helpers/validators_test.go

@@ -560,12 +560,24 @@ func TestActiveValidatorIndices(t *testing.T) {
 			},
 			want: []primitives.ValidatorIndex{0, 2, 3},
 		},
+		{
+			name: "impossible_zero_validators", // Regression test for issue #13051
+			args: args{
+				state: &ethpb.BeaconState{
+					RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+					Validators:  make([]*ethpb.Validator, 0),
+				},
+				epoch: 10,
+			},
+			wantedErr: "no active validator indices",
+		},
 	}
 	defer ClearCache()
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			s, err := state_native.InitializeFromProtoPhase0(tt.args.state)
 			require.NoError(t, err)
+			require.NoError(t, s.SetValidators(tt.args.state.Validators))
 			got, err := ActiveValidatorIndices(context.Background(), s, tt.args.epoch)
 			if tt.wantedErr != "" {
 				assert.ErrorContains(t, tt.wantedErr, err)

beacon-chain/core/signing/BUILD.bazel

@@ -16,7 +16,6 @@ go_library(
         "//crypto/bls:go_default_library",
         "//encoding/bytesutil:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
-        "//runtime/version:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_fastssz//:go_default_library",
     ],

beacon-chain/core/signing/signing_root.go

@@ -11,7 +11,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/crypto/bls"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
-	"github.com/prysmaticlabs/prysm/v4/runtime/version"
 )
 
 // ForkVersionByteLength length of fork version byte array.
@@ -57,18 +56,22 @@ const (
 
 // ComputeDomainAndSign computes the domain and signing root and sign it using the passed in private key.
 func ComputeDomainAndSign(st state.ReadOnlyBeaconState, epoch primitives.Epoch, obj fssz.HashRoot, domain [4]byte, key bls.SecretKey) ([]byte, error) {
-	fork := st.Fork()
+	return ComputeDomainAndSignWithoutState(st.Fork(), epoch, domain, st.GenesisValidatorsRoot(), obj, key)
+}
+
+// ComputeDomainAndSignWithoutState offers the same functionalit as ComputeDomainAndSign without the need to provide a BeaconState.
+// This is particularly helpful for signing values in tests.
+func ComputeDomainAndSignWithoutState(fork *ethpb.Fork, epoch primitives.Epoch, domain [4]byte, vr []byte, obj fssz.HashRoot, key bls.SecretKey) ([]byte, error) {
 	// EIP-7044: Beginning in Deneb, fix the fork version to Capella for signed exits.
 	// This allows for signed validator exits to be valid forever.
-	if st.Version() >= version.Deneb && domain == params.BeaconConfig().DomainVoluntaryExit {
+	if domain == params.BeaconConfig().DomainVoluntaryExit && epoch >= params.BeaconConfig().DenebForkEpoch {
 		fork = &ethpb.Fork{
 			PreviousVersion: params.BeaconConfig().CapellaForkVersion,
 			CurrentVersion:  params.BeaconConfig().CapellaForkVersion,
 			Epoch:           params.BeaconConfig().CapellaForkEpoch,
 		}
 	}
-
-	d, err := Domain(fork, epoch, domain, st.GenesisValidatorsRoot())
+	d, err := Domain(fork, epoch, domain, vr)
 	if err != nil {
 		return nil, err
 	}
@@ -92,18 +95,24 @@ func ComputeDomainAndSign(st state.ReadOnlyBeaconState, epoch primitives.Epoch,
 //	       domain=domain,
 //	   ))
 func ComputeSigningRoot(object fssz.HashRoot, domain []byte) ([32]byte, error) {
-	return SigningData(object.HashTreeRoot, domain)
+	return Data(object.HashTreeRoot, domain)
 }
 
-// SigningData computes the signing data by utilising the provided root function and then
+// Data computes the signing data by utilising the provided root function and then
 // returning the signing data of the container object.
-func SigningData(rootFunc func() ([32]byte, error), domain []byte) ([32]byte, error) {
+func Data(rootFunc func() ([32]byte, error), domain []byte) ([32]byte, error) {
 	objRoot, err := rootFunc()
 	if err != nil {
 		return [32]byte{}, err
 	}
+	return ComputeSigningRootForRoot(objRoot, domain)
+}
+
+// ComputeSigningRootForRoot works the same as ComputeSigningRoot,
+// except that gets the root from an argument instead of a callback.
+func ComputeSigningRootForRoot(root [32]byte, domain []byte) ([32]byte, error) {
 	container := &ethpb.SigningData{
-		ObjectRoot: objRoot[:],
+		ObjectRoot: root[:],
 		Domain:     domain,
 	}
 	return container.HashTreeRoot()
@@ -152,7 +161,7 @@ func VerifyBlockHeaderSigningRoot(blkHdr *ethpb.BeaconBlockHeader, pub, signatur
 	if err != nil {
 		return errors.Wrap(err, "could not convert bytes to signature")
 	}
-	root, err := SigningData(blkHdr.HashTreeRoot, domain)
+	root, err := Data(blkHdr.HashTreeRoot, domain)
 	if err != nil {
 		return errors.Wrap(err, "could not compute signing root")
 	}
@@ -191,7 +200,7 @@ func BlockSignatureBatch(pub, signature, domain []byte, rootFunc func() ([32]byt
 		return nil, errors.Wrap(err, "could not convert bytes to public key")
 	}
 	// utilize custom block hashing function
-	root, err := SigningData(rootFunc, domain)
+	root, err := Data(rootFunc, domain)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not compute signing root")
 	}

beacon-chain/core/transition/state.go

@@ -9,6 +9,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
 	state_native "github.com/prysmaticlabs/prysm/v4/beacon-chain/state/state-native"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state/stateutil"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/container/trie"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
@@ -221,6 +222,18 @@ func OptimizedGenesisBeaconState(genesisTime uint64, preState state.BeaconState,
 
 // EmptyGenesisState returns an empty beacon state object.
 func EmptyGenesisState() (state.BeaconState, error) {
+	blockRoots := make([][]byte, fieldparams.BlockRootsLength)
+	for i := range blockRoots {
+		blockRoots[i] = make([]byte, fieldparams.RootLength)
+	}
+	stateRoots := make([][]byte, fieldparams.StateRootsLength)
+	for i := range stateRoots {
+		stateRoots[i] = make([]byte, fieldparams.RootLength)
+	}
+	mixes := make([][]byte, fieldparams.RandaoMixesLength)
+	for i := range mixes {
+		mixes[i] = make([]byte, fieldparams.RootLength)
+	}
 	st := &ethpb.BeaconState{
 		// Misc fields.
 		Slot: 0,
@@ -229,6 +242,9 @@ func EmptyGenesisState() (state.BeaconState, error) {
 			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
 			Epoch:           0,
 		},
+		BlockRoots:  blockRoots,
+		StateRoots:  stateRoots,
+		RandaoMixes: mixes,
 		// Validator registry fields.
 		Validators: []*ethpb.Validator{},
 		Balances:   []uint64{},

beacon-chain/core/transition/state_test.go

@@ -103,8 +103,8 @@ func TestGenesisState_HashEquality(t *testing.T) {
 	pbstate, err := state_native.ProtobufBeaconStatePhase0(state.ToProto())
 	require.NoError(t, err)
 
-	root1, err1 := hash.HashProto(pbState1)
-	root2, err2 := hash.HashProto(pbstate)
+	root1, err1 := hash.Proto(pbState1)
+	root2, err2 := hash.Proto(pbstate)
 
 	if err1 != nil || err2 != nil {
 		t.Fatalf("Failed to marshal state to bytes: %v %v", err1, err2)

beacon-chain/core/transition/transition_test.go

@@ -382,10 +382,18 @@ func TestProcessEpochPrecompute_CanProcess(t *testing.T) {
 		FinalizedCheckpoint:        &ethpb.Checkpoint{Root: make([]byte, fieldparams.RootLength)},
 		JustificationBits:          bitfield.Bitvector4{0x00},
 		CurrentJustifiedCheckpoint: &ethpb.Checkpoint{Root: make([]byte, fieldparams.RootLength)},
+		Validators: []*ethpb.Validator{
+			{
+				ExitEpoch:        params.BeaconConfig().FarFutureEpoch,
+				EffectiveBalance: params.BeaconConfig().MinDepositAmount,
+			},
+		},
+		Balances: []uint64{
+			params.BeaconConfig().MinDepositAmount,
+		},
 	}
 	s, err := state_native.InitializeFromProtoPhase0(base)
 	require.NoError(t, err)
-	require.NoError(t, s.SetValidators([]*ethpb.Validator{}))
 	newState, err := transition.ProcessEpochPrecompute(context.Background(), s)
 	require.NoError(t, err)
 	assert.Equal(t, uint64(0), newState.Slashings()[2], "Unexpected slashed balance")

beacon-chain/core/validators/validator.go

@@ -21,9 +21,9 @@ import (
 // an already exited validator
 var ValidatorAlreadyExitedErr = errors.New("validator already exited")
 
-// ValidatorsMaxExitEpochAndChurn returns the maximum non-FAR_FUTURE_EPOCH exit
+// MaxExitEpochAndChurn returns the maximum non-FAR_FUTURE_EPOCH exit
 // epoch and the number of them
-func ValidatorsMaxExitEpochAndChurn(s state.BeaconState) (maxExitEpoch primitives.Epoch, churn uint64) {
+func MaxExitEpochAndChurn(s state.BeaconState) (maxExitEpoch primitives.Epoch, churn uint64) {
 	farFutureEpoch := params.BeaconConfig().FarFutureEpoch
 	err := s.ReadFromEveryValidator(func(idx int, val state.ReadOnlyValidator) error {
 		e := val.ExitEpoch()
@@ -134,7 +134,7 @@ func SlashValidator(
 	slashedIdx primitives.ValidatorIndex,
 	penaltyQuotient uint64,
 	proposerRewardQuotient uint64) (state.BeaconState, error) {
-	maxExitEpoch, churn := ValidatorsMaxExitEpochAndChurn(s)
+	maxExitEpoch, churn := MaxExitEpochAndChurn(s)
 	s, _, err := InitiateValidatorExit(ctx, s, slashedIdx, maxExitEpoch, churn)
 	if err != nil && !errors.Is(err, ValidatorAlreadyExitedErr) {
 		return nil, errors.Wrapf(err, "could not initiate validator %d exit", slashedIdx)

beacon-chain/core/validators/validator_test.go

@@ -410,7 +410,7 @@ func TestValidatorMaxExitEpochAndChurn(t *testing.T) {
 	for _, tt := range tests {
 		s, err := state_native.InitializeFromProtoPhase0(tt.state)
 		require.NoError(t, err)
-		epoch, churn := ValidatorsMaxExitEpochAndChurn(s)
+		epoch, churn := MaxExitEpochAndChurn(s)
 		require.Equal(t, tt.wantedEpoch, epoch)
 		require.Equal(t, tt.wantedChurn, churn)
 	}

beacon-chain/db/db.go

@@ -3,19 +3,12 @@
 package db
 
 import (
-	"context"
-
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/kv"
 )
 
-// NewDB initializes a new DB.
-func NewDB(ctx context.Context, dirPath string) (Database, error) {
-	return kv.NewKVStore(ctx, dirPath)
-}
-
-// NewDBFilename uses the KVStoreDatafilePath so that if this layer of
+// NewFileName uses the KVStoreDatafilePath so that if this layer of
 // indirection between db.NewDB->kv.NewKVStore ever changes, it will be easy to remember
 // to also change this filename indirection at the same time.
-func NewDBFilename(dirPath string) string {
-	return kv.KVStoreDatafilePath(dirPath)
+func NewFileName(dirPath string) string {
+	return kv.StoreDatafilePath(dirPath)
 }

beacon-chain/db/errors.go

@@ -1,6 +1,11 @@
 package db
 
-import "github.com/prysmaticlabs/prysm/v4/beacon-chain/db/kv"
+import (
+	"errors"
+	"os"
+
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/kv"
+)
 
 // ErrNotFound can be used to determine if an error from a method in the database package
 // represents a "not found" error. These often require different handling than a low-level
@@ -19,3 +24,9 @@ var ErrNotFoundBackfillBlockRoot = kv.ErrNotFoundBackfillBlockRoot
 
 // ErrNotFoundGenesisBlockRoot means no genesis block root was found, indicating the db was not initialized with genesis
 var ErrNotFoundGenesisBlockRoot = kv.ErrNotFoundGenesisBlockRoot
+
+// IsNotFound allows callers to treat errors from a flat-file database, where the file record is missing,
+// as equivalent to db.ErrNotFound.
+func IsNotFound(err error) bool {
+	return errors.Is(err, ErrNotFound) || os.IsNotExist(err)
+}

beacon-chain/db/filesystem/BUILD.bazel

@@ -0,0 +1,44 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "blob.go",
+        "ephemeral.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//beacon-chain/verification:go_default_library",
+        "//config/fieldparams:go_default_library",
+        "//config/params:go_default_library",
+        "//consensus-types/blocks:go_default_library",
+        "//consensus-types/primitives:go_default_library",
+        "//io/file:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//runtime/logging:go_default_library",
+        "//time/slots:go_default_library",
+        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+        "@com_github_spf13_afero//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["blob_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//beacon-chain/verification:go_default_library",
+        "//config/fieldparams:go_default_library",
+        "//consensus-types/primitives:go_default_library",
+        "//encoding/bytesutil:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//testing/require:go_default_library",
+        "//testing/util:go_default_library",
+        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
+        "@com_github_prysmaticlabs_fastssz//:go_default_library",
+        "@com_github_spf13_afero//:go_default_library",
+    ],
+)

beacon-chain/db/filesystem/blob.go

@@ -0,0 +1,274 @@
+package filesystem
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/verification"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
+	"github.com/prysmaticlabs/prysm/v4/config/params"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+	"github.com/prysmaticlabs/prysm/v4/io/file"
+	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/v4/runtime/logging"
+	"github.com/prysmaticlabs/prysm/v4/time/slots"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/afero"
+)
+
+var (
+	errIndexOutOfBounds = errors.New("blob index in file name > MaxBlobsPerBlock")
+)
+
+const (
+	sszExt  = "ssz"
+	partExt = "part"
+
+	bufferEpochs         = 2
+	directoryPermissions = 0700
+)
+
+// BlobStorageOption is a functional option for configuring a BlobStorage.
+type BlobStorageOption func(*BlobStorage)
+
+// WithBlobRetentionEpochs is an option that changes the number of epochs blobs will be persisted.
+func WithBlobRetentionEpochs(e primitives.Epoch) BlobStorageOption {
+	return func(b *BlobStorage) {
+		b.retentionEpochs = e
+	}
+}
+
+// NewBlobStorage creates a new instance of the BlobStorage object. Note that the implementation of BlobStorage may
+// attempt to hold a file lock to guarantee exclusive control of the blob storage directory, so this should only be
+// initialized once per beacon node.
+func NewBlobStorage(base string, opts ...BlobStorageOption) (*BlobStorage, error) {
+	base = path.Clean(base)
+	if err := file.MkdirAll(base); err != nil {
+		return nil, fmt.Errorf("failed to create blob storage at %s: %w", base, err)
+	}
+	fs := afero.NewBasePathFs(afero.NewOsFs(), base)
+	b := &BlobStorage{fs: fs, retentionEpochs: params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest}
+	for _, o := range opts {
+		o(b)
+	}
+	return b, nil
+}
+
+// BlobStorage is the concrete implementation of the filesystem backend for saving and retrieving BlobSidecars.
+type BlobStorage struct {
+	fs              afero.Fs
+	retentionEpochs primitives.Epoch
+}
+
+// Save saves blobs given a list of sidecars.
+func (bs *BlobStorage) Save(sidecar blocks.VerifiedROBlob) error {
+	fname := namerForSidecar(sidecar)
+	sszPath := fname.path()
+	exists, err := afero.Exists(bs.fs, sszPath)
+	if err != nil {
+		return err
+	}
+	if exists {
+		log.WithFields(logging.BlobFields(sidecar.ROBlob)).Debug("ignoring a duplicate blob sidecar Save attempt")
+		return nil
+	}
+
+	// Serialize the ethpb.BlobSidecar to binary data using SSZ.
+	sidecarData, err := sidecar.MarshalSSZ()
+	if err != nil {
+		return errors.Wrap(err, "failed to serialize sidecar data")
+	}
+	if err := bs.fs.MkdirAll(fname.dir(), directoryPermissions); err != nil {
+		return err
+	}
+	partPath := fname.partPath()
+	// Create a partial file and write the serialized data to it.
+	partialFile, err := bs.fs.Create(partPath)
+	if err != nil {
+		return errors.Wrap(err, "failed to create partial file")
+	}
+
+	_, err = partialFile.Write(sidecarData)
+	if err != nil {
+		closeErr := partialFile.Close()
+		if closeErr != nil {
+			return closeErr
+		}
+		return errors.Wrap(err, "failed to write to partial file")
+	}
+	err = partialFile.Close()
+	if err != nil {
+		return err
+	}
+
+	// Atomically rename the partial file to its final name.
+	err = bs.fs.Rename(partPath, sszPath)
+	if err != nil {
+		return errors.Wrap(err, "failed to rename partial file to final name")
+	}
+	return nil
+}
+
+// Get retrieves a single BlobSidecar by its root and index.
+// Since BlobStorage only writes blobs that have undergone full verification, the return
+// value is always a VerifiedROBlob.
+func (bs *BlobStorage) Get(root [32]byte, idx uint64) (blocks.VerifiedROBlob, error) {
+	expected := blobNamer{root: root, index: idx}
+	encoded, err := afero.ReadFile(bs.fs, expected.path())
+	var v blocks.VerifiedROBlob
+	if err != nil {
+		return v, err
+	}
+	s := &ethpb.BlobSidecar{}
+	if err := s.UnmarshalSSZ(encoded); err != nil {
+		return v, err
+	}
+	ro, err := blocks.NewROBlobWithRoot(s, root)
+	if err != nil {
+		return blocks.VerifiedROBlob{}, err
+	}
+	return verification.BlobSidecarNoop(ro)
+}
+
+// Indices generates a bitmap representing which BlobSidecar.Index values are present on disk for a given root.
+// This value can be compared to the commitments observed in a block to determine which indices need to be found
+// on the network to confirm data availability.
+func (bs *BlobStorage) Indices(root [32]byte) ([fieldparams.MaxBlobsPerBlock]bool, error) {
+	var mask [fieldparams.MaxBlobsPerBlock]bool
+	rootDir := blobNamer{root: root}.dir()
+	entries, err := afero.ReadDir(bs.fs, rootDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return mask, nil
+		}
+		return mask, err
+	}
+	for i := range entries {
+		if entries[i].IsDir() {
+			continue
+		}
+		name := entries[i].Name()
+		if !strings.HasSuffix(name, sszExt) {
+			continue
+		}
+		parts := strings.Split(name, ".")
+		if len(parts) != 2 {
+			continue
+		}
+		u, err := strconv.ParseUint(parts[0], 10, 64)
+		if err != nil {
+			return mask, errors.Wrapf(err, "unexpected directory entry breaks listing, %s", parts[0])
+		}
+		if u >= fieldparams.MaxBlobsPerBlock {
+			return mask, errIndexOutOfBounds
+		}
+		mask[u] = true
+	}
+	return mask, nil
+}
+
+type blobNamer struct {
+	root  [32]byte
+	index uint64
+}
+
+func namerForSidecar(sc blocks.VerifiedROBlob) blobNamer {
+	return blobNamer{root: sc.BlockRoot(), index: sc.Index}
+}
+
+func (p blobNamer) dir() string {
+	return fmt.Sprintf("%#x", p.root)
+}
+
+func (p blobNamer) fname(ext string) string {
+	return path.Join(p.dir(), fmt.Sprintf("%d.%s", p.index, ext))
+}
+
+func (p blobNamer) partPath() string {
+	return p.fname(partExt)
+}
+
+func (p blobNamer) path() string {
+	return p.fname(sszExt)
+}
+
+// Prune prunes blobs in the base directory based on the retention epoch.
+// It deletes blobs older than currentEpoch - (retentionEpochs+bufferEpochs).
+// This is so that we keep a slight buffer and blobs are deleted after n+2 epochs.
+func (bs *BlobStorage) Prune(currentSlot primitives.Slot) error {
+	retentionSlots, err := slots.EpochStart(bs.retentionEpochs + bufferEpochs)
+	if err != nil {
+		return err
+	}
+	if currentSlot < retentionSlots {
+		return nil // Overflow would occur
+	}
+
+	folders, err := afero.ReadDir(bs.fs, ".")
+	if err != nil {
+		return err
+	}
+	for _, folder := range folders {
+		if folder.IsDir() {
+			if err := bs.processFolder(folder, currentSlot, retentionSlots); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// processFolder will delete the folder of blobs if the blob slot is outside the
+// retention period. We determine the slot by looking at the first blob in the folder.
+func (bs *BlobStorage) processFolder(folder os.FileInfo, currentSlot, retentionSlots primitives.Slot) error {
+	f, err := bs.fs.Open(filepath.Join(folder.Name(), "0."+sszExt))
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			log.WithError(err).Errorf("Could not close blob file")
+		}
+	}()
+
+	slot, err := slotFromBlob(f)
+	if err != nil {
+		return err
+	}
+	if slot < (currentSlot - retentionSlots) {
+		if err = bs.fs.RemoveAll(folder.Name()); err != nil {
+			return errors.Wrapf(err, "failed to delete blob %s", f.Name())
+		}
+	}
+	return nil
+}
+
+// slotFromBlob reads the ssz data of a file at the specified offset (8 + 131072 + 48 + 48 = 131176 bytes),
+// which is calculated based on the size of the BlobSidecar struct and is based on the size of the fields
+// preceding the slot information within SignedBeaconBlockHeader.
+func slotFromBlob(at io.ReaderAt) (primitives.Slot, error) {
+	b := make([]byte, 8)
+	_, err := at.ReadAt(b, 131176)
+	if err != nil {
+		return 0, err
+	}
+	rawSlot := binary.LittleEndian.Uint64(b)
+	return primitives.Slot(rawSlot), nil
+}
+
+// Delete removes the directory matching the provided block root and all the blobs it contains.
+func (bs *BlobStorage) Delete(root [32]byte) error {
+	if err := bs.fs.RemoveAll(hexutil.Encode(root[:])); err != nil {
+		return fmt.Errorf("failed to delete blobs for root %#x: %w", root, err)
+	}
+	return nil
+}

beacon-chain/db/filesystem/blob_test.go

@@ -0,0 +1,208 @@
+package filesystem
+
+import (
+	"bytes"
+	"path"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	ssz "github.com/prysmaticlabs/fastssz"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/verification"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
+	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/v4/testing/require"
+	"github.com/prysmaticlabs/prysm/v4/testing/util"
+	"github.com/spf13/afero"
+)
+
+func TestBlobStorage_SaveBlobData(t *testing.T) {
+	_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, 1, fieldparams.MaxBlobsPerBlock)
+	testSidecars, err := verification.BlobSidecarSliceNoop(sidecars)
+	require.NoError(t, err)
+
+	t.Run("no error for duplicate", func(t *testing.T) {
+		fs, bs, err := NewEphemeralBlobStorageWithFs(t)
+		require.NoError(t, err)
+		existingSidecar := testSidecars[0]
+
+		blobPath := namerForSidecar(existingSidecar).path()
+		// Serialize the existing BlobSidecar to binary data.
+		existingSidecarData, err := ssz.MarshalSSZ(existingSidecar)
+		require.NoError(t, err)
+
+		require.NoError(t, bs.Save(existingSidecar))
+		// No error when attempting to write twice.
+		require.NoError(t, bs.Save(existingSidecar))
+
+		content, err := afero.ReadFile(fs, blobPath)
+		require.NoError(t, err)
+		require.Equal(t, true, bytes.Equal(existingSidecarData, content))
+
+		// Deserialize the BlobSidecar from the saved file data.
+		savedSidecar := &ethpb.BlobSidecar{}
+		err = savedSidecar.UnmarshalSSZ(content)
+		require.NoError(t, err)
+
+		// Compare the original Sidecar and the saved Sidecar.
+		require.DeepSSZEqual(t, existingSidecar.BlobSidecar, savedSidecar)
+
+	})
+	t.Run("indices", func(t *testing.T) {
+		bs := NewEphemeralBlobStorage(t)
+		sc := testSidecars[2]
+		require.NoError(t, bs.Save(sc))
+		actualSc, err := bs.Get(sc.BlockRoot(), sc.Index)
+		require.NoError(t, err)
+		expectedIdx := [fieldparams.MaxBlobsPerBlock]bool{false, false, true}
+		actualIdx, err := bs.Indices(actualSc.BlockRoot())
+		require.NoError(t, err)
+		require.Equal(t, expectedIdx, actualIdx)
+	})
+
+	t.Run("round trip write then read", func(t *testing.T) {
+		bs := NewEphemeralBlobStorage(t)
+		err := bs.Save(testSidecars[0])
+		require.NoError(t, err)
+
+		expected := testSidecars[0]
+		actual, err := bs.Get(expected.BlockRoot(), expected.Index)
+		require.NoError(t, err)
+		require.DeepSSZEqual(t, expected, actual)
+	})
+}
+
+func TestBlobIndicesBounds(t *testing.T) {
+	fs, bs, err := NewEphemeralBlobStorageWithFs(t)
+	require.NoError(t, err)
+	root := [32]byte{}
+
+	okIdx := uint64(fieldparams.MaxBlobsPerBlock - 1)
+	writeFakeSSZ(t, fs, root, okIdx)
+	indices, err := bs.Indices(root)
+	require.NoError(t, err)
+	var expected [fieldparams.MaxBlobsPerBlock]bool
+	expected[okIdx] = true
+	for i := range expected {
+		require.Equal(t, expected[i], indices[i])
+	}
+
+	oobIdx := uint64(fieldparams.MaxBlobsPerBlock)
+	writeFakeSSZ(t, fs, root, oobIdx)
+	_, err = bs.Indices(root)
+	require.ErrorIs(t, err, errIndexOutOfBounds)
+}
+
+func writeFakeSSZ(t *testing.T, fs afero.Fs, root [32]byte, idx uint64) {
+	namer := blobNamer{root: root, index: idx}
+	require.NoError(t, fs.MkdirAll(namer.dir(), 0700))
+	fh, err := fs.Create(namer.path())
+	require.NoError(t, err)
+	_, err = fh.Write([]byte("derp"))
+	require.NoError(t, err)
+	require.NoError(t, fh.Close())
+}
+
+func TestBlobStoragePrune(t *testing.T) {
+	currentSlot := primitives.Slot(200000)
+	fs, bs, err := NewEphemeralBlobStorageWithFs(t)
+	require.NoError(t, err)
+
+	t.Run("PruneOne", func(t *testing.T) {
+		_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, 300, fieldparams.MaxBlobsPerBlock)
+		testSidecars, err := verification.BlobSidecarSliceNoop(sidecars)
+		require.NoError(t, err)
+
+		for _, sidecar := range testSidecars {
+			require.NoError(t, bs.Save(sidecar))
+		}
+
+		require.NoError(t, bs.Prune(currentSlot))
+
+		remainingFolders, err := afero.ReadDir(fs, ".")
+		require.NoError(t, err)
+		require.Equal(t, 0, len(remainingFolders))
+	})
+	t.Run("PruneMany", func(t *testing.T) {
+		blockQty := 10
+		slot := primitives.Slot(0)
+
+		for j := 0; j <= blockQty; j++ {
+			root := bytesutil.ToBytes32(bytesutil.ToBytes(uint64(slot), 32))
+			_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, root, slot, fieldparams.MaxBlobsPerBlock)
+			testSidecars, err := verification.BlobSidecarSliceNoop(sidecars)
+			require.NoError(t, err)
+			require.NoError(t, bs.Save(testSidecars[0]))
+
+			slot += 10000
+		}
+
+		require.NoError(t, bs.Prune(currentSlot))
+
+		remainingFolders, err := afero.ReadDir(fs, ".")
+		require.NoError(t, err)
+		require.Equal(t, 4, len(remainingFolders))
+	})
+}
+
+func BenchmarkPruning(b *testing.B) {
+	var t *testing.T
+	_, bs, err := NewEphemeralBlobStorageWithFs(t)
+	require.NoError(t, err)
+
+	blockQty := 10000
+	currentSlot := primitives.Slot(150000)
+	slot := primitives.Slot(0)
+
+	for j := 0; j <= blockQty; j++ {
+		root := bytesutil.ToBytes32(bytesutil.ToBytes(uint64(slot), 32))
+		_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, root, slot, fieldparams.MaxBlobsPerBlock)
+		testSidecars, err := verification.BlobSidecarSliceNoop(sidecars)
+		require.NoError(t, err)
+		require.NoError(t, bs.Save(testSidecars[0]))
+
+		slot += 100
+	}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := bs.Prune(currentSlot)
+		require.NoError(b, err)
+	}
+}
+
+func TestBlobStorageDelete(t *testing.T) {
+	fs, bs, err := NewEphemeralBlobStorageWithFs(t)
+	require.NoError(t, err)
+	rawRoot := "0xcf9bb70c98f58092c9d6459227c9765f984d240be9690e85179bc5a6f60366ad"
+	blockRoot, err := hexutil.Decode(rawRoot)
+	require.NoError(t, err)
+
+	_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, 1, fieldparams.MaxBlobsPerBlock)
+	testSidecars, err := verification.BlobSidecarSliceNoop(sidecars)
+	require.NoError(t, err)
+	for _, sidecar := range testSidecars {
+		require.NoError(t, bs.Save(sidecar))
+	}
+
+	exists, err := afero.DirExists(fs, hexutil.Encode(blockRoot))
+	require.NoError(t, err)
+	require.Equal(t, true, exists)
+
+	// Delete the directory corresponding to the block root
+	require.NoError(t, bs.Delete(bytesutil.ToBytes32(blockRoot)))
+
+	// Ensure that the directory no longer exists after deletion
+	exists, err = afero.DirExists(fs, hexutil.Encode(blockRoot))
+	require.NoError(t, err)
+	require.Equal(t, false, exists)
+
+	// Deleting a non-existent root does not return an error.
+	require.NoError(t, bs.Delete(bytesutil.ToBytes32([]byte{0x1})))
+}
+
+func TestNewBlobStorage(t *testing.T) {
+	_, err := NewBlobStorage(path.Join(t.TempDir(), "good"))
+	require.NoError(t, err)
+}

beacon-chain/db/filesystem/ephemeral.go

@@ -0,0 +1,55 @@
+package filesystem
+
+import (
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/v4/config/params"
+	"github.com/spf13/afero"
+)
+
+// NewEphemeralBlobStorage should only be used for tests.
+// The instance of BlobStorage returned is backed by an in-memory virtual filesystem,
+// improving test performance and simplifying cleanup.
+func NewEphemeralBlobStorage(_ testing.TB) *BlobStorage {
+	return &BlobStorage{fs: afero.NewMemMapFs()}
+}
+
+// NewEphemeralBlobStorageWithFs can be used by tests that want access to the virtual filesystem
+// in order to interact with it outside the parameters of the BlobStorage api.
+func NewEphemeralBlobStorageWithFs(_ testing.TB) (afero.Fs, *BlobStorage, error) {
+	fs := afero.NewMemMapFs()
+	retentionEpoch := params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest
+	return fs, &BlobStorage{fs: fs, retentionEpochs: retentionEpoch}, nil
+}
+
+type BlobMocker struct {
+	fs afero.Fs
+	bs *BlobStorage
+}
+
+// CreateFakeIndices creates empty blob sidecar files at the expected path for the given
+// root and indices to influence the result of Indices().
+func (bm *BlobMocker) CreateFakeIndices(root [32]byte, indices []uint64) error {
+	for i := range indices {
+		n := blobNamer{root: root, index: indices[i]}
+		if err := bm.fs.MkdirAll(n.dir(), directoryPermissions); err != nil {
+			return err
+		}
+		f, err := bm.fs.Create(n.path())
+		if err != nil {
+			return err
+		}
+		if err := f.Close(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// NewEphemeralBlobStorageWithMocker returns a *BlobMocker value in addition to the BlobStorage value.
+// BlockMocker encapsulates things blob path construction to avoid leaking implementation details.
+func NewEphemeralBlobStorageWithMocker(_ testing.TB) (*BlobMocker, *BlobStorage) {
+	fs := afero.NewMemMapFs()
+	bs := &BlobStorage{fs: fs}
+	return &BlobMocker{fs: fs, bs: bs}, bs
+}

beacon-chain/db/iface/interface.go

@@ -56,8 +56,8 @@ type ReadOnlyDatabase interface {
 	RegistrationByValidatorID(ctx context.Context, id primitives.ValidatorIndex) (*ethpb.ValidatorRegistrationV1, error)
 
 	// Blob operations.
-	BlobSidecarsByRoot(ctx context.Context, beaconBlockRoot [32]byte, indices ...uint64) ([]*ethpb.BlobSidecar, error)
-	BlobSidecarsBySlot(ctx context.Context, slot primitives.Slot, indices ...uint64) ([]*ethpb.BlobSidecar, error)
+	BlobSidecarsByRoot(ctx context.Context, beaconBlockRoot [32]byte, indices ...uint64) ([]*ethpb.DeprecatedBlobSidecar, error)
+	BlobSidecarsBySlot(ctx context.Context, slot primitives.Slot, indices ...uint64) ([]*ethpb.DeprecatedBlobSidecar, error)
 
 	// origin checkpoint sync support
 	OriginCheckpointBlockRoot(ctx context.Context) ([32]byte, error)
@@ -95,8 +95,8 @@ type NoHeadAccessDatabase interface {
 	SaveRegistrationsByValidatorIDs(ctx context.Context, ids []primitives.ValidatorIndex, regs []*ethpb.ValidatorRegistrationV1) error
 
 	// Blob operations.
-	SaveBlobSidecar(ctx context.Context, sidecars []*ethpb.BlobSidecar) error
-	DeleteBlobSidecar(ctx context.Context, beaconBlockRoot [32]byte) error
+	SaveBlobSidecar(ctx context.Context, sidecars []*ethpb.DeprecatedBlobSidecar) error
+	DeleteBlobSidecars(ctx context.Context, beaconBlockRoot [32]byte) error
 
 	CleanUpDirtyStates(ctx context.Context, slotsPerArchivedPoint primitives.Slot) error
 }
@@ -170,7 +170,7 @@ type SlasherDatabase interface {
 // Database interface with full access.
 type Database interface {
 	io.Closer
-	backup.BackupExporter
+	backup.Exporter
 	HeadAccessDatabase
 
 	DatabasePath() string